Update:Watcher pending delay to 10s. Increase file mtime check interval to 3s and timeout to 600s. Remove file from pending scan if it times out.

server/Watcher.js

@@ -23,7 +23,7 @@ class FolderWatcher extends EventEmitter {
     this.libraryWatchers = []
     /** @type {PendingFileUpdate[]} */
     this.pendingFileUpdates = []
-    this.pendingDelay = 4000
+    this.pendingDelay = 10000
     /** @type {NodeJS.Timeout} */
     this.pendingTimeout = null
     /** @type {Task} */
@@ -42,11 +42,11 @@ class FolderWatcher extends EventEmitter {
   }
 
   get pendingFilePaths() {
-    return this.pendingFileUpdates.map(f => f.path)
+    return this.pendingFileUpdates.map((f) => f.path)
   }
 
   buildLibraryWatcher(library) {
-    if (this.libraryWatchers.find(w => w.id === library.id)) {
+    if (this.libraryWatchers.find((w) => w.id === library.id)) {
       Logger.warn('[Watcher] Already watching library', library.name)
       return
     }
@@ -67,17 +67,23 @@ class FolderWatcher extends EventEmitter {
     watcher
       .on('add', (path) => {
         this.onFileAdded(library.id, filePathToPOSIX(path))
-      }).on('change', (path) => {
+      })
+      .on('change', (path) => {
         // This is triggered from metadata changes, not what we want
-      }).on('unlink', path => {
+      })
+      .on('unlink', (path) => {
         this.onFileRemoved(library.id, filePathToPOSIX(path))
-      }).on('rename', (path, pathNext) => {
+      })
+      .on('rename', (path, pathNext) => {
         this.onFileRename(library.id, filePathToPOSIX(path), filePathToPOSIX(pathNext))
-      }).on('error', (error) => {
+      })
+      .on('error', (error) => {
         Logger.error(`[Watcher] ${error}`)
-      }).on('ready', () => {
+      })
+      .on('ready', () => {
         Logger.info(`[Watcher] "${library.name}" Ready`)
-      }).on('close', () => {
+      })
+      .on('close', () => {
         Logger.debug(`[Watcher] "${library.name}" Closed`)
       })
 
@@ -110,26 +116,26 @@ class FolderWatcher extends EventEmitter {
   updateLibrary(library) {
     if (this.disabled) return
 
-    const libwatcher = this.libraryWatchers.find(lib => lib.id === library.id)
+    const libwatcher = this.libraryWatchers.find((lib) => lib.id === library.id)
     if (libwatcher) {
       // Library watcher was disabled
       if (library.settings.disableWatcher) {
         Logger.info(`[Watcher] updateLibrary: Library "${library.name}" watcher disabled`)
         libwatcher.watcher.close()
-        this.libraryWatchers = this.libraryWatchers.filter(lw => lw.id !== libwatcher.id)
+        this.libraryWatchers = this.libraryWatchers.filter((lw) => lw.id !== libwatcher.id)
         return
       }
 
       libwatcher.name = library.name
 
       // If any folder paths were added or removed then re-init watcher
-      const pathsToAdd = library.folderPaths.filter(path => !libwatcher.paths.includes(path))
+      const pathsToAdd = library.folderPaths.filter((path) => !libwatcher.paths.includes(path))
-      const pathsRemoved = libwatcher.paths.filter(path => !library.folderPaths.includes(path))
+      const pathsRemoved = libwatcher.paths.filter((path) => !library.folderPaths.includes(path))
       if (pathsToAdd.length || pathsRemoved.length) {
         Logger.info(`[Watcher] Re-Initializing watcher for "${library.name}".`)
 
         libwatcher.watcher.close()
-        this.libraryWatchers = this.libraryWatchers.filter(lw => lw.id !== libwatcher.id)
+        this.libraryWatchers = this.libraryWatchers.filter((lw) => lw.id !== libwatcher.id)
         this.buildLibraryWatcher(library)
       }
     } else if (!library.settings.disableWatcher) {
@@ -141,18 +147,18 @@ class FolderWatcher extends EventEmitter {
 
   removeLibrary(library) {
     if (this.disabled) return
-    var libwatcher = this.libraryWatchers.find(lib => lib.id === library.id)
+    var libwatcher = this.libraryWatchers.find((lib) => lib.id === library.id)
     if (libwatcher) {
       Logger.info(`[Watcher] Removed watcher for "${library.name}"`)
       libwatcher.watcher.close()
-      this.libraryWatchers = this.libraryWatchers.filter(lib => lib.id !== library.id)
+      this.libraryWatchers = this.libraryWatchers.filter((lib) => lib.id !== library.id)
     } else {
       Logger.error(`[Watcher] Library watcher not found for "${library.name}"`)
     }
   }
 
   close() {
-    return this.libraryWatchers.map(lib => lib.watcher.close())
+    return this.libraryWatchers.map((lib) => lib.watcher.close())
   }
 
   /**
@@ -203,17 +209,18 @@ class FolderWatcher extends EventEmitter {
   }
 
   /**
-   * Get mtimeMs from an added file every second until it is no longer changing
+   * Get mtimeMs from an added file every 3 seconds until it is no longer changing
-   * Times out after 180s
+   * Times out after 600s
    *
    * @param {string} path
    * @param {number} [lastMTimeMs=0]
    * @param {number} [loop=0]
    */
   async waitForFileToAdd(path, lastMTimeMs = 0, loop = 0) {
-    // Safety to catch infinite loop (180s)
+    // Safety to catch infinite loop (600s)
-    if (loop >= 180) {
+    if (loop >= 200) {
-      Logger.warn(`[Watcher] Waiting to add file at "${path}" timeout (loop ${loop}) - proceeding`)
+      Logger.warn(`[Watcher] Waiting to add file at "${path}" timeout (loop ${loop}) - bailing`)
+      this.pendingFileUpdates = this.pendingFileUpdates.filter((pfu) => pfu.path !== path)
       return this.filesBeingAdded.delete(path)
     }
 
@@ -222,11 +229,11 @@ class FolderWatcher extends EventEmitter {
       if (lastMTimeMs) Logger.debug(`[Watcher] File finished adding at "${path}"`)
       return this.filesBeingAdded.delete(path)
     }
-    if (lastMTimeMs % 5 === 0) {
+    if (loop % 5 === 0) {
       Logger.debug(`[Watcher] Waiting to add file at "${path}". mtimeMs=${mtimeMs} lastMTimeMs=${lastMTimeMs} (loop ${loop})`)
     }
-    // Wait 1 second
+    // Wait 3 seconds
-    await new Promise((resolve) => setTimeout(resolve, 1000))
+    await new Promise((resolve) => setTimeout(resolve, 3000))
     this.waitForFileToAdd(path, mtimeMs, ++loop)
   }
 
@@ -241,14 +248,14 @@ class FolderWatcher extends EventEmitter {
     if (this.pendingFilePaths.includes(path)) return
 
     // Get file library
-    const libwatcher = this.libraryWatchers.find(lw => lw.id === libraryId)
+    const libwatcher = this.libraryWatchers.find((lw) => lw.id === libraryId)
     if (!libwatcher) {
       Logger.error(`[Watcher] Invalid library id from watcher ${libraryId}`)
       return
     }
 
     // Get file folder
-    const folder = libwatcher.folders.find(fold => isSameOrSubPath(fold.fullPath, path))
+    const folder = libwatcher.folders.find((fold) => isSameOrSubPath(fold.fullPath, path))
     if (!folder) {
       Logger.error(`[Watcher] New file folder not found in library "${libwatcher.name}" with path "${path}"`)
       return
@@ -264,7 +271,7 @@ class FolderWatcher extends EventEmitter {
     }
 
     // Ignore files/folders starting with "."
-    const hasDotPath = relPath.split('/').find(p => p.startsWith('.'))
+    const hasDotPath = relPath.split('/').find((p) => p.startsWith('.'))
     if (hasDotPath) {
       Logger.debug(`[Watcher] Ignoring dot path "${relPath}" | Piece "${hasDotPath}"`)
       return
@@ -298,12 +305,17 @@ class FolderWatcher extends EventEmitter {
     clearTimeout(this.pendingTimeout)
     this.pendingTimeout = setTimeout(() => {
       // Check that files are not still being added
-      if (this.pendingFileUpdates.some(pfu => this.filesBeingAdded.has(pfu.path))) {
+      if (this.pendingFileUpdates.some((pfu) => this.filesBeingAdded.has(pfu.path))) {
         Logger.debug(`[Watcher] Still waiting for pending files "${[...this.filesBeingAdded].join(', ')}"`)
         return this.handlePendingFileUpdatesTimeout()
       }
 
+      if (this.pendingFileUpdates.length) {
         LibraryScanner.scanFilesChanged(this.pendingFileUpdates, this.pendingTask)
+      } else {
+        this.pendingTask.setFinished('Scan abandoned. No files to scan.')
+        TaskManager.taskFinished(this.pendingTask)
+      }
       this.pendingTask = null
       this.pendingFileUpdates = []
       this.filesBeingAdded.clear()
@@ -311,7 +323,7 @@ class FolderWatcher extends EventEmitter {
   }
 
   checkShouldIgnorePath(path) {
-    return !!this.ignoreDirs.find(dirpath => {
+    return !!this.ignoreDirs.find((dirpath) => {
       return isSameOrSubPath(dirpath, path)
     })
   }
@@ -333,7 +345,7 @@ class FolderWatcher extends EventEmitter {
    */
   addIgnoreDir(path) {
     path = this.cleanDirPath(path)
-    this.pendingDirsToRemoveFromIgnore = this.pendingDirsToRemoveFromIgnore.filter(p => p !== path)
+    this.pendingDirsToRemoveFromIgnore = this.pendingDirsToRemoveFromIgnore.filter((p) => p !== path)
     if (this.ignoreDirs.includes(path)) {
       // Already ignoring dir
       return
@@ -364,9 +376,9 @@ class FolderWatcher extends EventEmitter {
     clearTimeout(this.removeFromIgnoreTimer)
     this.removeFromIgnoreTimer = setTimeout(() => {
       if (this.pendingDirsToRemoveFromIgnore.includes(path)) {
-        this.pendingDirsToRemoveFromIgnore = this.pendingDirsToRemoveFromIgnore.filter(p => p !== path)
+        this.pendingDirsToRemoveFromIgnore = this.pendingDirsToRemoveFromIgnore.filter((p) => p !== path)
         Logger.debug(`[Watcher] removeIgnoreDir: No longer ignoring directory "${path}"`)
-        this.ignoreDirs = this.ignoreDirs.filter(p => p !== path)
+        this.ignoreDirs = this.ignoreDirs.filter((p) => p !== path)
       }
     }, 5000)
   }