Auto add/update/remove audiobooks, update screenshots

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "audiobookshelf-client",
-  "version": "1.0.8",
+  "version": "1.1.0",
   "description": "Audiobook manager and player",
   "main": "index.js",
   "scripts": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "audiobookshelf",
-  "version": "1.0.8",
+  "version": "1.1.0",
   "description": "Self-hosted audiobook server for managing and playing audiobooks.",
   "main": "index.js",
   "scripts": {

readme.md

@@ -28,8 +28,6 @@ will store "With a Subtitle" as the subtitle
 
 #### Features coming soon:
 
-* Auto add and update audiobooks (currently you need to press scan)
-* User permissions & editing users
 * Support different views to see more details of each audiobook
 * Option to download all files in a zip file
 * iOS App (Android is in beta [here](https://play.google.com/store/apps/details?id=com.audiobookshelf.app))

server/Scanner.js

@@ -1,10 +1,13 @@
+const fs = require('fs-extra')
 const Logger = require('./Logger')
 const BookFinder = require('./BookFinder')
 const Audiobook = require('./objects/Audiobook')
 const audioFileScanner = require('./utils/audioFileScanner')
-const { getAllAudiobookFiles } = require('./utils/scandir')
+const { getAllAudiobookFileData, getAudiobookFileData } = require('./utils/scandir')
 const { comparePaths, getIno } = require('./utils/index')
 const { secondsToTimestamp } = require('./utils/fileUtils')
+const { ScanResult } = require('./utils/constants')
+
 
 class Scanner {
   constructor(AUDIOBOOK_PATH, METADATA_PATH, db, emitter) {
@@ -60,6 +63,110 @@ class Scanner {
     return audiobookDataAudioFiles.filter(abdFile => !!abdFile.ino)
   }
 
+  async scanAudiobookData(audiobookData) {
+    var existingAudiobook = this.audiobooks.find(a => a.ino === audiobookData.ino)
+    Logger.debug(`[Scanner] Scanning "${audiobookData.title}" (${audiobookData.ino}) - ${!!existingAudiobook ? 'Exists' : 'New'}`)
+
+    if (existingAudiobook) {
+
+      // REMOVE: No valid audio files
+      if (!audiobookData.audioFiles.length) {
+        Logger.error(`[Scanner] "${existingAudiobook.title}" no valid audio files found - removing audiobook`)
+
+        await this.db.removeEntity('audiobook', existingAudiobook.id)
+        this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
+
+        return ScanResult.REMOVED
+      }
+
+      audiobookData.audioFiles = await this.setAudioFileInos(audiobookData.audioFiles, existingAudiobook.audioFiles)
+
+      // Check for audio files that were removed
+      var abdAudioFileInos = audiobookData.audioFiles.map(af => af.ino)
+      var removedAudioFiles = existingAudiobook.audioFiles.filter(file => !abdAudioFileInos.includes(file.ino))
+      if (removedAudioFiles.length) {
+        Logger.info(`[Scanner] ${removedAudioFiles.length} audio files removed for audiobook "${existingAudiobook.title}"`)
+        removedAudioFiles.forEach((af) => existingAudiobook.removeAudioFile(af))
+      }
+
+      // Check for new audio files and sync existing audio files
+      var newAudioFiles = []
+      var hasUpdatedAudioFiles = false
+      audiobookData.audioFiles.forEach((file) => {
+        var existingAudioFile = existingAudiobook.getAudioFileByIno(file.ino)
+        if (existingAudioFile) { // Audio file exists, sync paths
+          if (existingAudiobook.syncAudioFile(existingAudioFile, file)) {
+            hasUpdatedAudioFiles = true
+          }
+        } else {
+          newAudioFiles.push(file)
+        }
+      })
+      if (newAudioFiles.length) {
+        Logger.info(`[Scanner] ${newAudioFiles.length} new audio files were found for audiobook "${existingAudiobook.title}"`)
+        // Scan new audio files found - sets tracks
+        await audioFileScanner.scanAudioFiles(existingAudiobook, newAudioFiles)
+      }
+
+
+      // REMOVE: No valid audio tracks
+      if (!existingAudiobook.tracks.length) {
+        Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`)
+
+        await this.db.removeEntity('audiobook', existingAudiobook.id)
+        this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
+        return ScanResult.REMOVED
+      }
+
+      var hasUpdates = removedAudioFiles.length || newAudioFiles.length || hasUpdatedAudioFiles
+
+      if (existingAudiobook.checkUpdateMissingParts()) {
+        Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`)
+        hasUpdates = true
+      }
+
+      if (existingAudiobook.syncOtherFiles(audiobookData.otherFiles)) {
+        hasUpdates = true
+      }
+
+      // Syncs path and fullPath
+      if (existingAudiobook.syncPaths(audiobookData)) {
+        hasUpdates = true
+      }
+
+      if (hasUpdates) {
+        Logger.info(`[Scanner] "${existingAudiobook.title}" was updated - saving`)
+        existingAudiobook.lastUpdate = Date.now()
+        await this.db.updateAudiobook(existingAudiobook)
+        this.emitter('audiobook_updated', existingAudiobook.toJSONMinified())
+
+        return ScanResult.UPDATED
+      }
+
+      return ScanResult.UPTODATE
+    }
+
+    // NEW: Check new audiobook
+    if (!audiobookData.audioFiles.length) {
+      Logger.error('[Scanner] No valid audio tracks for Audiobook', audiobookData.path)
+      return ScanResult.NOTHING
+    }
+
+    var audiobook = new Audiobook()
+    audiobook.setData(audiobookData)
+    await audioFileScanner.scanAudioFiles(audiobook, audiobookData.audioFiles)
+    if (!audiobook.tracks.length) {
+      Logger.warn('[Scanner] Invalid audiobook, no valid tracks', audiobook.title)
+      return ScanResult.NOTHING
+    }
+
+    audiobook.checkUpdateMissingParts()
+    Logger.info(`[Scanner] Audiobook "${audiobook.title}" Scanned (${audiobook.sizePretty}) [${audiobook.durationPretty}]`)
+    await this.db.insertAudiobook(audiobook)
+    this.emitter('audiobook_added', audiobook.toJSONMinified())
+    return ScanResult.ADDED
+  }
+
   async scan() {
     // TEMP - fix relative file paths
     // TEMP - update ino for each audiobook
@@ -80,7 +187,7 @@ class Scanner {
     }
 
     const scanStart = Date.now()
-    var audiobookDataFound = await getAllAudiobookFiles(this.AudiobookPath, this.db.serverSettings)
+    var audiobookDataFound = await getAllAudiobookFileData(this.AudiobookPath, this.db.serverSettings)
 
     // Set ino for each ab data as a string
     audiobookDataFound = await this.setAudiobookDataInos(audiobookDataFound)
@@ -112,97 +219,14 @@ class Scanner {
       }
     }
 
+    // Check for new and updated audiobooks
     for (let i = 0; i < audiobookDataFound.length; i++) {
       var audiobookData = audiobookDataFound[i]
-      var existingAudiobook = this.audiobooks.find(a => a.ino === audiobookData.ino)
-      Logger.debug(`[Scanner] Scanning "${audiobookData.title}" (${audiobookData.ino}) - ${!!existingAudiobook ? 'Exists' : 'New'}`)
+      var result = await this.scanAudiobookData(audiobookData)
+      if (result === ScanResult.ADDED) scanResults.added++
+      if (result === ScanResult.REMOVED) scanResults.removed++
+      if (result === ScanResult.UPDATED) scanResults.updated++
 
-      if (existingAudiobook) {
-        if (!audiobookData.audioFiles.length) {
-          Logger.error(`[Scanner] "${existingAudiobook.title}" no valid audio files found - removing audiobook`)
-
-          await this.db.removeEntity('audiobook', existingAudiobook.id)
-          this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
-          scanResults.removed++
-        } else {
-          audiobookData.audioFiles = await this.setAudioFileInos(audiobookData.audioFiles, existingAudiobook.audioFiles)
-          var abdAudioFileInos = audiobookData.audioFiles.map(af => af.ino)
-
-          // Check for audio files that were removed
-          var removedAudioFiles = existingAudiobook.audioFiles.filter(file => !abdAudioFileInos.includes(file.ino))
-          if (removedAudioFiles.length) {
-            Logger.info(`[Scanner] ${removedAudioFiles.length} audio files removed for audiobook "${existingAudiobook.title}"`)
-            removedAudioFiles.forEach((af) => existingAudiobook.removeAudioFile(af))
-          }
-
-          // Check for new audio files and sync existing audio files
-          var newAudioFiles = []
-          var hasUpdatedAudioFiles = false
-          audiobookData.audioFiles.forEach((file) => {
-            var existingAudioFile = existingAudiobook.getAudioFileByIno(file.ino)
-            if (existingAudioFile) { // Audio file exists, sync paths
-              if (existingAudiobook.syncAudioFile(existingAudioFile, file)) {
-                hasUpdatedAudioFiles = true
-              }
-            } else {
-              newAudioFiles.push(file)
-            }
-          })
-          if (newAudioFiles.length) {
-            Logger.info(`[Scanner] ${newAudioFiles.length} new audio files were found for audiobook "${existingAudiobook.title}"`)
-            // Scan new audio files found
-            await audioFileScanner.scanAudioFiles(existingAudiobook, newAudioFiles)
-          }
-
-          if (!existingAudiobook.tracks.length) {
-            Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`)
-
-            await this.db.removeEntity('audiobook', existingAudiobook.id)
-            this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
-          } else {
-            var hasUpdates = removedAudioFiles.length || newAudioFiles.length || hasUpdatedAudioFiles
-
-            if (existingAudiobook.checkUpdateMissingParts()) {
-              Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`)
-              hasUpdates = true
-            }
-
-            if (existingAudiobook.syncOtherFiles(audiobookData.otherFiles)) {
-              hasUpdates = true
-            }
-
-            // Syncs path and fullPath
-            if (existingAudiobook.syncPaths(audiobookData)) {
-              hasUpdates = true
-            }
-
-            if (hasUpdates) {
-              Logger.info(`[Scanner] "${existingAudiobook.title}" was updated - saving`)
-              existingAudiobook.lastUpdate = Date.now()
-              await this.db.updateAudiobook(existingAudiobook)
-              this.emitter('audiobook_updated', existingAudiobook.toJSONMinified())
-              scanResults.updated++
-            }
-          }
-        } // end if update existing
-      } else {
-        if (!audiobookData.audioFiles.length) {
-          Logger.error('[Scanner] No valid audio tracks for Audiobook', audiobookData.path)
-        } else {
-          var audiobook = new Audiobook()
-          audiobook.setData(audiobookData)
-          await audioFileScanner.scanAudioFiles(audiobook, audiobookData.audioFiles)
-          if (!audiobook.tracks.length) {
-            Logger.warn('[Scanner] Invalid audiobook, no valid tracks', audiobook.title)
-          } else {
-            audiobook.checkUpdateMissingParts()
-            Logger.info(`[Scanner] Audiobook "${audiobook.title}" Scanned (${audiobook.sizePretty}) [${audiobook.durationPretty}]`)
-            await this.db.insertAudiobook(audiobook)
-            this.emitter('audiobook_added', audiobook.toJSONMinified())
-            scanResults.added++
-          }
-        } // end if add new
-      }
       var progress = Math.round(100 * (i + 1) / audiobookDataFound.length)
       this.emitter('scan_progress', {
         scanType: 'files',
@@ -222,6 +246,29 @@ class Scanner {
     return scanResults
   }
 
+  async scanAudiobook(audiobookPath) {
+    var exists = await fs.pathExists(audiobookPath)
+    if (!exists) {
+      // Audiobook was deleted, TODO: Should confirm this better
+      var audiobook = this.db.audiobooks.find(ab => ab.fullPath === audiobookPath)
+      if (audiobook) {
+        var audiobookJSON = audiobook.toJSONMinified()
+        await this.db.removeEntity('audiobook', audiobook.id)
+        this.emitter('audiobook_removed', audiobookJSON)
+        return ScanResult.REMOVED
+      }
+      Logger.warn('Path was deleted but no audiobook found', audiobookPath)
+      return ScanResult.NOTHING
+    }
+
+    var audiobookData = await getAudiobookFileData(this.AudiobookPath, audiobookPath, this.db.serverSettings)
+    if (!audiobookData) {
+      return ScanResult.NOTHING
+    }
+    audiobookData.ino = await getIno(audiobookData.fullPath)
+    return this.scanAudiobookData(audiobookData)
+  }
+
   async fetchMetadata(id, trackIndex = 0) {
     var audiobook = this.audiobooks.find(a => a.id === id)
     if (!audiobook) {

server/Server.js

@@ -14,6 +14,7 @@ const StreamManager = require('./StreamManager')
 const RssFeeds = require('./RssFeeds')
 const DownloadManager = require('./DownloadManager')
 const Logger = require('./Logger')
+const { ScanResult } = require('./utils/constants')
 
 class Server {
   constructor(PORT, CONFIG_PATH, METADATA_PATH, AUDIOBOOK_PATH) {
@@ -75,8 +76,21 @@ class Server {
     })
   }
 
-  async fileAddedUpdated({ path, fullPath }) { }
-  async fileRemoved({ path, fullPath }) { }
+  async newFilesAdded({ dir, files }) {
+    Logger.info(files.length, 'New Files Added in dir', dir)
+    var result = await this.scanner.scanAudiobook(dir)
+    Logger.info('New Files Added result', result)
+  }
+  async filesRemoved({ dir, files }) {
+    Logger.info(files.length, 'Files Removed in dir', dir)
+    var result = await this.scanner.scanAudiobook(dir)
+    Logger.info('Files Removed result', result)
+  }
+  async filesRenamed({ dir, files }) {
+    Logger.info(files.length, 'Files Renamed in dir', dir)
+    var result = await this.scanner.scanAudiobook(dir)
+    Logger.info('Files Renamed result', result)
+  }
 
   async scan() {
     Logger.info('[Server] Starting Scan')
@@ -112,9 +126,9 @@ class Server {
     this.auth.init()
 
     this.watcher.initWatcher()
-    this.watcher.on('file_added', this.fileAddedUpdated.bind(this))
-    this.watcher.on('file_removed', this.fileRemoved.bind(this))
-    this.watcher.on('file_updated', this.fileAddedUpdated.bind(this))
+    this.watcher.on('new_files', this.newFilesAdded.bind(this))
+    this.watcher.on('removed_files', this.filesRemoved.bind(this))
+    this.watcher.on('renamed_files', this.filesRenamed.bind(this))
   }
 
   authMiddleware(req, res, next) {

server/Watcher.js

@@ -1,6 +1,8 @@
-var EventEmitter = require('events')
-var Logger = require('./Logger')
-var Watcher = require('watcher')
+const Path = require('path')
+const EventEmitter = require('events')
+const Watcher = require('watcher')
+const Logger = require('./Logger')
+const { getIno } = require('./utils/index')
 
 class FolderWatcher extends EventEmitter {
   constructor(audiobookPath) {
@@ -8,6 +10,11 @@ class FolderWatcher extends EventEmitter {
     this.AudiobookPath = audiobookPath
     this.folderMap = {}
     this.watcher = null
+
+    this.pendingBatchDelay = 4000
+
+    // Audiobook paths with changes
+    this.pendingBatch = {}
   }
 
   initWatcher() {
@@ -46,32 +53,69 @@ class FolderWatcher extends EventEmitter {
     return this.watcher.close()
   }
 
-  onNewFile(path) {
+  // After [pendingBatchDelay] seconds emit batch
+  async onNewFile(path) {
     Logger.debug('FolderWatcher: New File', path)
-    this.emit('file_added', {
-      path: path.replace(this.AudiobookPath, ''),
-      fullPath: path
-    })
+
+    var dir = Path.dirname(path)
+    if (this.pendingBatch[dir]) {
+      this.pendingBatch[dir].files.push(path)
+      clearTimeout(this.pendingBatch[dir].timeout)
+    } else {
+      this.pendingBatch[dir] = {
+        dir,
+        files: [path]
+      }
+    }
+
+    this.pendingBatch[dir].timeout = setTimeout(() => {
+      this.emit('new_files', this.pendingBatch[dir])
+      delete this.pendingBatch[dir]
+    }, this.pendingBatchDelay)
   }
 
   onFileRemoved(path) {
     Logger.debug('[FolderWatcher] File Removed', path)
-    this.emit('file_removed', {
-      path: path.replace(this.AudiobookPath, ''),
-      fullPath: path
-    })
+
+    var dir = Path.dirname(path)
+    if (this.pendingBatch[dir]) {
+      this.pendingBatch[dir].files.push(path)
+      clearTimeout(this.pendingBatch[dir].timeout)
+    } else {
+      this.pendingBatch[dir] = {
+        dir,
+        files: [path]
+      }
+    }
+
+    this.pendingBatch[dir].timeout = setTimeout(() => {
+      this.emit('removed_files', this.pendingBatch[dir])
+      delete this.pendingBatch[dir]
+    }, this.pendingBatchDelay)
   }
 
   onFileUpdated(path) {
     Logger.debug('[FolderWatcher] Updated File', path)
-    this.emit('file_updated', {
-      path: path.replace(this.AudiobookPath, ''),
-      fullPath: path
-    })
   }
 
   onRename(pathFrom, pathTo) {
     Logger.debug(`[FolderWatcher] Rename ${pathFrom} => ${pathTo}`)
+
+    var dir = Path.dirname(pathTo)
+    if (this.pendingBatch[dir]) {
+      this.pendingBatch[dir].files.push(pathTo)
+      clearTimeout(this.pendingBatch[dir].timeout)
+    } else {
+      this.pendingBatch[dir] = {
+        dir,
+        files: [pathTo]
+      }
+    }
+
+    this.pendingBatch[dir].timeout = setTimeout(() => {
+      this.emit('renamed_files', this.pendingBatch[dir])
+      delete this.pendingBatch[dir]
+    }, this.pendingBatchDelay)
   }
 }
 module.exports = FolderWatcher

server/utils/constants.js

@@ -0,0 +1,7 @@
+module.exports.ScanResult = {
+  NOTHING: 0,
+  ADDED: 1,
+  UPDATED: 2,
+  REMOVED: 3,
+  UPTODATE: 4
+}

server/utils/index.js

@@ -32,17 +32,20 @@ module.exports.levenshteinDistance = levenshteinDistance
 const cleanString = (str) => {
   if (!str) return ''
 
+  // Now supporting all utf-8 characters, can remove this method in future
+
   // replace accented characters: https://stackoverflow.com/a/49901740/7431543
-  str = str.normalize('NFD').replace(/[\u0300-\u036f]/g, "")
+  // str = str.normalize('NFD').replace(/[\u0300-\u036f]/g, "")
 
-  const availableChars = " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
-  const cleanChar = (char) => availableChars.indexOf(char) < 0 ? '?' : char
+  // const availableChars = " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
+  // const cleanChar = (char) => availableChars.indexOf(char) < 0 ? '?' : char
 
-  var cleaned = ''
-  for (let i = 0; i < str.length; i++) {
-    cleaned += cleanChar(str[i])
-  }
-  return cleaned
+  // var cleaned = ''
+  // for (let i = 0; i < str.length; i++) {
+  //   cleaned += cleanChar(str[i])
+  // }
+
+  return cleaned.trim()
 }
 module.exports.cleanString = cleanString
 

server/utils/scandir.js

@@ -30,24 +30,18 @@ function getFileType(ext) {
   return 'unknown'
 }
 
-async function getAllAudiobookFiles(abRootPath, serverSettings = {}) {
-  var parseSubtitle = !!serverSettings.scannerParseSubtitle
-
-  var paths = await getPaths(abRootPath)
-  var audiobooks = {}
-
-  paths.files.forEach((filepath) => {
-    var relpath = Path.normalize(filepath).replace(abRootPath, '').slice(1)
+// Input relative filepath, output all details that can be parsed
+function getAudiobookDataFromFilepath(abRootPath, relpath, parseSubtitle = false) {
   var pathformat = Path.parse(relpath)
   var path = pathformat.dir
 
   if (!path) {
-      Logger.error('Ignoring file in root dir', filepath)
-      return
+    Logger.error('Ignoring file in root dir', relpath)
+    return null
   }
 
   // If relative file directory has 3 folders, then the middle folder will be series
-    var splitDir = pathformat.dir.split(Path.sep)
+  var splitDir = path.split(Path.sep)
   var author = null
   if (splitDir.length > 1) author = splitDir.shift()
   var series = null
@@ -72,25 +66,45 @@ async function getAllAudiobookFiles(abRootPath, serverSettings = {}) {
     subtitle = splitOnSubtitle.join(' - ')
   }
 
-    if (!audiobooks[path]) {
-      audiobooks[path] = {
+  return {
     author,
     title,
     subtitle,
-        series: cleanString(series),
-        publishYear: publishYear,
-        path: path,
-        fullPath: Path.join(abRootPath, path),
+    series,
+    publishYear,
+    path, // relative audiobook path i.e. /Author Name/Book Name/..
+    fullPath: Path.join(abRootPath, path) // i.e. /audiobook/Author Name/Book Name/..
+  }
+}
+
+async function getAllAudiobookFileData(abRootPath, serverSettings = {}) {
+  var parseSubtitle = !!serverSettings.scannerParseSubtitle
+
+  var paths = await getPaths(abRootPath)
+  var audiobooks = {}
+
+  paths.files.forEach((filepath) => {
+    var relpath = Path.normalize(filepath).replace(abRootPath, '').slice(1)
+    var parsed = Path.parse(relpath)
+    var path = parsed.dir
+
+    if (!audiobooks[path]) {
+      var audiobookData = getAudiobookDataFromFilepath(abRootPath, relpath, parseSubtitle)
+      if (!audiobookData) return
+
+      audiobooks[path] = {
+        ...audiobookData,
         audioFiles: [],
         otherFiles: []
       }
     }
+
     var fileObj = {
-      filetype: getFileType(pathformat.ext),
-      filename: pathformat.base,
+      filetype: getFileType(parsed.ext),
+      filename: parsed.base,
       path: relpath,
       fullPath: filepath,
-      ext: pathformat.ext
+      ext: parsed.ext
     }
     if (fileObj.filetype === 'audio') {
       audiobooks[path].audioFiles.push(fileObj)
@@ -100,4 +114,44 @@ async function getAllAudiobookFiles(abRootPath, serverSettings = {}) {
   })
   return Object.values(audiobooks)
 }
-module.exports.getAllAudiobookFiles = getAllAudiobookFiles
+module.exports.getAllAudiobookFileData = getAllAudiobookFileData
+
+
+async function getAudiobookFileData(abRootPath, audiobookPath, serverSettings = {}) {
+  var parseSubtitle = !!serverSettings.scannerParseSubtitle
+
+  var paths = await getPaths(audiobookPath)
+  var audiobook = null
+
+  paths.files.forEach((filepath) => {
+    var relpath = Path.normalize(filepath).replace(abRootPath, '').slice(1)
+
+    if (!audiobook) {
+      var audiobookData = getAudiobookDataFromFilepath(abRootPath, relpath, parseSubtitle)
+      if (!audiobookData) return
+
+      audiobook = {
+        ...audiobookData,
+        audioFiles: [],
+        otherFiles: []
+      }
+    }
+
+    var extname = Path.extname(filepath)
+    var basename = Path.basename(filepath)
+    var fileObj = {
+      filetype: getFileType(extname),
+      filename: basename,
+      path: relpath,
+      fullPath: filepath,
+      ext: extname
+    }
+    if (fileObj.filetype === 'audio') {
+      audiobook.audioFiles.push(fileObj)
+    } else {
+      audiobook.otherFiles.push(fileObj)
+    }
+  })
+  return audiobook
+}
+module.exports.getAudiobookFileData = getAudiobookFileData