Fix: Scanner check path and inode value for removed books, scanner v5 outlined

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "audiobookshelf-client",
-  "version": "1.6.25",
+  "version": "1.6.26",
   "description": "Audiobook manager and player",
   "main": "index.js",
   "scripts": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "audiobookshelf",
-  "version": "1.6.25",
+  "version": "1.6.26",
   "description": "Self-hosted audiobook server for managing and playing audiobooks",
   "main": "index.js",
   "scripts": {

server/Scanner.js

@@ -454,7 +454,6 @@ class Scanner {
     var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryId)
 
     // TODO: This temporary fix from pre-release should be removed soon, "checkUpdateInos"
-    // TEMP - update ino for each audiobook
     if (audiobooksInLibrary.length) {
       for (let i = 0; i < audiobooksInLibrary.length; i++) {
         var ab = audiobooksInLibrary[i]
@@ -463,7 +462,7 @@ class Scanner {
         if (shouldUpdateIno) {
           var filesWithMissingIno = ab.getFilesWithMissingIno()
 
-          Logger.debug(`\n\Updating inos for "${ab.title}"`)
+          Logger.debug(`\nUpdating inos for "${ab.title}"`)
           Logger.debug(`In Scan, Files with missing inode`, filesWithMissingIno)
 
           var hasUpdates = await ab.checkUpdateInos()
@@ -504,7 +503,7 @@ class Scanner {
     // Check for removed audiobooks
     for (let i = 0; i < audiobooksInLibrary.length; i++) {
       var audiobook = audiobooksInLibrary[i]
-      var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino)
+      var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path))
       if (!dataFound) {
         Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`)
         audiobook.isMissing = true

server/objects/AudioFile.js

@@ -153,6 +153,17 @@ class AudioFile {
     this.metadata.setData(data)
   }
 
+  // New scanner creates AudioFile from AudioFileScanner
+  setData2(fileData, probeData) {
+    this.index = fileData.index || null
+    this.ino = fileData.ino || null
+    this.filename = fileData.filename
+    this.ext = fileData.ext
+    this.path = fileData.path
+    this.fullPath = fileData.fullPath
+    this.addedAt = Date.now()
+  }
+
   syncChapters(updatedChapters) {
     if (this.chapters.length !== updatedChapters.length) {
       this.chapters = updatedChapters.map(ch => ({ ...ch }))

server/objects/Audiobook.js

@@ -823,5 +823,141 @@ class Audiobook {
     var audioFile = this.audioFiles[0]
     return this.book.setDetailsFromFileMetadata(audioFile.metadata)
   }
+
+  // Returns null if file not found, true if file was updated, false if up to date
+  checkFileFound(fileFound, isAudioFile) {
+    var hasUpdated = false
+
+    const arrayToCheck = isAudioFile ? this.audioFiles : this.otherFiles
+
+    var existingFile = arrayToCheck.find(_af => _af.ino === fileFound.ino)
+    if (!existingFile) {
+      existingFile = arrayToCheck.find(_af => _af.path === fileFound.path)
+      if (existingFile) {
+        // file inode was updated
+        existingFile.ino = fileFound.ino
+        hasUpdated = true
+      } else {
+        // file not found
+        return null
+      }
+    }
+
+    if (existingFile.filename !== fileFound.filename) {
+      existingFile.filename = fileFound.filename
+      existingFile.ext = fileFound.ext
+      hasUpdated = true
+    }
+
+    if (existingFile.path !== fileFound.path) {
+      existingFile.path = fileFound.path
+      existingFile.fullPath = fileFound.fullPath
+      hasUpdated = true
+    } else if (existingFile.fullPath !== fileFound.fullPath) {
+      existingFile.fullPath = fileFound.fullPath
+      hasUpdated = true
+    }
+
+    if (!isAudioFile && existingFile.filetype !== fileFound.filetype) {
+      existingFile.filetype = fileFound.filetype
+      hasUpdated = true
+    }
+
+    return hasUpdated
+  }
+
+  checkShouldScan(dataFound) {
+    var hasUpdated = false
+
+    if (dataFound.ino !== this.ino) {
+      this.ino = dataFound.ino
+      hasUpdated = true
+    }
+
+    if (dataFound.folderId !== this.folderId) {
+      Logger.warn(`[Audiobook] Check scan audiobook changed folder ${this.folderId} -> ${dataFound.folderId}`)
+      this.folderId = dataFound.folderId
+      hasUpdated = true
+    }
+
+    if (dataFound.path !== this.path) {
+      Logger.warn(`[Audiobook] Check scan audiobook changed path "${this.path}" -> "${dataFound.path}"`)
+      this.path = dataFound.path
+      this.fullPath = dataFound.fullPath
+      hasUpdated = true
+    } else if (dataFound.fullPath !== this.fullPath) {
+      Logger.warn(`[Audiobook] Check scan audiobook changed fullpath "${this.fullPath}" -> "${dataFound.fullPath}"`)
+      this.fullPath = dataFound.fullPath
+      hasUpdated = true
+    }
+
+    var newAudioFileData = []
+    var newOtherFileData = []
+
+    dataFound.audioFiles.forEach((af) => {
+      var audioFileFoundCheck = this.checkFileFound(af, true)
+      if (audioFileFoundCheck === null) {
+        newAudioFileData.push(af)
+      } else if (audioFileFoundCheck === true) {
+        hasUpdated = true
+      }
+    })
+
+    dataFound.otherFiles.forEach((otherFileData) => {
+      var fileFoundCheck = this.checkFileFound(otherFileData, false)
+      if (fileFoundCheck === null) {
+        newOtherFileData.push(otherFileData)
+      } else if (fileFoundCheck === true) {
+        hasUpdated = true
+      }
+    })
+
+    const audioFilesRemoved = []
+    const otherFilesRemoved = []
+
+    // inodes will all be up to date at this point
+    this.audioFiles = this.audioFiles.filter(af => {
+      if (!dataFound.audioFiles.find(_af => _af.ino === af.ino)) {
+        audioFilesRemoved.push(af.toJSON())
+        return false
+      }
+      return true
+    })
+
+    // Remove all tracks that were associated with removed audio files
+    if (audioFilesRemoved.length) {
+      const audioFilesRemovedInodes = audioFilesRemoved.map(afr => afr.ino)
+      this.tracks = this.tracks.filter(t => !audioFilesRemovedInodes.includes(t.ino))
+      this.checkUpdateMissingParts()
+      hasUpdated = true
+    }
+
+    this.otherFiles = this.otherFiles.filter(otherFile => {
+      if (!dataFound.otherFiles.find(_otherFile => _otherFile.ino === otherFile.ino)) {
+        otherFilesRemoved.push(otherFile.toJSON())
+
+        // Check remove cover
+        if (otherFile.fullPath === this.book.coverFullPath) {
+          Logger.debug(`[Audiobook] "${this.title}" Check scan book cover removed`)
+          this.book.removeCover()
+        }
+
+        return false
+      }
+      return true
+    })
+
+    if (otherFilesRemoved.length) {
+      hasUpdated = true
+    }
+
+    return {
+      updated: hasUpdated,
+      newAudioFileData,
+      newOtherFileData,
+      audioFilesRemoved,
+      otherFilesRemoved
+    }
+  }
 }
 module.exports = Audiobook

server/scanner/AudioFileScanner.js

@@ -0,0 +1,22 @@
+const AudioFile = require('../objects/AudioFile')
+const AudioProbeData = require('./AudioProbeData')
+
+const prober = require('../utils/prober')
+const Logger = require('../Logger')
+
+class AudioFileScanner {
+  constructor() { }
+
+  async scan(audioFileData, verbose = false) {
+    var probeData = await prober.probe2(audioFileData.fullPath, verbose)
+    if (probeData.error) {
+      Logger.error(`[AudioFileScanner] ${probeData.error} : "${audioFileData.fullPath}"`)
+      return null
+    }
+
+    var audioFile = new AudioFile()
+    // TODO: Build audio file
+    return audioFile
+  }
+}
+module.exports = new AudioFileScanner()

server/scanner/AudioProbeData.js

@@ -0,0 +1,74 @@
+const AudioFileMetadata = require('../objects/AudioFileMetadata')
+
+class AudioProbeData {
+  constructor() {
+    this.embeddedCoverArt = null
+    this.format = null
+    this.duration = null
+    this.size = null
+    this.bitRate = null
+    this.codec = null
+    this.timeBase = null
+    this.language = null
+    this.channelLayout = null
+    this.channels = null
+    this.sampleRate = null
+    this.chapters = []
+
+    this.audioFileMetadata = null
+
+    this.trackNumber = null
+    this.trackTotal = null
+  }
+
+  getDefaultAudioStream(audioStreams) {
+    if (audioStreams.length === 1) return audioStreams[0]
+    var defaultStream = audioStreams.find(a => a.is_default)
+    if (!defaultStream) return audioStreams[0]
+    return defaultStream
+  }
+
+  getEmbeddedCoverArt(videoStream) {
+    const ImageCodecs = ['mjpeg', 'jpeg', 'png']
+    return ImageCodecs.includes(videoStream.codec) ? videoStream.codec : null
+  }
+
+  setData(data) {
+    var audioStream = getDefaultAudioStream(data.audio_streams)
+
+    this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : false
+    this.format = data.format
+    this.duration = data.duration
+    this.size = data.size
+    this.bitRate = audioStream.bit_rate || data.bit_rate
+    this.codec = audioStream.codec
+    this.timeBase = audioStream.time_base
+    this.language = audioStream.language
+    this.channelLayout = audioStream.channel_layout
+    this.channels = audioStream.channels
+    this.sampleRate = audioStream.sample_rate
+    this.chapters = data.chapters || []
+
+    var metatags = {}
+    for (const key in data) {
+      if (data[key] && key.startsWith('file_tag')) {
+        metatags[key] = data[key]
+      }
+    }
+
+    this.audioFileMetadata = new AudioFileMetadata()
+    this.audioFileMetadata.setData(metatags)
+
+    // Track ID3 tag might be "3/10" or just "3"
+    if (this.audioFileMetadata.tagTrack) {
+      var trackParts = this.audioFileMetadata.tagTrack.split('/').map(part => Number(part))
+      if (trackParts.length > 0) {
+        this.trackNumber = trackParts[0]
+      }
+      if (trackParts.length > 1) {
+        this.trackTotal = trackParts[1]
+      }
+    }
+  }
+}
+module.exports = AudioProbeData

server/scanner/LibraryScan.js

@@ -0,0 +1,34 @@
+const Folder = require('../objects/Folder')
+
+const { getId } = require('../utils/index')
+
+class LibraryScan {
+  constructor() {
+    this.id = null
+    this.libraryId = null
+    this.libraryName = null
+    this.folders = null
+
+    this.scanOptions = null
+
+    this.startedAt = null
+    this.finishedAt = null
+
+    this.folderScans = []
+  }
+
+  get _scanOptions() { return this.scanOptions || {} }
+  get forceRescan() { return !!this._scanOptions.forceRescan }
+
+  setData(library, scanOptions) {
+    this.id = getId('lscan')
+    this.libraryId = library.id
+    this.libraryName = library.name
+    this.folders = library.folders.map(folder => Folder(folder.toJSON()))
+
+    this.scanOptions = scanOptions
+
+    this.startedAt = Date.now()
+  }
+}
+module.exports = LibraryScan

server/scanner/ScanOptions.js

@@ -0,0 +1,68 @@
+const { CoverDestination } = require('../utils/constants')
+
+class ScanOptions {
+  constructor(options) {
+    this.forceRescan = false
+
+    this.metadataPrecedence = [
+      {
+        id: 'directory',
+        include: true
+      },
+      {
+        id: 'reader-desc-txt',
+        include: true
+      },
+      {
+        id: 'audio-file-metadata',
+        include: true
+      },
+      {
+        id: 'metadata-opf',
+        include: true
+      },
+      {
+        id: 'external-source',
+        include: false
+      }
+    ]
+
+    // Server settings
+    this.parseSubtitles = false
+    this.findCovers = false
+    this.coverDestination = CoverDestination.METADATA
+
+    if (options) {
+      this.construct(options)
+    }
+  }
+
+  construct(options) {
+    for (const key in options) {
+      if (key === 'metadataPrecedence' && options[key].length) {
+        this.metadataPrecedence = [...options[key]]
+      } else if (this[key] !== undefined) {
+        this[key] = options[key]
+      }
+    }
+  }
+
+  toJSON() {
+    return {
+      forceRescan: this.forceRescan,
+      metadataPrecedence: this.metadataPrecedence,
+      parseSubtitles: this.parseSubtitles,
+      findCovers: this.findCovers,
+      coverDestination: this.coverDestination
+    }
+  }
+
+  setData(options, serverSettings) {
+    this.forceRescan = !!options.forceRescan
+
+    this.parseSubtitles = !!serverSettings.scannerParseSubtitle
+    this.findCovers = !!serverSettings.scannerFindCovers
+    this.coverDestination = serverSettings.coverDestination
+  }
+}
+module.exports = ScanOptions

server/scanner/Scanner.js

@@ -0,0 +1,172 @@
+const fs = require('fs-extra')
+const Path = require('path')
+
+// Utils
+const Logger = require('../Logger')
+const { version } = require('../../package.json')
+const audioFileScanner = require('../utils/audioFileScanner')
+const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('../utils/scandir')
+const { comparePaths, getIno, getId } = require('../utils/index')
+const { secondsToTimestamp } = require('../utils/fileUtils')
+const { ScanResult, CoverDestination } = require('../utils/constants')
+
+const AudioFileScanner = require('./AudioFileScanner')
+const BookFinder = require('../BookFinder')
+const Audiobook = require('../objects/Audiobook')
+const LibraryScan = require('./LibraryScan')
+const ScanOptions = require('./ScanOptions')
+
+class Scanner {
+  constructor(AUDIOBOOK_PATH, METADATA_PATH, db, coverController, emitter) {
+    this.AudiobookPath = AUDIOBOOK_PATH
+    this.MetadataPath = METADATA_PATH
+    this.BookMetadataPath = Path.posix.join(this.MetadataPath.replace(/\\/g, '/'), 'books')
+
+    this.db = db
+    this.coverController = coverController
+    this.emitter = emitter
+
+    this.cancelScan = false
+    this.cancelLibraryScan = {}
+    this.librariesScanning = []
+
+    this.bookFinder = new BookFinder()
+  }
+
+  async scan(libraryId, options = {}) {
+    if (this.librariesScanning.includes(libraryId)) {
+      Logger.error(`[Scanner] Already scanning ${libraryId}`)
+      return
+    }
+
+    var library = this.db.libraries.find(lib => lib.id === libraryId)
+    if (!library) {
+      Logger.error(`[Scanner] Library not found for scan ${libraryId}`)
+      return
+    } else if (!library.folders.length) {
+      Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`)
+      return
+    }
+
+    var scanOptions = new ScanOptions()
+    scanOptions.setData(options, this.db.serverSettings)
+
+    var libraryScan = new LibraryScan()
+    libraryScan.setData(library, scanOptions)
+
+    Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
+
+    var results = await this.scanLibrary(libraryScan)
+
+    Logger.info(`[Scanner] Library scan ${libraryScan.id} complete`)
+
+    return results
+  }
+
+  async scanLibrary(libraryScan) {
+    var audiobookDataFound = []
+    for (let i = 0; i < libraryScan.folders.length; i++) {
+      var folder = libraryScan.folders[i]
+      var abDataFoundInFolder = await scanRootDir(folder, this.db.serverSettings)
+      Logger.debug(`[Scanner] ${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`)
+      audiobookDataFound = audiobookDataFound.concat(abDataFoundInFolder)
+    }
+
+    // Remove audiobooks with no inode
+    audiobookDataFound = audiobookDataFound.filter(abd => abd.ino)
+
+    var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryScan.libraryId)
+
+    const audiobooksToUpdate = []
+    const audiobooksToRescan = []
+    const newAudiobookData = []
+
+    // Check for existing & removed audiobooks
+    for (let i = 0; i < audiobooksInLibrary.length; i++) {
+      var audiobook = audiobooksInLibrary[i]
+      var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path))
+      if (!dataFound) {
+        Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`)
+        audiobook.isMissing = true
+        audiobook.lastUpdate = Date.now()
+        scanResults.missing++
+        audiobooksToUpdate.push(audiobook)
+      } else {
+        var checkRes = audiobook.checkShouldRescan(dataFound)
+        if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) {
+          // existing audiobook has new files
+          checkRes.audiobook = audiobook
+          audiobooksToRescan.push(checkRes)
+        } else if (checkRes.updated) {
+          audiobooksToUpdate.push(audiobook)
+        }
+
+        // Remove this abf
+        audiobookDataFound = audiobookDataFound.filter(abf => abf.ino !== dataFound.ino)
+      }
+    }
+
+    // Potential NEW Audiobooks
+    for (let i = 0; i < audiobookDataFound.length; i++) {
+      var dataFound = audiobookDataFound[i]
+      var hasEbook = dataFound.otherFiles.find(otherFile => otherFile.filetype === 'ebook')
+      if (!hasEbook && !dataFound.audioFiles.length) {
+        Logger.info(`[Scanner] Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
+      } else {
+        newAudiobookData.push(dataFound)
+      }
+    }
+
+    var rescans = []
+    for (let i = 0; i < audiobooksToRescan.length; i++) {
+      var rescan = this.rescanAudiobook(audiobooksToRescan[i])
+      rescans.push(rescan)
+    }
+    var newscans = []
+    for (let i = 0; i < newAudiobookData.length; i++) {
+      var newscan = this.scanNewAudiobook(newAudiobookData[i])
+      newscans.push(newscan)
+    }
+
+    var rescanResults = await Promise.all(rescans)
+
+    var newscanResults = await Promise.all(newscans)
+
+    // TODO: Return report
+    return {
+      updates: 0,
+      additions: 0
+    }
+  }
+
+  // Return scan result payload
+  async rescanAudiobook(audiobookCheckData) {
+    const { newAudioFileData, newOtherFileData, audiobook } = audiobookCheckData
+    if (newAudioFileData.length) {
+      var newAudioFiles = await this.scanAudioFiles(newAudioFileData)
+      // TODO: Update audiobook tracks
+    }
+    if (newOtherFileData.length) {
+      // TODO: Check other files
+    }
+
+    return {
+      updated: true
+    }
+  }
+
+  async scanNewAudiobook(audiobookData) {
+    // TODO: Return new audiobook
+    return null
+  }
+
+  async scanAudioFiles(audioFileData) {
+    var proms = []
+    for (let i = 0; i < audioFileData.length; i++) {
+      var prom = AudioFileScanner.scan(audioFileData[i])
+      proms.push(prom)
+    }
+    return Promise.all(proms)
+  }
+}
+module.exports = Scanner

server/utils/audioFileScanner.js

@@ -13,7 +13,7 @@ function getDefaultAudioStream(audioStreams) {
 
 async function scan(path, verbose = false) {
   Logger.debug(`Scanning path "${path}"`)
-  var probeData = await prober(path, verbose)
+  var probeData = await prober.probe(path, verbose)
   if (!probeData || !probeData.audio_streams || !probeData.audio_streams.length) {
     return {
       error: 'Invalid audio file'

server/utils/prober.js

@@ -1,5 +1,8 @@
 var Ffmpeg = require('fluent-ffmpeg')
 const Path = require('path')
+
+const AudioProbeData = require('../scanner/AudioProbeData')
+
 const Logger = require('../Logger')
 
 function tryGrabBitRate(stream, all_streams, total_bit_rate) {
@@ -241,4 +244,31 @@ function probe(filepath, verbose = false) {
     })
   })
 }
-module.exports = probe
+module.exports.probe = probe
+
+// Updated probe returns AudioProbeData object
+function probe2(filepath, verbose = false) {
+  return new Promise((resolve) => {
+    Ffmpeg.ffprobe(filepath, ['-show_chapters'], (err, raw) => {
+      if (err) {
+        console.error(err)
+        var errorMsg = err ? err.message : null
+        resolve({
+          error: errorMsg || 'Probe Error'
+        })
+      } else {
+        var rawProbeData = parseProbeData(raw, verbose)
+        if (!rawProbeData || !rawProbeData.audio_streams.length) {
+          resolve({
+            error: rawProbeData ? 'Invalid audio file: no audio streams found' : 'Probe Failed'
+          })
+        } else {
+          var probeData = new AudioProbeData()
+          probeData.setData(rawProbeData)
+          resolve(probeData)
+        }
+      }
+    })
+  })
+}
+module.exports.probe2 = probe2