audiobookshelf/server/scanner/LibraryScanner.js

const Path = require('path')
const packageJson = require('../../package.json')
const Logger = require('../Logger')
const SocketAuthority = require('../SocketAuthority')
const Database = require('../Database')
const fs = require('../libs/fsExtra')
const fileUtils = require('../utils/fileUtils')
const scanUtils = require('../utils/scandir')
const { ScanResult, LogLevel } = require('../utils/constants')
const globals = require('../utils/globals')
const libraryFilters = require('../utils/queries/libraryFilters')
const AudioFileScanner = require('./AudioFileScanner')
const ScanOptions = require('./ScanOptions')
const LibraryScan = require('./LibraryScan')
const LibraryItemScanData = require('./LibraryItemScanData')
const AudioFile = require('../objects/files/AudioFile')
const Book = require('../models/Book')
const BookScanner = require('./BookScanner')

class LibraryScanner {
  constructor(coverManager, taskManager) {
    this.coverManager = coverManager
    this.taskManager = taskManager

    this.cancelLibraryScan = {}
    this.librariesScanning = []
  }

  /**
   * @param {string} libraryId 
   * @returns {boolean}
   */
  isLibraryScanning(libraryId) {
    return this.librariesScanning.some(ls => ls.id === libraryId)
  }

  /**
   * 
   * @param {import('../objects/Library')} library 
   * @param {*} options 
   */
  async scan(library, options = {}) {
    if (this.isLibraryScanning(library.id)) {
      Logger.error(`[Scanner] Already scanning ${library.id}`)
      return
    }

    if (!library.folders.length) {
      Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`)
      return
    }

    const scanOptions = new ScanOptions()
    scanOptions.setData(options, Database.serverSettings)

    const libraryScan = new LibraryScan()
    libraryScan.setData(library, scanOptions)
    libraryScan.verbose = true
    this.librariesScanning.push(libraryScan.getScanEmitData)

    SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)

    Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)

    const canceled = await this.scanLibrary(libraryScan)

    if (canceled) {
      Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
      delete this.cancelLibraryScan[libraryScan.libraryId]
    }

    libraryScan.setComplete()

    Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
    this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)

    if (canceled && !libraryScan.totalResults) {
      const emitData = libraryScan.getScanEmitData
      emitData.results = null
      SocketAuthority.emitter('scan_complete', emitData)
      return
    }

    SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)

    if (libraryScan.totalResults) {
      libraryScan.saveLog()
    }
  }

  /**
   * 
   * @param {import('./LibraryScan')} libraryScan 
   */
  async scanLibrary(libraryScan) {
    // Make sure library filter data is set
    //   this is used to check for existing authors & series
    await libraryFilters.getFilterData(libraryScan.library)

    /** @type {LibraryItemScanData[]} */
    let libraryItemDataFound = []

    // Scan each library folder
    for (let i = 0; i < libraryScan.folders.length; i++) {
      const folder = libraryScan.folders[i]
      const itemDataFoundInFolder = await this.scanFolder(libraryScan.library, folder)
      libraryScan.addLog(LogLevel.INFO, `${itemDataFoundInFolder.length} item data found in folder "${folder.fullPath}"`)
      libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder)
    }

    if (this.cancelLibraryScan[libraryScan.libraryId]) return true

    const existingLibraryItems = await Database.libraryItemModel.findAll({
      where: {
        libraryId: libraryScan.libraryId
      },
      attributes: ['id', 'mediaId', 'mediaType', 'path', 'relPath', 'ino', 'isMissing', 'isFile', 'mtime', 'ctime', 'birthtime', 'libraryFiles', 'libraryFolderId', 'size']
    })

    const libraryItemIdsMissing = []
    for (const existingLibraryItem of existingLibraryItems) {
      // First try to find matching library item with exact file path
      let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path)
      if (!libraryItemData) {
        // Fallback to finding matching library item with matching inode value
        libraryItemData = libraryItemDataFound.find(lid => lid.ino === existingLibraryItem.ino)
        if (libraryItemData) {
          libraryScan.addLog(LogLevel.INFO, `Library item with path "${existingLibraryItem.path}" was not found, but library item inode "${existingLibraryItem.ino}" was found at path "${libraryItemData.path}"`)
        }
      }

      if (!libraryItemData) {
        // Podcast folder can have no episodes and still be valid
        if (libraryScan.libraryMediaType === 'podcast' && await fs.pathExists(existingLibraryItem.path)) {
          libraryScan.addLog(LogLevel.INFO, `Library item "${existingLibraryItem.relPath}" folder exists but has no episodes`)
        } else {
          libraryScan.addLog(LogLevel.WARN, `Library Item "${existingLibraryItem.path}" (inode: ${existingLibraryItem.ino}) is missing`)
          libraryScan.resultsMissing++
          if (!existingLibraryItem.isMissing) {
            libraryItemIdsMissing.push(existingLibraryItem.id)
          }
        }
      } else {
        libraryItemDataFound = libraryItemDataFound.filter(lidf => lidf !== libraryItemData)
        await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)
        if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
          await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
        }
      }
    }

    // Update missing library items
    if (libraryItemIdsMissing.length) {
      libraryScan.addLog(LogLevel.INFO, `Updating ${libraryItemIdsMissing.length} library items missing`)
      await Database.libraryItemModel.update({
        isMissing: true,
        lastScan: Date.now(),
        lastScanVersion: packageJson.version
      }, {
        where: {
          id: libraryItemIdsMissing
        }
      })
    }

    // Add new library items
    if (libraryItemDataFound.length) {
      for (const libraryItemData of libraryItemDataFound) {
        const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan)
        if (newLibraryItem) {
          libraryScan.resultsAdded++
        }
      }
    }

    // TODO: Socket emitter
  }

  /**
   * Get scan data for library folder
   * @param {import('../objects/Library')} library 
   * @param {import('../objects/Folder')} folder 
   * @returns {LibraryItemScanData[]}
   */
  async scanFolder(library, folder) {
    const folderPath = fileUtils.filePathToPOSIX(folder.fullPath)

    const pathExists = await fs.pathExists(folderPath)
    if (!pathExists) {
      Logger.error(`[scandir] Invalid folder path does not exist "${folderPath}"`)
      return []
    }

    const fileItems = await fileUtils.recurseFiles(folderPath)
    const libraryItemGrouping = scanUtils.groupFileItemsIntoLibraryItemDirs(library.mediaType, fileItems, library.settings.audiobooksOnly)

    if (!Object.keys(libraryItemGrouping).length) {
      Logger.error(`Root path has no media folders: ${folderPath}`)
      return []
    }

    const items = []
    for (const libraryItemPath in libraryItemGrouping) {
      let isFile = false // item is not in a folder
      let libraryItemData = null
      let fileObjs = []
      if (libraryItemPath === libraryItemGrouping[libraryItemPath]) {
        // Media file in root only get title
        libraryItemData = {
          mediaMetadata: {
            title: Path.basename(libraryItemPath, Path.extname(libraryItemPath))
          },
          path: Path.posix.join(folderPath, libraryItemPath),
          relPath: libraryItemPath
        }
        fileObjs = await scanUtils.buildLibraryFile(folderPath, [libraryItemPath])
        isFile = true
      } else {
        libraryItemData = scanUtils.getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath)
        fileObjs = await scanUtils.buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath])
      }

      const libraryItemFolderStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path)

      if (!libraryItemFolderStats.ino) {
        Logger.warn(`[LibraryScanner] Library item folder "${libraryItemData.path}" has no inode value`)
        continue
      }

      items.push(new LibraryItemScanData({
        libraryFolderId: folder.id,
        libraryId: folder.libraryId,
        mediaType: library.mediaType,
        ino: libraryItemFolderStats.ino,
        mtimeMs: libraryItemFolderStats.mtimeMs || 0,
        ctimeMs: libraryItemFolderStats.ctimeMs || 0,
        birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
        path: libraryItemData.path,
        relPath: libraryItemData.relPath,
        isFile,
        mediaMetadata: libraryItemData.mediaMetadata || null,
        libraryFiles: fileObjs
      }))
    }
    return items
  }

  /**
   * 
   * @param {import('../models/LibraryItem')} existingLibraryItem 
   * @param {LibraryItemScanData} libraryItemData 
   * @param {LibraryScan} libraryScan
   */
  async rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) {
    if (existingLibraryItem.mediaType === 'book') {
      /** @type {Book} */
      const media = await existingLibraryItem.getMedia({
        include: [
          {
            model: Database.authorModel,
            through: {
              attributes: ['createdAt']
            }
          },
          {
            model: Database.seriesModel,
            through: {
              attributes: ['sequence', 'createdAt']
            }
          }
        ]
      })

      let hasMediaChanges = libraryItemData.hasAudioFileChanges
      if (libraryItemData.hasAudioFileChanges || libraryItemData.audioLibraryFiles.length !== media.audioFiles.length) {
        // Filter out audio files that were removed
        media.audioFiles = media.audioFiles.filter(af => libraryItemData.checkAudioFileRemoved(af))

        // Update audio files that were modified
        if (libraryItemData.audioLibraryFilesModified.length) {
          let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified)
          media.audioFiles = media.audioFiles.map((audioFileObj) => {
            let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === audioFileObj.metadata.path)
            if (!matchedScannedAudioFile) {
              matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.ino === audioFileObj.ino)
            }

            if (matchedScannedAudioFile) {
              scannedAudioFiles = scannedAudioFiles.filter(saf => saf !== matchedScannedAudioFile)
              const audioFile = new AudioFile(audioFileObj)
              audioFile.updateFromScan(matchedScannedAudioFile)
              return audioFile.toJSON()
            }
            return audioFileObj
          })
          // Modified audio files that were not found on the book
          if (scannedAudioFiles.length) {
            media.audioFiles.push(...scannedAudioFiles)
          }
        }

        // Add new audio files scanned in
        if (libraryItemData.audioLibraryFilesAdded.length) {
          const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesAdded)
          media.audioFiles.push(...scannedAudioFiles)
        }

        // Add audio library files that are not already set on the book (safety check)
        let audioLibraryFilesToAdd = []
        for (const audioLibraryFile of libraryItemData.audioLibraryFiles) {
          if (!media.audioFiles.some(af => af.ino === audioLibraryFile.ino)) {
            libraryScan.addLog(LogLevel.DEBUG, `Existing audio library file "${audioLibraryFile.metadata.relPath}" was not set on book "${media.title}" so setting it now`)
            audioLibraryFilesToAdd.push(audioLibraryFile)
          }
        }
        if (audioLibraryFilesToAdd.length) {
          const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, audioLibraryFilesToAdd)
          media.audioFiles.push(...scannedAudioFiles)
        }

        media.audioFiles = AudioFileScanner.runSmartTrackOrder(existingLibraryItem.relPath, media.audioFiles)

        media.duration = 0
        media.audioFiles.forEach((af) => {
          if (!isNaN(af.duration)) {
            media.duration += af.duration
          }
        })

        media.changed('audioFiles', true)
      }

      // Check if cover was removed
      if (media.coverPath && !libraryItemData.imageLibraryFiles.some(lf => lf.metadata.path === media.coverPath)) {
        media.coverPath = null
        hasMediaChanges = true
      }

      // Check if cover is not set and image files were found
      if (!media.coverPath && libraryItemData.imageLibraryFiles.length) {
        // Prefer using a cover image with the name "cover" otherwise use the first image
        const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path))
        media.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path
        hasMediaChanges = true
      }

      // Check if ebook was removed
      if (media.ebookFile && (libraryScan.library.settings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) {
        media.ebookFile = null
        hasMediaChanges = true
      }

      // Check if ebook is not set and ebooks were found
      if (!media.ebookFile && !libraryScan.library.settings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) {
        // Prefer to use an epub ebook then fallback to the first ebook found
        let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub')
        if (!ebookLibraryFile) ebookLibraryFile = libraryItemData.ebookLibraryFiles[0]
        // Ebook file is the same as library file except for additional `ebookFormat`
        ebookLibraryFile.ebookFormat = ebookLibraryFile.metadata.ext.slice(1).toLowerCase()
        media.ebookFile = ebookLibraryFile
        media.changed('ebookFile', true)
        hasMediaChanges = true
      }

      // Check/update the isSupplementary flag on libraryFiles for the LibraryItem
      let libraryItemUpdated = false
      for (const libraryFile of existingLibraryItem.libraryFiles) {
        if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) {
          if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) {
            if (libraryFile.isSupplementary !== false) {
              libraryFile.isSupplementary = false
              libraryItemUpdated = true
            }
          } else if (libraryFile.isSupplementary !== true) {
            libraryFile.isSupplementary = true
            libraryItemUpdated = true
          }
        }
      }
      if (libraryItemUpdated) {
        existingLibraryItem.changed('libraryFiles', true)
        await existingLibraryItem.save()
      }

      // TODO: Update chapters & metadata

      if (hasMediaChanges) {
        await media.save()
      }
    } else {
      // TODO: Scan updated podcast
    }
  }

  /**
   * 
   * @param {LibraryItemScanData} libraryItemData 
   * @param {LibraryScan} libraryScan
   */
  async scanNewLibraryItem(libraryItemData, libraryScan) {
    if (libraryScan.libraryMediaType === 'book') {
      const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, libraryScan)
      if (newLibraryItem) {
        libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
      }
      return newLibraryItem
    } else {
      // TODO: Scan new podcast
      return null
    }
  }
}
module.exports = LibraryScanner