Updates for new book scanner

client/pages/item/_id/index.vue

@@ -761,6 +761,7 @@ export default {
     if (this.libraryId) {
       this.$store.commit('libraries/setCurrentLibrary', this.libraryId)
     }
+    this.$eventBus.$on(`${this.libraryItem.id}_updated`, this.libraryItemUpdated)
     this.$root.socket.on('item_updated', this.libraryItemUpdated)
     this.$root.socket.on('rss_feed_open', this.rssFeedOpen)
     this.$root.socket.on('rss_feed_closed', this.rssFeedClosed)
@@ -769,6 +770,7 @@ export default {
     this.$root.socket.on('episode_download_finished', this.episodeDownloadFinished)
   },
   beforeDestroy() {
+    this.$eventBus.$off(`${this.libraryItem.id}_updated`, this.libraryItemUpdated)
     this.$root.socket.off('item_updated', this.libraryItemUpdated)
     this.$root.socket.off('rss_feed_open', this.rssFeedOpen)
     this.$root.socket.off('rss_feed_closed', this.rssFeedClosed)

server/controllers/AuthorController.js

@@ -234,8 +234,8 @@ class AuthorController {
     return this.cacheManager.handleAuthorCache(res, author, options)
   }
 
-  middleware(req, res, next) {
+  async middleware(req, res, next) {
-    const author = Database.authors.find(au => au.id === req.params.id)
+    const author = await Database.authorModel.getOldById(req.params.id)
     if (!author) return res.sendStatus(404)
 
     if (req.method == 'DELETE' && !req.user.canDelete) {

server/controllers/LibraryController.js

@@ -573,18 +573,19 @@ class LibraryController {
    * rssfeed: adds `rssFeed` to series object if a feed is open
    * progress: adds `progress` to series object with { libraryItemIds:Array<llid>, libraryItemIdsFinished:Array<llid>, isFinished:boolean }
    * 
-   * @param {*} req 
+   * @param {import('express').Request} req 
-   * @param {*} res - Series
+   * @param {import('express').Response} res - Series
    */
   async getSeriesForLibrary(req, res) {
     const include = (req.query.include || '').split(',').map(v => v.trim().toLowerCase()).filter(v => !!v)
 
-    const series = Database.series.find(se => se.id === req.params.seriesId)
+    const series = await Database.seriesModel.findByPk(req.params.seriesId)
     if (!series) return res.sendStatus(404)
+    const oldSeries = series.getOldSeries()
 
-    const libraryItemsInSeries = await libraryItemsBookFilters.getLibraryItemsForSeries(series, req.user)
+    const libraryItemsInSeries = await libraryItemsBookFilters.getLibraryItemsForSeries(oldSeries, req.user)
 
-    const seriesJson = series.toJSON()
+    const seriesJson = oldSeries.toJSON()
     if (include.includes('progress')) {
       const libraryItemsFinished = libraryItemsInSeries.filter(li => !!req.user.getMediaProgress(li.id)?.isFinished)
       seriesJson.progress = {

server/managers/CoverManager.js

@@ -279,7 +279,7 @@ class CoverManager {
     if (global.ServerSettings.storeCoverWithItem && libraryItemPath) {
       coverDirPath = libraryItemPath
     } else {
-      coverDirPath = Path.posix.join(this.ItemMetadataPath, libraryItemId)
+      coverDirPath = Path.posix.join(global.MetadataPath, 'items', libraryItemId)
     }
     await fs.ensureDir(coverDirPath)
 

server/models/Author.js

@@ -83,6 +83,17 @@ class Author extends Model {
     })
   }
 
+  /**
+   * Get oldAuthor by id
+   * @param {string} authorId 
+   * @returns {oldAuthor}
+   */
+  static async getOldById(authorId) {
+    const author = await this.findByPk(authorId)
+    if (!author) return null
+    return author.getOldAuthor()
+  }
+
   /**
    * Check if author exists
    * @param {string} authorId 

server/objects/files/AudioFile.js

@@ -118,7 +118,12 @@ class AudioFile {
   setDataFromProbe(libraryFile, probeData) {
     this.ino = libraryFile.ino || null
 
-    this.metadata = libraryFile.metadata.clone()
+    if (libraryFile.metadata instanceof FileMetadata) {
+      this.metadata = libraryFile.metadata.clone()
+    } else {
+      this.metadata = new FileMetadata(libraryFile.metadata)
+    }
+
     this.addedAt = Date.now()
     this.updatedAt = Date.now()
 

server/scanner/BookScanner.js

@@ -1,15 +1,18 @@
 const uuidv4 = require("uuid").v4
+const { Sequelize } = require('sequelize')
 const { LogLevel } = require('../utils/constants')
-const { getTitleIgnorePrefix } = require('../utils/index')
+const { getTitleIgnorePrefix, areEquivalent } = require('../utils/index')
 const { parseOpfMetadataXML } = require('../utils/parsers/parseOpfMetadata')
 const { parseOverdriveMediaMarkersAsChapters } = require('../utils/parsers/parseOverdriveMediaMarkers')
 const abmetadataGenerator = require('../utils/generators/abmetadataGenerator')
 const parseNameString = require('../utils/parsers/parseNameString')
+const globals = require('../utils/globals')
 const AudioFileScanner = require('./AudioFileScanner')
 const Database = require('../Database')
 const { readTextFile } = require('../utils/fileUtils')
 const AudioFile = require('../objects/files/AudioFile')
 const CoverManager = require('../managers/CoverManager')
+const fsExtra = require("../libs/fsExtra")
 
 /**
  * Metadata for books pulled from files
@@ -37,6 +40,313 @@ const CoverManager = require('../managers/CoverManager')
 class BookScanner {
   constructor() { }
 
+  /**
+   * @param {import('../models/LibraryItem')} existingLibraryItem 
+   * @param {import('./LibraryItemScanData')} libraryItemData 
+   * @param {import('./LibraryScan')} libraryScan 
+   * @returns {import('../models/LibraryItem')}
+   */
+  async rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan) {
+    /** @type {import('../models/Book')} */
+    const media = await existingLibraryItem.getMedia({
+      include: [
+        {
+          model: Database.authorModel,
+          through: {
+            attributes: ['id', 'createdAt']
+          }
+        },
+        {
+          model: Database.seriesModel,
+          through: {
+            attributes: ['id', 'sequence', 'createdAt']
+          }
+        }
+      ],
+      order: [
+        [Database.authorModel, Database.bookAuthorModel, 'createdAt', 'ASC'],
+        [Database.seriesModel, 'bookSeries', 'createdAt', 'ASC']
+      ]
+    })
+
+    let hasMediaChanges = libraryItemData.hasAudioFileChanges
+    if (libraryItemData.hasAudioFileChanges || libraryItemData.audioLibraryFiles.length !== media.audioFiles.length) {
+      // Filter out audio files that were removed
+      media.audioFiles = media.audioFiles.filter(af => !libraryItemData.checkAudioFileRemoved(af))
+
+      // Update audio files that were modified
+      if (libraryItemData.audioLibraryFilesModified.length) {
+        let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified)
+        media.audioFiles = media.audioFiles.map((audioFileObj) => {
+          let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === audioFileObj.metadata.path)
+          if (!matchedScannedAudioFile) {
+            matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.ino === audioFileObj.ino)
+          }
+
+          if (matchedScannedAudioFile) {
+            scannedAudioFiles = scannedAudioFiles.filter(saf => saf !== matchedScannedAudioFile)
+            const audioFile = new AudioFile(audioFileObj)
+            audioFile.updateFromScan(matchedScannedAudioFile)
+            return audioFile.toJSON()
+          }
+          return audioFileObj
+        })
+        // Modified audio files that were not found on the book
+        if (scannedAudioFiles.length) {
+          media.audioFiles.push(...scannedAudioFiles)
+        }
+      }
+
+      // Add new audio files scanned in
+      if (libraryItemData.audioLibraryFilesAdded.length) {
+        const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesAdded)
+        media.audioFiles.push(...scannedAudioFiles)
+      }
+
+      // Add audio library files that are not already set on the book (safety check)
+      let audioLibraryFilesToAdd = []
+      for (const audioLibraryFile of libraryItemData.audioLibraryFiles) {
+        if (!media.audioFiles.some(af => af.ino === audioLibraryFile.ino)) {
+          libraryScan.addLog(LogLevel.DEBUG, `Existing audio library file "${audioLibraryFile.metadata.relPath}" was not set on book "${media.title}" so setting it now`)
+
+          audioLibraryFilesToAdd.push(audioLibraryFile)
+        }
+      }
+      if (audioLibraryFilesToAdd.length) {
+        const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, audioLibraryFilesToAdd)
+        media.audioFiles.push(...scannedAudioFiles)
+      }
+
+      media.audioFiles = AudioFileScanner.runSmartTrackOrder(existingLibraryItem.relPath, media.audioFiles)
+
+      media.duration = 0
+      media.audioFiles.forEach((af) => {
+        if (!isNaN(af.duration)) {
+          media.duration += af.duration
+        }
+      })
+
+      media.changed('audioFiles', true)
+    }
+
+    // Check if cover was removed
+    if (media.coverPath && !libraryItemData.imageLibraryFiles.some(lf => lf.metadata.path === media.coverPath)) {
+      media.coverPath = null
+      hasMediaChanges = true
+    }
+
+    // Check if cover is not set and image files were found
+    if (!media.coverPath && libraryItemData.imageLibraryFiles.length) {
+      // Prefer using a cover image with the name "cover" otherwise use the first image
+      const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path))
+      media.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path
+      hasMediaChanges = true
+    }
+
+    // Check if ebook was removed
+    if (media.ebookFile && (libraryScan.library.settings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) {
+      media.ebookFile = null
+      hasMediaChanges = true
+    }
+
+    // Check if ebook is not set and ebooks were found
+    if (!media.ebookFile && !libraryScan.library.settings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) {
+      // Prefer to use an epub ebook then fallback to the first ebook found
+      let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub')
+      if (!ebookLibraryFile) ebookLibraryFile = libraryItemData.ebookLibraryFiles[0]
+      // Ebook file is the same as library file except for additional `ebookFormat`
+      ebookLibraryFile.ebookFormat = ebookLibraryFile.metadata.ext.slice(1).toLowerCase()
+      media.ebookFile = ebookLibraryFile
+      media.changed('ebookFile', true)
+      hasMediaChanges = true
+    }
+
+    // Check/update the isSupplementary flag on libraryFiles for the LibraryItem
+    let libraryItemUpdated = false
+    for (const libraryFile of existingLibraryItem.libraryFiles) {
+      if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) {
+        if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) {
+          if (libraryFile.isSupplementary !== false) {
+            libraryFile.isSupplementary = false
+            libraryItemUpdated = true
+          }
+        } else if (libraryFile.isSupplementary !== true) {
+          libraryFile.isSupplementary = true
+          libraryItemUpdated = true
+        }
+      }
+    }
+    if (libraryItemUpdated) {
+      existingLibraryItem.changed('libraryFiles', true)
+      await existingLibraryItem.save()
+    }
+
+    // TODO: When metadata file is stored in /metadata/items/{libraryItemId}.[abs|json] we should load this
+    // TODO: store an additional array of metadata keys that the user has changed manually so we know what not to override
+    const bookMetadata = await this.getBookMetadataFromScanData(media.audioFiles, libraryItemData, libraryScan)
+    let authorsUpdated = false
+    const bookAuthorsRemoved = []
+    let seriesUpdated = false
+    const bookSeriesRemoved = []
+
+    for (const key in bookMetadata) {
+      // Ignore unset metadata and empty arrays
+      if (bookMetadata[key] === undefined || (Array.isArray(bookMetadata[key]) && !bookMetadata[key].length)) continue
+
+      if (key === 'authors') {
+        // Check for authors added
+        for (const authorName of bookMetadata.authors) {
+          if (!media.authors.some(au => au.name === authorName)) {
+            const existingAuthor = Database.libraryFilterData[libraryScan.libraryId].authors.find(au => au.name === authorName)
+            if (existingAuthor) {
+              await Database.bookAuthorModel.create({
+                bookId: media.id,
+                authorId: existingAuthor.id
+              })
+              libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added author "${authorName}"`)
+              authorsUpdated = true
+            } else {
+              const newAuthor = await Database.authorModel.create({
+                name: authorName,
+                lastFirst: parseNameString.nameToLastFirst(authorName),
+                libraryId: libraryScan.libraryId
+              })
+              await media.addAuthor(newAuthor)
+              Database.addAuthorToFilterData(libraryScan.libraryId, newAuthor.name, newAuthor.id)
+              libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added new author "${authorName}"`)
+              authorsUpdated = true
+            }
+          }
+        }
+        // Check for authors removed
+        for (const author of media.authors) {
+          if (!bookMetadata.authors.includes(author.name)) {
+            await author.bookAuthor.destroy()
+            libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" removed author "${author.name}"`)
+            authorsUpdated = true
+            bookAuthorsRemoved.push(author.id)
+          }
+        }
+      } else if (key === 'series') {
+        // Check for series added
+        for (const seriesObj of bookMetadata.series) {
+          if (!media.series.some(se => se.name === seriesObj.name)) {
+            const existingSeries = Database.libraryFilterData[libraryScan.libraryId].series.find(se => se.name === seriesObj.name)
+            if (existingSeries) {
+              await Database.bookSeriesModel.create({
+                bookId: media.id,
+                seriesId: existingSeries.id,
+                sequence: seriesObj.sequence
+              })
+              libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added series "${seriesObj.name}"${seriesObj.sequence ? ` with sequence "${seriesObj.sequence}"` : ''}`)
+              seriesUpdated = true
+            } else {
+              const newSeries = await Database.seriesModel.create({
+                name: seriesObj.name,
+                nameIgnorePrefix: getTitleIgnorePrefix(seriesObj.name),
+                libraryId: libraryScan.libraryId
+              })
+              await media.addSeries(newSeries)
+              Database.addSeriesToFilterData(libraryScan.libraryId, newSeries.name, newSeries.id)
+              libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added new series "${seriesObj.name}"${seriesObj.sequence ? ` with sequence "${seriesObj.sequence}"` : ''}`)
+              seriesUpdated = true
+            }
+          }
+        }
+        // Check for series removed
+        for (const series of media.series) {
+          if (!bookMetadata.series.some(se => se.name === series.name)) {
+            await series.bookSeries.destroy()
+            libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" removed series "${series.name}"`)
+            seriesUpdated = true
+            bookSeriesRemoved.push(series.id)
+          }
+        }
+      } else if (key === 'genres') {
+        const existingGenres = media.genres || []
+        if (bookMetadata.genres.some(g => !existingGenres.includes(g)) || existingGenres.some(g => !bookMetadata.genres.includes(g))) {
+          libraryScan.addLog(LogLevel.DEBUG, `Updating book genres "${existingGenres.join(',')}" => "${bookMetadata.genres.join(',')}" for book "${bookMetadata.title}"`)
+          media.genres = bookMetadata.genres
+          hasMediaChanges = true
+        }
+      } else if (key === 'tags') {
+        const existingTags = media.tags || []
+        if (bookMetadata.tags.some(t => !existingTags.includes(t)) || existingTags.some(t => !bookMetadata.tags.includes(t))) {
+          libraryScan.addLog(LogLevel.DEBUG, `Updating book tags "${existingTags.join(',')}" => "${bookMetadata.tags.join(',')}" for book "${bookMetadata.title}"`)
+          media.tags = bookMetadata.tags
+          hasMediaChanges = true
+        }
+      } else if (key === 'narrators') {
+        const existingNarrators = media.narrators || []
+        if (bookMetadata.narrators.some(t => !existingNarrators.includes(t)) || existingNarrators.some(t => !bookMetadata.narrators.includes(t))) {
+          libraryScan.addLog(LogLevel.DEBUG, `Updating book narrators "${existingNarrators.join(',')}" => "${bookMetadata.narrators.join(',')}" for book "${bookMetadata.title}"`)
+          media.narrators = bookMetadata.narrators
+          hasMediaChanges = true
+        }
+      } else if (key === 'chapters') {
+        if (!areEquivalent(media.chapters, bookMetadata.chapters)) {
+          libraryScan.addLog(LogLevel.DEBUG, `Updating book chapters for book "${bookMetadata.title}"`)
+          media.chapters = bookMetadata.chapters
+          hasMediaChanges = true
+        }
+      } else if (key === 'coverPath') {
+        if (media.coverPath && media.coverPath !== bookMetadata.coverPath && !(await fsExtra.pathExists(media.coverPath))) {
+          libraryScan.addLog(LogLevel.DEBUG, `Updating book cover "${media.coverPath}" => "${bookMetadata.coverPath}" for book "${bookMetadata.title}" - original cover path does not exist`)
+          media.coverPath = bookMetadata.coverPath
+          hasMediaChanges = true
+        } else if (!media.coverPath) {
+          libraryScan.addLog(LogLevel.DEBUG, `Updating book cover "unset" => "${bookMetadata.coverPath}" for book "${bookMetadata.title}"`)
+          media.coverPath = bookMetadata.coverPath
+          hasMediaChanges = true
+        }
+      } else if (bookMetadata[key] !== media[key]) {
+        libraryScan.addLog(LogLevel.DEBUG, `Updating book ${key} "${media[key]}" => "${bookMetadata[key]}" for book "${bookMetadata.title}"`)
+        media[key] = bookMetadata[key]
+        hasMediaChanges = true
+      }
+    }
+
+    // If no cover then extract cover from audio file if available
+    if (!media.coverPath && media.audioFiles.length) {
+      const libraryItemDir = existingLibraryItem.isFile ? null : existingLibraryItem.path
+      const extractedCoverPath = await CoverManager.saveEmbeddedCoverArtNew(media.audioFiles, existingLibraryItem.id, libraryItemDir)
+      if (extractedCoverPath) {
+        libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" extracted embedded cover art from audio file to path "${extractedCoverPath}"`)
+        media.coverPath = extractedCoverPath
+        hasMediaChanges = true
+      }
+    }
+
+    // Save Book changes to db
+    if (hasMediaChanges) {
+      await media.save()
+    }
+
+    // Load authors/series again if updated (for sending back to client)
+    if (authorsUpdated) {
+      media.authors = await media.getAuthors({
+        joinTableAttributes: ['createdAt'],
+        order: [
+          Sequelize.literal(`bookAuthor.createdAt ASC`)
+        ]
+      })
+    }
+    if (seriesUpdated) {
+      media.series = await media.getSeries({
+        joinTableAttributes: ['sequence', 'createdAt'],
+        order: [
+          Sequelize.literal(`bookSeries.createdAt ASC`)
+        ]
+      })
+    }
+
+    libraryScan.seriesRemovedFromBooks.push(...bookSeriesRemoved)
+    libraryScan.authorsRemovedFromBooks.push(...bookAuthorsRemoved)
+
+    existingLibraryItem.media = media
+    return existingLibraryItem
+  }
+
   /**
    * 
    * @param {import('./LibraryItemScanData')} libraryItemData 
@@ -49,7 +359,7 @@ class BookScanner {
     scannedAudioFiles = AudioFileScanner.runSmartTrackOrder(libraryItemData.relPath, scannedAudioFiles)
 
     // Find ebook file (prefer epub)
-    let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') || libraryItemData.ebookLibraryFiles[0]
+    let ebookLibraryFile = libraryScan.library.settings.audiobooksOnly ? null : libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') || libraryItemData.ebookLibraryFiles[0]
 
     // Do not add library items that have no valid audio files and no ebook file
     if (!ebookLibraryFile && !scannedAudioFiles.length) {
@@ -62,6 +372,8 @@ class BookScanner {
     }
 
     const bookMetadata = await this.getBookMetadataFromScanData(scannedAudioFiles, libraryItemData, libraryScan)
+    bookMetadata.explicit = !!bookMetadata.explicit // Ensure boolean
+    bookMetadata.abridged = !!bookMetadata.abridged // Ensure boolean
 
     let duration = 0
     scannedAudioFiles.forEach((af) => duration += (!isNaN(af.duration) ? Number(af.duration) : 0))
@@ -116,6 +428,15 @@ class BookScanner {
 
     const libraryItemObj = libraryItemData.libraryItemObject
     libraryItemObj.id = uuidv4() // Generate library item id ahead of time to use for saving extracted cover image
+    libraryItemObj.isMissing = false
+    libraryItemObj.isInvalid = false
+
+    // Set isSupplementary flag on ebook library files
+    for (const libraryFile of libraryItemObj.libraryFiles) {
+      if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) {
+        libraryFile.isSupplementary = libraryFile.ino !== ebookLibraryFile?.ino
+      }
+    }
 
     // If cover was not found in folder then check embedded covers in audio files
     if (!bookObject.coverPath && scannedAudioFiles.length) {
@@ -166,37 +487,59 @@ class BookScanner {
     Database.addPublisherToFilterData(libraryScan.libraryId, libraryItem.book.publisher)
     Database.addLanguageToFilterData(libraryScan.libraryId, libraryItem.book.language)
 
+    // Load for emitting to client
+    libraryItem.media = await libraryItem.getMedia({
+      include: [
+        {
+          model: Database.authorModel,
+          through: {
+            attributes: ['id', 'createdAt']
+          }
+        },
+        {
+          model: Database.seriesModel,
+          through: {
+            attributes: ['id', 'sequence', 'createdAt']
+          }
+        }
+      ],
+      order: [
+        [Database.authorModel, Database.bookAuthorModel, 'createdAt', 'ASC'],
+        [Database.seriesModel, 'bookSeries', 'createdAt', 'ASC']
+      ]
+    })
+
     return libraryItem
   }
 
   /**
    * 
-   * @param {import('../objects/files/AudioFile')[]} scannedAudioFiles 
+   * @param {import('../models/Book').AudioFileObject[]} audioFiles 
    * @param {import('./LibraryItemScanData')} libraryItemData 
    * @param {import('./LibraryScan')} libraryScan 
    * @returns {Promise<BookMetadataObject>}
    */
-  async getBookMetadataFromScanData(scannedAudioFiles, libraryItemData, libraryScan) {
+  async getBookMetadataFromScanData(audioFiles, libraryItemData, libraryScan) {
     // First set book metadata from folder/file names
     const bookMetadata = {
       title: libraryItemData.mediaMetadata.title,
       titleIgnorePrefix: getTitleIgnorePrefix(libraryItemData.mediaMetadata.title),
-      subtitle: libraryItemData.mediaMetadata.subtitle,
+      subtitle: libraryItemData.mediaMetadata.subtitle || undefined,
-      publishedYear: libraryItemData.mediaMetadata.publishedYear,
+      publishedYear: libraryItemData.mediaMetadata.publishedYear || undefined,
-      publisher: null,
+      publisher: undefined,
-      description: null,
+      description: undefined,
-      isbn: null,
+      isbn: undefined,
-      asin: null,
+      asin: undefined,
-      language: null,
+      language: undefined,
       narrators: parseNameString.parse(libraryItemData.mediaMetadata.narrators)?.names || [],
       genres: [],
       tags: [],
       authors: parseNameString.parse(libraryItemData.mediaMetadata.author)?.names || [],
       series: [],
       chapters: [],
-      explicit: false,
+      explicit: undefined,
-      abridged: false,
+      abridged: undefined,
-      coverPath: null
+      coverPath: undefined
     }
     if (libraryItemData.mediaMetadata.series) {
       bookMetadata.series.push({
@@ -206,7 +549,7 @@ class BookScanner {
     }
 
     // Fill in or override book metadata from audio file meta tags
-    if (scannedAudioFiles.length) {
+    if (audioFiles.length) {
       const MetadataMapArray = [
         {
           tag: 'tagComposer',
@@ -261,7 +604,7 @@ class BookScanner {
         }
       ]
       const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata
-      const firstScannedFile = scannedAudioFiles[0]
+      const firstScannedFile = audioFiles[0]
       const audioFileMetaTags = firstScannedFile.metaTags
       MetadataMapArray.forEach((mapping) => {
         let value = audioFileMetaTags[mapping.tag]
@@ -372,7 +715,7 @@ class BookScanner {
 
     // Set chapters from audio files if not already set
     if (!bookMetadata.chapters.length) {
-      bookMetadata.chapters = this.getChaptersFromAudioFiles(bookMetadata.title, scannedAudioFiles, libraryScan)
+      bookMetadata.chapters = this.getChaptersFromAudioFiles(bookMetadata.title, audioFiles, libraryScan)
     }
 
     // Set cover from library file if one is found otherwise check audiofile

server/scanner/LibraryItemScanData.js

@@ -56,7 +56,7 @@ class LibraryItemScanData {
       mediaType: this.mediaType,
       isFile: this.isFile,
       mtime: this.mtimeMs,
-      ctime: this.ctime,
+      ctime: this.ctimeMs,
       birthtime: this.birthtimeMs,
       lastScan: Date.now(),
       lastScanVersion: packageJson.version,
@@ -74,7 +74,7 @@ class LibraryItemScanData {
 
   /** @type {boolean} */
   get hasAudioFileChanges() {
-    return this.audioLibraryFilesRemoved.length + this.audioLibraryFilesAdded.length + this.audioLibraryFilesModified
+    return (this.audioLibraryFilesRemoved.length + this.audioLibraryFilesAdded.length + this.audioLibraryFilesModified.length) > 0
   }
 
   /** @type {LibraryItem.LibraryFileObject[]} */
@@ -136,6 +136,7 @@ class LibraryItemScanData {
    * 
    * @param {LibraryItem} existingLibraryItem 
    * @param {import('./LibraryScan')} libraryScan
+   * @returns {boolean} true if changes found
    */
   async checkLibraryItemData(existingLibraryItem, libraryScan) {
     const keysToCompare = ['libraryFolderId', 'ino', 'path', 'relPath', 'isFile']
@@ -154,18 +155,18 @@ class LibraryItemScanData {
     }
 
     // Check mtime, ctime and birthtime
-    if (existingLibraryItem.mtime.valueOf() !== this.mtimeMs) {
+    if (existingLibraryItem.mtime?.valueOf() !== this.mtimeMs) {
-      libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "mtime" changed from "${existingLibraryItem.mtime.valueOf()}" to "${this.mtimeMs}"`)
+      libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "mtime" changed from "${existingLibraryItem.mtime?.valueOf()}" to "${this.mtimeMs}"`)
       existingLibraryItem.mtime = this.mtimeMs
       this.hasChanges = true
     }
-    if (existingLibraryItem.birthtime.valueOf() !== this.birthtimeMs) {
+    if (existingLibraryItem.birthtime?.valueOf() !== this.birthtimeMs) {
-      libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "birthtime" changed from "${existingLibraryItem.birthtime.valueOf()}" to "${this.birthtimeMs}"`)
+      libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "birthtime" changed from "${existingLibraryItem.birthtime?.valueOf()}" to "${this.birthtimeMs}"`)
       existingLibraryItem.birthtime = this.birthtimeMs
       this.hasChanges = true
     }
-    if (existingLibraryItem.ctime.valueOf() !== this.ctimeMs) {
+    if (existingLibraryItem.ctime?.valueOf() !== this.ctimeMs) {
-      libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "ctime" changed from "${existingLibraryItem.ctime.valueOf()}" to "${this.ctimeMs}"`)
+      libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "ctime" changed from "${existingLibraryItem.ctime?.valueOf()}" to "${this.ctimeMs}"`)
       existingLibraryItem.ctime = this.ctimeMs
       this.hasChanges = true
     }
@@ -221,14 +222,15 @@ class LibraryItemScanData {
       existingLibraryItem.lastScanVersion = packageJson.version
 
       libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" changed: [${existingLibraryItem.changed()?.join(',') || ''}]`)
-      libraryScan.resultsUpdated++
 
       if (this.hasLibraryFileChanges) {
         existingLibraryItem.changed('libraryFiles', true)
       }
       await existingLibraryItem.save()
+      return true
     } else {
       libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" is up-to-date`)
+      return false
     }
   }
 
@@ -249,10 +251,6 @@ class LibraryItemScanData {
     }
 
     for (const key in existingLibraryFile.metadata) {
-      if (existingLibraryFile.metadata.relPath === 'metadata.json' || existingLibraryFile.metadata.relPath === 'metadata.abs') {
-        if (key === 'mtimeMs' || key === 'size') continue
-      }
-
       if (existingLibraryFile.metadata[key] !== scannedLibraryFile.metadata[key]) {
         if (key !== 'path' && key !== 'relPath') {
           libraryScan.addLog(LogLevel.DEBUG, `Library file "${existingLibraryFile.metadata.path}" for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`)

server/scanner/LibraryScan.js

@@ -27,6 +27,11 @@ class LibraryScan {
     this.resultsAdded = 0
     this.resultsUpdated = 0
 
+    /** @type {string[]} */
+    this.authorsRemovedFromBooks = []
+    /** @type {string[]} */
+    this.seriesRemovedFromBooks = []
+
     this.logs = []
   }
 

server/scanner/LibraryScanner.js

@@ -1,3 +1,4 @@
+const sequelize = require('sequelize')
 const Path = require('path')
 const packageJson = require('../../package.json')
 const Logger = require('../Logger')
@@ -7,14 +8,10 @@ const fs = require('../libs/fsExtra')
 const fileUtils = require('../utils/fileUtils')
 const scanUtils = require('../utils/scandir')
 const { ScanResult, LogLevel } = require('../utils/constants')
-const globals = require('../utils/globals')
 const libraryFilters = require('../utils/queries/libraryFilters')
-const AudioFileScanner = require('./AudioFileScanner')
 const ScanOptions = require('./ScanOptions')
 const LibraryScan = require('./LibraryScan')
 const LibraryItemScanData = require('./LibraryItemScanData')
-const AudioFile = require('../objects/files/AudioFile')
-const Book = require('../models/Book')
 const BookScanner = require('./BookScanner')
 
 class LibraryScanner {
@@ -91,6 +88,7 @@ class LibraryScanner {
   /**
    * 
    * @param {import('./LibraryScan')} libraryScan 
+   * @returns {boolean} true if scan canceled
    */
   async scanLibrary(libraryScan) {
     // Make sure library filter data is set
@@ -113,11 +111,13 @@ class LibraryScanner {
     const existingLibraryItems = await Database.libraryItemModel.findAll({
       where: {
         libraryId: libraryScan.libraryId
-      },
+      }
-      attributes: ['id', 'mediaId', 'mediaType', 'path', 'relPath', 'ino', 'isMissing', 'isFile', 'mtime', 'ctime', 'birthtime', 'libraryFiles', 'libraryFolderId', 'size']
     })
 
+    if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+
     const libraryItemIdsMissing = []
+    let oldLibraryItemsUpdated = []
     for (const existingLibraryItem of existingLibraryItems) {
       // First try to find matching library item with exact file path
       let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path)
@@ -138,15 +138,81 @@ class LibraryScanner {
           libraryScan.resultsMissing++
           if (!existingLibraryItem.isMissing) {
             libraryItemIdsMissing.push(existingLibraryItem.id)
+
+            // TODO: Temporary while using old model to socket emit
+            const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id)
+            oldLibraryItem.isMissing = true
+            oldLibraryItem.updatedAt = Date.now()
+            oldLibraryItemsUpdated.push(oldLibraryItem)
           }
         }
       } else {
         libraryItemDataFound = libraryItemDataFound.filter(lidf => lidf !== libraryItemData)
-        await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)
+        if (await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)) {
-        if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
+          libraryScan.resultsUpdated++
-          await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
+          if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
+            const libraryItem = await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
+            const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
+            await oldLibraryItem.saveMetadata() // Save metadata.json
+            oldLibraryItemsUpdated.push(oldLibraryItem)
+          } else {
+            // TODO: Temporary while using old model to socket emit
+            const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id)
+            oldLibraryItemsUpdated.push(oldLibraryItem)
+          }
         }
       }
+
+      // Emit item updates in chunks of 10 to client
+      if (oldLibraryItemsUpdated.length === 10) {
+        // TODO: Should only emit to clients where library item is accessible
+        SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded()))
+        oldLibraryItemsUpdated = []
+      }
+
+      if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+    }
+    // Emit item updates to client
+    if (oldLibraryItemsUpdated.length) {
+      // TODO: Should only emit to clients where library item is accessible
+      SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded()))
+    }
+
+    // Check authors that were removed from a book and remove them if they no longer have any books
+    //  keep authors without books that have a asin, description or imagePath
+    if (libraryScan.authorsRemovedFromBooks.length) {
+      const bookAuthorsToRemove = (await Database.authorModel.findAll({
+        where: [
+          {
+            id: libraryScan.authorsRemovedFromBooks,
+            asin: {
+              [sequelize.Op.or]: [null, ""]
+            },
+            description: {
+              [sequelize.Op.or]: [null, ""]
+            },
+            imagePath: {
+              [sequelize.Op.or]: [null, ""]
+            }
+          },
+          sequelize.where(sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 0)
+        ],
+        attributes: ['id'],
+        raw: true
+      })).map(au => au.id)
+      if (bookAuthorsToRemove.length) {
+        await Database.authorModel.destroy({
+          where: {
+            id: bookAuthorsToRemove
+          }
+        })
+        bookAuthorsToRemove.forEach((authorId) => {
+          Database.removeAuthorFromFilterData(libraryScan.libraryId, authorId)
+          // TODO: Clients were expecting full author in payload but its unnecessary
+          SocketAuthority.emitter('author_removed', { id: authorId })
+        })
+        libraryScan.addLog(LogLevel.INFO, `Removed ${bookAuthorsToRemove.length} authors`)
+      }
     }
 
     // Update missing library items
@@ -163,17 +229,36 @@ class LibraryScanner {
       })
     }
 
+    if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+
     // Add new library items
     if (libraryItemDataFound.length) {
+      let newOldLibraryItems = []
       for (const libraryItemData of libraryItemDataFound) {
         const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan)
         if (newLibraryItem) {
+          const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem)
+          await oldLibraryItem.saveMetadata() // Save metadata.json
+          newOldLibraryItems.push(oldLibraryItem)
+
           libraryScan.resultsAdded++
         }
+
+        // Emit new items in chunks of 10 to client
+        if (newOldLibraryItems.length === 10) {
+          // TODO: Should only emit to clients where library item is accessible
+          SocketAuthority.emitter('items_added', newOldLibraryItems.map(li => li.toJSONExpanded()))
+          newOldLibraryItems = []
+        }
+
+        if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+      }
+      // Emit new items to client
+      if (newOldLibraryItems.length) {
+        // TODO: Should only emit to clients where library item is accessible
+        SocketAuthority.emitter('items_added', newOldLibraryItems.map(li => li.toJSONExpanded()))
       }
     }
-
-    // TODO: Socket emitter
   }
 
   /**
@@ -253,140 +338,8 @@ class LibraryScanner {
    */
   async rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) {
     if (existingLibraryItem.mediaType === 'book') {
-      /** @type {Book} */
+      const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
-      const media = await existingLibraryItem.getMedia({
+      return libraryItem
-        include: [
-          {
-            model: Database.authorModel,
-            through: {
-              attributes: ['createdAt']
-            }
-          },
-          {
-            model: Database.seriesModel,
-            through: {
-              attributes: ['sequence', 'createdAt']
-            }
-          }
-        ]
-      })
-
-      let hasMediaChanges = libraryItemData.hasAudioFileChanges
-      if (libraryItemData.hasAudioFileChanges || libraryItemData.audioLibraryFiles.length !== media.audioFiles.length) {
-        // Filter out audio files that were removed
-        media.audioFiles = media.audioFiles.filter(af => libraryItemData.checkAudioFileRemoved(af))
-
-        // Update audio files that were modified
-        if (libraryItemData.audioLibraryFilesModified.length) {
-          let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified)
-          media.audioFiles = media.audioFiles.map((audioFileObj) => {
-            let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === audioFileObj.metadata.path)
-            if (!matchedScannedAudioFile) {
-              matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.ino === audioFileObj.ino)
-            }
-
-            if (matchedScannedAudioFile) {
-              scannedAudioFiles = scannedAudioFiles.filter(saf => saf !== matchedScannedAudioFile)
-              const audioFile = new AudioFile(audioFileObj)
-              audioFile.updateFromScan(matchedScannedAudioFile)
-              return audioFile.toJSON()
-            }
-            return audioFileObj
-          })
-          // Modified audio files that were not found on the book
-          if (scannedAudioFiles.length) {
-            media.audioFiles.push(...scannedAudioFiles)
-          }
-        }
-
-        // Add new audio files scanned in
-        if (libraryItemData.audioLibraryFilesAdded.length) {
-          const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesAdded)
-          media.audioFiles.push(...scannedAudioFiles)
-        }
-
-        // Add audio library files that are not already set on the book (safety check)
-        let audioLibraryFilesToAdd = []
-        for (const audioLibraryFile of libraryItemData.audioLibraryFiles) {
-          if (!media.audioFiles.some(af => af.ino === audioLibraryFile.ino)) {
-            libraryScan.addLog(LogLevel.DEBUG, `Existing audio library file "${audioLibraryFile.metadata.relPath}" was not set on book "${media.title}" so setting it now`)
-            audioLibraryFilesToAdd.push(audioLibraryFile)
-          }
-        }
-        if (audioLibraryFilesToAdd.length) {
-          const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, audioLibraryFilesToAdd)
-          media.audioFiles.push(...scannedAudioFiles)
-        }
-
-        media.audioFiles = AudioFileScanner.runSmartTrackOrder(existingLibraryItem.relPath, media.audioFiles)
-
-        media.duration = 0
-        media.audioFiles.forEach((af) => {
-          if (!isNaN(af.duration)) {
-            media.duration += af.duration
-          }
-        })
-
-        media.changed('audioFiles', true)
-      }
-
-      // Check if cover was removed
-      if (media.coverPath && !libraryItemData.imageLibraryFiles.some(lf => lf.metadata.path === media.coverPath)) {
-        media.coverPath = null
-        hasMediaChanges = true
-      }
-
-      // Check if cover is not set and image files were found
-      if (!media.coverPath && libraryItemData.imageLibraryFiles.length) {
-        // Prefer using a cover image with the name "cover" otherwise use the first image
-        const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path))
-        media.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path
-        hasMediaChanges = true
-      }
-
-      // Check if ebook was removed
-      if (media.ebookFile && (libraryScan.library.settings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) {
-        media.ebookFile = null
-        hasMediaChanges = true
-      }
-
-      // Check if ebook is not set and ebooks were found
-      if (!media.ebookFile && !libraryScan.library.settings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) {
-        // Prefer to use an epub ebook then fallback to the first ebook found
-        let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub')
-        if (!ebookLibraryFile) ebookLibraryFile = libraryItemData.ebookLibraryFiles[0]
-        // Ebook file is the same as library file except for additional `ebookFormat`
-        ebookLibraryFile.ebookFormat = ebookLibraryFile.metadata.ext.slice(1).toLowerCase()
-        media.ebookFile = ebookLibraryFile
-        media.changed('ebookFile', true)
-        hasMediaChanges = true
-      }
-
-      // Check/update the isSupplementary flag on libraryFiles for the LibraryItem
-      let libraryItemUpdated = false
-      for (const libraryFile of existingLibraryItem.libraryFiles) {
-        if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) {
-          if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) {
-            if (libraryFile.isSupplementary !== false) {
-              libraryFile.isSupplementary = false
-              libraryItemUpdated = true
-            }
-          } else if (libraryFile.isSupplementary !== true) {
-            libraryFile.isSupplementary = true
-            libraryItemUpdated = true
-          }
-        }
-      }
-      if (libraryItemUpdated) {
-        existingLibraryItem.changed('libraryFiles', true)
-        await existingLibraryItem.save()
-      }
-
-      // TODO: Update chapters & metadata
-
-      if (hasMediaChanges) {
-        await media.save()
-      }
     } else {
       // TODO: Scan updated podcast
     }

server/scanner/MediaProbeData.js

@@ -42,13 +42,8 @@ class MediaProbeData {
     }
   }
 
-  getEmbeddedCoverArt(videoStream) {
-    const ImageCodecs = ['mjpeg', 'jpeg', 'png']
-    return ImageCodecs.includes(videoStream.codec) ? videoStream.codec : null
-  }
-
   setData(data) {
-    this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : null
+    this.embeddedCoverArt = data.video_stream?.codec || null
     this.format = data.format
     this.duration = data.duration
     this.size = data.size

server/utils/generators/abmetadataGenerator.js

@@ -324,6 +324,10 @@ function parseAbMetadataText(text, mediaType) {
 
   mediaDetails.chapters.sort((a, b) => a.start - b.start)
 
+  if (mediaDetails.chapters.length) {
+    mediaDetails.chapters = cleanChaptersArray(mediaDetails.chapters, mediaDetails.metadata.title) || []
+  }
+
   return mediaDetails
 }
 module.exports.parse = parseAbMetadataText
@@ -425,9 +429,8 @@ function parseJsonMetadataText(text) {
     if (abmetadataData.tags?.length) {
       abmetadataData.tags = [...new Set(abmetadataData.tags.map(t => t?.trim()).filter(t => t))]
     }
-    // TODO: Clean chapters
     if (abmetadataData.chapters?.length) {
-
+      abmetadataData.chapters = cleanChaptersArray(abmetadataData.chapters, abmetadataData.metadata.title)
     }
     // clean remove dupes
     if (abmetadataData.metadata.authors?.length) {