Update new library scanner to handle metadata file changes

server/Database.js

@@ -325,9 +325,9 @@ class Database {
     return Promise.all(oldUsers.map(u => this.updateUser(u)))
   }
 
-  async removeUser(userId) {
+  removeUser(userId) {
     if (!this.sequelize) return false
-    await this.models.user.removeById(userId)
+    return this.models.user.removeById(userId)
   }
 
   upsertMediaProgress(oldMediaProgress) {
@@ -345,9 +345,9 @@ class Database {
     return Promise.all(oldBooks.map(oldBook => this.models.book.saveFromOld(oldBook)))
   }
 
-  async createLibrary(oldLibrary) {
+  createLibrary(oldLibrary) {
     if (!this.sequelize) return false
-    await this.models.library.createFromOld(oldLibrary)
+    return this.models.library.createFromOld(oldLibrary)
   }
 
   updateLibrary(oldLibrary) {
@@ -355,9 +355,9 @@ class Database {
     return this.models.library.updateFromOld(oldLibrary)
   }
 
-  async removeLibrary(libraryId) {
+  removeLibrary(libraryId) {
     if (!this.sequelize) return false
-    await this.models.library.removeById(libraryId)
+    return this.models.library.removeById(libraryId)
   }
 
   createBulkCollectionBooks(collectionBooks) {

server/objects/LibraryItem.js

@@ -10,7 +10,7 @@ const Podcast = require('./mediaTypes/Podcast')
 const Video = require('./mediaTypes/Video')
 const Music = require('./mediaTypes/Music')
 const { areEquivalent, copyValue, cleanStringForSearch } = require('../utils/index')
-const { filePathToPOSIX } = require('../utils/fileUtils')
+const { filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils')
 
 class LibraryItem {
   constructor(libraryItem = null) {
@@ -40,6 +40,7 @@ class LibraryItem {
     this.mediaType = null
     this.media = null
 
+    /** @type {LibraryFile[]} */
     this.libraryFiles = []
 
     if (libraryItem) {
@@ -525,19 +526,20 @@ class LibraryItem {
 
   /**
    * Save metadata.json/metadata.abs file
-   * @returns {boolean} true if saved
+   * @returns {Promise<LibraryFile>} null if not saved
    */
   async saveMetadata() {
-    if (this.mediaType === 'video' || this.mediaType === 'music') return
+    if (this.isSavingMetadata) return null
 
-    if (this.isSavingMetadata) return
     this.isSavingMetadata = true
 
     let metadataPath = Path.join(global.MetadataPath, 'items', this.id)
-    if (global.ServerSettings.storeMetadataWithItem && !this.isFile) {
+    let storeMetadataWithItem = global.ServerSettings.storeMetadataWithItem
+    if (storeMetadataWithItem && !this.isFile) {
       metadataPath = this.path
     } else {
       // Make sure metadata book dir exists
+      storeMetadataWithItem = false
       await fs.ensureDir(metadataPath)
     }
 
@@ -552,20 +554,29 @@ class LibraryItem {
       }
 
       return fs.writeFile(metadataFilePath, JSON.stringify(this.media.toJSONForMetadataFile(), null, 2)).then(async () => {
-        this.isSavingMetadata = false
         // Add metadata.json to libraryFiles array if it is new
-        if (global.ServerSettings.storeMetadataWithItem && !this.libraryFiles.some(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))) {
+        let metadataLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
-          const newLibraryFile = new LibraryFile()
+        if (storeMetadataWithItem && !metadataLibraryFile) {
-          await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
+          metadataLibraryFile = new LibraryFile()
-          this.libraryFiles.push(newLibraryFile)
+          await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
+          this.libraryFiles.push(metadataLibraryFile)
+        } else if (storeMetadataWithItem) {
+          const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
+          if (fileTimestamps) {
+            metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
+            metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
+            metadataLibraryFile.metadata.size = fileTimestamps.size
+            metadataLibraryFile.ino = fileTimestamps.ino
+          }
         }
         Logger.debug(`[LibraryItem] Success saving abmetadata to "${metadataFilePath}"`)
 
-        return true
+        return metadataLibraryFile
       }).catch((error) => {
-        this.isSavingMetadata = false
         Logger.error(`[LibraryItem] Failed to save json file at "${metadataFilePath}"`, error)
-        return false
+        return null
+      }).finally(() => {
+        this.isSavingMetadata = false
       })
     } else {
       // Remove metadata.json if it exists
@@ -576,19 +587,30 @@ class LibraryItem {
       }
 
       return abmetadataGenerator.generate(this, metadataFilePath).then(async (success) => {
-        this.isSavingMetadata = false
+        if (!success) {
-        if (!success) Logger.error(`[LibraryItem] Failed saving abmetadata to "${metadataFilePath}"`)
+          Logger.error(`[LibraryItem] Failed saving abmetadata to "${metadataFilePath}"`)
-        else {
+          return null
-          // Add metadata.abs to libraryFiles array if it is new
-          if (global.ServerSettings.storeMetadataWithItem && !this.libraryFiles.some(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))) {
-            const newLibraryFile = new LibraryFile()
-            await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
-            this.libraryFiles.push(newLibraryFile)
-          }
-
-          Logger.debug(`[LibraryItem] Success saving abmetadata to "${metadataFilePath}"`)
         }
-        return success
+        // Add metadata.abs to libraryFiles array if it is new
+        let metadataLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
+        if (storeMetadataWithItem && !metadataLibraryFile) {
+          metadataLibraryFile = new LibraryFile()
+          await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
+          this.libraryFiles.push(metadataLibraryFile)
+        } else if (storeMetadataWithItem) {
+          const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
+          if (fileTimestamps) {
+            metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
+            metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
+            metadataLibraryFile.metadata.size = fileTimestamps.size
+            metadataLibraryFile.ino = fileTimestamps.ino
+          }
+        }
+
+        Logger.debug(`[LibraryItem] Success saving abmetadata to "${metadataFilePath}"`)
+        return metadataLibraryFile
+      }).finally(() => {
+        this.isSavingMetadata = false
       })
     }
   }

server/scanner/BookScanner.js

@@ -1,4 +1,5 @@
 const uuidv4 = require("uuid").v4
+const Path = require('path')
 const { Sequelize } = require('sequelize')
 const { LogLevel } = require('../utils/constants')
 const { getTitleIgnorePrefix, areEquivalent } = require('../utils/index')
@@ -9,9 +10,10 @@ const parseNameString = require('../utils/parsers/parseNameString')
 const globals = require('../utils/globals')
 const AudioFileScanner = require('./AudioFileScanner')
 const Database = require('../Database')
-const { readTextFile } = require('../utils/fileUtils')
+const { readTextFile, filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils')
 const AudioFile = require('../objects/files/AudioFile')
 const CoverManager = require('../managers/CoverManager')
+const LibraryFile = require('../objects/files/LibraryFile')
 const fsExtra = require("../libs/fsExtra")
 
 /**
@@ -161,26 +163,6 @@ class BookScanner {
       hasMediaChanges = true
     }
 
-    // Check/update the isSupplementary flag on libraryFiles for the LibraryItem
-    let libraryItemUpdated = false
-    for (const libraryFile of existingLibraryItem.libraryFiles) {
-      if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) {
-        if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) {
-          if (libraryFile.isSupplementary !== false) {
-            libraryFile.isSupplementary = false
-            libraryItemUpdated = true
-          }
-        } else if (libraryFile.isSupplementary !== true) {
-          libraryFile.isSupplementary = true
-          libraryItemUpdated = true
-        }
-      }
-    }
-    if (libraryItemUpdated) {
-      existingLibraryItem.changed('libraryFiles', true)
-      await existingLibraryItem.save()
-    }
-
     // TODO: When metadata file is stored in /metadata/items/{libraryItemId}.[abs|json] we should load this
     // TODO: store an additional array of metadata keys that the user has changed manually so we know what not to override
     const bookMetadata = await this.getBookMetadataFromScanData(media.audioFiles, libraryItemData, libraryScan)
@@ -317,11 +299,6 @@ class BookScanner {
       }
     }
 
-    // Save Book changes to db
-    if (hasMediaChanges) {
-      await media.save()
-    }
-
     // Load authors/series again if updated (for sending back to client)
     if (authorsUpdated) {
       media.authors = await media.getAuthors({
@@ -340,10 +317,39 @@ class BookScanner {
       })
     }
 
+    existingLibraryItem.media = media
+
+    let libraryItemUpdated = false
+
+    // Save Book changes to db
+    if (hasMediaChanges) {
+      await media.save()
+      await this.saveMetadataFile(existingLibraryItem, libraryScan)
+      libraryItemUpdated = global.ServerSettings.storeMetadataWithItem && !existingLibraryItem.isFile
+    }
+
+    // Check/update the isSupplementary flag on libraryFiles for the LibraryItem
+    for (const libraryFile of existingLibraryItem.libraryFiles) {
+      if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) {
+        if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) {
+          if (libraryFile.isSupplementary !== false) {
+            libraryFile.isSupplementary = false
+            libraryItemUpdated = true
+          }
+        } else if (libraryFile.isSupplementary !== true) {
+          libraryFile.isSupplementary = true
+          libraryItemUpdated = true
+        }
+      }
+    }
+    if (libraryItemUpdated) {
+      existingLibraryItem.changed('libraryFiles', true)
+      await existingLibraryItem.save()
+    }
+
     libraryScan.seriesRemovedFromBooks.push(...bookSeriesRemoved)
     libraryScan.authorsRemovedFromBooks.push(...bookAuthorsRemoved)
 
-    existingLibraryItem.media = media
     return existingLibraryItem
   }
 
@@ -509,6 +515,12 @@ class BookScanner {
       ]
     })
 
+    await this.saveMetadataFile(libraryItem, libraryScan)
+    if (global.ServerSettings.storeMetadataWithItem && !libraryItem.isFile) {
+      libraryItem.changed('libraryFiles', true)
+      await libraryItem.save()
+    }
+
     return libraryItem
   }
 
@@ -691,7 +703,7 @@ class BookScanner {
     const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile
     const metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
     if (metadataText) {
-      libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataLibraryFile.metadata.relPath}" - preferring`)
+      libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataLibraryFile.metadata.path}" - preferring`)
       let abMetadata = null
       if (!!libraryItemData.metadataJsonLibraryFile) {
         abMetadata = abmetadataGenerator.parseJson(metadataText)
@@ -707,7 +719,7 @@ class BookScanner {
           bookMetadata.chapters = abMetadata.chapters
         }
         for (const key in abMetadata.metadata) {
-          if (bookMetadata[key] === undefined || abMetadata.metadata[key] === undefined) continue
+          if (abMetadata.metadata[key] === undefined) continue
           bookMetadata[key] = abMetadata.metadata[key]
         }
       }
@@ -803,7 +815,7 @@ class BookScanner {
       // Build chapters from audio files
       let currChapterId = 0
       let currStartTime = 0
-      includedAudioFiles.forEach((file) => {
+      audioFiles.forEach((file) => {
         if (file.duration) {
           let title = file.metadata.filename ? Path.basename(file.metadata.filename, Path.extname(file.metadata.filename)) : `Chapter ${currChapterId}`
 
@@ -824,5 +836,118 @@ class BookScanner {
     }
     return chapters
   }
+
+  /**
+   * 
+   * @param {import('../models/LibraryItem')} libraryItem 
+   * @param {import('./LibraryScan')} libraryScan
+   * @returns {Promise}
+   */
+  async saveMetadataFile(libraryItem, libraryScan) {
+    let metadataPath = Path.join(global.MetadataPath, 'items', libraryItem.id)
+    let storeMetadataWithItem = global.ServerSettings.storeMetadataWithItem
+    if (storeMetadataWithItem && !libraryItem.isFile) {
+      metadataPath = libraryItem.path
+    } else {
+      // Make sure metadata book dir exists
+      storeMetadataWithItem = false
+      await fsExtra.ensureDir(metadataPath)
+    }
+
+    const metadataFileFormat = global.ServerSettings.metadataFileFormat
+    const metadataFilePath = Path.join(metadataPath, `metadata.${metadataFileFormat}`)
+    if (metadataFileFormat === 'json') {
+      // Remove metadata.abs if it exists
+      if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.abs`))) {
+        libraryScan.addLog(LogLevel.DEBUG, `Removing metadata.abs for item "${libraryItem.media.title}"`)
+        await fsExtra.remove(Path.join(metadataPath, `metadata.abs`))
+        libraryItem.libraryFiles = libraryItem.libraryFiles.filter(lf => lf.metadata.path !== filePathToPOSIX(Path.join(metadataPath, `metadata.abs`)))
+      }
+
+      // TODO: Update to not use `metadata` so it fits the updated model
+      const jsonObject = {
+        tags: libraryItem.media.tags || [],
+        chapters: libraryItem.media.chapters?.map(c => ({ ...c })) || [],
+        metadata: {
+          title: libraryItem.media.title,
+          subtitle: libraryItem.media.subtitle,
+          authors: libraryItem.media.authors.map(a => a.name),
+          narrators: libraryItem.media.narrators,
+          series: libraryItem.media.series.map(se => {
+            const sequence = se.bookSeries?.sequence || ''
+            if (!sequence) return se.name
+            return `${se.name} #${sequence}`
+          }),
+          genres: libraryItem.media.genres || [],
+          publishedYear: libraryItem.media.publishedYear,
+          publishedDate: libraryItem.media.publishedDate,
+          publisher: libraryItem.media.publisher,
+          description: libraryItem.media.description,
+          isbn: libraryItem.media.isbn,
+          asin: libraryItem.media.asin,
+          language: libraryItem.media.language,
+          explicit: !!libraryItem.media.explicit,
+          abridged: !!libraryItem.media.abridged
+        }
+      }
+      return fsExtra.writeFile(metadataFilePath, JSON.stringify(jsonObject, null, 2)).then(async () => {
+        // Add metadata.json to libraryFiles array if it is new
+        let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
+        if (storeMetadataWithItem && !metadataLibraryFile) {
+          const newLibraryFile = new LibraryFile()
+          await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
+          metadataLibraryFile = newLibraryFile.toJSON()
+          libraryItem.libraryFiles.push(metadataLibraryFile)
+        } else if (storeMetadataWithItem) {
+          const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
+          if (fileTimestamps) {
+            metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
+            metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
+            metadataLibraryFile.metadata.size = fileTimestamps.size
+            metadataLibraryFile.ino = fileTimestamps.ino
+          }
+        }
+        libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`)
+
+        return metadataLibraryFile
+      }).catch((error) => {
+        libraryScan.addLog(LogLevel.ERROR, `Failed to save json file at "${metadataFilePath}"`, error)
+        return null
+      })
+    } else {
+      // Remove metadata.json if it exists
+      if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.json`))) {
+        libraryScan.addLog(LogLevel.DEBUG, `Removing metadata.json for item "${libraryItem.media.title}"`)
+        await fsExtra.remove(Path.join(metadataPath, `metadata.json`))
+        libraryItem.libraryFiles = libraryItem.libraryFiles.filter(lf => lf.metadata.path !== filePathToPOSIX(Path.join(metadataPath, `metadata.json`)))
+      }
+
+      return abmetadataGenerator.generateFromNewModel(libraryItem, metadataFilePath).then(async (success) => {
+        if (!success) {
+          libraryScan.addLog(LogLevel.ERROR, `Failed saving abmetadata to "${metadataFilePath}"`)
+          return null
+        }
+        // Add metadata.abs to libraryFiles array if it is new
+        let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
+        if (storeMetadataWithItem && !metadataLibraryFile) {
+          const newLibraryFile = new LibraryFile()
+          await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
+          metadataLibraryFile = newLibraryFile.toJSON()
+          libraryItem.libraryFiles.push(metadataLibraryFile)
+        } else if (storeMetadataWithItem) {
+          const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
+          if (fileTimestamps) {
+            metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
+            metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
+            metadataLibraryFile.metadata.size = fileTimestamps.size
+            metadataLibraryFile.ino = fileTimestamps.ino
+          }
+        }
+
+        libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`)
+        return metadataLibraryFile
+      })
+    }
+  }
 }
 module.exports = new BookScanner()

server/scanner/LibraryItemScanData.js

@@ -253,7 +253,7 @@ class LibraryItemScanData {
     for (const key in existingLibraryFile.metadata) {
       if (existingLibraryFile.metadata[key] !== scannedLibraryFile.metadata[key]) {
         if (key !== 'path' && key !== 'relPath') {
-          libraryScan.addLog(LogLevel.DEBUG, `Library file "${existingLibraryFile.metadata.path}" for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`)
+          libraryScan.addLog(LogLevel.DEBUG, `Library file "${existingLibraryFile.metadata.relPath}" for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`)
         } else {
           libraryScan.addLog(LogLevel.DEBUG, `Library file for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`)
         }

server/scanner/LibraryScanner.js

@@ -153,7 +153,6 @@ class LibraryScanner {
           if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
             const libraryItem = await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
             const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
-            await oldLibraryItem.saveMetadata() // Save metadata.json
             oldLibraryItemsUpdated.push(oldLibraryItem)
           } else {
             // TODO: Temporary while using old model to socket emit
@@ -264,7 +263,6 @@ class LibraryScanner {
         const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan)
         if (newLibraryItem) {
           const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem)
-          await oldLibraryItem.saveMetadata() // Save metadata.json
           newOldLibraryItems.push(oldLibraryItem)
 
           libraryScan.resultsAdded++

server/utils/generators/abmetadataGenerator.js

@@ -41,7 +41,7 @@ const podcastMetadataMapper = {
     from: (v) => v || null
   },
   genres: {
-    to: (m) => m.genres.join(', '),
+    to: (m) => m.genres?.join(', ') || '',
     from: (v) => commaSeparatedToArray(v)
   },
   feedUrl: {
@@ -68,11 +68,15 @@ const bookMetadataMapper = {
     from: (v) => v || null
   },
   authors: {
-    to: (m) => m.authorName || '',
+    to: (m) => {
+      if (m.authorName !== undefined) return m.authorName
+      if (!m.authors?.length) return ''
+      return m.authors.map(au => au.name).join(', ')
+    },
     from: (v) => commaSeparatedToArray(v)
   },
   narrators: {
-    to: (m) => m.narratorName || '',
+    to: (m) => m.narrators?.join(', ') || '',
     from: (v) => commaSeparatedToArray(v)
   },
   publishedYear: {
@@ -96,11 +100,19 @@ const bookMetadataMapper = {
     from: (v) => v || null
   },
   genres: {
-    to: (m) => m.genres.join(', '),
+    to: (m) => m.genres?.join(', ') || '',
     from: (v) => commaSeparatedToArray(v)
   },
   series: {
-    to: (m) => m.seriesName,
+    to: (m) => {
+      if (m.seriesName !== undefined) return m.seriesName
+      if (!m.series?.length) return ''
+      return m.series.map((se) => {
+        const sequence = se.bookSeries?.sequence || ''
+        if (!sequence) return se.name
+        return `${se.name} #${sequence}`
+      }).join(', ')
+    },
     from: (v) => {
       return commaSeparatedToArray(v).map(series => { // Return array of { name, sequence }
         let sequence = null
@@ -174,6 +186,45 @@ function generate(libraryItem, outputPath) {
 }
 module.exports.generate = generate
 
+function generateFromNewModel(libraryItem, outputPath) {
+  let fileString = `;ABMETADATA${CurrentAbMetadataVersion}\n`
+  fileString += `#audiobookshelf v${package.version}\n\n`
+
+  const mediaType = libraryItem.mediaType
+
+  fileString += `media=${mediaType}\n`
+  fileString += `tags=${JSON.stringify(libraryItem.media.tags || '')}\n`
+
+  const metadataMapper = metadataMappers[mediaType]
+  for (const key in metadataMapper) {
+    fileString += `${key}=${metadataMapper[key].to(libraryItem.media)}\n`
+  }
+
+  // Description block
+  if (libraryItem.media.description) {
+    fileString += '\n[DESCRIPTION]\n'
+    fileString += libraryItem.media.description + '\n'
+  }
+
+  // Book chapters
+  if (mediaType == 'book' && libraryItem.media.chapters?.length) {
+    fileString += '\n'
+    libraryItem.media.chapters.forEach((chapter) => {
+      fileString += `[CHAPTER]\n`
+      fileString += `start=${chapter.start}\n`
+      fileString += `end=${chapter.end}\n`
+      fileString += `title=${chapter.title}\n`
+    })
+  }
+  return fs.writeFile(outputPath, fileString).then(() => {
+    return filePerms.setDefault(outputPath, true).then(() => true)
+  }).catch((error) => {
+    Logger.error(`[absMetaFileGenerator] Failed to save abs file`, error)
+    return false
+  })
+}
+module.exports.generateFromNewModel = generateFromNewModel
+
 function parseSections(lines) {
   if (!lines || !lines.length || !lines[0].startsWith('[')) { // First line must be section start
     return []