From 2df95c17123f56f3c0627f9f0121b1a999831ee8 Mon Sep 17 00:00:00 2001 From: advplyr Date: Sat, 2 Sep 2023 17:49:28 -0500 Subject: [PATCH] Updates for new book scanner --- client/pages/item/_id/index.vue | 2 + server/controllers/AuthorController.js | 4 +- server/controllers/LibraryController.js | 11 +- server/managers/CoverManager.js | 2 +- server/models/Author.js | 11 + server/objects/files/AudioFile.js | 7 +- server/scanner/BookScanner.js | 377 +++++++++++++++++- server/scanner/LibraryItemScanData.js | 24 +- server/scanner/LibraryScan.js | 5 + server/scanner/LibraryScanner.js | 243 +++++------ server/scanner/MediaProbeData.js | 7 +- .../utils/generators/abmetadataGenerator.js | 7 +- 12 files changed, 508 insertions(+), 192 deletions(-) diff --git a/client/pages/item/_id/index.vue b/client/pages/item/_id/index.vue index ac4e3d8a..902b6835 100644 --- a/client/pages/item/_id/index.vue +++ b/client/pages/item/_id/index.vue @@ -761,6 +761,7 @@ export default { if (this.libraryId) { this.$store.commit('libraries/setCurrentLibrary', this.libraryId) } + this.$eventBus.$on(`${this.libraryItem.id}_updated`, this.libraryItemUpdated) this.$root.socket.on('item_updated', this.libraryItemUpdated) this.$root.socket.on('rss_feed_open', this.rssFeedOpen) this.$root.socket.on('rss_feed_closed', this.rssFeedClosed) @@ -769,6 +770,7 @@ export default { this.$root.socket.on('episode_download_finished', this.episodeDownloadFinished) }, beforeDestroy() { + this.$eventBus.$off(`${this.libraryItem.id}_updated`, this.libraryItemUpdated) this.$root.socket.off('item_updated', this.libraryItemUpdated) this.$root.socket.off('rss_feed_open', this.rssFeedOpen) this.$root.socket.off('rss_feed_closed', this.rssFeedClosed) diff --git a/server/controllers/AuthorController.js b/server/controllers/AuthorController.js index 1b434d45..ee364b68 100644 --- a/server/controllers/AuthorController.js +++ b/server/controllers/AuthorController.js @@ -234,8 +234,8 @@ class AuthorController { return this.cacheManager.handleAuthorCache(res, author, options) } - middleware(req, res, next) { - const author = Database.authors.find(au => au.id === req.params.id) + async middleware(req, res, next) { + const author = await Database.authorModel.getOldById(req.params.id) if (!author) return res.sendStatus(404) if (req.method == 'DELETE' && !req.user.canDelete) { diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index d821a49d..140eb84f 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -573,18 +573,19 @@ class LibraryController { * rssfeed: adds `rssFeed` to series object if a feed is open * progress: adds `progress` to series object with { libraryItemIds:Array, libraryItemIdsFinished:Array, isFinished:boolean } * - * @param {*} req - * @param {*} res - Series + * @param {import('express').Request} req + * @param {import('express').Response} res - Series */ async getSeriesForLibrary(req, res) { const include = (req.query.include || '').split(',').map(v => v.trim().toLowerCase()).filter(v => !!v) - const series = Database.series.find(se => se.id === req.params.seriesId) + const series = await Database.seriesModel.findByPk(req.params.seriesId) if (!series) return res.sendStatus(404) + const oldSeries = series.getOldSeries() - const libraryItemsInSeries = await libraryItemsBookFilters.getLibraryItemsForSeries(series, req.user) + const libraryItemsInSeries = await libraryItemsBookFilters.getLibraryItemsForSeries(oldSeries, req.user) - const seriesJson = series.toJSON() + const seriesJson = oldSeries.toJSON() if (include.includes('progress')) { const libraryItemsFinished = libraryItemsInSeries.filter(li => !!req.user.getMediaProgress(li.id)?.isFinished) seriesJson.progress = { diff --git a/server/managers/CoverManager.js b/server/managers/CoverManager.js index 74fe2f03..821eb453 100644 --- a/server/managers/CoverManager.js +++ b/server/managers/CoverManager.js @@ -279,7 +279,7 @@ class CoverManager { if (global.ServerSettings.storeCoverWithItem && libraryItemPath) { coverDirPath = libraryItemPath } else { - coverDirPath = Path.posix.join(this.ItemMetadataPath, libraryItemId) + coverDirPath = Path.posix.join(global.MetadataPath, 'items', libraryItemId) } await fs.ensureDir(coverDirPath) diff --git a/server/models/Author.js b/server/models/Author.js index 9eeda5bf..ea8fc9db 100644 --- a/server/models/Author.js +++ b/server/models/Author.js @@ -83,6 +83,17 @@ class Author extends Model { }) } + /** + * Get oldAuthor by id + * @param {string} authorId + * @returns {oldAuthor} + */ + static async getOldById(authorId) { + const author = await this.findByPk(authorId) + if (!author) return null + return author.getOldAuthor() + } + /** * Check if author exists * @param {string} authorId diff --git a/server/objects/files/AudioFile.js b/server/objects/files/AudioFile.js index 6e43e73b..0de03606 100644 --- a/server/objects/files/AudioFile.js +++ b/server/objects/files/AudioFile.js @@ -118,7 +118,12 @@ class AudioFile { setDataFromProbe(libraryFile, probeData) { this.ino = libraryFile.ino || null - this.metadata = libraryFile.metadata.clone() + if (libraryFile.metadata instanceof FileMetadata) { + this.metadata = libraryFile.metadata.clone() + } else { + this.metadata = new FileMetadata(libraryFile.metadata) + } + this.addedAt = Date.now() this.updatedAt = Date.now() diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js index 70ab5bca..be408a46 100644 --- a/server/scanner/BookScanner.js +++ b/server/scanner/BookScanner.js @@ -1,15 +1,18 @@ const uuidv4 = require("uuid").v4 +const { Sequelize } = require('sequelize') const { LogLevel } = require('../utils/constants') -const { getTitleIgnorePrefix } = require('../utils/index') +const { getTitleIgnorePrefix, areEquivalent } = require('../utils/index') const { parseOpfMetadataXML } = require('../utils/parsers/parseOpfMetadata') const { parseOverdriveMediaMarkersAsChapters } = require('../utils/parsers/parseOverdriveMediaMarkers') const abmetadataGenerator = require('../utils/generators/abmetadataGenerator') const parseNameString = require('../utils/parsers/parseNameString') +const globals = require('../utils/globals') const AudioFileScanner = require('./AudioFileScanner') const Database = require('../Database') const { readTextFile } = require('../utils/fileUtils') const AudioFile = require('../objects/files/AudioFile') const CoverManager = require('../managers/CoverManager') +const fsExtra = require("../libs/fsExtra") /** * Metadata for books pulled from files @@ -37,6 +40,313 @@ const CoverManager = require('../managers/CoverManager') class BookScanner { constructor() { } + /** + * @param {import('../models/LibraryItem')} existingLibraryItem + * @param {import('./LibraryItemScanData')} libraryItemData + * @param {import('./LibraryScan')} libraryScan + * @returns {import('../models/LibraryItem')} + */ + async rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan) { + /** @type {import('../models/Book')} */ + const media = await existingLibraryItem.getMedia({ + include: [ + { + model: Database.authorModel, + through: { + attributes: ['id', 'createdAt'] + } + }, + { + model: Database.seriesModel, + through: { + attributes: ['id', 'sequence', 'createdAt'] + } + } + ], + order: [ + [Database.authorModel, Database.bookAuthorModel, 'createdAt', 'ASC'], + [Database.seriesModel, 'bookSeries', 'createdAt', 'ASC'] + ] + }) + + let hasMediaChanges = libraryItemData.hasAudioFileChanges + if (libraryItemData.hasAudioFileChanges || libraryItemData.audioLibraryFiles.length !== media.audioFiles.length) { + // Filter out audio files that were removed + media.audioFiles = media.audioFiles.filter(af => !libraryItemData.checkAudioFileRemoved(af)) + + // Update audio files that were modified + if (libraryItemData.audioLibraryFilesModified.length) { + let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified) + media.audioFiles = media.audioFiles.map((audioFileObj) => { + let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === audioFileObj.metadata.path) + if (!matchedScannedAudioFile) { + matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.ino === audioFileObj.ino) + } + + if (matchedScannedAudioFile) { + scannedAudioFiles = scannedAudioFiles.filter(saf => saf !== matchedScannedAudioFile) + const audioFile = new AudioFile(audioFileObj) + audioFile.updateFromScan(matchedScannedAudioFile) + return audioFile.toJSON() + } + return audioFileObj + }) + // Modified audio files that were not found on the book + if (scannedAudioFiles.length) { + media.audioFiles.push(...scannedAudioFiles) + } + } + + // Add new audio files scanned in + if (libraryItemData.audioLibraryFilesAdded.length) { + const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesAdded) + media.audioFiles.push(...scannedAudioFiles) + } + + // Add audio library files that are not already set on the book (safety check) + let audioLibraryFilesToAdd = [] + for (const audioLibraryFile of libraryItemData.audioLibraryFiles) { + if (!media.audioFiles.some(af => af.ino === audioLibraryFile.ino)) { + libraryScan.addLog(LogLevel.DEBUG, `Existing audio library file "${audioLibraryFile.metadata.relPath}" was not set on book "${media.title}" so setting it now`) + + audioLibraryFilesToAdd.push(audioLibraryFile) + } + } + if (audioLibraryFilesToAdd.length) { + const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, audioLibraryFilesToAdd) + media.audioFiles.push(...scannedAudioFiles) + } + + media.audioFiles = AudioFileScanner.runSmartTrackOrder(existingLibraryItem.relPath, media.audioFiles) + + media.duration = 0 + media.audioFiles.forEach((af) => { + if (!isNaN(af.duration)) { + media.duration += af.duration + } + }) + + media.changed('audioFiles', true) + } + + // Check if cover was removed + if (media.coverPath && !libraryItemData.imageLibraryFiles.some(lf => lf.metadata.path === media.coverPath)) { + media.coverPath = null + hasMediaChanges = true + } + + // Check if cover is not set and image files were found + if (!media.coverPath && libraryItemData.imageLibraryFiles.length) { + // Prefer using a cover image with the name "cover" otherwise use the first image + const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path)) + media.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path + hasMediaChanges = true + } + + // Check if ebook was removed + if (media.ebookFile && (libraryScan.library.settings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) { + media.ebookFile = null + hasMediaChanges = true + } + + // Check if ebook is not set and ebooks were found + if (!media.ebookFile && !libraryScan.library.settings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) { + // Prefer to use an epub ebook then fallback to the first ebook found + let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') + if (!ebookLibraryFile) ebookLibraryFile = libraryItemData.ebookLibraryFiles[0] + // Ebook file is the same as library file except for additional `ebookFormat` + ebookLibraryFile.ebookFormat = ebookLibraryFile.metadata.ext.slice(1).toLowerCase() + media.ebookFile = ebookLibraryFile + media.changed('ebookFile', true) + hasMediaChanges = true + } + + // Check/update the isSupplementary flag on libraryFiles for the LibraryItem + let libraryItemUpdated = false + for (const libraryFile of existingLibraryItem.libraryFiles) { + if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) { + if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) { + if (libraryFile.isSupplementary !== false) { + libraryFile.isSupplementary = false + libraryItemUpdated = true + } + } else if (libraryFile.isSupplementary !== true) { + libraryFile.isSupplementary = true + libraryItemUpdated = true + } + } + } + if (libraryItemUpdated) { + existingLibraryItem.changed('libraryFiles', true) + await existingLibraryItem.save() + } + + // TODO: When metadata file is stored in /metadata/items/{libraryItemId}.[abs|json] we should load this + // TODO: store an additional array of metadata keys that the user has changed manually so we know what not to override + const bookMetadata = await this.getBookMetadataFromScanData(media.audioFiles, libraryItemData, libraryScan) + let authorsUpdated = false + const bookAuthorsRemoved = [] + let seriesUpdated = false + const bookSeriesRemoved = [] + + for (const key in bookMetadata) { + // Ignore unset metadata and empty arrays + if (bookMetadata[key] === undefined || (Array.isArray(bookMetadata[key]) && !bookMetadata[key].length)) continue + + if (key === 'authors') { + // Check for authors added + for (const authorName of bookMetadata.authors) { + if (!media.authors.some(au => au.name === authorName)) { + const existingAuthor = Database.libraryFilterData[libraryScan.libraryId].authors.find(au => au.name === authorName) + if (existingAuthor) { + await Database.bookAuthorModel.create({ + bookId: media.id, + authorId: existingAuthor.id + }) + libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added author "${authorName}"`) + authorsUpdated = true + } else { + const newAuthor = await Database.authorModel.create({ + name: authorName, + lastFirst: parseNameString.nameToLastFirst(authorName), + libraryId: libraryScan.libraryId + }) + await media.addAuthor(newAuthor) + Database.addAuthorToFilterData(libraryScan.libraryId, newAuthor.name, newAuthor.id) + libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added new author "${authorName}"`) + authorsUpdated = true + } + } + } + // Check for authors removed + for (const author of media.authors) { + if (!bookMetadata.authors.includes(author.name)) { + await author.bookAuthor.destroy() + libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" removed author "${author.name}"`) + authorsUpdated = true + bookAuthorsRemoved.push(author.id) + } + } + } else if (key === 'series') { + // Check for series added + for (const seriesObj of bookMetadata.series) { + if (!media.series.some(se => se.name === seriesObj.name)) { + const existingSeries = Database.libraryFilterData[libraryScan.libraryId].series.find(se => se.name === seriesObj.name) + if (existingSeries) { + await Database.bookSeriesModel.create({ + bookId: media.id, + seriesId: existingSeries.id, + sequence: seriesObj.sequence + }) + libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added series "${seriesObj.name}"${seriesObj.sequence ? ` with sequence "${seriesObj.sequence}"` : ''}`) + seriesUpdated = true + } else { + const newSeries = await Database.seriesModel.create({ + name: seriesObj.name, + nameIgnorePrefix: getTitleIgnorePrefix(seriesObj.name), + libraryId: libraryScan.libraryId + }) + await media.addSeries(newSeries) + Database.addSeriesToFilterData(libraryScan.libraryId, newSeries.name, newSeries.id) + libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added new series "${seriesObj.name}"${seriesObj.sequence ? ` with sequence "${seriesObj.sequence}"` : ''}`) + seriesUpdated = true + } + } + } + // Check for series removed + for (const series of media.series) { + if (!bookMetadata.series.some(se => se.name === series.name)) { + await series.bookSeries.destroy() + libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" removed series "${series.name}"`) + seriesUpdated = true + bookSeriesRemoved.push(series.id) + } + } + } else if (key === 'genres') { + const existingGenres = media.genres || [] + if (bookMetadata.genres.some(g => !existingGenres.includes(g)) || existingGenres.some(g => !bookMetadata.genres.includes(g))) { + libraryScan.addLog(LogLevel.DEBUG, `Updating book genres "${existingGenres.join(',')}" => "${bookMetadata.genres.join(',')}" for book "${bookMetadata.title}"`) + media.genres = bookMetadata.genres + hasMediaChanges = true + } + } else if (key === 'tags') { + const existingTags = media.tags || [] + if (bookMetadata.tags.some(t => !existingTags.includes(t)) || existingTags.some(t => !bookMetadata.tags.includes(t))) { + libraryScan.addLog(LogLevel.DEBUG, `Updating book tags "${existingTags.join(',')}" => "${bookMetadata.tags.join(',')}" for book "${bookMetadata.title}"`) + media.tags = bookMetadata.tags + hasMediaChanges = true + } + } else if (key === 'narrators') { + const existingNarrators = media.narrators || [] + if (bookMetadata.narrators.some(t => !existingNarrators.includes(t)) || existingNarrators.some(t => !bookMetadata.narrators.includes(t))) { + libraryScan.addLog(LogLevel.DEBUG, `Updating book narrators "${existingNarrators.join(',')}" => "${bookMetadata.narrators.join(',')}" for book "${bookMetadata.title}"`) + media.narrators = bookMetadata.narrators + hasMediaChanges = true + } + } else if (key === 'chapters') { + if (!areEquivalent(media.chapters, bookMetadata.chapters)) { + libraryScan.addLog(LogLevel.DEBUG, `Updating book chapters for book "${bookMetadata.title}"`) + media.chapters = bookMetadata.chapters + hasMediaChanges = true + } + } else if (key === 'coverPath') { + if (media.coverPath && media.coverPath !== bookMetadata.coverPath && !(await fsExtra.pathExists(media.coverPath))) { + libraryScan.addLog(LogLevel.DEBUG, `Updating book cover "${media.coverPath}" => "${bookMetadata.coverPath}" for book "${bookMetadata.title}" - original cover path does not exist`) + media.coverPath = bookMetadata.coverPath + hasMediaChanges = true + } else if (!media.coverPath) { + libraryScan.addLog(LogLevel.DEBUG, `Updating book cover "unset" => "${bookMetadata.coverPath}" for book "${bookMetadata.title}"`) + media.coverPath = bookMetadata.coverPath + hasMediaChanges = true + } + } else if (bookMetadata[key] !== media[key]) { + libraryScan.addLog(LogLevel.DEBUG, `Updating book ${key} "${media[key]}" => "${bookMetadata[key]}" for book "${bookMetadata.title}"`) + media[key] = bookMetadata[key] + hasMediaChanges = true + } + } + + // If no cover then extract cover from audio file if available + if (!media.coverPath && media.audioFiles.length) { + const libraryItemDir = existingLibraryItem.isFile ? null : existingLibraryItem.path + const extractedCoverPath = await CoverManager.saveEmbeddedCoverArtNew(media.audioFiles, existingLibraryItem.id, libraryItemDir) + if (extractedCoverPath) { + libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" extracted embedded cover art from audio file to path "${extractedCoverPath}"`) + media.coverPath = extractedCoverPath + hasMediaChanges = true + } + } + + // Save Book changes to db + if (hasMediaChanges) { + await media.save() + } + + // Load authors/series again if updated (for sending back to client) + if (authorsUpdated) { + media.authors = await media.getAuthors({ + joinTableAttributes: ['createdAt'], + order: [ + Sequelize.literal(`bookAuthor.createdAt ASC`) + ] + }) + } + if (seriesUpdated) { + media.series = await media.getSeries({ + joinTableAttributes: ['sequence', 'createdAt'], + order: [ + Sequelize.literal(`bookSeries.createdAt ASC`) + ] + }) + } + + libraryScan.seriesRemovedFromBooks.push(...bookSeriesRemoved) + libraryScan.authorsRemovedFromBooks.push(...bookAuthorsRemoved) + + existingLibraryItem.media = media + return existingLibraryItem + } + /** * * @param {import('./LibraryItemScanData')} libraryItemData @@ -49,7 +359,7 @@ class BookScanner { scannedAudioFiles = AudioFileScanner.runSmartTrackOrder(libraryItemData.relPath, scannedAudioFiles) // Find ebook file (prefer epub) - let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') || libraryItemData.ebookLibraryFiles[0] + let ebookLibraryFile = libraryScan.library.settings.audiobooksOnly ? null : libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') || libraryItemData.ebookLibraryFiles[0] // Do not add library items that have no valid audio files and no ebook file if (!ebookLibraryFile && !scannedAudioFiles.length) { @@ -62,6 +372,8 @@ class BookScanner { } const bookMetadata = await this.getBookMetadataFromScanData(scannedAudioFiles, libraryItemData, libraryScan) + bookMetadata.explicit = !!bookMetadata.explicit // Ensure boolean + bookMetadata.abridged = !!bookMetadata.abridged // Ensure boolean let duration = 0 scannedAudioFiles.forEach((af) => duration += (!isNaN(af.duration) ? Number(af.duration) : 0)) @@ -116,6 +428,15 @@ class BookScanner { const libraryItemObj = libraryItemData.libraryItemObject libraryItemObj.id = uuidv4() // Generate library item id ahead of time to use for saving extracted cover image + libraryItemObj.isMissing = false + libraryItemObj.isInvalid = false + + // Set isSupplementary flag on ebook library files + for (const libraryFile of libraryItemObj.libraryFiles) { + if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) { + libraryFile.isSupplementary = libraryFile.ino !== ebookLibraryFile?.ino + } + } // If cover was not found in folder then check embedded covers in audio files if (!bookObject.coverPath && scannedAudioFiles.length) { @@ -166,37 +487,59 @@ class BookScanner { Database.addPublisherToFilterData(libraryScan.libraryId, libraryItem.book.publisher) Database.addLanguageToFilterData(libraryScan.libraryId, libraryItem.book.language) + // Load for emitting to client + libraryItem.media = await libraryItem.getMedia({ + include: [ + { + model: Database.authorModel, + through: { + attributes: ['id', 'createdAt'] + } + }, + { + model: Database.seriesModel, + through: { + attributes: ['id', 'sequence', 'createdAt'] + } + } + ], + order: [ + [Database.authorModel, Database.bookAuthorModel, 'createdAt', 'ASC'], + [Database.seriesModel, 'bookSeries', 'createdAt', 'ASC'] + ] + }) + return libraryItem } /** * - * @param {import('../objects/files/AudioFile')[]} scannedAudioFiles + * @param {import('../models/Book').AudioFileObject[]} audioFiles * @param {import('./LibraryItemScanData')} libraryItemData * @param {import('./LibraryScan')} libraryScan * @returns {Promise} */ - async getBookMetadataFromScanData(scannedAudioFiles, libraryItemData, libraryScan) { + async getBookMetadataFromScanData(audioFiles, libraryItemData, libraryScan) { // First set book metadata from folder/file names const bookMetadata = { title: libraryItemData.mediaMetadata.title, titleIgnorePrefix: getTitleIgnorePrefix(libraryItemData.mediaMetadata.title), - subtitle: libraryItemData.mediaMetadata.subtitle, - publishedYear: libraryItemData.mediaMetadata.publishedYear, - publisher: null, - description: null, - isbn: null, - asin: null, - language: null, + subtitle: libraryItemData.mediaMetadata.subtitle || undefined, + publishedYear: libraryItemData.mediaMetadata.publishedYear || undefined, + publisher: undefined, + description: undefined, + isbn: undefined, + asin: undefined, + language: undefined, narrators: parseNameString.parse(libraryItemData.mediaMetadata.narrators)?.names || [], genres: [], tags: [], authors: parseNameString.parse(libraryItemData.mediaMetadata.author)?.names || [], series: [], chapters: [], - explicit: false, - abridged: false, - coverPath: null + explicit: undefined, + abridged: undefined, + coverPath: undefined } if (libraryItemData.mediaMetadata.series) { bookMetadata.series.push({ @@ -206,7 +549,7 @@ class BookScanner { } // Fill in or override book metadata from audio file meta tags - if (scannedAudioFiles.length) { + if (audioFiles.length) { const MetadataMapArray = [ { tag: 'tagComposer', @@ -261,7 +604,7 @@ class BookScanner { } ] const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata - const firstScannedFile = scannedAudioFiles[0] + const firstScannedFile = audioFiles[0] const audioFileMetaTags = firstScannedFile.metaTags MetadataMapArray.forEach((mapping) => { let value = audioFileMetaTags[mapping.tag] @@ -372,7 +715,7 @@ class BookScanner { // Set chapters from audio files if not already set if (!bookMetadata.chapters.length) { - bookMetadata.chapters = this.getChaptersFromAudioFiles(bookMetadata.title, scannedAudioFiles, libraryScan) + bookMetadata.chapters = this.getChaptersFromAudioFiles(bookMetadata.title, audioFiles, libraryScan) } // Set cover from library file if one is found otherwise check audiofile diff --git a/server/scanner/LibraryItemScanData.js b/server/scanner/LibraryItemScanData.js index 54facc4e..f8bc29c0 100644 --- a/server/scanner/LibraryItemScanData.js +++ b/server/scanner/LibraryItemScanData.js @@ -56,7 +56,7 @@ class LibraryItemScanData { mediaType: this.mediaType, isFile: this.isFile, mtime: this.mtimeMs, - ctime: this.ctime, + ctime: this.ctimeMs, birthtime: this.birthtimeMs, lastScan: Date.now(), lastScanVersion: packageJson.version, @@ -74,7 +74,7 @@ class LibraryItemScanData { /** @type {boolean} */ get hasAudioFileChanges() { - return this.audioLibraryFilesRemoved.length + this.audioLibraryFilesAdded.length + this.audioLibraryFilesModified + return (this.audioLibraryFilesRemoved.length + this.audioLibraryFilesAdded.length + this.audioLibraryFilesModified.length) > 0 } /** @type {LibraryItem.LibraryFileObject[]} */ @@ -136,6 +136,7 @@ class LibraryItemScanData { * * @param {LibraryItem} existingLibraryItem * @param {import('./LibraryScan')} libraryScan + * @returns {boolean} true if changes found */ async checkLibraryItemData(existingLibraryItem, libraryScan) { const keysToCompare = ['libraryFolderId', 'ino', 'path', 'relPath', 'isFile'] @@ -154,18 +155,18 @@ class LibraryItemScanData { } // Check mtime, ctime and birthtime - if (existingLibraryItem.mtime.valueOf() !== this.mtimeMs) { - libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "mtime" changed from "${existingLibraryItem.mtime.valueOf()}" to "${this.mtimeMs}"`) + if (existingLibraryItem.mtime?.valueOf() !== this.mtimeMs) { + libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "mtime" changed from "${existingLibraryItem.mtime?.valueOf()}" to "${this.mtimeMs}"`) existingLibraryItem.mtime = this.mtimeMs this.hasChanges = true } - if (existingLibraryItem.birthtime.valueOf() !== this.birthtimeMs) { - libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "birthtime" changed from "${existingLibraryItem.birthtime.valueOf()}" to "${this.birthtimeMs}"`) + if (existingLibraryItem.birthtime?.valueOf() !== this.birthtimeMs) { + libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "birthtime" changed from "${existingLibraryItem.birthtime?.valueOf()}" to "${this.birthtimeMs}"`) existingLibraryItem.birthtime = this.birthtimeMs this.hasChanges = true } - if (existingLibraryItem.ctime.valueOf() !== this.ctimeMs) { - libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "ctime" changed from "${existingLibraryItem.ctime.valueOf()}" to "${this.ctimeMs}"`) + if (existingLibraryItem.ctime?.valueOf() !== this.ctimeMs) { + libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "ctime" changed from "${existingLibraryItem.ctime?.valueOf()}" to "${this.ctimeMs}"`) existingLibraryItem.ctime = this.ctimeMs this.hasChanges = true } @@ -221,14 +222,15 @@ class LibraryItemScanData { existingLibraryItem.lastScanVersion = packageJson.version libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" changed: [${existingLibraryItem.changed()?.join(',') || ''}]`) - libraryScan.resultsUpdated++ if (this.hasLibraryFileChanges) { existingLibraryItem.changed('libraryFiles', true) } await existingLibraryItem.save() + return true } else { libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" is up-to-date`) + return false } } @@ -249,10 +251,6 @@ class LibraryItemScanData { } for (const key in existingLibraryFile.metadata) { - if (existingLibraryFile.metadata.relPath === 'metadata.json' || existingLibraryFile.metadata.relPath === 'metadata.abs') { - if (key === 'mtimeMs' || key === 'size') continue - } - if (existingLibraryFile.metadata[key] !== scannedLibraryFile.metadata[key]) { if (key !== 'path' && key !== 'relPath') { libraryScan.addLog(LogLevel.DEBUG, `Library file "${existingLibraryFile.metadata.path}" for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`) diff --git a/server/scanner/LibraryScan.js b/server/scanner/LibraryScan.js index 1be4cc66..5ec1c09b 100644 --- a/server/scanner/LibraryScan.js +++ b/server/scanner/LibraryScan.js @@ -27,6 +27,11 @@ class LibraryScan { this.resultsAdded = 0 this.resultsUpdated = 0 + /** @type {string[]} */ + this.authorsRemovedFromBooks = [] + /** @type {string[]} */ + this.seriesRemovedFromBooks = [] + this.logs = [] } diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index 59c0d17c..01dc1d2d 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -1,3 +1,4 @@ +const sequelize = require('sequelize') const Path = require('path') const packageJson = require('../../package.json') const Logger = require('../Logger') @@ -7,14 +8,10 @@ const fs = require('../libs/fsExtra') const fileUtils = require('../utils/fileUtils') const scanUtils = require('../utils/scandir') const { ScanResult, LogLevel } = require('../utils/constants') -const globals = require('../utils/globals') const libraryFilters = require('../utils/queries/libraryFilters') -const AudioFileScanner = require('./AudioFileScanner') const ScanOptions = require('./ScanOptions') const LibraryScan = require('./LibraryScan') const LibraryItemScanData = require('./LibraryItemScanData') -const AudioFile = require('../objects/files/AudioFile') -const Book = require('../models/Book') const BookScanner = require('./BookScanner') class LibraryScanner { @@ -91,6 +88,7 @@ class LibraryScanner { /** * * @param {import('./LibraryScan')} libraryScan + * @returns {boolean} true if scan canceled */ async scanLibrary(libraryScan) { // Make sure library filter data is set @@ -113,11 +111,13 @@ class LibraryScanner { const existingLibraryItems = await Database.libraryItemModel.findAll({ where: { libraryId: libraryScan.libraryId - }, - attributes: ['id', 'mediaId', 'mediaType', 'path', 'relPath', 'ino', 'isMissing', 'isFile', 'mtime', 'ctime', 'birthtime', 'libraryFiles', 'libraryFolderId', 'size'] + } }) + if (this.cancelLibraryScan[libraryScan.libraryId]) return true + const libraryItemIdsMissing = [] + let oldLibraryItemsUpdated = [] for (const existingLibraryItem of existingLibraryItems) { // First try to find matching library item with exact file path let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path) @@ -138,15 +138,81 @@ class LibraryScanner { libraryScan.resultsMissing++ if (!existingLibraryItem.isMissing) { libraryItemIdsMissing.push(existingLibraryItem.id) + + // TODO: Temporary while using old model to socket emit + const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id) + oldLibraryItem.isMissing = true + oldLibraryItem.updatedAt = Date.now() + oldLibraryItemsUpdated.push(oldLibraryItem) } } } else { libraryItemDataFound = libraryItemDataFound.filter(lidf => lidf !== libraryItemData) - await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan) - if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) { - await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) + if (await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)) { + libraryScan.resultsUpdated++ + if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) { + const libraryItem = await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) + const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem) + await oldLibraryItem.saveMetadata() // Save metadata.json + oldLibraryItemsUpdated.push(oldLibraryItem) + } else { + // TODO: Temporary while using old model to socket emit + const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id) + oldLibraryItemsUpdated.push(oldLibraryItem) + } } } + + // Emit item updates in chunks of 10 to client + if (oldLibraryItemsUpdated.length === 10) { + // TODO: Should only emit to clients where library item is accessible + SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded())) + oldLibraryItemsUpdated = [] + } + + if (this.cancelLibraryScan[libraryScan.libraryId]) return true + } + // Emit item updates to client + if (oldLibraryItemsUpdated.length) { + // TODO: Should only emit to clients where library item is accessible + SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded())) + } + + // Check authors that were removed from a book and remove them if they no longer have any books + // keep authors without books that have a asin, description or imagePath + if (libraryScan.authorsRemovedFromBooks.length) { + const bookAuthorsToRemove = (await Database.authorModel.findAll({ + where: [ + { + id: libraryScan.authorsRemovedFromBooks, + asin: { + [sequelize.Op.or]: [null, ""] + }, + description: { + [sequelize.Op.or]: [null, ""] + }, + imagePath: { + [sequelize.Op.or]: [null, ""] + } + }, + sequelize.where(sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 0) + ], + attributes: ['id'], + raw: true + })).map(au => au.id) + if (bookAuthorsToRemove.length) { + await Database.authorModel.destroy({ + where: { + id: bookAuthorsToRemove + } + }) + bookAuthorsToRemove.forEach((authorId) => { + Database.removeAuthorFromFilterData(libraryScan.libraryId, authorId) + // TODO: Clients were expecting full author in payload but its unnecessary + SocketAuthority.emitter('author_removed', { id: authorId }) + }) + libraryScan.addLog(LogLevel.INFO, `Removed ${bookAuthorsToRemove.length} authors`) + } } // Update missing library items @@ -163,17 +229,36 @@ class LibraryScanner { }) } + if (this.cancelLibraryScan[libraryScan.libraryId]) return true + // Add new library items if (libraryItemDataFound.length) { + let newOldLibraryItems = [] for (const libraryItemData of libraryItemDataFound) { const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan) if (newLibraryItem) { + const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem) + await oldLibraryItem.saveMetadata() // Save metadata.json + newOldLibraryItems.push(oldLibraryItem) + libraryScan.resultsAdded++ } + + // Emit new items in chunks of 10 to client + if (newOldLibraryItems.length === 10) { + // TODO: Should only emit to clients where library item is accessible + SocketAuthority.emitter('items_added', newOldLibraryItems.map(li => li.toJSONExpanded())) + newOldLibraryItems = [] + } + + if (this.cancelLibraryScan[libraryScan.libraryId]) return true + } + // Emit new items to client + if (newOldLibraryItems.length) { + // TODO: Should only emit to clients where library item is accessible + SocketAuthority.emitter('items_added', newOldLibraryItems.map(li => li.toJSONExpanded())) } } - - // TODO: Socket emitter } /** @@ -253,140 +338,8 @@ class LibraryScanner { */ async rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) { if (existingLibraryItem.mediaType === 'book') { - /** @type {Book} */ - const media = await existingLibraryItem.getMedia({ - include: [ - { - model: Database.authorModel, - through: { - attributes: ['createdAt'] - } - }, - { - model: Database.seriesModel, - through: { - attributes: ['sequence', 'createdAt'] - } - } - ] - }) - - let hasMediaChanges = libraryItemData.hasAudioFileChanges - if (libraryItemData.hasAudioFileChanges || libraryItemData.audioLibraryFiles.length !== media.audioFiles.length) { - // Filter out audio files that were removed - media.audioFiles = media.audioFiles.filter(af => libraryItemData.checkAudioFileRemoved(af)) - - // Update audio files that were modified - if (libraryItemData.audioLibraryFilesModified.length) { - let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified) - media.audioFiles = media.audioFiles.map((audioFileObj) => { - let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === audioFileObj.metadata.path) - if (!matchedScannedAudioFile) { - matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.ino === audioFileObj.ino) - } - - if (matchedScannedAudioFile) { - scannedAudioFiles = scannedAudioFiles.filter(saf => saf !== matchedScannedAudioFile) - const audioFile = new AudioFile(audioFileObj) - audioFile.updateFromScan(matchedScannedAudioFile) - return audioFile.toJSON() - } - return audioFileObj - }) - // Modified audio files that were not found on the book - if (scannedAudioFiles.length) { - media.audioFiles.push(...scannedAudioFiles) - } - } - - // Add new audio files scanned in - if (libraryItemData.audioLibraryFilesAdded.length) { - const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesAdded) - media.audioFiles.push(...scannedAudioFiles) - } - - // Add audio library files that are not already set on the book (safety check) - let audioLibraryFilesToAdd = [] - for (const audioLibraryFile of libraryItemData.audioLibraryFiles) { - if (!media.audioFiles.some(af => af.ino === audioLibraryFile.ino)) { - libraryScan.addLog(LogLevel.DEBUG, `Existing audio library file "${audioLibraryFile.metadata.relPath}" was not set on book "${media.title}" so setting it now`) - audioLibraryFilesToAdd.push(audioLibraryFile) - } - } - if (audioLibraryFilesToAdd.length) { - const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, audioLibraryFilesToAdd) - media.audioFiles.push(...scannedAudioFiles) - } - - media.audioFiles = AudioFileScanner.runSmartTrackOrder(existingLibraryItem.relPath, media.audioFiles) - - media.duration = 0 - media.audioFiles.forEach((af) => { - if (!isNaN(af.duration)) { - media.duration += af.duration - } - }) - - media.changed('audioFiles', true) - } - - // Check if cover was removed - if (media.coverPath && !libraryItemData.imageLibraryFiles.some(lf => lf.metadata.path === media.coverPath)) { - media.coverPath = null - hasMediaChanges = true - } - - // Check if cover is not set and image files were found - if (!media.coverPath && libraryItemData.imageLibraryFiles.length) { - // Prefer using a cover image with the name "cover" otherwise use the first image - const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path)) - media.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path - hasMediaChanges = true - } - - // Check if ebook was removed - if (media.ebookFile && (libraryScan.library.settings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) { - media.ebookFile = null - hasMediaChanges = true - } - - // Check if ebook is not set and ebooks were found - if (!media.ebookFile && !libraryScan.library.settings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) { - // Prefer to use an epub ebook then fallback to the first ebook found - let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') - if (!ebookLibraryFile) ebookLibraryFile = libraryItemData.ebookLibraryFiles[0] - // Ebook file is the same as library file except for additional `ebookFormat` - ebookLibraryFile.ebookFormat = ebookLibraryFile.metadata.ext.slice(1).toLowerCase() - media.ebookFile = ebookLibraryFile - media.changed('ebookFile', true) - hasMediaChanges = true - } - - // Check/update the isSupplementary flag on libraryFiles for the LibraryItem - let libraryItemUpdated = false - for (const libraryFile of existingLibraryItem.libraryFiles) { - if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) { - if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) { - if (libraryFile.isSupplementary !== false) { - libraryFile.isSupplementary = false - libraryItemUpdated = true - } - } else if (libraryFile.isSupplementary !== true) { - libraryFile.isSupplementary = true - libraryItemUpdated = true - } - } - } - if (libraryItemUpdated) { - existingLibraryItem.changed('libraryFiles', true) - await existingLibraryItem.save() - } - - // TODO: Update chapters & metadata - - if (hasMediaChanges) { - await media.save() - } + const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan) + return libraryItem } else { // TODO: Scan updated podcast } diff --git a/server/scanner/MediaProbeData.js b/server/scanner/MediaProbeData.js index 43906781..a9ccbe53 100644 --- a/server/scanner/MediaProbeData.js +++ b/server/scanner/MediaProbeData.js @@ -42,13 +42,8 @@ class MediaProbeData { } } - getEmbeddedCoverArt(videoStream) { - const ImageCodecs = ['mjpeg', 'jpeg', 'png'] - return ImageCodecs.includes(videoStream.codec) ? videoStream.codec : null - } - setData(data) { - this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : null + this.embeddedCoverArt = data.video_stream?.codec || null this.format = data.format this.duration = data.duration this.size = data.size diff --git a/server/utils/generators/abmetadataGenerator.js b/server/utils/generators/abmetadataGenerator.js index 3c8e7b54..2e64f91a 100644 --- a/server/utils/generators/abmetadataGenerator.js +++ b/server/utils/generators/abmetadataGenerator.js @@ -324,6 +324,10 @@ function parseAbMetadataText(text, mediaType) { mediaDetails.chapters.sort((a, b) => a.start - b.start) + if (mediaDetails.chapters.length) { + mediaDetails.chapters = cleanChaptersArray(mediaDetails.chapters, mediaDetails.metadata.title) || [] + } + return mediaDetails } module.exports.parse = parseAbMetadataText @@ -425,9 +429,8 @@ function parseJsonMetadataText(text) { if (abmetadataData.tags?.length) { abmetadataData.tags = [...new Set(abmetadataData.tags.map(t => t?.trim()).filter(t => t))] } - // TODO: Clean chapters if (abmetadataData.chapters?.length) { - + abmetadataData.chapters = cleanChaptersArray(abmetadataData.chapters, abmetadataData.metadata.title) } // clean remove dupes if (abmetadataData.metadata.authors?.length) {