From 0ecfdab463775e48ff0d3e44d8c2a02458159337 Mon Sep 17 00:00:00 2001 From: advplyr Date: Fri, 1 Sep 2023 18:01:17 -0500 Subject: [PATCH] Update new library scanner for scanning in new books --- server/Database.js | 62 ++- server/controllers/LibraryController.js | 3 +- server/controllers/MiscController.js | 6 +- server/managers/CoverManager.js | 28 + server/objects/metadata/BookMetadata.js | 4 - server/scanner/AudioFileScanner.js | 10 +- server/scanner/BookScanner.js | 485 ++++++++++++++++++ server/scanner/LibraryItemScanData.js | 53 +- server/scanner/LibraryScanner.js | 31 +- server/utils/fileUtils.js | 5 + .../utils/generators/abmetadataGenerator.js | 26 +- server/utils/index.js | 12 + server/utils/parsers/parseOpfMetadata.js | 4 +- 13 files changed, 694 insertions(+), 35 deletions(-) create mode 100644 server/scanner/BookScanner.js diff --git a/server/Database.js b/server/Database.js index 4bad018d..bf1c1851 100644 --- a/server/Database.js +++ b/server/Database.js @@ -551,16 +551,35 @@ class Database { return this.models.device.createFromOld(oldDevice) } + replaceTagInFilterData(oldTag, newTag) { + for (const libraryId in this.libraryFilterData) { + const indexOf = this.libraryFilterData[libraryId].tags.findIndex(n => n === oldTag) + if (indexOf >= 0) { + this.libraryFilterData[libraryId].tags.splice(indexOf, 1, newTag) + } + } + } + removeTagFromFilterData(tag) { for (const libraryId in this.libraryFilterData) { this.libraryFilterData[libraryId].tags = this.libraryFilterData[libraryId].tags.filter(t => t !== tag) } } - addTagToFilterData(tag) { + addTagsToFilterData(libraryId, tags) { + if (!this.libraryFilterData[libraryId] || !tags?.length) return + tags.forEach((t) => { + if (!this.libraryFilterData[libraryId].tags.includes(t)) { + this.libraryFilterData[libraryId].tags.push(t) + } + }) + } + + replaceGenreInFilterData(oldGenre, newGenre) { for (const libraryId in this.libraryFilterData) { - if (!this.libraryFilterData[libraryId].tags.includes(tag)) { - this.libraryFilterData[libraryId].tags.push(tag) + const indexOf = this.libraryFilterData[libraryId].genres.findIndex(n => n === oldGenre) + if (indexOf >= 0) { + this.libraryFilterData[libraryId].genres.splice(indexOf, 1, newGenre) } } } @@ -571,10 +590,20 @@ class Database { } } - addGenreToFilterData(genre) { + addGenresToFilterData(libraryId, genres) { + if (!this.libraryFilterData[libraryId] || !genres?.length) return + genres.forEach((g) => { + if (!this.libraryFilterData[libraryId].genres.includes(g)) { + this.libraryFilterData[libraryId].genres.push(g) + } + }) + } + + replaceNarratorInFilterData(oldNarrator, newNarrator) { for (const libraryId in this.libraryFilterData) { - if (!this.libraryFilterData[libraryId].genres.includes(genre)) { - this.libraryFilterData[libraryId].genres.push(genre) + const indexOf = this.libraryFilterData[libraryId].narrators.findIndex(n => n === oldNarrator) + if (indexOf >= 0) { + this.libraryFilterData[libraryId].narrators.splice(indexOf, 1, newNarrator) } } } @@ -585,12 +614,13 @@ class Database { } } - addNarratorToFilterData(narrator) { - for (const libraryId in this.libraryFilterData) { - if (!this.libraryFilterData[libraryId].narrators.includes(narrator)) { - this.libraryFilterData[libraryId].narrators.push(narrator) + addNarratorsToFilterData(libraryId, narrators) { + if (!this.libraryFilterData[libraryId] || !narrators?.length) return + narrators.forEach((n) => { + if (!this.libraryFilterData[libraryId].narrators.includes(n)) { + this.libraryFilterData[libraryId].narrators.push(n) } - } + }) } removeSeriesFromFilterData(libraryId, seriesId) { @@ -623,6 +653,16 @@ class Database { }) } + addPublisherToFilterData(libraryId, publisher) { + if (!this.libraryFilterData[libraryId] || !publisher || this.libraryFilterData[libraryId].publishers.includes(publisher)) return + this.libraryFilterData[libraryId].publishers.push(publisher) + } + + addLanguageToFilterData(libraryId, language) { + if (!this.libraryFilterData[libraryId] || !language || this.libraryFilterData[libraryId].languages.includes(language)) return + this.libraryFilterData[libraryId].languages.push(language) + } + /** * Used when updating items to make sure author id exists * If library filter data is set then use that for check diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 176b7f2d..d821a49d 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -889,8 +889,7 @@ class LibraryController { } // Update filter data - Database.removeNarratorFromFilterData(narratorName) - Database.addNarratorToFilterData(updatedName) + Database.replaceNarratorInFilterData(narratorName, updatedName) const itemsUpdated = [] diff --git a/server/controllers/MiscController.js b/server/controllers/MiscController.js index 94ab9ea4..942cda70 100644 --- a/server/controllers/MiscController.js +++ b/server/controllers/MiscController.js @@ -230,8 +230,7 @@ class MiscController { let numItemsUpdated = 0 // Update filter data - Database.removeTagFromFilterData(tag) - Database.addTagToFilterData(newTag) + Database.replaceTagInFilterData(tag, newTag) const libraryItemsWithTag = await libraryItemFilters.getAllLibraryItemsWithTags([tag, newTag]) for (const libraryItem of libraryItemsWithTag) { @@ -364,8 +363,7 @@ class MiscController { let numItemsUpdated = 0 // Update filter data - Database.removeGenreFromFilterData(genre) - Database.addGenreToFilterData(newGenre) + Database.replaceGenreInFilterData(genre, newGenre) const libraryItemsWithGenre = await libraryItemFilters.getAllLibraryItemsWithGenres([genre, newGenre]) for (const libraryItem of libraryItemsWithGenre) { diff --git a/server/managers/CoverManager.js b/server/managers/CoverManager.js index 1f36d545..74fe2f03 100644 --- a/server/managers/CoverManager.js +++ b/server/managers/CoverManager.js @@ -270,5 +270,33 @@ class CoverManager { } return false } + + static async saveEmbeddedCoverArtNew(audioFiles, libraryItemId, libraryItemPath) { + let audioFileWithCover = audioFiles.find(af => af.embeddedCoverArt) + if (!audioFileWithCover) return null + + let coverDirPath = null + if (global.ServerSettings.storeCoverWithItem && libraryItemPath) { + coverDirPath = libraryItemPath + } else { + coverDirPath = Path.posix.join(this.ItemMetadataPath, libraryItemId) + } + await fs.ensureDir(coverDirPath) + + const coverFilename = audioFileWithCover.embeddedCoverArt === 'png' ? 'cover.png' : 'cover.jpg' + const coverFilePath = Path.join(coverDirPath, coverFilename) + + const coverAlreadyExists = await fs.pathExists(coverFilePath) + if (coverAlreadyExists) { + Logger.warn(`[CoverManager] Extract embedded cover art but cover already exists for "${libraryItemPath}" - bail`) + return null + } + + const success = await extractCoverArt(audioFileWithCover.metadata.path, coverFilePath) + if (success) { + return coverFilePath + } + return null + } } module.exports = CoverManager \ No newline at end of file diff --git a/server/objects/metadata/BookMetadata.js b/server/objects/metadata/BookMetadata.js index 2740d25f..9fb07bc8 100644 --- a/server/objects/metadata/BookMetadata.js +++ b/server/objects/metadata/BookMetadata.js @@ -330,10 +330,6 @@ class BookMetadata { { tag: 'tagASIN', key: 'asin' - }, - { - tag: 'tagOverdriveMediaMarker', - key: 'overdriveMediaMarker' } ] diff --git a/server/scanner/AudioFileScanner.js b/server/scanner/AudioFileScanner.js index 01251bfd..75b157e9 100644 --- a/server/scanner/AudioFileScanner.js +++ b/server/scanner/AudioFileScanner.js @@ -41,11 +41,13 @@ class AudioFileScanner { /** * Order audio files by track/disc number - * @param {import('../models/Book')} book + * @param {string} libraryItemRelPath * @param {import('../models/Book').AudioFileObject[]} audioFiles * @returns {import('../models/Book').AudioFileObject[]} */ - runSmartTrackOrder(book, audioFiles) { + runSmartTrackOrder(libraryItemRelPath, audioFiles) { + if (!audioFiles.length) return [] + let discsFromFilename = [] let tracksFromFilename = [] let discsFromMeta = [] @@ -79,14 +81,14 @@ class AudioFileScanner { } if (discKey !== null) { - Logger.debug(`[AudioFileScanner] Smart track order for "${book.title}" using disc key ${discKey} and track key ${trackKey}`) + Logger.debug(`[AudioFileScanner] Smart track order for "${libraryItemRelPath}" using disc key ${discKey} and track key ${trackKey}`) audioFiles.sort((a, b) => { let Dx = a[discKey] - b[discKey] if (Dx === 0) Dx = a[trackKey] - b[trackKey] return Dx }) } else { - Logger.debug(`[AudioFileScanner] Smart track order for "${book.title}" using track key ${trackKey}`) + Logger.debug(`[AudioFileScanner] Smart track order for "${libraryItemRelPath}" using track key ${trackKey}`) audioFiles.sort((a, b) => a[trackKey] - b[trackKey]) } diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js new file mode 100644 index 00000000..70ab5bca --- /dev/null +++ b/server/scanner/BookScanner.js @@ -0,0 +1,485 @@ +const uuidv4 = require("uuid").v4 +const { LogLevel } = require('../utils/constants') +const { getTitleIgnorePrefix } = require('../utils/index') +const { parseOpfMetadataXML } = require('../utils/parsers/parseOpfMetadata') +const { parseOverdriveMediaMarkersAsChapters } = require('../utils/parsers/parseOverdriveMediaMarkers') +const abmetadataGenerator = require('../utils/generators/abmetadataGenerator') +const parseNameString = require('../utils/parsers/parseNameString') +const AudioFileScanner = require('./AudioFileScanner') +const Database = require('../Database') +const { readTextFile } = require('../utils/fileUtils') +const AudioFile = require('../objects/files/AudioFile') +const CoverManager = require('../managers/CoverManager') + +/** + * Metadata for books pulled from files + * @typedef BookMetadataObject + * @property {string} title + * @property {string} titleIgnorePrefix + * @property {string} subtitle + * @property {string} publishedYear + * @property {string} publisher + * @property {string} description + * @property {string} isbn + * @property {string} asin + * @property {string} language + * @property {string[]} narrators + * @property {string[]} genres + * @property {string[]} tags + * @property {string[]} authors + * @property {{name:string, sequence:string}[]} series + * @property {{id:number, start:number, end:number, title:string}[]} chapters + * @property {boolean} explicit + * @property {boolean} abridged + * @property {string} coverPath + */ + +class BookScanner { + constructor() { } + + /** + * + * @param {import('./LibraryItemScanData')} libraryItemData + * @param {import('./LibraryScan')} libraryScan + * @returns {import('../models/LibraryItem')} + */ + async scanNewBookLibraryItem(libraryItemData, libraryScan) { + // Scan audio files found + let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(libraryScan.libraryMediaType, libraryItemData, libraryItemData.audioLibraryFiles) + scannedAudioFiles = AudioFileScanner.runSmartTrackOrder(libraryItemData.relPath, scannedAudioFiles) + + // Find ebook file (prefer epub) + let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') || libraryItemData.ebookLibraryFiles[0] + + // Do not add library items that have no valid audio files and no ebook file + if (!ebookLibraryFile && !scannedAudioFiles.length) { + libraryScan.addLog(LogLevel.WARN, `Library item at path "${libraryItemData.relPath}" has no audio files and no ebook file - ignoring`) + return null + } + + if (ebookLibraryFile) { + ebookLibraryFile.ebookFormat = ebookLibraryFile.metadata.ext.slice(1).toLowerCase() + } + + const bookMetadata = await this.getBookMetadataFromScanData(scannedAudioFiles, libraryItemData, libraryScan) + + let duration = 0 + scannedAudioFiles.forEach((af) => duration += (!isNaN(af.duration) ? Number(af.duration) : 0)) + const bookObject = { + ...bookMetadata, + audioFiles: scannedAudioFiles, + ebookFile: ebookLibraryFile || null, + duration, + bookAuthors: [], + bookSeries: [] + } + if (bookMetadata.authors.length) { + for (const authorName of bookMetadata.authors) { + const matchingAuthor = Database.libraryFilterData[libraryScan.libraryId].authors.find(au => au.name === authorName) + if (matchingAuthor) { + bookObject.bookAuthors.push({ + authorId: matchingAuthor.id + }) + } else { + // New author + bookObject.bookAuthors.push({ + author: { + libraryId: libraryScan.libraryId, + name: authorName, + lastFirst: parseNameString.nameToLastFirst(authorName) + } + }) + } + } + } + if (bookMetadata.series.length) { + for (const seriesObj of bookMetadata.series) { + if (!seriesObj.name) continue + const matchingSeries = Database.libraryFilterData[libraryScan.libraryId].series.find(se => se.name === seriesObj.name) + if (matchingSeries) { + bookObject.bookSeries.push({ + seriesId: matchingSeries.id, + sequence: seriesObj.sequence + }) + } else { + bookObject.bookSeries.push({ + sequence: seriesObj.sequence, + series: { + name: seriesObj.name, + nameIgnorePrefix: getTitleIgnorePrefix(seriesObj.name), + libraryId: libraryScan.libraryId + } + }) + } + } + } + + const libraryItemObj = libraryItemData.libraryItemObject + libraryItemObj.id = uuidv4() // Generate library item id ahead of time to use for saving extracted cover image + + // If cover was not found in folder then check embedded covers in audio files + if (!bookObject.coverPath && scannedAudioFiles.length) { + const libraryItemDir = libraryItemObj.isFile ? null : libraryItemObj.path + // Extract and save embedded cover art + bookObject.coverPath = await CoverManager.saveEmbeddedCoverArtNew(scannedAudioFiles, libraryItemObj.id, libraryItemDir) + } + + libraryItemObj.book = bookObject + const libraryItem = await Database.libraryItemModel.create(libraryItemObj, { + include: { + model: Database.bookModel, + include: [ + { + model: Database.bookSeriesModel, + include: { + model: Database.seriesModel + } + }, + { + model: Database.bookAuthorModel, + include: { + model: Database.authorModel + } + } + ] + } + }) + + // Update library filter data + if (libraryItem.book.bookSeries?.length) { + for (const bs of libraryItem.book.bookSeries) { + if (bs.series) { + Database.addSeriesToFilterData(libraryScan.libraryId, bs.series.name, bs.series.id) + } + } + } + if (libraryItem.book.bookAuthors?.length) { + for (const ba of libraryItem.book.bookAuthors) { + if (ba.author) { + Database.addAuthorToFilterData(libraryScan.libraryId, ba.author.name, ba.author.id) + } + } + } + Database.addNarratorsToFilterData(libraryScan.libraryId, libraryItem.book.narrators) + Database.addGenresToFilterData(libraryScan.libraryId, libraryItem.book.genres) + Database.addTagsToFilterData(libraryScan.libraryId, libraryItem.book.tags) + Database.addPublisherToFilterData(libraryScan.libraryId, libraryItem.book.publisher) + Database.addLanguageToFilterData(libraryScan.libraryId, libraryItem.book.language) + + return libraryItem + } + + /** + * + * @param {import('../objects/files/AudioFile')[]} scannedAudioFiles + * @param {import('./LibraryItemScanData')} libraryItemData + * @param {import('./LibraryScan')} libraryScan + * @returns {Promise} + */ + async getBookMetadataFromScanData(scannedAudioFiles, libraryItemData, libraryScan) { + // First set book metadata from folder/file names + const bookMetadata = { + title: libraryItemData.mediaMetadata.title, + titleIgnorePrefix: getTitleIgnorePrefix(libraryItemData.mediaMetadata.title), + subtitle: libraryItemData.mediaMetadata.subtitle, + publishedYear: libraryItemData.mediaMetadata.publishedYear, + publisher: null, + description: null, + isbn: null, + asin: null, + language: null, + narrators: parseNameString.parse(libraryItemData.mediaMetadata.narrators)?.names || [], + genres: [], + tags: [], + authors: parseNameString.parse(libraryItemData.mediaMetadata.author)?.names || [], + series: [], + chapters: [], + explicit: false, + abridged: false, + coverPath: null + } + if (libraryItemData.mediaMetadata.series) { + bookMetadata.series.push({ + name: libraryItemData.mediaMetadata.series, + sequence: libraryItemData.mediaMetadata.sequence || null + }) + } + + // Fill in or override book metadata from audio file meta tags + if (scannedAudioFiles.length) { + const MetadataMapArray = [ + { + tag: 'tagComposer', + key: 'narrators' + }, + { + tag: 'tagDescription', + altTag: 'tagComment', + key: 'description' + }, + { + tag: 'tagPublisher', + key: 'publisher' + }, + { + tag: 'tagDate', + key: 'publishedYear' + }, + { + tag: 'tagSubtitle', + key: 'subtitle' + }, + { + tag: 'tagAlbum', + altTag: 'tagTitle', + key: 'title', + }, + { + tag: 'tagArtist', + altTag: 'tagAlbumArtist', + key: 'authors' + }, + { + tag: 'tagGenre', + key: 'genres' + }, + { + tag: 'tagSeries', + key: 'series' + }, + { + tag: 'tagIsbn', + key: 'isbn' + }, + { + tag: 'tagLanguage', + key: 'language' + }, + { + tag: 'tagASIN', + key: 'asin' + } + ] + const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata + const firstScannedFile = scannedAudioFiles[0] + const audioFileMetaTags = firstScannedFile.metaTags + MetadataMapArray.forEach((mapping) => { + let value = audioFileMetaTags[mapping.tag] + if (!value && mapping.altTag) { + value = audioFileMetaTags[mapping.altTag] + } + + if (value && typeof value === 'string') { + value = value.trim() // Trim whitespace + + if (mapping.key === 'narrators' && (!bookMetadata.narrators.length || overrideExistingDetails)) { + bookMetadata.narrators = parseNameString.parse(value)?.names || [] + } else if (mapping.key === 'authors' && (!bookMetadata.authors.length || overrideExistingDetails)) { + bookMetadata.authors = parseNameString.parse(value)?.names || [] + } else if (mapping.key === 'genres' && (!bookMetadata.genres.length || overrideExistingDetails)) { + bookMetadata.genres = this.parseGenresString(value) + } else if (mapping.key === 'series' && (!bookMetadata.series.length || overrideExistingDetails)) { + bookMetadata.series = [ + { + name: value, + sequence: audioFileMetaTags.tagSeriesPart || null + } + ] + } else if (!bookMetadata[mapping.key] || overrideExistingDetails) { + bookMetadata[mapping.key] = value + } + } + }) + } + + // If desc.txt in library item folder then use this for description + if (libraryItemData.descTxtLibraryFile) { + const description = await readTextFile(libraryItemData.descTxtLibraryFile.metadata.path) + if (description.trim()) bookMetadata.description = description.trim() + } + + // If reader.txt in library item folder then use this for narrator + if (libraryItemData.readerTxtLibraryFile) { + let narrator = await readTextFile(libraryItemData.readerTxtLibraryFile.metadata.path) + narrator = narrator.split(/\r?\n/)[0]?.trim() || '' // Only use first line + if (narrator) { + bookMetadata.narrators = parseNameString.parse(narrator)?.names || [] + } + } + + // If opf file is found look for metadata + if (libraryItemData.metadataOpfLibraryFile) { + const xmlText = await readTextFile(libraryItemData.metadataOpfLibraryFile.metadata.path) + const opfMetadata = xmlText ? await parseOpfMetadataXML(xmlText) : null + if (opfMetadata) { + const opfMetadataOverrideDetails = Database.serverSettings.scannerPreferOpfMetadata + for (const key in opfMetadata) { + if (key === 'tags') { // Add tags only if tags are empty + if (opfMetadata.tags.length && (!bookMetadata.tags.length || opfMetadataOverrideDetails)) { + bookMetadata.tags = opfMetadata.tags + } + } else if (key === 'genres') { // Add genres only if genres are empty + if (opfMetadata.genres.length && (!bookMetadata.genres.length || opfMetadataOverrideDetails)) { + bookMetadata.genres = opfMetadata.genres + } + } else if (key === 'authors') { + if (opfMetadata.authors?.length && (!bookMetadata.authors.length || opfMetadataOverrideDetails)) { + bookMetadata.authors = opfMetadata.authors + } + } else if (key === 'narrators') { + if (opfMetadata.narrators?.length && (!bookMetadata.narrators.length || opfMetadataOverrideDetails)) { + bookMetadata.narrators = opfMetadata.narrators + } + } else if (key === 'series') { + if (opfMetadata.series && (!bookMetadata.series.length || opfMetadataOverrideDetails)) { + bookMetadata.series = [{ + name: opfMetadata.series, + sequence: opfMetadata.sequence || null + }] + } + } else if (opfMetadata[key] && (!bookMetadata[key] || opfMetadataOverrideDetails)) { + bookMetadata[key] = opfMetadata[key] + } + } + } + } + + // If metadata.json or metadata.abs use this for metadata + const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile + const metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null + if (metadataText) { + libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataLibraryFile.metadata.relPath}" - preferring`) + let abMetadata = null + if (!!libraryItemData.metadataJsonLibraryFile) { + abMetadata = abmetadataGenerator.parseJson(metadataText) + } else { + abMetadata = abmetadataGenerator.parse(metadataText, 'book') + } + + if (abMetadata) { + if (abMetadata.tags?.length) { + bookMetadata.tags = abMetadata.tags + } + if (abMetadata.chapters?.length) { + bookMetadata.chapters = abMetadata.chapters + } + for (const key in abMetadata.metadata) { + if (bookMetadata[key] === undefined || abMetadata.metadata[key] === undefined) continue + bookMetadata[key] = abMetadata.metadata[key] + } + } + } + + // Set chapters from audio files if not already set + if (!bookMetadata.chapters.length) { + bookMetadata.chapters = this.getChaptersFromAudioFiles(bookMetadata.title, scannedAudioFiles, libraryScan) + } + + // Set cover from library file if one is found otherwise check audiofile + if (libraryItemData.imageLibraryFiles.length) { + const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path)) + bookMetadata.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path + } + + return bookMetadata + } + + /** + * Parse a genre string into multiple genres + * @example "Fantasy;Sci-Fi;History" => ["Fantasy", "Sci-Fi", "History"] + * @param {string} genreTag + * @returns {string[]} + */ + parseGenresString(genreTag) { + if (!genreTag?.length) return [] + const separators = ['/', '//', ';'] + for (let i = 0; i < separators.length; i++) { + if (genreTag.includes(separators[i])) { + return genreTag.split(separators[i]).map(genre => genre.trim()).filter(g => !!g) + } + } + return [genreTag] + } + + /** + * @param {string} bookTitle + * @param {AudioFile[]} audioFiles + * @param {import('./LibraryScan')} libraryScan + * @returns {import('../models/Book').ChapterObject[]} + */ + getChaptersFromAudioFiles(bookTitle, audioFiles, libraryScan) { + if (!audioFiles.length) return [] + + // If overdrive media markers are present and preferred, use those instead + if (Database.serverSettings.scannerPreferOverdriveMediaMarker) { + const overdriveChapters = parseOverdriveMediaMarkersAsChapters(audioFiles) + if (overdriveChapters) { + libraryScan.addLog(LogLevel.DEBUG, 'Overdrive Media Markers and preference found! Using these for chapter definitions') + + return overdriveChapters + } + } + + let chapters = [] + + // If first audio file has embedded chapters then use embedded chapters + if (audioFiles[0].chapters?.length) { + // If all files chapters are the same, then only make chapters for the first file + if ( + audioFiles.length === 1 || + audioFiles.length > 1 && + audioFiles[0].chapters.length === audioFiles[1].chapters?.length && + audioFiles[0].chapters.every((c, i) => c.title === audioFiles[1].chapters[i].title) + ) { + libraryScan.addLog(LogLevel.DEBUG, `setChapters: Using embedded chapters in first audio file ${audioFiles[0].metadata?.path}`) + chapters = audioFiles[0].chapters.map((c) => ({ ...c })) + } else { + libraryScan.addLog(LogLevel.DEBUG, `setChapters: Using embedded chapters from all audio files ${audioFiles[0].metadata?.path}`) + let currChapterId = 0 + let currStartTime = 0 + + audioFiles.forEach((file) => { + if (file.duration) { + const afChapters = file.chapters?.map((c) => ({ + ...c, + id: c.id + currChapterId, + start: c.start + currStartTime, + end: c.end + currStartTime, + })) ?? [] + chapters = chapters.concat(afChapters) + + currChapterId += file.chapters?.length ?? 0 + currStartTime += file.duration + } + }) + return chapters + } + } else if (audioFiles.length > 1) { + const preferAudioMetadata = !!Database.serverSettings.scannerPreferAudioMetadata + + // Build chapters from audio files + let currChapterId = 0 + let currStartTime = 0 + includedAudioFiles.forEach((file) => { + if (file.duration) { + let title = file.metadata.filename ? Path.basename(file.metadata.filename, Path.extname(file.metadata.filename)) : `Chapter ${currChapterId}` + + // When prefer audio metadata server setting is set then use ID3 title tag as long as it is not the same as the book title + if (preferAudioMetadata && file.metaTags?.tagTitle && file.metaTags?.tagTitle !== bookTitle) { + title = file.metaTags.tagTitle + } + + chapters.push({ + id: currChapterId++, + start: currStartTime, + end: currStartTime + file.duration, + title + }) + currStartTime += file.duration + } + }) + } + return chapters + } +} +module.exports = new BookScanner() \ No newline at end of file diff --git a/server/scanner/LibraryItemScanData.js b/server/scanner/LibraryItemScanData.js index 88af6a7d..54facc4e 100644 --- a/server/scanner/LibraryItemScanData.js +++ b/server/scanner/LibraryItemScanData.js @@ -10,6 +10,8 @@ class LibraryItemScanData { /** @type {string} */ this.libraryId = data.libraryId /** @type {string} */ + this.mediaType = data.mediaType + /** @type {string} */ this.ino = data.ino /** @type {number} */ this.mtimeMs = data.mtimeMs @@ -23,7 +25,7 @@ class LibraryItemScanData { this.relPath = data.relPath /** @type {boolean} */ this.isFile = data.isFile - /** @type {{title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} */ + /** @type {{author:string, title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} */ this.mediaMetadata = data.mediaMetadata /** @type {import('../objects/files/LibraryFile')[]} */ this.libraryFiles = data.libraryFiles @@ -41,6 +43,30 @@ class LibraryItemScanData { this.libraryFilesModified = [] } + /** + * Used to create a library item + */ + get libraryItemObject() { + let size = 0 + this.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0)) + return { + ino: this.ino, + path: this.path, + relPath: this.relPath, + mediaType: this.mediaType, + isFile: this.isFile, + mtime: this.mtimeMs, + ctime: this.ctime, + birthtime: this.birthtimeMs, + lastScan: Date.now(), + lastScanVersion: packageJson.version, + libraryFiles: this.libraryFiles, + libraryId: this.libraryId, + libraryFolderId: this.libraryFolderId, + size + } + } + /** @type {boolean} */ get hasLibraryFileChanges() { return this.libraryFilesRemoved.length + this.libraryFilesModified.length + this.libraryFilesAdded.length @@ -81,6 +107,31 @@ class LibraryItemScanData { return this.libraryFiles.filter(lf => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || '')) } + /** @type {LibraryItem.LibraryFileObject} */ + get descTxtLibraryFile() { + return this.libraryFiles.find(lf => lf.metadata.filename === 'desc.txt') + } + + /** @type {LibraryItem.LibraryFileObject} */ + get readerTxtLibraryFile() { + return this.libraryFiles.find(lf => lf.metadata.filename === 'reader.txt') + } + + /** @type {LibraryItem.LibraryFileObject} */ + get metadataAbsLibraryFile() { + return this.libraryFiles.find(lf => lf.metadata.filename === 'metadata.abs') + } + + /** @type {LibraryItem.LibraryFileObject} */ + get metadataJsonLibraryFile() { + return this.libraryFiles.find(lf => lf.metadata.filename === 'metadata.json') + } + + /** @type {LibraryItem.LibraryFileObject} */ + get metadataOpfLibraryFile() { + return this.libraryFiles.find(lf => lf.metadata.ext.toLowerCase() === '.opf') + } + /** * * @param {LibraryItem} existingLibraryItem diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index 19a642f2..59c0d17c 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -8,12 +8,14 @@ const fileUtils = require('../utils/fileUtils') const scanUtils = require('../utils/scandir') const { ScanResult, LogLevel } = require('../utils/constants') const globals = require('../utils/globals') +const libraryFilters = require('../utils/queries/libraryFilters') const AudioFileScanner = require('./AudioFileScanner') const ScanOptions = require('./ScanOptions') const LibraryScan = require('./LibraryScan') const LibraryItemScanData = require('./LibraryItemScanData') const AudioFile = require('../objects/files/AudioFile') const Book = require('../models/Book') +const BookScanner = require('./BookScanner') class LibraryScanner { constructor(coverManager, taskManager) { @@ -91,6 +93,10 @@ class LibraryScanner { * @param {import('./LibraryScan')} libraryScan */ async scanLibrary(libraryScan) { + // Make sure library filter data is set + // this is used to check for existing authors & series + await libraryFilters.getFilterData(libraryScan.library) + /** @type {LibraryItemScanData[]} */ let libraryItemDataFound = [] @@ -159,8 +165,15 @@ class LibraryScanner { // Add new library items if (libraryItemDataFound.length) { - + for (const libraryItemData of libraryItemDataFound) { + const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan) + if (newLibraryItem) { + libraryScan.resultsAdded++ + } + } } + + // TODO: Socket emitter } /** @@ -217,6 +230,7 @@ class LibraryScanner { items.push(new LibraryItemScanData({ libraryFolderId: folder.id, libraryId: folder.libraryId, + mediaType: library.mediaType, ino: libraryItemFolderStats.ino, mtimeMs: libraryItemFolderStats.mtimeMs || 0, ctimeMs: libraryItemFolderStats.ctimeMs || 0, @@ -304,7 +318,7 @@ class LibraryScanner { media.audioFiles.push(...scannedAudioFiles) } - media.audioFiles = AudioFileScanner.runSmartTrackOrder(media, media.audioFiles) + media.audioFiles = AudioFileScanner.runSmartTrackOrder(existingLibraryItem.relPath, media.audioFiles) media.duration = 0 media.audioFiles.forEach((af) => { @@ -373,6 +387,8 @@ class LibraryScanner { if (hasMediaChanges) { await media.save() } + } else { + // TODO: Scan updated podcast } } @@ -382,10 +398,15 @@ class LibraryScanner { * @param {LibraryScan} libraryScan */ async scanNewLibraryItem(libraryItemData, libraryScan) { - if (libraryScan.libraryMediaType === 'book') { - let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(libraryScan.libraryMediaType, libraryItemData, libraryItemData.audioLibraryFiles) - // TODO: Create new book + const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, libraryScan) + if (newLibraryItem) { + libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`) + } + return newLibraryItem + } else { + // TODO: Scan new podcast + return null } } } diff --git a/server/utils/fileUtils.js b/server/utils/fileUtils.js index 87bacac0..b13bb67e 100644 --- a/server/utils/fileUtils.js +++ b/server/utils/fileUtils.js @@ -82,6 +82,11 @@ function getIno(path) { } module.exports.getIno = getIno +/** + * Read contents of file + * @param {string} path + * @returns {string} + */ async function readTextFile(path) { try { var data = await fs.readFile(path) diff --git a/server/utils/generators/abmetadataGenerator.js b/server/utils/generators/abmetadataGenerator.js index da9db2c9..3c8e7b54 100644 --- a/server/utils/generators/abmetadataGenerator.js +++ b/server/utils/generators/abmetadataGenerator.js @@ -24,7 +24,7 @@ const CurrentAbMetadataVersion = 2 const commaSeparatedToArray = (v) => { if (!v) return [] - return v.split(',').map(_v => _v.trim()).filter(_v => _v) + return [...new Set(v.split(',').map(_v => _v.trim()).filter(_v => _v))] } const podcastMetadataMapper = { @@ -401,7 +401,10 @@ function checkArraysChanged(abmetadataArray, mediaArray) { function parseJsonMetadataText(text) { try { const abmetadataData = JSON.parse(text) - if (abmetadataData.metadata?.series?.length) { + if (!abmetadataData.metadata) abmetadataData.metadata = {} + + if (abmetadataData.metadata.series?.length) { + abmetadataData.metadata.series = [...new Set(abmetadataData.metadata.series.map(t => t?.trim()).filter(t => t))] abmetadataData.metadata.series = abmetadataData.metadata.series.map(series => { let sequence = null let name = series @@ -418,12 +421,31 @@ function parseJsonMetadataText(text) { } }) } + // clean tags & remove dupes + if (abmetadataData.tags?.length) { + abmetadataData.tags = [...new Set(abmetadataData.tags.map(t => t?.trim()).filter(t => t))] + } + // TODO: Clean chapters + if (abmetadataData.chapters?.length) { + + } + // clean remove dupes + if (abmetadataData.metadata.authors?.length) { + abmetadataData.metadata.authors = [...new Set(abmetadataData.metadata.authors.map(t => t?.trim()).filter(t => t))] + } + if (abmetadataData.metadata.narrators?.length) { + abmetadataData.metadata.narrators = [...new Set(abmetadataData.metadata.narrators.map(t => t?.trim()).filter(t => t))] + } + if (abmetadataData.metadata.genres?.length) { + abmetadataData.metadata.genres = [...new Set(abmetadataData.metadata.genres.map(t => t?.trim()).filter(t => t))] + } return abmetadataData } catch (error) { Logger.error(`[abmetadataGenerator] Invalid metadata.json JSON`, error) return null } } +module.exports.parseJson = parseJsonMetadataText function cleanChaptersArray(chaptersArray, mediaTitle) { const chapters = [] diff --git a/server/utils/index.js b/server/utils/index.js index 69e6dcb4..5797b0b5 100644 --- a/server/utils/index.js +++ b/server/utils/index.js @@ -147,10 +147,22 @@ const getTitleParts = (title) => { return [title, null] } +/** + * Remove sortingPrefixes from title + * @example "The Good Book" => "Good Book" + * @param {string} title + * @returns {string} + */ module.exports.getTitleIgnorePrefix = (title) => { return getTitleParts(title)[0] } +/** + * Put sorting prefix at the end of title + * @example "The Good Book" => "Good Book, The" + * @param {string} title + * @returns {string} + */ module.exports.getTitlePrefixAtEnd = (title) => { let [sort, prefix] = getTitleParts(title) return prefix ? `${sort}, ${prefix}` : title diff --git a/server/utils/parsers/parseOpfMetadata.js b/server/utils/parsers/parseOpfMetadata.js index 3a34db57..22be7712 100644 --- a/server/utils/parsers/parseOpfMetadata.js +++ b/server/utils/parsers/parseOpfMetadata.js @@ -159,8 +159,8 @@ module.exports.parseOpfMetadataXML = async (xml) => { } const creators = parseCreators(metadata) - const authors = (fetchCreators(creators, 'aut') || []).filter(au => au && au.trim()) - const narrators = (fetchNarrators(creators, metadata) || []).filter(nrt => nrt && nrt.trim()) + const authors = (fetchCreators(creators, 'aut') || []).map(au => au?.trim()).filter(au => au) + const narrators = (fetchNarrators(creators, metadata) || []).map(nrt => nrt?.trim()).filter(nrt => nrt) const data = { title: fetchTitle(metadata), subtitle: fetchSubtitle(metadata),