diff --git a/server/scanner/AbsMetadataFileScanner.js b/server/scanner/AbsMetadataFileScanner.js index 2718fb15..d9d077c0 100644 --- a/server/scanner/AbsMetadataFileScanner.js +++ b/server/scanner/AbsMetadataFileScanner.js @@ -61,5 +61,65 @@ class AbsMetadataFileScanner { } } } + + /** + * Check for metadata.json or metadata.abs file and set podcast metadata + * + * @param {import('./LibraryScan')} libraryScan + * @param {import('./LibraryItemScanData')} libraryItemData + * @param {Object} podcastMetadata + * @param {string} [existingLibraryItemId] + */ + async scanPodcastMetadataFile(libraryScan, libraryItemData, podcastMetadata, existingLibraryItemId = null) { + const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile + let metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null + let metadataFilePath = metadataLibraryFile?.metadata.path + let metadataFileFormat = libraryItemData.metadataJsonLibraryFile ? 'json' : 'abs' + + // When metadata file is not stored with library item then check in the /metadata/items folder for it + if (!metadataText && existingLibraryItemId) { + let metadataPath = Path.join(global.MetadataPath, 'items', existingLibraryItemId) + + let altFormat = global.ServerSettings.metadataFileFormat === 'json' ? 'abs' : 'json' + // First check the metadata format set in server settings, fallback to the alternate + metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`) + metadataFileFormat = global.ServerSettings.metadataFileFormat + if (await fsExtra.pathExists(metadataFilePath)) { + metadataText = await readTextFile(metadataFilePath) + } else if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.${altFormat}`))) { + metadataFilePath = Path.join(metadataPath, `metadata.${altFormat}`) + metadataFileFormat = altFormat + metadataText = await readTextFile(metadataFilePath) + } + } + + if (metadataText) { + libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataFilePath}" - preferring`) + let abMetadata = null + if (metadataFileFormat === 'json') { + abMetadata = abmetadataGenerator.parseJson(metadataText) + } else { + abMetadata = abmetadataGenerator.parse(metadataText, 'podcast') + } + + if (abMetadata) { + if (abMetadata.tags?.length) { + podcastMetadata.tags = abMetadata.tags + } + for (const key in abMetadata.metadata) { + if (abMetadata.metadata[key] === undefined) continue + + // TODO: New podcast model changed some keys, need to update the abmetadataGenerator + let newModelKey = key + if (key === 'feedUrl') newModelKey = 'feedURL' + else if (key === 'imageUrl') newModelKey = 'imageURL' + else if (key === 'itunesPageUrl') newModelKey = 'itunesPageURL' + else if (key === 'type') newModelKey = 'podcastType' + + podcastMetadata[newModelKey] = abMetadata.metadata[key] + } + } + } + } } module.exports = new AbsMetadataFileScanner() \ No newline at end of file diff --git a/server/scanner/AudioFileScanner.js b/server/scanner/AudioFileScanner.js index 4aad56e7..ddd994d0 100644 --- a/server/scanner/AudioFileScanner.js +++ b/server/scanner/AudioFileScanner.js @@ -215,7 +215,7 @@ class AudioFileScanner { * @param {string} bookTitle * @param {import('../models/Book').AudioFileObject} audioFile * @param {Object} bookMetadata - * @param {LibraryScan} libraryScan + * @param {import('./LibraryScan')} libraryScan */ setBookMetadataFromAudioMetaTags(bookTitle, audioFiles, bookMetadata, libraryScan) { const MetadataMapArray = [ @@ -309,10 +309,145 @@ class AudioFileScanner { } } + /** + * Set podcast metadata from first audio file + * + * @param {import('../models/Book').AudioFileObject} audioFile + * @param {Object} podcastMetadata + * @param {import('./LibraryScan')} libraryScan + */ + setPodcastMetadataFromAudioMetaTags(audioFile, podcastMetadata, libraryScan) { + const audioFileMetaTags = audioFile.metaTags + + const MetadataMapArray = [ + { + tag: 'tagAlbum', + altTag: 'tagSeries', + key: 'title' + }, + { + tag: 'tagArtist', + key: 'author' + }, + { + tag: 'tagGenre', + key: 'genres' + }, + { + tag: 'tagLanguage', + key: 'language' + }, + { + tag: 'tagItunesId', + key: 'itunesId' + }, + { + tag: 'tagPodcastType', + key: 'podcastType', + } + ] + + MetadataMapArray.forEach((mapping) => { + let value = audioFileMetaTags[mapping.tag] + let tagToUse = mapping.tag + if (!value && mapping.altTag) { + value = audioFileMetaTags[mapping.altTag] + tagToUse = mapping.altTag + } + + if (value && typeof value === 'string') { + value = value.trim() // Trim whitespace + + if (mapping.key === 'genres') { + podcastMetadata.genres = this.parseGenresString(value) + libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata.genres.join(', ')}`) + } else { + podcastMetadata[mapping.key] = value + libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata[mapping.key]}`) + } + } + }) + } + + /** + * + * @param {import('../models/PodcastEpisode')} podcastEpisode Not the model when creating new podcast + * @param {import('./ScanLogger')} scanLogger + */ + setPodcastEpisodeMetadataFromAudioMetaTags(podcastEpisode, scanLogger) { + const MetadataMapArray = [ + { + tag: 'tagComment', + altTag: 'tagSubtitle', + key: 'description' + }, + { + tag: 'tagSubtitle', + key: 'subtitle' + }, + { + tag: 'tagDate', + key: 'pubDate' + }, + { + tag: 'tagDisc', + key: 'season', + }, + { + tag: 'tagTrack', + altTag: 'tagSeriesPart', + key: 'episode' + }, + { + tag: 'tagTitle', + key: 'title' + }, + { + tag: 'tagEpisodeType', + key: 'episodeType' + } + ] + + const audioFileMetaTags = podcastEpisode.audioFile.metaTags + MetadataMapArray.forEach((mapping) => { + let value = audioFileMetaTags[mapping.tag] + let tagToUse = mapping.tag + if (!value && mapping.altTag) { + tagToUse = mapping.altTag + value = audioFileMetaTags[mapping.altTag] + } + + if (value && typeof value === 'string') { + value = value.trim() // Trim whitespace + + if (mapping.key === 'pubDate') { + const pubJsDate = new Date(value) + if (pubJsDate && !isNaN(pubJsDate)) { + podcastEpisode.publishedAt = pubJsDate.valueOf() + podcastEpisode.pubDate = value + scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`) + } else { + scanLogger.addLog(LogLevel.WARN, `Mapping pubDate with tag ${tagToUse} has invalid date "${value}"`) + } + } else if (mapping.key === 'episodeType') { + if (['full', 'trailer', 'bonus'].includes(value)) { + podcastEpisode.episodeType = value + scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`) + } else { + scanLogger.addLog(LogLevel.WARN, `Mapping episodeType with invalid value "${value}". Must be one of [full, trailer, bonus].`) + } + } else { + podcastEpisode[mapping.key] = value + scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`) + } + } + }) + } + /** * @param {string} bookTitle * @param {AudioFile[]} audioFiles - * @param {LibraryScan} libraryScan + * @param {import('./LibraryScan')} libraryScan * @returns {import('../models/Book').ChapterObject[]} */ getBookChaptersFromAudioFiles(bookTitle, audioFiles, libraryScan) { diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js index 68388c3e..e7d49d35 100644 --- a/server/scanner/BookScanner.js +++ b/server/scanner/BookScanner.js @@ -580,9 +580,10 @@ class BookScanner { } const bookMetadataSourceHandler = new BookScanner.BookMetadataSourceHandler(bookMetadata, audioFiles, libraryItemData, libraryScan, existingLibraryItemId) - for (const metadataSource of librarySettings.metadataPrecedence) { + const metadataPrecedence = librarySettings.metadataPrecedence || ['folderStructure', 'audioMetatags', 'txtFiles', 'opfFile', 'absMetadata'] + libraryScan.addLog(LogLevel.DEBUG, `"${bookMetadata.title}" Getting metadata with precedence [${metadataPrecedence.join(', ')}]`) + for (const metadataSource of metadataPrecedence) { if (bookMetadataSourceHandler[metadataSource]) { - libraryScan.addLog(LogLevel.DEBUG, `Getting metadata from source "${metadataSource}"`) await bookMetadataSourceHandler[metadataSource]() } else { libraryScan.addLog(LogLevel.ERROR, `Invalid metadata source "${metadataSource}"`) diff --git a/server/scanner/LibraryItemScanner.js b/server/scanner/LibraryItemScanner.js index 8caca557..e9ca3302 100644 --- a/server/scanner/LibraryItemScanner.js +++ b/server/scanner/LibraryItemScanner.js @@ -50,28 +50,25 @@ class LibraryItemScanner { const scanLogger = new ScanLogger() scanLogger.verbose = true - scanLogger.setData('libraryItem', libraryItemId) + scanLogger.setData('libraryItem', libraryItem.relPath) const libraryItemPath = fileUtils.filePathToPOSIX(libraryItem.path) const folder = library.libraryFolders[0] const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, false) - if (await libraryItemScanData.checkLibraryItemData(libraryItem, scanLogger)) { - if (libraryItemScanData.hasLibraryFileChanges || libraryItemScanData.hasPathChange) { - const { libraryItem: expandedLibraryItem } = await this.rescanLibraryItemMedia(libraryItem, libraryItemScanData, library.settings, scanLogger) - const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(expandedLibraryItem) - SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded()) + let libraryItemDataUpdated = await libraryItemScanData.checkLibraryItemData(libraryItem, scanLogger) - await this.checkAuthorsAndSeriesRemovedFromBooks(library.id, scanLogger) - } else { - // TODO: Temporary while using old model to socket emit - const oldLibraryItem = await Database.libraryItemModel.getOldById(libraryItem.id) - SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded()) - } + const { libraryItem: expandedLibraryItem, wasUpdated } = await this.rescanLibraryItemMedia(libraryItem, libraryItemScanData, library.settings, scanLogger) + if (libraryItemDataUpdated || wasUpdated) { + const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(expandedLibraryItem) + SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded()) + + await this.checkAuthorsAndSeriesRemovedFromBooks(library.id, scanLogger) return ScanResult.UPDATED } - libraryScan.addLog(LogLevel.DEBUG, `Library item "${libraryItem.relPath}" is up-to-date`) + + scanLogger.addLog(LogLevel.DEBUG, `Library item is up-to-date`) return ScanResult.UPTODATE } diff --git a/server/scanner/PodcastScanner.js b/server/scanner/PodcastScanner.js index d7290bc8..53d4ad1f 100644 --- a/server/scanner/PodcastScanner.js +++ b/server/scanner/PodcastScanner.js @@ -5,12 +5,13 @@ const { getTitleIgnorePrefix } = require('../utils/index') const abmetadataGenerator = require('../utils/generators/abmetadataGenerator') const AudioFileScanner = require('./AudioFileScanner') const Database = require('../Database') -const { readTextFile, filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils') +const { filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils') const AudioFile = require('../objects/files/AudioFile') const CoverManager = require('../managers/CoverManager') const LibraryFile = require('../objects/files/LibraryFile') const fsExtra = require("../libs/fsExtra") const PodcastEpisode = require("../models/PodcastEpisode") +const AbsMetadataFileScanner = require("./AbsMetadataFileScanner") /** * Metadata for podcasts pulled from files @@ -87,7 +88,7 @@ class PodcastScanner { podcastEpisode.changed('audioFile', true) // Set metadata and save episode - this.setPodcastEpisodeMetadataFromAudioFile(podcastEpisode, libraryScan) + AudioFileScanner.setPodcastEpisodeMetadataFromAudioMetaTags(podcastEpisode, libraryScan) libraryScan.addLog(LogLevel.INFO, `Podcast episode "${podcastEpisode.title}" keys changed [${podcastEpisode.changed()?.join(', ')}]`) await podcastEpisode.save() } @@ -122,7 +123,7 @@ class PodcastScanner { } const newPodcastEpisode = Database.podcastEpisodeModel.build(newEpisode) // Set metadata and save new episode - this.setPodcastEpisodeMetadataFromAudioFile(newPodcastEpisode, libraryScan) + AudioFileScanner.setPodcastEpisodeMetadataFromAudioMetaTags(newPodcastEpisode, libraryScan) libraryScan.addLog(LogLevel.INFO, `New Podcast episode "${newPodcastEpisode.title}" added`) await newPodcastEpisode.save() existingPodcastEpisodes.push(newPodcastEpisode) @@ -242,7 +243,7 @@ class PodcastScanner { } // Set metadata and save new episode - this.setPodcastEpisodeMetadataFromAudioFile(newEpisode, libraryScan) + AudioFileScanner.setPodcastEpisodeMetadataFromAudioMetaTags(newEpisode, libraryScan) libraryScan.addLog(LogLevel.INFO, `New Podcast episode "${newEpisode.title}" found`) newPodcastEpisodes.push(newEpisode) } @@ -320,7 +321,7 @@ class PodcastScanner { async getPodcastMetadataFromScanData(podcastEpisodes, libraryItemData, libraryScan, existingLibraryItemId = null) { const podcastMetadata = { title: libraryItemData.mediaMetadata.title, - titleIgnorePrefix: getTitleIgnorePrefix(libraryItemData.mediaMetadata.title), + titleIgnorePrefix: undefined, author: undefined, releaseDate: undefined, feedURL: undefined, @@ -336,132 +337,19 @@ class PodcastScanner { genres: [] } + // Use audio meta tags if (podcastEpisodes.length) { - const audioFileMetaTags = podcastEpisodes[0].audioFile.metaTags - - const MetadataMapArray = [ - { - tag: 'tagAlbum', - altTag: 'tagSeries', - key: 'title' - }, - { - tag: 'tagArtist', - key: 'author' - }, - { - tag: 'tagGenre', - key: 'genres' - }, - { - tag: 'tagLanguage', - key: 'language' - }, - { - tag: 'tagItunesId', - key: 'itunesId' - }, - { - tag: 'tagPodcastType', - key: 'podcastType', - } - ] - - MetadataMapArray.forEach((mapping) => { - let value = audioFileMetaTags[mapping.tag] - let tagToUse = mapping.tag - if (!value && mapping.altTag) { - value = audioFileMetaTags[mapping.altTag] - tagToUse = mapping.altTag - } - - if (value && typeof value === 'string') { - value = value.trim() // Trim whitespace - - if (mapping.key === 'genres') { - podcastMetadata.genres = this.parseGenresString(value) - libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata.genres.join(', ')}`) - } else { - podcastMetadata[mapping.key] = value - libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata[mapping.key]}`) - } - } - }) + AudioFileScanner.setPodcastMetadataFromAudioMetaTags(podcastEpisodes[0].audioFile, podcastMetadata, libraryScan) } - // If metadata.json or metadata.abs use this for metadata - const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile - let metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null - let metadataFilePath = metadataLibraryFile?.metadata.path - let metadataFileFormat = libraryItemData.metadataJsonLibraryFile ? 'json' : 'abs' - - // When metadata file is not stored with library item then check in the /metadata/items folder for it - if (!metadataText && existingLibraryItemId) { - let metadataPath = Path.join(global.MetadataPath, 'items', existingLibraryItemId) - - let altFormat = global.ServerSettings.metadataFileFormat === 'json' ? 'abs' : 'json' - // First check the metadata format set in server settings, fallback to the alternate - metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`) - metadataFileFormat = global.ServerSettings.metadataFileFormat - if (await fsExtra.pathExists(metadataFilePath)) { - metadataText = await readTextFile(metadataFilePath) - } else if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.${altFormat}`))) { - metadataFilePath = Path.join(metadataPath, `metadata.${altFormat}`) - metadataFileFormat = altFormat - metadataText = await readTextFile(metadataFilePath) - } - } - - if (metadataText) { - libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataFilePath}" - preferring`) - let abMetadata = null - if (metadataFileFormat === 'json') { - abMetadata = abmetadataGenerator.parseJson(metadataText) - } else { - abMetadata = abmetadataGenerator.parse(metadataText, 'podcast') - } - - if (abMetadata) { - if (abMetadata.tags?.length) { - podcastMetadata.tags = abMetadata.tags - } - for (const key in abMetadata.metadata) { - if (abMetadata.metadata[key] === undefined) continue - - // TODO: New podcast model changed some keys, need to update the abmetadataGenerator - let newModelKey = key - if (key === 'feedUrl') newModelKey = 'feedURL' - else if (key === 'imageUrl') newModelKey = 'imageURL' - else if (key === 'itunesPageUrl') newModelKey = 'itunesPageURL' - else if (key === 'type') newModelKey = 'podcastType' - - podcastMetadata[newModelKey] = abMetadata.metadata[key] - } - } - } + // Use metadata.json or metadata.abs file + await AbsMetadataFileScanner.scanPodcastMetadataFile(libraryScan, libraryItemData, podcastMetadata, existingLibraryItemId) podcastMetadata.titleIgnorePrefix = getTitleIgnorePrefix(podcastMetadata.title) return podcastMetadata } - /** - * Parse a genre string into multiple genres - * @example "Fantasy;Sci-Fi;History" => ["Fantasy", "Sci-Fi", "History"] - * @param {string} genreTag - * @returns {string[]} - */ - parseGenresString(genreTag) { - if (!genreTag?.length) return [] - const separators = ['/', '//', ';'] - for (let i = 0; i < separators.length; i++) { - if (genreTag.includes(separators[i])) { - return genreTag.split(separators[i]).map(genre => genre.trim()).filter(g => !!g) - } - } - return [genreTag] - } - /** * * @param {import('../models/LibraryItem')} libraryItem @@ -589,80 +477,5 @@ class PodcastScanner { }) } } - - /** - * - * @param {PodcastEpisode} podcastEpisode Not the model when creating new podcast - * @param {import('./ScanLogger')} scanLogger - */ - setPodcastEpisodeMetadataFromAudioFile(podcastEpisode, scanLogger) { - const MetadataMapArray = [ - { - tag: 'tagComment', - altTag: 'tagSubtitle', - key: 'description' - }, - { - tag: 'tagSubtitle', - key: 'subtitle' - }, - { - tag: 'tagDate', - key: 'pubDate' - }, - { - tag: 'tagDisc', - key: 'season', - }, - { - tag: 'tagTrack', - altTag: 'tagSeriesPart', - key: 'episode' - }, - { - tag: 'tagTitle', - key: 'title' - }, - { - tag: 'tagEpisodeType', - key: 'episodeType' - } - ] - - const audioFileMetaTags = podcastEpisode.audioFile.metaTags - MetadataMapArray.forEach((mapping) => { - let value = audioFileMetaTags[mapping.tag] - let tagToUse = mapping.tag - if (!value && mapping.altTag) { - tagToUse = mapping.altTag - value = audioFileMetaTags[mapping.altTag] - } - - if (value && typeof value === 'string') { - value = value.trim() // Trim whitespace - - if (mapping.key === 'pubDate') { - const pubJsDate = new Date(value) - if (pubJsDate && !isNaN(pubJsDate)) { - podcastEpisode.publishedAt = pubJsDate.valueOf() - podcastEpisode.pubDate = value - scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`) - } else { - scanLogger.addLog(LogLevel.WARN, `Mapping pubDate with tag ${tagToUse} has invalid date "${value}"`) - } - } else if (mapping.key === 'episodeType') { - if (['full', 'trailer', 'bonus'].includes(value)) { - podcastEpisode.episodeType = value - scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`) - } else { - scanLogger.addLog(LogLevel.WARN, `Mapping episodeType with invalid value "${value}". Must be one of [full, trailer, bonus].`) - } - } else { - podcastEpisode[mapping.key] = value - scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`) - } - } - }) - } } module.exports = new PodcastScanner() \ No newline at end of file