Mirrors/audiobookshelf
Mirrors/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2025-01-22 00:07:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

Podcast scanner refactor/cleanup

Browse Source
This commit is contained in:
advplyr 2023-10-09 16:41:43 -05:00
parent 347b49f564
commit 89821b91b0
5 changed files with 220 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
server/scanner/AbsMetadataFileScanner.js
View File

@ -61,5 +61,65 @@ class AbsMetadataFileScanner {
}
}
}
/**
* Check for metadata.json or metadata.abs file and set podcast metadata
*
* @param {import('./LibraryScan')} libraryScan
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {Object} podcastMetadata
* @param {string} [existingLibraryItemId]
*/
async scanPodcastMetadataFile(libraryScan, libraryItemData, podcastMetadata, existingLibraryItemId = null) {
const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile
let metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
let metadataFilePath = metadataLibraryFile?.metadata.path
let metadataFileFormat = libraryItemData.metadataJsonLibraryFile ? 'json' : 'abs'
// When metadata file is not stored with library item then check in the /metadata/items folder for it
if (!metadataText && existingLibraryItemId) {
let metadataPath = Path.join(global.MetadataPath, 'items', existingLibraryItemId)
let altFormat = global.ServerSettings.metadataFileFormat === 'json' ? 'abs' : 'json'
// First check the metadata format set in server settings, fallback to the alternate
metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`)
metadataFileFormat = global.ServerSettings.metadataFileFormat
if (await fsExtra.pathExists(metadataFilePath)) {
metadataText = await readTextFile(metadataFilePath)
} else if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.${altFormat}`))) {
metadataFilePath = Path.join(metadataPath, `metadata.${altFormat}`)
metadataFileFormat = altFormat
metadataText = await readTextFile(metadataFilePath)
}
}
if (metadataText) {
libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataFilePath}" - preferring`)
let abMetadata = null
if (metadataFileFormat === 'json') {
abMetadata = abmetadataGenerator.parseJson(metadataText)
} else {
abMetadata = abmetadataGenerator.parse(metadataText, 'podcast')
}
if (abMetadata) {
if (abMetadata.tags?.length) {
podcastMetadata.tags = abMetadata.tags
}
for (const key in abMetadata.metadata) {
if (abMetadata.metadata[key] === undefined) continue
// TODO: New podcast model changed some keys, need to update the abmetadataGenerator
let newModelKey = key
if (key === 'feedUrl') newModelKey = 'feedURL'
else if (key === 'imageUrl') newModelKey = 'imageURL'
else if (key === 'itunesPageUrl') newModelKey = 'itunesPageURL'
else if (key === 'type') newModelKey = 'podcastType'
podcastMetadata[newModelKey] = abMetadata.metadata[key]
}
}
}
}
}
module.exports = new AbsMetadataFileScanner()

139
server/scanner/AudioFileScanner.js
View File

@ -215,7 +215,7 @@ class AudioFileScanner {
* @param {string} bookTitle
* @param {import('../models/Book').AudioFileObject} audioFile
* @param {Object} bookMetadata
* @param {LibraryScan} libraryScan
* @param {import('./LibraryScan')} libraryScan
*/
setBookMetadataFromAudioMetaTags(bookTitle, audioFiles, bookMetadata, libraryScan) {
const MetadataMapArray = [
@ -309,10 +309,145 @@ class AudioFileScanner {
}
}
/**
* Set podcast metadata from first audio file
*
* @param {import('../models/Book').AudioFileObject} audioFile
* @param {Object} podcastMetadata
* @param {import('./LibraryScan')} libraryScan
*/
setPodcastMetadataFromAudioMetaTags(audioFile, podcastMetadata, libraryScan) {
const audioFileMetaTags = audioFile.metaTags
const MetadataMapArray = [
{
tag: 'tagAlbum',
altTag: 'tagSeries',
key: 'title'
},
{
tag: 'tagArtist',
key: 'author'
},
{
tag: 'tagGenre',
key: 'genres'
},
{
tag: 'tagLanguage',
key: 'language'
},
{
tag: 'tagItunesId',
key: 'itunesId'
},
{
tag: 'tagPodcastType',
key: 'podcastType',
}
]
MetadataMapArray.forEach((mapping) => {
let value = audioFileMetaTags[mapping.tag]
let tagToUse = mapping.tag
if (!value && mapping.altTag) {
value = audioFileMetaTags[mapping.altTag]
tagToUse = mapping.altTag
}
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'genres') {
podcastMetadata.genres = this.parseGenresString(value)
libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata.genres.join(', ')}`)
} else {
podcastMetadata[mapping.key] = value
libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata[mapping.key]}`)
}
}
})
}
/**
*
* @param {import('../models/PodcastEpisode')} podcastEpisode Not the model when creating new podcast
* @param {import('./ScanLogger')} scanLogger
*/
setPodcastEpisodeMetadataFromAudioMetaTags(podcastEpisode, scanLogger) {
const MetadataMapArray = [
{
tag: 'tagComment',
altTag: 'tagSubtitle',
key: 'description'
},
{
tag: 'tagSubtitle',
key: 'subtitle'
},
{
tag: 'tagDate',
key: 'pubDate'
},
{
tag: 'tagDisc',
key: 'season',
},
{
tag: 'tagTrack',
altTag: 'tagSeriesPart',
key: 'episode'
},
{
tag: 'tagTitle',
key: 'title'
},
{
tag: 'tagEpisodeType',
key: 'episodeType'
}
]
const audioFileMetaTags = podcastEpisode.audioFile.metaTags
MetadataMapArray.forEach((mapping) => {
let value = audioFileMetaTags[mapping.tag]
let tagToUse = mapping.tag
if (!value && mapping.altTag) {
tagToUse = mapping.altTag
value = audioFileMetaTags[mapping.altTag]
}
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'pubDate') {
const pubJsDate = new Date(value)
if (pubJsDate && !isNaN(pubJsDate)) {
podcastEpisode.publishedAt = pubJsDate.valueOf()
podcastEpisode.pubDate = value
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
} else {
scanLogger.addLog(LogLevel.WARN, `Mapping pubDate with tag ${tagToUse} has invalid date "${value}"`)
}
} else if (mapping.key === 'episodeType') {
if (['full', 'trailer', 'bonus'].includes(value)) {
podcastEpisode.episodeType = value
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
} else {
scanLogger.addLog(LogLevel.WARN, `Mapping episodeType with invalid value "${value}". Must be one of [full, trailer, bonus].`)
}
} else {
podcastEpisode[mapping.key] = value
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
}
}
})
}
/**
* @param {string} bookTitle
* @param {AudioFile[]} audioFiles
* @param {LibraryScan} libraryScan
* @param {import('./LibraryScan')} libraryScan
* @returns {import('../models/Book').ChapterObject[]}
*/
getBookChaptersFromAudioFiles(bookTitle, audioFiles, libraryScan) {

5
server/scanner/BookScanner.js
View File

@ -580,9 +580,10 @@ class BookScanner {
}
const bookMetadataSourceHandler = new BookScanner.BookMetadataSourceHandler(bookMetadata, audioFiles, libraryItemData, libraryScan, existingLibraryItemId)
for (const metadataSource of librarySettings.metadataPrecedence) {
const metadataPrecedence = librarySettings.metadataPrecedence || ['folderStructure', 'audioMetatags', 'txtFiles', 'opfFile', 'absMetadata']
libraryScan.addLog(LogLevel.DEBUG, `"${bookMetadata.title}" Getting metadata with precedence [${metadataPrecedence.join(', ')}]`)
for (const metadataSource of metadataPrecedence) {
if (bookMetadataSourceHandler[metadataSource]) {
libraryScan.addLog(LogLevel.DEBUG, `Getting metadata from source "${metadataSource}"`)
await bookMetadataSourceHandler[metadataSource]()
} else {
libraryScan.addLog(LogLevel.ERROR, `Invalid metadata source "${metadataSource}"`)

23
server/scanner/LibraryItemScanner.js
View File

@ -50,28 +50,25 @@ class LibraryItemScanner {
const scanLogger = new ScanLogger()
scanLogger.verbose = true
scanLogger.setData('libraryItem', libraryItemId)
scanLogger.setData('libraryItem', libraryItem.relPath)
const libraryItemPath = fileUtils.filePathToPOSIX(libraryItem.path)
const folder = library.libraryFolders[0]
const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, false)
if (await libraryItemScanData.checkLibraryItemData(libraryItem, scanLogger)) {
if (libraryItemScanData.hasLibraryFileChanges || libraryItemScanData.hasPathChange) {
const { libraryItem: expandedLibraryItem } = await this.rescanLibraryItemMedia(libraryItem, libraryItemScanData, library.settings, scanLogger)
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(expandedLibraryItem)
SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded())
let libraryItemDataUpdated = await libraryItemScanData.checkLibraryItemData(libraryItem, scanLogger)
await this.checkAuthorsAndSeriesRemovedFromBooks(library.id, scanLogger)
} else {
// TODO: Temporary while using old model to socket emit
const oldLibraryItem = await Database.libraryItemModel.getOldById(libraryItem.id)
SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded())
}
const { libraryItem: expandedLibraryItem, wasUpdated } = await this.rescanLibraryItemMedia(libraryItem, libraryItemScanData, library.settings, scanLogger)
if (libraryItemDataUpdated || wasUpdated) {
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(expandedLibraryItem)
SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded())
await this.checkAuthorsAndSeriesRemovedFromBooks(library.id, scanLogger)
return ScanResult.UPDATED
}
libraryScan.addLog(LogLevel.DEBUG, `Library item "${libraryItem.relPath}" is up-to-date`)
scanLogger.addLog(LogLevel.DEBUG, `Library item is up-to-date`)
return ScanResult.UPTODATE
}

207
server/scanner/PodcastScanner.js
View File

@ -5,12 +5,13 @@ const { getTitleIgnorePrefix } = require('../utils/index')
const abmetadataGenerator = require('../utils/generators/abmetadataGenerator')
const AudioFileScanner = require('./AudioFileScanner')
const Database = require('../Database')
const { readTextFile, filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils')
const { filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils')
const AudioFile = require('../objects/files/AudioFile')
const CoverManager = require('../managers/CoverManager')
const LibraryFile = require('../objects/files/LibraryFile')
const fsExtra = require("../libs/fsExtra")
const PodcastEpisode = require("../models/PodcastEpisode")
const AbsMetadataFileScanner = require("./AbsMetadataFileScanner")
/**
* Metadata for podcasts pulled from files
@ -87,7 +88,7 @@ class PodcastScanner {
podcastEpisode.changed('audioFile', true)
// Set metadata and save episode
this.setPodcastEpisodeMetadataFromAudioFile(podcastEpisode, libraryScan)
AudioFileScanner.setPodcastEpisodeMetadataFromAudioMetaTags(podcastEpisode, libraryScan)
libraryScan.addLog(LogLevel.INFO, `Podcast episode "${podcastEpisode.title}" keys changed [${podcastEpisode.changed()?.join(', ')}]`)
await podcastEpisode.save()
}
@ -122,7 +123,7 @@ class PodcastScanner {
}
const newPodcastEpisode = Database.podcastEpisodeModel.build(newEpisode)
// Set metadata and save new episode
this.setPodcastEpisodeMetadataFromAudioFile(newPodcastEpisode, libraryScan)
AudioFileScanner.setPodcastEpisodeMetadataFromAudioMetaTags(newPodcastEpisode, libraryScan)
libraryScan.addLog(LogLevel.INFO, `New Podcast episode "${newPodcastEpisode.title}" added`)
await newPodcastEpisode.save()
existingPodcastEpisodes.push(newPodcastEpisode)
@ -242,7 +243,7 @@ class PodcastScanner {
}
// Set metadata and save new episode
this.setPodcastEpisodeMetadataFromAudioFile(newEpisode, libraryScan)
AudioFileScanner.setPodcastEpisodeMetadataFromAudioMetaTags(newEpisode, libraryScan)
libraryScan.addLog(LogLevel.INFO, `New Podcast episode "${newEpisode.title}" found`)
newPodcastEpisodes.push(newEpisode)
}
@ -320,7 +321,7 @@ class PodcastScanner {
async getPodcastMetadataFromScanData(podcastEpisodes, libraryItemData, libraryScan, existingLibraryItemId = null) {
const podcastMetadata = {
title: libraryItemData.mediaMetadata.title,
titleIgnorePrefix: getTitleIgnorePrefix(libraryItemData.mediaMetadata.title),
titleIgnorePrefix: undefined,
author: undefined,
releaseDate: undefined,
feedURL: undefined,
@ -336,132 +337,19 @@ class PodcastScanner {
genres: []
}
// Use audio meta tags
if (podcastEpisodes.length) {
const audioFileMetaTags = podcastEpisodes[0].audioFile.metaTags
const MetadataMapArray = [
{
tag: 'tagAlbum',
altTag: 'tagSeries',
key: 'title'
},
{
tag: 'tagArtist',
key: 'author'
},
{
tag: 'tagGenre',
key: 'genres'
},
{
tag: 'tagLanguage',
key: 'language'
},
{
tag: 'tagItunesId',
key: 'itunesId'
},
{
tag: 'tagPodcastType',
key: 'podcastType',
}
]
MetadataMapArray.forEach((mapping) => {
let value = audioFileMetaTags[mapping.tag]
let tagToUse = mapping.tag
if (!value && mapping.altTag) {
value = audioFileMetaTags[mapping.altTag]
tagToUse = mapping.altTag
}
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'genres') {
podcastMetadata.genres = this.parseGenresString(value)
libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata.genres.join(', ')}`)
} else {
podcastMetadata[mapping.key] = value
libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata[mapping.key]}`)
}
}
})
AudioFileScanner.setPodcastMetadataFromAudioMetaTags(podcastEpisodes[0].audioFile, podcastMetadata, libraryScan)
}
// If metadata.json or metadata.abs use this for metadata
const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile
let metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
let metadataFilePath = metadataLibraryFile?.metadata.path
let metadataFileFormat = libraryItemData.metadataJsonLibraryFile ? 'json' : 'abs'
// When metadata file is not stored with library item then check in the /metadata/items folder for it
if (!metadataText && existingLibraryItemId) {
let metadataPath = Path.join(global.MetadataPath, 'items', existingLibraryItemId)
let altFormat = global.ServerSettings.metadataFileFormat === 'json' ? 'abs' : 'json'
// First check the metadata format set in server settings, fallback to the alternate
metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`)
metadataFileFormat = global.ServerSettings.metadataFileFormat
if (await fsExtra.pathExists(metadataFilePath)) {
metadataText = await readTextFile(metadataFilePath)
} else if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.${altFormat}`))) {
metadataFilePath = Path.join(metadataPath, `metadata.${altFormat}`)
metadataFileFormat = altFormat
metadataText = await readTextFile(metadataFilePath)
}
}
if (metadataText) {
libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataFilePath}" - preferring`)
let abMetadata = null
if (metadataFileFormat === 'json') {
abMetadata = abmetadataGenerator.parseJson(metadataText)
} else {
abMetadata = abmetadataGenerator.parse(metadataText, 'podcast')
}
if (abMetadata) {
if (abMetadata.tags?.length) {
podcastMetadata.tags = abMetadata.tags
}
for (const key in abMetadata.metadata) {
if (abMetadata.metadata[key] === undefined) continue
// TODO: New podcast model changed some keys, need to update the abmetadataGenerator
let newModelKey = key
if (key === 'feedUrl') newModelKey = 'feedURL'
else if (key === 'imageUrl') newModelKey = 'imageURL'
else if (key === 'itunesPageUrl') newModelKey = 'itunesPageURL'
else if (key === 'type') newModelKey = 'podcastType'
podcastMetadata[newModelKey] = abMetadata.metadata[key]
}
}
}
// Use metadata.json or metadata.abs file
await AbsMetadataFileScanner.scanPodcastMetadataFile(libraryScan, libraryItemData, podcastMetadata, existingLibraryItemId)
podcastMetadata.titleIgnorePrefix = getTitleIgnorePrefix(podcastMetadata.title)
return podcastMetadata
}
/**
* Parse a genre string into multiple genres
* @example "Fantasy;Sci-Fi;History" => ["Fantasy", "Sci-Fi", "History"]
* @param {string} genreTag
* @returns {string[]}
*/
parseGenresString(genreTag) {
if (!genreTag?.length) return []
const separators = ['/', '//', ';']
for (let i = 0; i < separators.length; i++) {
if (genreTag.includes(separators[i])) {
return genreTag.split(separators[i]).map(genre => genre.trim()).filter(g => !!g)
}
}
return [genreTag]
}
/**
*
* @param {import('../models/LibraryItem')} libraryItem
@ -589,80 +477,5 @@ class PodcastScanner {
})
}
}
/**
*
* @param {PodcastEpisode} podcastEpisode Not the model when creating new podcast
* @param {import('./ScanLogger')} scanLogger
*/
setPodcastEpisodeMetadataFromAudioFile(podcastEpisode, scanLogger) {
const MetadataMapArray = [
{
tag: 'tagComment',
altTag: 'tagSubtitle',
key: 'description'
},
{
tag: 'tagSubtitle',
key: 'subtitle'
},
{
tag: 'tagDate',
key: 'pubDate'
},
{
tag: 'tagDisc',
key: 'season',
},
{
tag: 'tagTrack',
altTag: 'tagSeriesPart',
key: 'episode'
},
{
tag: 'tagTitle',
key: 'title'
},
{
tag: 'tagEpisodeType',
key: 'episodeType'
}
]
const audioFileMetaTags = podcastEpisode.audioFile.metaTags
MetadataMapArray.forEach((mapping) => {
let value = audioFileMetaTags[mapping.tag]
let tagToUse = mapping.tag
if (!value && mapping.altTag) {
tagToUse = mapping.altTag
value = audioFileMetaTags[mapping.altTag]
}
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'pubDate') {
const pubJsDate = new Date(value)
if (pubJsDate && !isNaN(pubJsDate)) {
podcastEpisode.publishedAt = pubJsDate.valueOf()
podcastEpisode.pubDate = value
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
} else {
scanLogger.addLog(LogLevel.WARN, `Mapping pubDate with tag ${tagToUse} has invalid date "${value}"`)
}
} else if (mapping.key === 'episodeType') {
if (['full', 'trailer', 'bonus'].includes(value)) {
podcastEpisode.episodeType = value
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
} else {
scanLogger.addLog(LogLevel.WARN, `Mapping episodeType with invalid value "${value}". Must be one of [full, trailer, bonus].`)
}
} else {
podcastEpisode[mapping.key] = value
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
}
}
})
}
}
module.exports = new PodcastScanner()