From b3d9323f66e75583e90d0bd44bf3d9474d82a5b7 Mon Sep 17 00:00:00 2001 From: jmt-gh Date: Sat, 11 Jun 2022 23:17:22 -0700 Subject: [PATCH] Initial commit for server side approach This is the first commit for bringing this over to the server side. It works! Right now it fails if the autoscanner or or the manual individual book scanner try to do it's thing. I'll need to update those --- server/controllers/PodcastController.js | 2 +- server/objects/mediaTypes/Book.js | 187 +++++++++++++++++----- server/objects/settings/ServerSettings.js | 3 + server/scanner/MediaFileScanner.js | 14 +- server/scanner/ScanOptions.js | 5 +- server/scanner/Scanner.js | 6 + 6 files changed, 175 insertions(+), 42 deletions(-) diff --git a/server/controllers/PodcastController.js b/server/controllers/PodcastController.js index 81029c95..9d1aa2c8 100644 --- a/server/controllers/PodcastController.js +++ b/server/controllers/PodcastController.js @@ -103,7 +103,7 @@ class PodcastController { Logger.error('Invalid podcast feed request response') return res.status(500).send('Bad response from feed request') } - Logger.debug(`[PdocastController] Podcast feed size ${(data.data.length / 1024 / 1024).toFixed(2)}MB`) + Logger.debug(`[PodcastController] Podcast feed size ${(data.data.length / 1024 / 1024).toFixed(2)}MB`) var payload = await parsePodcastRssFeedXml(data.data, false, includeRaw) if (!payload) { return res.status(500).send('Invalid podcast RSS feed') diff --git a/server/objects/mediaTypes/Book.js b/server/objects/mediaTypes/Book.js index 6f0f4a1d..d8ee7528 100644 --- a/server/objects/mediaTypes/Book.js +++ b/server/objects/mediaTypes/Book.js @@ -360,10 +360,12 @@ class Book { this.rebuildTracks() } - rebuildTracks() { + rebuildTracks(preferOverdriveMediaMarker = false) { + Logger.debug(`[Book] we are rebuilding the tracks!`) + Logger.debug(`[Book] preferOverdriveMediaMarker: ${preferOverdriveMediaMarker}`) this.audioFiles.sort((a, b) => a.index - b.index) this.missingParts = [] - this.setChapters() + this.setChapters(preferOverdriveMediaMarker) this.checkUpdateMissingTracks() } @@ -395,7 +397,103 @@ class Book { return wasUpdated } - setChapters() { + generateChaptersFromOverdriveMediaMarkers(overdriveMediaMarkers, includedAudioFiles) { + var parseString = require('xml2js').parseString; // function to convert xml to JSON + + var parsedOverdriveMediaMarkers = [] // an array of objects. each object being a chapter with a name and time key. the values are arrays of strings + + overdriveMediaMarkers.forEach(function (item, index) { + var parsed_result + parseString(item, function (err, result) { + // result.Markers.Marker is the result of parsing the XML for the MediaMarker tags for the MP3 file (Part##.mp3) + // it is shaped like this: + // [ + // { + // "Name": [ + // "Chapter 1: " + // ], + // "Time": [ + // "0:00.000" + // ] + // }, + // { + // "Name": [ + // "Chapter 2: " + // ], + // "Time": [ + // "15:51.000" + // ] + // } + // ] + + parsed_result = result.Markers.Marker + + // The values for Name and Time in parsed_results are returned as Arrays from parseString + // update them to be strings + parsed_result.forEach((item, index) => { + Object.keys(item).forEach(key => { + item[key] = item[key].toString() + }) + }) + }) + + parsedOverdriveMediaMarkers.push(parsed_result) + }) + + // go from an array of arrays of objects to an array of objects + // end result looks like: + // [ + // { + // "Name": "Chapter 1: The Worst Birthday", + // "Time": "0:00.000" + // }, + // { + // "Name": "Chapter 2: Dobby's Warning", + // "Time": "15:51.000" + // }, + // { redacted } + // ] + parsedOverdriveMediaMarkers = parsedOverdriveMediaMarkers + + var index = 0 + + var time = 0.0 + + + // actually generate the chapter object + // logic ported over from benonymity's OverdriveChapterizer: + // https://github.com/benonymity/OverdriveChapterizer/blob/main/chapters.py + var length = 0.0 + var newOChapters = [] + const weirdChapterFilterRegex = /([(]\d|[cC]ontinued)/ + includedAudioFiles.forEach((track, track_index) => { + parsedOverdriveMediaMarkers[track_index].forEach((chapter) => { + Logger.debug(`[Book] Attempting regex check for ${chapter.Name}!`) + if (weirdChapterFilterRegex.test(chapter.Name)) { + Logger.debug(`[Book] That shit weird yo`) + return + } + time = chapter.Time.split(":") + time = length + parseFloat(time[0]) * 60 + parseFloat(time[1]) + newOChapters.push( + { + id: index++, + start: time, + end: length, + title: chapter.Name + } + ) + }) + length += track.duration + }) + + Logger.debug(`[Book] newOChapters: ${JSON.stringify(newOChapters)}`) + return newOChapters + } + + + setChapters(preferOverdriveMediaMarker = false) { + Logger.debug('[Book] inside setChapters!') // If 1 audio file without chapters, then no chapters will be set var includedAudioFiles = this.audioFiles.filter(af => !af.exclude) if (includedAudioFiles.length === 1) { @@ -407,44 +505,55 @@ class Book { this.chapters = [] var currChapterId = 0 var currStartTime = 0 - includedAudioFiles.forEach((file) => { - // If audio file has chapters use chapters - if (file.chapters && file.chapters.length) { - file.chapters.forEach((chapter) => { - if (chapter.start > this.duration) { - Logger.warn(`[Book] Invalid chapter start time > duration`) - } else { - var chapterAlreadyExists = this.chapters.find(ch => ch.start === chapter.start) - if (!chapterAlreadyExists) { - var chapterDuration = chapter.end - chapter.start - if (chapterDuration > 0) { - var title = `Chapter ${currChapterId}` - if (chapter.title) { - title += ` (${chapter.title})` + var overdriveMediaMarkers = includedAudioFiles.map((af) => af.metaTags.tagOverdriveMediaMarker).filter(notUndefined => notUndefined !== undefined) || [] + Logger.debug(`[setChapters] overdriveMediaMarkers: ${JSON.stringify(overdriveMediaMarkers)}`) + + // If preferOverdriveMediaMarker is set, try and use that first + if (preferOverdriveMediaMarker) { + Logger.debug(`[Book] preferring overdrive media markers! Lets generate em.`) + this.chapters = this.generateChaptersFromOverdriveMediaMarkers(overdriveMediaMarkers, includedAudioFiles) + + } else { + includedAudioFiles.forEach((file) => { + //console.log(`audiofile MetaTags Overdrive: ${JSON.stringify(file.metaTags.tagOverdriveMediaMarker)}}`) + // If audio file has chapters use chapters + if (file.chapters && file.chapters.length) { + file.chapters.forEach((chapter) => { + if (chapter.start > this.duration) { + Logger.warn(`[Book] Invalid chapter start time > duration`) + } else { + var chapterAlreadyExists = this.chapters.find(ch => ch.start === chapter.start) + if (!chapterAlreadyExists) { + var chapterDuration = chapter.end - chapter.start + if (chapterDuration > 0) { + var title = `Chapter ${currChapterId}` + if (chapter.title) { + title += ` (${chapter.title})` + } + var endTime = Math.min(this.duration, currStartTime + chapterDuration) + this.chapters.push({ + id: currChapterId++, + start: currStartTime, + end: endTime, + title + }) + currStartTime += chapterDuration + } } - var endTime = Math.min(this.duration, currStartTime + chapterDuration) - this.chapters.push({ - id: currChapterId++, - start: currStartTime, - end: endTime, - title - }) - currStartTime += chapterDuration } - } - } - }) - } else if (file.duration) { - // Otherwise just use track has chapter - this.chapters.push({ - id: currChapterId++, - start: currStartTime, - end: currStartTime + file.duration, - title: file.metadata.filename ? Path.basename(file.metadata.filename, Path.extname(file.metadata.filename)) : `Chapter ${currChapterId}` - }) - currStartTime += file.duration - } - }) + }) + } else if (file.duration) { + // Otherwise just use track has chapter + this.chapters.push({ + id: currChapterId++, + start: currStartTime, + end: currStartTime + file.duration, + title: file.metadata.filename ? Path.basename(file.metadata.filename, Path.extname(file.metadata.filename)) : `Chapter ${currChapterId}` + }) + currStartTime += file.duration + } + }) + } } } diff --git a/server/objects/settings/ServerSettings.js b/server/objects/settings/ServerSettings.js index 286e15c0..00c56b2a 100644 --- a/server/objects/settings/ServerSettings.js +++ b/server/objects/settings/ServerSettings.js @@ -13,6 +13,7 @@ class ServerSettings { this.scannerPreferOpfMetadata = false this.scannerPreferMatchedMetadata = false this.scannerDisableWatcher = false + this.scannerPreferOverdriveMediaMarker = false // Metadata - choose to store inside users library item folder this.storeCoverWithItem = false @@ -65,6 +66,7 @@ class ServerSettings { this.scannerPreferOpfMetadata = !!settings.scannerPreferOpfMetadata this.scannerPreferMatchedMetadata = !!settings.scannerPreferMatchedMetadata this.scannerDisableWatcher = !!settings.scannerDisableWatcher + this.scannerPreferOverdriveMediaMarker = !!settings.scannerPreferOverdriveMediaMarker this.storeCoverWithItem = !!settings.storeCoverWithItem if (settings.storeCoverWithBook != undefined) { // storeCoverWithBook was old name of setting < v2 @@ -111,6 +113,7 @@ class ServerSettings { scannerPreferOpfMetadata: this.scannerPreferOpfMetadata, scannerPreferMatchedMetadata: this.scannerPreferMatchedMetadata, scannerDisableWatcher: this.scannerDisableWatcher, + scannerPreferOverdriveMediaMarker: this.scannerPreferOverdriveMediaMarker, storeCoverWithItem: this.storeCoverWithItem, storeMetadataWithItem: this.storeMetadataWithItem, rateLimitLoginRequests: this.rateLimitLoginRequests, diff --git a/server/scanner/MediaFileScanner.js b/server/scanner/MediaFileScanner.js index bccbae26..d10e185f 100644 --- a/server/scanner/MediaFileScanner.js +++ b/server/scanner/MediaFileScanner.js @@ -196,6 +196,9 @@ class MediaFileScanner { } async scanMediaFiles(mediaLibraryFiles, scanData, libraryItem, preferAudioMetadata, libraryScan = null) { + Logger.debug('[scanMediaFiles] inside scan media files!') + Logger.debug(`[scanMediaFiles] libraryScan: ${JSON.stringify(libraryScan)}`) + var hasUpdated = false var mediaScanResult = await this.executeMediaFileScans(libraryItem.mediaType, mediaLibraryFiles, scanData) @@ -208,6 +211,7 @@ class MediaFileScanner { } else if (mediaScanResult.audioFiles.length) { if (libraryScan) { libraryScan.addLog(LogLevel.DEBUG, `Library Item "${scanData.path}" Audio file scan took ${mediaScanResult.elapsed}ms for ${mediaScanResult.audioFiles.length} with average time of ${mediaScanResult.averageScanDuration}ms`) + Logger.debug(`Library Item "${scanData.path}" Audio file scan took ${mediaScanResult.elapsed}ms for ${mediaScanResult.audioFiles.length} with average time of ${mediaScanResult.averageScanDuration}ms`) } var totalAudioFilesToInclude = mediaScanResult.audioFiles.length @@ -217,18 +221,23 @@ class MediaFileScanner { // Book: Adding audio files to book media if (libraryItem.mediaType === 'book') { + Logger.debug('Its a book!') if (newAudioFiles.length) { + Logger.debug('[MediaFileScanner] newAudioFiles.length was true?') // Single Track Audiobooks if (totalAudioFilesToInclude === 1) { + Logger.debug('[MediaFileScanner] totalAudioFilesToInclude === 1') var af = mediaScanResult.audioFiles[0] af.index = 1 libraryItem.media.addAudioFile(af) hasUpdated = true } else { + Logger.debug('[MediaFileScanner] totalAudioFilesToInclude === 1 WAS FALSE') this.runSmartTrackOrder(libraryItem, mediaScanResult.audioFiles) hasUpdated = true } } else { + Logger.debug('[MediaFileScanner] Only updating metadata?') // Only update metadata not index mediaScanResult.audioFiles.forEach((af) => { var existingAF = libraryItem.media.findFileWithInode(af.ino) @@ -247,7 +256,9 @@ class MediaFileScanner { } if (hasUpdated) { - libraryItem.media.rebuildTracks() + Logger.debug('[MediaFileScanner] hasUpdated is true! Going to rebuild tracks now...') + Logger.debug(`[MediaFileScanner] libraryScan: ${JSON.stringify(libraryScan)}`) + libraryItem.media.rebuildTracks(libraryScan.scanOptions.preferOverdriveMediaMarker) } } else { // Podcast Media Type var existingAudioFiles = mediaScanResult.audioFiles.filter(af => libraryItem.media.findFileWithInode(af.ino)) @@ -264,6 +275,7 @@ class MediaFileScanner { // Update audio file metadata for audio files already there existingAudioFiles.forEach((af) => { var peAudioFile = libraryItem.media.findFileWithInode(af.ino) + Logger.debug(`[MediaFileScanner] peAudioFile: ${JSON.stringify(peAudioFile)}`) if (peAudioFile.updateFromScan && peAudioFile.updateFromScan(af)) { hasUpdated = true } diff --git a/server/scanner/ScanOptions.js b/server/scanner/ScanOptions.js index 3d9d4556..968f0723 100644 --- a/server/scanner/ScanOptions.js +++ b/server/scanner/ScanOptions.js @@ -9,6 +9,7 @@ class ScanOptions { this.preferAudioMetadata = false this.preferOpfMetadata = false this.preferMatchedMetadata = false + this.preferOverdriveMediaMarker = false if (options) { this.construct(options) @@ -34,7 +35,8 @@ class ScanOptions { storeCoverWithItem: this.storeCoverWithItem, preferAudioMetadata: this.preferAudioMetadata, preferOpfMetadata: this.preferOpfMetadata, - preferMatchedMetadata: this.preferMatchedMetadata + preferMatchedMetadata: this.preferMatchedMetadata, + preferOverdriveMediaMarker: this.preferOverdriveMediaMarker } } @@ -47,6 +49,7 @@ class ScanOptions { this.preferAudioMetadata = serverSettings.scannerPreferAudioMetadata this.preferOpfMetadata = serverSettings.scannerPreferOpfMetadata this.scannerPreferMatchedMetadata = serverSettings.scannerPreferMatchedMetadata + this.preferOverdriveMediaMarker = serverSettings.scannerPreferOverdriveMediaMarker } } module.exports = ScanOptions \ No newline at end of file diff --git a/server/scanner/Scanner.js b/server/scanner/Scanner.js index 677585e1..6832e27a 100644 --- a/server/scanner/Scanner.js +++ b/server/scanner/Scanner.js @@ -62,6 +62,7 @@ class Scanner { } async scanLibraryItem(libraryMediaType, folder, libraryItem) { + Logger.debug(`[Scanner] SCANNING ITEMS JOE`) // TODO: Support for single media item var libraryItemData = await getLibraryItemFileData(libraryMediaType, folder, libraryItem.path, false, this.db.serverSettings) if (!libraryItemData) { @@ -114,6 +115,7 @@ class Scanner { } async scan(library, options = {}) { + Logger.debug('[scan] inside of scan') if (this.isLibraryScanning(library.id)) { Logger.error(`[Scanner] Already scanning ${library.id}`) return @@ -126,6 +128,7 @@ class Scanner { var scanOptions = new ScanOptions() scanOptions.setData(options, this.db.serverSettings) + Logger.debug(`[Scanner] scanOptions: ${JSON.stringify(scanOptions)}`) var libraryScan = new LibraryScan() libraryScan.setData(library, scanOptions) @@ -165,6 +168,8 @@ class Scanner { async scanLibrary(libraryScan) { var libraryItemDataFound = [] + Logger.debug(`[scanLibrary] libraryScan: ${JSON.stringify(libraryScan)}`) + // Scan each library for (let i = 0; i < libraryScan.folders.length; i++) { var folder = libraryScan.folders[i] @@ -750,6 +755,7 @@ class Scanner { } async matchLibraryItems(library) { + Logger.debug("SCANNING!") if (library.mediaType === 'podcast') { Logger.error(`[Scanner] matchLibraryItems: Match all not supported for podcasts yet`) return