From a5fc382cadb5cc2130818d8136fda782fc933b9a Mon Sep 17 00:00:00 2001 From: advplyr Date: Thu, 25 Nov 2021 18:39:02 -0600 Subject: [PATCH] Add:New scanner and scanner server settings --- client/layouts/default.vue | 26 ++ client/pages/config/index.vue | 31 +- server/Scanner.js | 13 +- server/Server.js | 15 +- server/Watcher.js | 7 +- server/objects/AudioFile.js | 40 ++- server/objects/AudioFileMetadata.js | 8 + server/objects/Audiobook.js | 71 +++-- server/objects/Book.js | 15 +- server/objects/ServerSettings.js | 5 + server/scanner/AudioFileScanner.js | 69 ++++- server/scanner/AudioProbeData.js | 1 - server/scanner/LibraryScan.js | 74 ++++- server/scanner/ScanOptions.js | 23 -- server/scanner/Scanner.js | 449 +++++++++++++++++++++++----- server/utils/constants.js | 8 - server/utils/scandir.js | 2 +- 17 files changed, 681 insertions(+), 176 deletions(-) diff --git a/client/layouts/default.vue b/client/layouts/default.vue index 25d6b23e..e0fed353 100644 --- a/client/layouts/default.vue +++ b/client/layouts/default.vue @@ -104,11 +104,23 @@ export default { if (payload.serverSettings) { this.$store.commit('setServerSettings', payload.serverSettings) } + + // Start scans currently running if (payload.librariesScanning) { payload.librariesScanning.forEach((libraryScan) => { this.scanStart(libraryScan) }) } + + // Remove any current scans that are no longer running + var currentScans = [...this.$store.state.scanners.libraryScans] + currentScans.forEach((ls) => { + if (!payload.librariesScanning || !payload.librariesScanning.find((_ls) => _ls.id === ls.id)) { + this.$toast.dismiss(ls.toastId) + this.$store.commit('scanners/remove', ls) + } + }) + if (payload.backups && payload.backups.length) { this.$store.commit('setBackups', payload.backups) } @@ -152,6 +164,16 @@ export default { } this.$store.commit('audiobooks/remove', audiobook) }, + audiobooksAdded(audiobooks) { + audiobooks.forEach((ab) => { + this.$store.commit('audiobooks/addUpdate', ab) + }) + }, + audiobooksUpdated(audiobooks) { + audiobooks.forEach((ab) => { + this.$store.commit('audiobooks/addUpdate', ab) + }) + }, libraryAdded(library) { this.$store.commit('libraries/addUpdate', library) }, @@ -162,6 +184,8 @@ export default { this.$store.commit('libraries/remove', library) }, scanComplete(data) { + console.log('Scan complete received', data) + var message = `Scan "${data.name}" complete!` if (data.results) { var scanResultMsgs = [] @@ -337,6 +361,8 @@ export default { this.socket.on('audiobook_updated', this.audiobookUpdated) this.socket.on('audiobook_added', this.audiobookAdded) this.socket.on('audiobook_removed', this.audiobookRemoved) + this.socket.on('audiobooks_updated', this.audiobooksUpdated) + this.socket.on('audiobooks_added', this.audiobooksAdded) // Library Listeners this.socket.on('library_updated', this.libraryUpdated) diff --git a/client/pages/config/index.vue b/client/pages/config/index.vue index 9b0ff432..49fb8c5d 100644 --- a/client/pages/config/index.vue +++ b/client/pages/config/index.vue @@ -21,6 +21,20 @@ +
+ + +

Scanner prefer audio metadata info_outlined

+ +
+ +
+ + +

Scanner prefer OPF metadata info_outlined

+ +
+
@@ -83,6 +97,12 @@ export default { } }, computed: { + scannerPreferAudioMetaTooltip() { + return 'Audio file ID3 meta tags will be used for book details over folder & filenames' + }, + scannerPreferOpfMetaTooltip() { + return 'OPF file metadata will be used for book details over folder & filenames' + }, saveMetadataTooltip() { return 'This will write a "metadata.nfo" file in all of your audiobook directories.' }, @@ -127,6 +147,16 @@ export default { scannerParseSubtitle: !!val }) }, + updateScannerPreferAudioMeta(val) { + this.updateServerSettings({ + scannerPreferAudioMetadata: !!val + }) + }, + updateScannerPreferOpfMeta(val) { + this.updateServerSettings({ + scannerPreferOpfMetadata: !!val + }) + }, updateServerSettings(payload) { this.updatingServerSettings = true this.$store @@ -144,7 +174,6 @@ export default { this.newServerSettings = this.serverSettings ? { ...this.serverSettings } : {} this.storeCoversInAudiobookDir = this.newServerSettings.coverDestination === this.$constants.CoverDestination.AUDIOBOOK - this.storeCoversInAudiobookDir = this.newServerSettings.coverDestination === this.$constants.CoverDestination.AUDIOBOOK }, resetAudiobooks() { if (confirm('WARNING! This action will remove all audiobooks from the database including any updates or matches you have made. This does not do anything to your actual files. Shall we continue?')) { diff --git a/server/Scanner.js b/server/Scanner.js index 60e2c015..cf2e057e 100644 --- a/server/Scanner.js +++ b/server/Scanner.js @@ -196,7 +196,7 @@ class Scanner { // Sync other files (all files that are not audio files) - Updates cover path var hasOtherFileUpdates = false - var otherFilesUpdated = await existingAudiobook.syncOtherFiles(audiobookData.otherFiles, this.MetadataPath, forceAudioFileScan) + var otherFilesUpdated = await existingAudiobook.syncOtherFiles(audiobookData.otherFiles, this.MetadataPath, false, forceAudioFileScan) if (otherFilesUpdated) { hasOtherFileUpdates = true } @@ -250,7 +250,7 @@ class Scanner { var hasUpdates = hasOtherFileUpdates || hasUpdatedIno || hasUpdatedLibraryOrFolder || removedAudioFiles.length || removedAudioTracks.length || newAudioFiles.length || hasUpdatedAudioFiles // Check that audio tracks are in sequential order with no gaps - if (existingAudiobook.checkUpdateMissingParts()) { + if (existingAudiobook.checkUpdateMissingTracks()) { Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`) hasUpdates = true } @@ -299,7 +299,7 @@ class Scanner { } // Look for desc.txt and reader.txt and update - await audiobook.saveDataFromTextFiles() + await audiobook.saveDataFromTextFiles(false) // Extract embedded cover art if cover is not already in directory if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) { @@ -314,7 +314,7 @@ class Scanner { audiobook.setDetailsFromFileMetadata() // Check for gaps in track numbers - audiobook.checkUpdateMissingParts() + audiobook.checkUpdateMissingTracks() // Set chapters from audio files audiobook.setChapters() @@ -671,11 +671,6 @@ class Scanner { var folder = library.getFolderById(folderId) if (!folder) { Logger.error(`[Scanner] Folder is not in library in files changed "${folderId}", Library "${library.name}"`) - - Logger.debug(`Looking at folders in library "${library.name}" for folderid ${folderId}`) - library.folders.forEach((fold) => { - Logger.debug(`Folder "${fold.id}" "${fold.fullPath}"`) - }) continue; } diff --git a/server/Server.js b/server/Server.js index 7a6c6710..8d76dd6e 100644 --- a/server/Server.js +++ b/server/Server.js @@ -311,19 +311,18 @@ class Server { async filesChanged(fileUpdates) { Logger.info('[Server]', fileUpdates.length, 'Files Changed') - await this.scanner.filesChanged(fileUpdates) - // Logger.debug('[Server] Files changed result', result) + await this.scanner2.scanFilesChanged(fileUpdates) } - async scan(libraryId, forceAudioFileScan = false) { + async scan(libraryId) { Logger.info('[Server] Starting Scan') - // await this.scanner2.scan(libraryId) - await this.scanner(libraryId, forceAudioFileScan) + await this.scanner2.scan(libraryId) + // await this.scanner.scan(libraryId) Logger.info('[Server] Scan complete') } async scanAudiobook(socket, audiobookId) { - var result = await this.scanner.scanAudiobookById(audiobookId) + var result = await this.scanner2.scanAudiobookById(audiobookId) var scanResultName = '' for (const key in ScanResult) { if (ScanResult[key] === result) { @@ -335,7 +334,7 @@ class Server { cancelScan(id) { Logger.debug('[Server] Cancel scan', id) - this.scanner.cancelLibraryScan[id] = true + this.scanner2.cancelLibraryScan[id] = true } // Generates an NFO metadata file, if no audiobookId is passed then all audiobooks are done @@ -624,7 +623,7 @@ class Server { configPath: this.ConfigPath, user: client.user.toJSONForBrowser(), stream: client.stream || null, - librariesScanning: this.scanner.librariesScanning, + librariesScanning: this.scanner2.librariesScanning, backups: (this.backupManager.backups || []).map(b => b.toJSON()) } if (user.type === 'root') { diff --git a/server/Watcher.js b/server/Watcher.js index 0de3bc29..c9fede87 100644 --- a/server/Watcher.js +++ b/server/Watcher.js @@ -24,8 +24,11 @@ class FolderWatcher extends EventEmitter { Logger.warn('[Watcher] Already watching library', library.name) return } - Logger.info(`[Watcher] Initializing watcher for "${library.name}"..`) + Logger.info(`[Watcher] Initializing watcher for "${library.name}".`) var folderPaths = library.folderPaths + folderPaths.forEach((fp) => { + Logger.debug(`[Watcher] Init watcher for library folder path "${fp}"`) + }) var watcher = new Watcher(folderPaths, { ignored: /(^|[\/\\])\../, // ignore dotfiles renameDetection: true, @@ -48,6 +51,8 @@ class FolderWatcher extends EventEmitter { Logger.error(`[Watcher] ${error}`) }).on('ready', () => { Logger.info(`[Watcher] "${library.name}" Ready`) + }).on('close', () => { + Logger.debug(`[Watcher] "${library.name}" Closed`) }) this.libraryWatchers.push({ diff --git a/server/objects/AudioFile.js b/server/objects/AudioFile.js index da7be888..b448d217 100644 --- a/server/objects/AudioFile.js +++ b/server/objects/AudioFile.js @@ -165,9 +165,9 @@ class AudioFile { this.fullPath = fileData.fullPath this.addedAt = Date.now() - this.trackNumFromMeta = fileData.trackNumFromMeta || null - this.trackNumFromFilename = fileData.trackNumFromFilename || null - this.cdNumFromFilename = fileData.cdNumFromFilename || null + this.trackNumFromMeta = fileData.trackNumFromMeta + this.trackNumFromFilename = fileData.trackNumFromFilename + this.cdNumFromFilename = fileData.cdNumFromFilename this.format = probeData.format this.duration = probeData.duration @@ -180,15 +180,13 @@ class AudioFile { this.channelLayout = probeData.channelLayout this.chapters = probeData.chapters || [] this.metadata = probeData.audioFileMetadata + this.embeddedCoverArt = probeData.embeddedCoverArt } - validateTrackIndex(isSingleTrack) { + validateTrackIndex() { var numFromMeta = isNullOrNaN(this.trackNumFromMeta) ? null : Number(this.trackNumFromMeta) var numFromFilename = isNullOrNaN(this.trackNumFromFilename) ? null : Number(this.trackNumFromFilename) - if (isSingleTrack) { // Single audio track audiobook only use metadata tag and default to 1 - return numFromMeta ? numFromMeta : 1 - } if (numFromMeta !== null) return numFromMeta if (numFromFilename !== null) return numFromFilename @@ -284,5 +282,33 @@ class AudioFile { }) return hasUpdates } + + updateFromScan(scannedAudioFile) { + var hasUpdated = false + + var newjson = scannedAudioFile.toJSON() + if (this.manuallyVerified) newjson.manuallyVerified = true + if (this.exclude) newjson.exclude = true + newjson.addedAt = this.addedAt + + for (const key in newjson) { + if (key === 'metadata') { + if (!this.metadata || !this.metadata.isEqual(scannedAudioFile.metadata)) { + this.metadata = scannedAudioFile.metadata + hasUpdated = true + // console.log('metadata updated for audio file') + } + } else if (key === 'chapters') { + if (this.syncChapters(newjson.chapters || [])) { + hasUpdated = true + } + } else if (this[key] !== newjson[key]) { + this[key] = newjson[key] + hasUpdated = true + // console.log('key', key, 'updated', this[key], newjson[key]) + } + } + return hasUpdated + } } module.exports = AudioFile \ No newline at end of file diff --git a/server/objects/AudioFileMetadata.js b/server/objects/AudioFileMetadata.js index bd47538c..3a82ceab 100644 --- a/server/objects/AudioFileMetadata.js +++ b/server/objects/AudioFileMetadata.js @@ -101,5 +101,13 @@ class AudioFileMetadata { } return hasUpdates } + + isEqual(audioFileMetadata) { + if (!audioFileMetadata || !audioFileMetadata.toJSON) return false + for (const key in audioFileMetadata.toJSON()) { + if (audioFileMetadata[key] !== this[key]) return false + } + return true + } } module.exports = AudioFileMetadata \ No newline at end of file diff --git a/server/objects/Audiobook.js b/server/objects/Audiobook.js index 9ffbe455..6a03b1f9 100644 --- a/server/objects/Audiobook.js +++ b/server/objects/Audiobook.js @@ -353,6 +353,11 @@ class Audiobook { this.lastUpdate = Date.now() } + setInvalid() { + this.isInvalid = true + this.lastUpdate = Date.now() + } + setBook(data) { // Use first image file as cover if (this.otherFiles && this.otherFiles.length) { @@ -400,6 +405,11 @@ class Audiobook { } } + updateAudioFile(updatedAudioFile) { + var audioFile = this.audioFiles.find(af => af.ino === updatedAudioFile.ino) + return audioFile.updateFromScan(updatedAudioFile) + } + addOtherFile(fileData) { var file = new AudiobookFile() file.setData(fileData) @@ -437,8 +447,8 @@ class Audiobook { return this.book.updateCover(cover, coverFullPath) } - checkHasTrackNum(trackNum) { - return this.tracks.find(t => t.index === trackNum) + checkHasTrackNum(trackNum, excludeIno) { + return this._audioFiles.find(t => t.index === trackNum && t.ino !== excludeIno) } updateAudioTracks(orderedFileData) { @@ -473,6 +483,7 @@ class Audiobook { } }) this.setChapters() + this.checkUpdateMissingTracks() this.lastUpdate = Date.now() } @@ -486,7 +497,7 @@ class Audiobook { this.audioFiles = this.audioFiles.filter(f => f.ino !== track.ino) } - checkUpdateMissingParts() { + checkUpdateMissingTracks() { var currMissingParts = (this.missingParts || []).join(',') || '' var current_index = 1 @@ -515,13 +526,14 @@ class Audiobook { } // On scan check other files found with other files saved - async syncOtherFiles(newOtherFiles, metadataPath, forceRescan = false) { + async syncOtherFiles(newOtherFiles, metadataPath, opfMetadataOverrideDetails, forceRescan = false) { var hasUpdates = false var currOtherFileNum = this.otherFiles.length - var alreadyHasDescTxt = this.otherFiles.find(of => of.filename === 'desc.txt') - var alreadyHasReaderTxt = this.otherFiles.find(of => of.filename === 'reader.txt') + var otherFilenamesAlreadyInBook = this.otherFiles.map(ofile => ofile.filename) + var alreadyHasDescTxt = otherFilenamesAlreadyInBook.includes('desc.txt') + var alreadyHasReaderTxt = otherFilenamesAlreadyInBook.includes('reader.txt') var newOtherFilePaths = newOtherFiles.map(f => f.path) this.otherFiles = this.otherFiles.filter(f => newOtherFilePaths.includes(f.path)) @@ -553,21 +565,22 @@ class Audiobook { } } + // If OPF file and was not already there var metadataOpf = newOtherFiles.find(file => file.ext === '.opf' || file.filename === 'metadata.xml') - if (metadataOpf) { + if (metadataOpf && (!otherFilenamesAlreadyInBook.includes(metadataOpf.filename) || opfMetadataOverrideDetails)) { var xmlText = await readTextFile(metadataOpf.fullPath) if (xmlText) { var opfMetadata = await parseOpfMetadataXML(xmlText) - Logger.debug(`[Audiobook] Sync Other File "${metadataOpf.filename}" parsed:`, opfMetadata) + // Logger.debug(`[Audiobook] Sync Other File "${metadataOpf.filename}" parsed:`, opfMetadata) if (opfMetadata) { const bookUpdatePayload = {} for (const key in opfMetadata) { // Add genres only if genres are empty if (key === 'genres') { - if (opfMetadata.genres.length && !this.book._genres.length) { + if (opfMetadata.genres.length && (!this.book._genres.length || opfMetadataOverrideDetails)) { bookUpdatePayload[key] = opfMetadata.genres } - } else if (opfMetadata[key] && !this.book[key]) { + } else if (opfMetadata[key] && (!this.book[key] || opfMetadataOverrideDetails)) { bookUpdatePayload[key] = opfMetadata[key] } } @@ -789,7 +802,7 @@ class Audiobook { } // Look for desc.txt and reader.txt and update details if found - async saveDataFromTextFiles() { + async saveDataFromTextFiles(opfMetadataOverrideDetails) { var bookUpdatePayload = {} var descriptionText = await this.fetchTextFromTextFile('desc.txt') if (descriptionText) { @@ -807,15 +820,15 @@ class Audiobook { var xmlText = await readTextFile(metadataOpf.fullPath) if (xmlText) { var opfMetadata = await parseOpfMetadataXML(xmlText) - Logger.debug(`[Audiobook] "${this.title}" found "${metadataOpf.filename}" parsed:`, opfMetadata) + // Logger.debug(`[Audiobook] "${this.title}" found "${metadataOpf.filename}" parsed:`, opfMetadata) if (opfMetadata) { for (const key in opfMetadata) { // Add genres only if genres are empty if (key === 'genres') { - if (opfMetadata.genres.length && !this.book._genres.length) { + if (opfMetadata.genres.length && (!this.book._genres.length || opfMetadataOverrideDetails)) { bookUpdatePayload[key] = opfMetadata.genres } - } else if (opfMetadata[key] && !this.book[key] && !bookUpdatePayload[key]) { + } else if (opfMetadata[key] && ((!this.book[key] && !bookUpdatePayload[key]) || opfMetadataOverrideDetails)) { bookUpdatePayload[key] = opfMetadata[key] } } @@ -836,10 +849,10 @@ class Audiobook { } // Audio file metadata tags map to book details (will not overwrite) - setDetailsFromFileMetadata() { + setDetailsFromFileMetadata(overrideExistingDetails = false) { if (!this.audioFiles.length) return false var audioFile = this.audioFiles[0] - return this.book.setDetailsFromFileMetadata(audioFile.metadata) + return this.book.setDetailsFromFileMetadata(audioFile.metadata, overrideExistingDetails) } // Returns null if file not found, true if file was updated, false if up to date @@ -884,9 +897,15 @@ class Audiobook { return hasUpdated } - checkScanData(dataFound) { + checkScanData(dataFound, version) { var hasUpdated = false + if (this.isMissing) { + // Audiobook no longer missing + this.isMissing = false + hasUpdated = true + } + if (dataFound.ino !== this.ino) { this.ino = dataFound.ino hasUpdated = true @@ -916,7 +935,7 @@ class Audiobook { var audioFileFoundCheck = this.checkFileFound(af, true) if (audioFileFoundCheck === null) { newAudioFileData.push(af) - } else if (audioFileFoundCheck === true) { + } else if (audioFileFoundCheck) { hasUpdated = true } }) @@ -925,7 +944,7 @@ class Audiobook { var fileFoundCheck = this.checkFileFound(otherFileData, false) if (fileFoundCheck === null) { newOtherFileData.push(otherFileData) - } else if (fileFoundCheck === true) { + } else if (fileFoundCheck) { hasUpdated = true } }) @@ -933,7 +952,7 @@ class Audiobook { const audioFilesRemoved = [] const otherFilesRemoved = [] - // inodes will all be up to date at this point + // Remove audio files not found (inodes will all be up to date at this point) this.audioFiles = this.audioFiles.filter(af => { if (!dataFound.audioFiles.find(_af => _af.ino === af.ino)) { audioFilesRemoved.push(af.toJSON()) @@ -946,10 +965,11 @@ class Audiobook { if (audioFilesRemoved.length) { const audioFilesRemovedInodes = audioFilesRemoved.map(afr => afr.ino) this.tracks = this.tracks.filter(t => !audioFilesRemovedInodes.includes(t.ino)) - this.checkUpdateMissingParts() + this.checkUpdateMissingTracks() hasUpdated = true } + // Remove other files not found this.otherFiles = this.otherFiles.filter(otherFile => { if (!dataFound.otherFiles.find(_otherFile => _otherFile.ino === otherFile.ino)) { otherFilesRemoved.push(otherFile.toJSON()) @@ -969,6 +989,15 @@ class Audiobook { hasUpdated = true } + // Check if invalid (has no audio files or ebooks) + if (!this.audioFilesToInclude.length && !this.ebooks.length && !newAudioFileData.length && !newOtherFileData.length) { + this.isInvalid = true + } + + if (hasUpdated) { + this.setLastScan(version) + } + return { updated: hasUpdated, newAudioFileData, diff --git a/server/objects/Book.js b/server/objects/Book.js index 63fbf2e9..97e6bb3f 100644 --- a/server/objects/Book.js +++ b/server/objects/Book.js @@ -43,6 +43,7 @@ class Book { get _genres() { return this.genres || [] } get shouldSearchForCover() { + if (this.cover) return false if (this.authorFL !== this.lastCoverSearchAuthor || this.title !== this.lastCoverSearchTitle || !this.lastCoverSearch) return true var timeSinceLastSearch = Date.now() - this.lastCoverSearch return timeSinceLastSearch > 1000 * 60 * 60 * 24 * 7 // every 7 days do another lookup @@ -297,7 +298,7 @@ class Book { return [genreTag] } - setDetailsFromFileMetadata(audioFileMetadata) { + setDetailsFromFileMetadata(audioFileMetadata, overrideExistingDetails = false) { const MetadataMapArray = [ { tag: 'tagComposer', @@ -319,6 +320,10 @@ class Book { tag: 'tagSubtitle', key: 'subtitle' }, + { + tag: 'tagAlbum', + key: 'title', + }, { tag: 'tagArtist', key: 'author' @@ -342,12 +347,12 @@ class Book { MetadataMapArray.forEach((mapping) => { if (audioFileMetadata[mapping.tag]) { // Genres can contain multiple - if (mapping.key === 'genres' && (!this[mapping.key].length || !this[mapping.key])) { + if (mapping.key === 'genres' && (!this[mapping.key].length || !this[mapping.key] || overrideExistingDetails)) { updatePayload[mapping.key] = this.parseGenresTag(audioFileMetadata[mapping.tag]) - Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key].join(',')}`) - } else if (!this[mapping.key]) { + // Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key].join(',')}`) + } else if (!this[mapping.key] || overrideExistingDetails) { updatePayload[mapping.key] = audioFileMetadata[mapping.tag] - Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key]}`) + // Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key]}`) } } }) diff --git a/server/objects/ServerSettings.js b/server/objects/ServerSettings.js index 70e04122..1c805316 100644 --- a/server/objects/ServerSettings.js +++ b/server/objects/ServerSettings.js @@ -46,6 +46,9 @@ class ServerSettings { this.newTagExpireDays = settings.newTagExpireDays this.scannerFindCovers = !!settings.scannerFindCovers this.scannerParseSubtitle = settings.scannerParseSubtitle + this.scannerPreferAudioMetadata = !!settings.scannerPreferAudioMetadata + this.scannerPreferOpfMetadata = !!settings.scannerPreferOpfMetadata + this.coverDestination = settings.coverDestination || CoverDestination.METADATA this.saveMetadataFile = !!settings.saveMetadataFile this.rateLimitLoginRequests = !isNaN(settings.rateLimitLoginRequests) ? Number(settings.rateLimitLoginRequests) : 10 @@ -73,6 +76,8 @@ class ServerSettings { newTagExpireDays: this.newTagExpireDays, scannerFindCovers: this.scannerFindCovers, scannerParseSubtitle: this.scannerParseSubtitle, + scannerPreferAudioMetadata: this.scannerPreferAudioMetadata, + scannerPreferOpfMetadata: this.scannerPreferOpfMetadata, coverDestination: this.coverDestination, saveMetadataFile: !!this.saveMetadataFile, rateLimitLoginRequests: this.rateLimitLoginRequests, diff --git a/server/scanner/AudioFileScanner.js b/server/scanner/AudioFileScanner.js index e73cbcfc..b52a1415 100644 --- a/server/scanner/AudioFileScanner.js +++ b/server/scanner/AudioFileScanner.js @@ -4,7 +4,7 @@ const AudioFile = require('../objects/AudioFile') const prober = require('../utils/prober') const Logger = require('../Logger') -const { msToTimestamp } = require('../utils') +const { LogLevel } = require('../utils/constants') class AudioFileScanner { constructor() { } @@ -80,6 +80,9 @@ class AudioFileScanner { audioFileData.trackNumFromFilename = this.getTrackNumberFromFilename(bookScanData, audioFileData.filename) audioFileData.cdNumFromFilename = this.getCdNumberFromFilename(bookScanData, audioFileData.filename) audioFile.setDataFromProbe(audioFileData, probeData) + if (audioFile.embeddedCoverArt) { + + } return { audioFile, elapsed: Date.now() - probeStart @@ -87,12 +90,11 @@ class AudioFileScanner { } - // Returns array of { AudioFile, elapsed } from audio file scan objects - async scanAudioFiles(audioFileDataArray, bookScanData) { + // Returns array of { AudioFile, elapsed, averageScanDuration } from audio file scan objects + async executeAudioFileScans(audioFileDataArray, bookScanData) { var proms = [] for (let i = 0; i < audioFileDataArray.length; i++) { - var prom = this.scan(audioFileDataArray[i], bookScanData) - proms.push(prom) + proms.push(this.scan(audioFileDataArray[i], bookScanData)) } var scanStart = Date.now() var results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr)) @@ -102,5 +104,62 @@ class AudioFileScanner { averageScanDuration: this.getAverageScanDurationMs(results) } } + + async scanAudioFiles(audioFileDataArray, bookScanData, audiobook, preferAudioMetadata, libraryScan = null) { + var hasUpdated = false + + var audioScanResult = await this.executeAudioFileScans(audioFileDataArray, bookScanData) + if (audioScanResult.audioFiles.length) { + if (libraryScan) { + libraryScan.addLog(LogLevel.DEBUG, `Book "${bookScanData.path}" Audio file scan took ${audioScanResult.elapsed}ms for ${audioScanResult.audioFiles.length} with average time of ${audioScanResult.averageScanDuration}ms`) + } + + var totalAudioFilesToInclude = audiobook.audioFilesToInclude.filter(af => !audioScanResult.audioFiles.find(_af => _af.ino === af.ino)).length + audioScanResult.audioFiles.length + + // validate & add/update audio files to audiobook + for (let i = 0; i < audioScanResult.audioFiles.length; i++) { + var newAF = audioScanResult.audioFiles[i] + var existingAF = audiobook.getAudioFileByIno(newAF.ino) + + var trackIndex = null + if (totalAudioFilesToInclude === 1) { // Single track audiobooks + trackIndex = 1 + } else if (existingAF && existingAF.manuallyVerified) { // manually verified audio files use existing index + trackIndex = existingAF.index + } else { + trackIndex = newAF.validateTrackIndex() + } + + if (trackIndex !== null) { + if (audiobook.checkHasTrackNum(trackIndex, newAF.ino)) { + newAF.setDuplicateTrackNumber(trackIndex) + } else { + newAF.index = trackIndex + } + } + if (existingAF) { + if (audiobook.updateAudioFile(newAF)) { + // console.log('update dauido file') + hasUpdated = true + } + } else { + audiobook.addAudioFile(newAF) + // console.log('added auido file') + hasUpdated = true + } + } + + if (hasUpdated) { + audiobook.rebuildTracks() + } + + // Set book details from audio file ID3 tags, optional prefer + if (audiobook.setDetailsFromFileMetadata(preferAudioMetadata)) { + hasUpdated = true + } + + } + return hasUpdated + } } module.exports = new AudioFileScanner() \ No newline at end of file diff --git a/server/scanner/AudioProbeData.js b/server/scanner/AudioProbeData.js index 758cb47a..c6513ace 100644 --- a/server/scanner/AudioProbeData.js +++ b/server/scanner/AudioProbeData.js @@ -35,7 +35,6 @@ class AudioProbeData { setData(data) { var audioStream = this.getDefaultAudioStream(data.audio_streams) - this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : false this.format = data.format this.duration = data.duration diff --git a/server/scanner/LibraryScan.js b/server/scanner/LibraryScan.js index 5e32876b..a7a0809d 100644 --- a/server/scanner/LibraryScan.js +++ b/server/scanner/LibraryScan.js @@ -1,6 +1,10 @@ -const Folder = require('../objects/Folder') -const Constants = require('../utils/constants') +const Path = require('path') +const fs = require('fs-extra') +const date = require('date-and-time') +const Logger = require('../Logger') +const Folder = require('../objects/Folder') +const { LogLevel } = require('../utils/constants') const { getId, secondsToTimestamp } = require('../utils/index') class LibraryScan { @@ -9,6 +13,7 @@ class LibraryScan { this.libraryId = null this.libraryName = null this.folders = null + this.verbose = false this.scanOptions = null @@ -16,14 +21,21 @@ class LibraryScan { this.finishedAt = null this.elapsed = null - this.status = Constants.ScanStatus.NOTHING this.resultsMissing = 0 this.resultsAdded = 0 this.resultsUpdated = 0 + + this.logs = [] } get _scanOptions() { return this.scanOptions || {} } get forceRescan() { return !!this._scanOptions.forceRescan } + get preferAudioMetadata() { return !!this._scanOptions.preferAudioMetadata } + get preferOpfMetadata() { return !!this._scanOptions.preferOpfMetadata } + get findCovers() { return !!this._scanOptions.findCovers } + get timestamp() { + return (new Date()).toISOString() + } get resultStats() { return `${this.resultsAdded} Added | ${this.resultsUpdated} Updated | ${this.resultsMissing} Missing` @@ -42,6 +54,28 @@ class LibraryScan { } } } + get totalResults() { + return this.resultsAdded + this.resultsUpdated + this.resultsMissing + } + get getLogFilename() { + return date.format(new Date(), 'YYYY-MM-DD') + '_' + this.id + '.txt' + } + + toJSON() { + return { + id: this.id, + libraryId: this.libraryId, + libraryName: this.libraryName, + folders: this.folders.map(f => f.toJSON()), + scanOptions: this.scanOptions.toJSON(), + startedAt: this.startedAt, + finishedAt: this.finishedAt, + elapsed: this.elapsed, + resultsAdded: this.resultsAdded, + resultsUpdated: this.resultsUpdated, + resultsMissing: this.resultsMissing + } + } setData(library, scanOptions) { this.id = getId('lscan') @@ -58,5 +92,39 @@ class LibraryScan { this.finishedAt = Date.now() this.elapsed = this.finishedAt - this.startedAt } + + getLogLevelString(level) { + for (const key in LogLevel) { + if (LogLevel[key] === level) { + return key + } + } + return 'UNKNOWN' + } + + addLog(level, ...args) { + const logObj = { + timestamp: this.timestamp, + message: args.join(' '), + levelName: this.getLogLevelString(level), + level + } + + if (this.verbose) { + Logger.debug(`[LibraryScan] "${this.libraryName}":`, args) + } + this.logs.push(logObj) + } + + async saveLog(logDir) { + await fs.ensureDir(logDir) + var outputPath = Path.join(logDir, this.getLogFilename) + var logLines = [JSON.stringify(this.toJSON())] + this.logs.forEach(l => { + logLines.push(JSON.stringify(l)) + }) + await fs.writeFile(outputPath, logLines.join('\n') + '\n') + Logger.info(`[LibraryScan] Scan log saved "${outputPath}"`) + } } module.exports = LibraryScan \ No newline at end of file diff --git a/server/scanner/ScanOptions.js b/server/scanner/ScanOptions.js index 851cff01..40b09fbb 100644 --- a/server/scanner/ScanOptions.js +++ b/server/scanner/ScanOptions.js @@ -4,29 +4,6 @@ class ScanOptions { constructor(options) { this.forceRescan = false - // this.metadataPrecedence = [ - // { - // id: 'directory', - // include: true - // }, - // { - // id: 'reader-desc-txt', - // include: true - // }, - // { - // id: 'audio-file-metadata', - // include: true - // }, - // { - // id: 'metadata-opf', - // include: true - // }, - // { - // id: 'external-source', - // include: false - // } - // ] - // Server settings this.parseSubtitles = false this.findCovers = false diff --git a/server/scanner/Scanner.js b/server/scanner/Scanner.js index ddd4c21f..a3203087 100644 --- a/server/scanner/Scanner.js +++ b/server/scanner/Scanner.js @@ -4,10 +4,9 @@ const Path = require('path') // Utils const Logger = require('../Logger') const { version } = require('../../package.json') -const audioFileScanner = require('../utils/audioFileScanner') const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('../utils/scandir') const { comparePaths, getIno, getId, msToTimestamp } = require('../utils/index') -const { ScanResult, CoverDestination } = require('../utils/constants') +const { ScanResult, CoverDestination, LogLevel } = require('../utils/constants') const AudioFileScanner = require('./AudioFileScanner') const BookFinder = require('../BookFinder') @@ -20,6 +19,8 @@ class Scanner { this.AudiobookPath = AUDIOBOOK_PATH this.MetadataPath = METADATA_PATH this.BookMetadataPath = Path.posix.join(this.MetadataPath.replace(/\\/g, '/'), 'books') + var LogDirPath = Path.join(this.MetadataPath, 'logs') + this.ScanLogPath = Path.join(LogDirPath, 'scans') this.db = db this.coverController = coverController @@ -46,8 +47,82 @@ class Scanner { } } + isLibraryScanning(libraryId) { + return this.librariesScanning.find(ls => ls.id === libraryId) + } + + async scanAudiobookById(audiobookId) { + var audiobook = this.db.audiobooks.find(ab => ab.id === audiobookId) + if (!audiobook) { + Logger.error(`[Scanner] Scan audiobook by id not found ${audiobookId}`) + return ScanResult.NOTHING + } + const library = this.db.libraries.find(lib => lib.id === audiobook.libraryId) + if (!library) { + Logger.error(`[Scanner] Scan audiobook by id library not found "${audiobook.libraryId}"`) + return ScanResult.NOTHING + } + const folder = library.folders.find(f => f.id === audiobook.folderId) + if (!folder) { + Logger.error(`[Scanner] Scan audiobook by id folder not found "${audiobook.folderId}" in library "${library.name}"`) + return ScanResult.NOTHING + } + Logger.info(`[Scanner] Scanning Audiobook "${audiobook.title}"`) + return this.scanAudiobook(folder, audiobook) + } + + async scanAudiobook(folder, audiobook) { + var audiobookData = await getAudiobookFileData(folder, audiobook.fullPath, this.db.serverSettings) + if (!audiobookData) { + return ScanResult.NOTHING + } + var hasUpdated = false + + var checkRes = audiobook.checkScanData(audiobookData, version) + if (checkRes.updated) hasUpdated = true + + // Sync other files first so that local images are used as cover art + // TODO: Cleanup other file sync + var allOtherFiles = checkRes.newOtherFileData.concat(audiobook._otherFiles) + if (await audiobook.syncOtherFiles(allOtherFiles, this.MetadataPath, this.db.serverSettings.scannerPreferOpfMetadata)) { + hasUpdated = true + } + + // Scan all audio files + if (audiobookData.audioFiles.length) { + if (await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData, audiobook, this.db.serverSettings.scannerPreferAudioMetadata)) { + hasUpdated = true + } + + // Extract embedded cover art if cover is not already in directory + if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) { + var outputCoverDirs = this.getCoverDirectory(audiobook) + var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath) + if (relativeDir) { + Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`) + hasUpdated = true + } + } + } + + if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) { // Audiobook is invalid + audiobook.setInvalid() + hasUpdated = true + } else if (audiobook.isInvalid) { + audiobook.isInvalid = false + hasUpdated = true + } + + if (hasUpdated) { + this.emitter('audiobook_updated', audiobook.toJSONExpanded()) + await this.db.updateEntity('audiobook', audiobook) + return ScanResult.UPDATED + } + return ScanResult.UPTODATE + } + async scan(libraryId, options = {}) { - if (this.librariesScanning.includes(libraryId)) { + if (this.isLibraryScanning(libraryId)) { Logger.error(`[Scanner] Already scanning ${libraryId}`) return } @@ -66,172 +141,380 @@ class Scanner { var libraryScan = new LibraryScan() libraryScan.setData(library, scanOptions) - this.librariesScanning.push(libraryScan) + libraryScan.verbose = false + this.librariesScanning.push(libraryScan.getScanEmitData) this.emitter('scan_start', libraryScan.getScanEmitData) Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`) - await this.scanLibrary(libraryScan) + var canceled = await this.scanLibrary(libraryScan) + + if (canceled) { + Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`) + delete this.cancelLibraryScan[libraryScan.libraryId] + } libraryScan.setComplete() - Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp}. ${libraryScan.resultStats}`) + Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`) this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id) + + if (canceled && !libraryScan.totalResults) { + var emitData = libraryScan.getScanEmitData + emitData.results = null + this.emitter('scan_complete', emitData) + return + } + this.emitter('scan_complete', libraryScan.getScanEmitData) + + if (libraryScan.totalResults) { + libraryScan.saveLog(this.ScanLogPath) + } } async scanLibrary(libraryScan) { var audiobookDataFound = [] + + // Scan each library for (let i = 0; i < libraryScan.folders.length; i++) { var folder = libraryScan.folders[i] var abDataFoundInFolder = await scanRootDir(folder, this.db.serverSettings) - Logger.debug(`[Scanner] ${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`) + libraryScan.addLog(LogLevel.INFO, `${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`) audiobookDataFound = audiobookDataFound.concat(abDataFoundInFolder) } + if (this.cancelLibraryScan[libraryScan.libraryId]) return true + // Remove audiobooks with no inode audiobookDataFound = audiobookDataFound.filter(abd => abd.ino) var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryScan.libraryId) + const NumScansPerChunk = 25 + const audiobooksToUpdateChunks = [] + const audiobookDataToRescanChunks = [] + const newAudiobookDataToScanChunks = [] var audiobooksToUpdate = [] - var audiobookRescans = [] - var newAudiobookScans = [] + var audiobookDataToRescan = [] + var newAudiobookDataToScan = [] + var audiobooksToFindCovers = [] // Check for existing & removed audiobooks for (let i = 0; i < audiobooksInLibrary.length; i++) { var audiobook = audiobooksInLibrary[i] var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path)) if (!dataFound) { - Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`) + libraryScan.addLog(LogLevel.WARN, `Audiobook "${audiobook.title}" is missing`) + libraryScan.resultsMissing++ audiobook.setMissing() audiobooksToUpdate.push(audiobook) + if (audiobooksToUpdate.length === NumScansPerChunk) { + audiobooksToUpdateChunks.push(audiobooksToUpdate) + audiobooksToUpdate = [] + } } else { - var checkRes = audiobook.checkScanData(dataFound) - if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) { - // existing audiobook has new files + var checkRes = audiobook.checkScanData(dataFound, version) + if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) { // Audiobook has new files checkRes.audiobook = audiobook checkRes.bookScanData = dataFound - audiobookRescans.push(this.rescanAudiobook(checkRes, libraryScan)) - libraryScan.resultsMissing++ - } else if (checkRes.updated) { - audiobooksToUpdate.push(audiobook) + audiobookDataToRescan.push(checkRes) + if (audiobookDataToRescan.length === NumScansPerChunk) { + audiobookDataToRescanChunks.push(audiobookDataToRescan) + audiobookDataToRescan = [] + } + } else if (libraryScan.findCovers && audiobook.book.shouldSearchForCover) { libraryScan.resultsUpdated++ + audiobooksToFindCovers.push(audiobook) + audiobooksToUpdate.push(audiobook) + if (audiobooksToUpdate.length === NumScansPerChunk) { + audiobooksToUpdateChunks.push(audiobooksToUpdate) + audiobooksToUpdate = [] + } + } else if (checkRes.updated) { // Updated but no scan required + libraryScan.resultsUpdated++ + audiobooksToUpdate.push(audiobook) + if (audiobooksToUpdate.length === NumScansPerChunk) { + audiobooksToUpdateChunks.push(audiobooksToUpdate) + audiobooksToUpdate = [] + } } audiobookDataFound = audiobookDataFound.filter(abf => abf.ino !== dataFound.ino) } } + if (audiobooksToUpdate.length) audiobooksToUpdateChunks.push(audiobooksToUpdate) + if (audiobookDataToRescan.length) audiobookDataToRescanChunks.push(audiobookDataToRescan) // Potential NEW Audiobooks for (let i = 0; i < audiobookDataFound.length; i++) { var dataFound = audiobookDataFound[i] var hasEbook = dataFound.otherFiles.find(otherFile => otherFile.filetype === 'ebook') if (!hasEbook && !dataFound.audioFiles.length) { - Logger.info(`[Scanner] Directory found "${audiobookDataFound.path}" has no ebook or audio files`) + libraryScan.addLog(LogLevel.WARN, `Directory found "${audiobookDataFound.path}" has no ebook or audio files`) } else { - newAudiobookScans.push(this.scanNewAudiobook(dataFound, libraryScan)) + newAudiobookDataToScan.push(dataFound) + if (newAudiobookDataToScan.length === NumScansPerChunk) { + newAudiobookDataToScanChunks.push(newAudiobookDataToScan) + newAudiobookDataToScan = [] + } } } + if (newAudiobookDataToScan.length) newAudiobookDataToScanChunks.push(newAudiobookDataToScan) + + // console.log('Num chunks to update', audiobooksToUpdateChunks.length) + // console.log('Num chunks to rescan', audiobookDataToRescanChunks.length) + // console.log('Num chunks to new scan', newAudiobookDataToScanChunks.length) + + // Audiobooks not requiring a scan but require a search for cover + for (let i = 0; i < audiobooksToFindCovers.length; i++) { + var audiobook = audiobooksToFindCovers[i] + var updatedCover = await this.searchForCover(audiobook, libraryScan) + audiobook.book.updateLastCoverSearch(updatedCover) + } - if (audiobookRescans.length) { - var updatedAudiobooks = (await Promise.all(audiobookRescans)).filter(ab => !!ab) - if (updatedAudiobooks.length) { - audiobooksToUpdate = audiobooksToUpdate.concat(updatedAudiobooks) - libraryScan.resultsUpdated += updatedAudiobooks.length - } + for (let i = 0; i < audiobooksToUpdateChunks.length; i++) { + await this.updateAudiobooksChunk(audiobooksToUpdateChunks[i]) + if (this.cancelLibraryScan[libraryScan.libraryId]) return true + // console.log('Update chunk done', i, 'of', audiobooksToUpdateChunks.length) } - if (audiobooksToUpdate.length) { - Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" updating ${audiobooksToUpdate.length} books`) - await this.db.updateEntities('audiobook', audiobooksToUpdate) + for (let i = 0; i < audiobookDataToRescanChunks.length; i++) { + await this.rescanAudiobookDataChunk(audiobookDataToRescanChunks[i], libraryScan) + if (this.cancelLibraryScan[libraryScan.libraryId]) return true + // console.log('Rescan chunk done', i, 'of', audiobookDataToRescanChunks.length) } + for (let i = 0; i < newAudiobookDataToScanChunks.length; i++) { + await this.scanNewAudiobookDataChunk(newAudiobookDataToScanChunks[i], libraryScan) + // console.log('New scan chunk done', i, 'of', newAudiobookDataToScanChunks.length) + if (this.cancelLibraryScan[libraryScan.libraryId]) return true + } + } - if (newAudiobookScans.length) { - var newAudiobooks = (await Promise.all(newAudiobookScans)).filter(ab => !!ab) - if (newAudiobooks.length) { - Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" inserting ${newAudiobooks.length} books`) - await this.db.insertEntities('audiobook', newAudiobooks) - libraryScan.resultsAdded = newAudiobooks.length - } - } + async updateAudiobooksChunk(audiobooksToUpdate) { + await this.db.updateEntities('audiobook', audiobooksToUpdate) + this.emitter('audiobooks_updated', audiobooksToUpdate.map(ab => ab.toJSONExpanded())) + } + + async rescanAudiobookDataChunk(audiobookDataToRescan, libraryScan) { + var audiobooksUpdated = await Promise.all(audiobookDataToRescan.map((abd) => { + return this.rescanAudiobook(abd, libraryScan) + })) + audiobooksUpdated = audiobooksUpdated.filter(ab => ab) // Filter out nulls + libraryScan.resultsUpdated += audiobooksUpdated.length + await this.db.updateEntities('audiobook', audiobooksUpdated) + this.emitter('audiobooks_updated', audiobooksUpdated.map(ab => ab.toJSONExpanded())) + } + + async scanNewAudiobookDataChunk(newAudiobookDataToScan, libraryScan) { + var newAudiobooks = await Promise.all(newAudiobookDataToScan.map((abd) => { + return this.scanNewAudiobook(abd, libraryScan.preferAudioMetadata, libraryScan.preferOpfMetadata, libraryScan.findCovers, libraryScan) + })) + newAudiobooks = newAudiobooks.filter(ab => ab) // Filter out nulls + libraryScan.resultsAdded += newAudiobooks.length + await this.db.insertEntities('audiobook', newAudiobooks) + this.emitter('audiobooks_added', newAudiobooks.map(ab => ab.toJSONExpanded())) } async rescanAudiobook(audiobookCheckData, libraryScan) { const { newAudioFileData, newOtherFileData, audiobook, bookScanData } = audiobookCheckData - Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`) + libraryScan.addLog(LogLevel.DEBUG, `Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`) + + // Sync other files first to use local images as cover before extracting audio file cover + if (newOtherFileData.length) { + // TODO: Cleanup other file sync + var allOtherFiles = newOtherFileData.concat(audiobook._otherFiles) + await audiobook.syncOtherFiles(allOtherFiles, this.MetadataPath, libraryScan.preferOpfMetadata) + } if (newAudioFileData.length) { - var audioScanResult = await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData) - Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobook.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`) - if (audioScanResult.audioFiles.length) { - var totalAudioFilesToInclude = audiobook.audioFilesToInclude.length + audioScanResult.audioFiles.length + await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData, audiobook, libraryScan.preferAudioMetadata, libraryScan) - // validate & add audio files to audiobook - for (let i = 0; i < audioScanResult.audioFiles.length; i++) { - var newAF = audioScanResult.audioFiles[i] - var trackIndex = newAF.validateTrackIndex(totalAudioFilesToInclude === 1) - if (trackIndex !== null) { - if (audiobook.checkHasTrackNum(trackIndex)) { - newAF.setDuplicateTrackNumber(trackIndex) - } else { - newAF.index = trackIndex - } - } - audiobook.addAudioFile(newAF) + // Extract embedded cover art if cover is not already in directory + if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) { + var outputCoverDirs = this.getCoverDirectory(audiobook) + var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath) + if (relativeDir) { + libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${relativeDir}"`) } - - audiobook.rebuildTracks() } } - if (newOtherFileData.length) { - await audiobook.syncOtherFiles(newOtherFileData, this.MetadataPath) + + if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) { // Audiobook is invalid + audiobook.setInvalid() + } else if (audiobook.isInvalid) { + audiobook.isInvalid = false } + + // Scan for cover if enabled and has no cover + if (audiobook && libraryScan.findCovers && !audiobook.cover && audiobook.book.shouldSearchForCover) { + var updatedCover = await this.searchForCover(audiobook, libraryScan) + audiobook.book.updateLastCoverSearch(updatedCover) + } + return audiobook } - async scanNewAudiobook(audiobookData, libraryScan) { - Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Scanning new "${audiobookData.path}"`) + async scanNewAudiobook(audiobookData, preferAudioMetadata, preferOpfMetadata, findCovers, libraryScan = null) { + if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Scanning new book "${audiobookData.path}"`) + else Logger.debug(`[Scanner] Scanning new book "${audiobookData.path}"`) + var audiobook = new Audiobook() audiobook.setData(audiobookData) if (audiobookData.audioFiles.length) { - var audioScanResult = await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData) - Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobookData.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`) - if (audioScanResult.audioFiles.length) { - // validate & add audio files to audiobook - for (let i = 0; i < audioScanResult.audioFiles.length; i++) { - var newAF = audioScanResult.audioFiles[i] - var trackIndex = newAF.validateTrackIndex(audioScanResult.audioFiles.length === 1) - if (trackIndex !== null) { - if (audiobook.checkHasTrackNum(trackIndex)) { - newAF.setDuplicateTrackNumber(trackIndex) - } else { - newAF.index = trackIndex - } - } - audiobook.addAudioFile(newAF) - } - audiobook.rebuildTracks() - } else if (!audiobook.ebooks.length) { - // Audiobook has no ebooks and no valid audio tracks do not continue - Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`) - return null - } + await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData, audiobook, preferAudioMetadata, libraryScan) + } + + if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) { + // Audiobook has no ebooks and no valid audio tracks do not continue + Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`) + return null } // Look for desc.txt and reader.txt and update - await audiobook.saveDataFromTextFiles() + await audiobook.saveDataFromTextFiles(preferOpfMetadata) // Extract embedded cover art if cover is not already in directory if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) { var outputCoverDirs = this.getCoverDirectory(audiobook) var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath) if (relativeDir) { - Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`) + if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${relativeDir}"`) + else Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`) } } + // Scan for cover if enabled and has no cover + if (audiobook && findCovers && !audiobook.cover && audiobook.book.shouldSearchForCover) { + var updatedCover = await this.searchForCover(audiobook, libraryScan) + audiobook.book.updateLastCoverSearch(updatedCover) + } + return audiobook } + + getFileUpdatesGrouped(fileUpdates) { + var folderGroups = {} + fileUpdates.forEach((file) => { + if (folderGroups[file.folderId]) { + folderGroups[file.folderId].fileUpdates.push(file) + } else { + folderGroups[file.folderId] = { + libraryId: file.libraryId, + folderId: file.folderId, + fileUpdates: [file] + } + } + }) + return folderGroups + } + + async scanFilesChanged(fileUpdates) { + if (!fileUpdates.length) return + // files grouped by folder + var folderGroups = this.getFileUpdatesGrouped(fileUpdates) + + for (const folderId in folderGroups) { + var libraryId = folderGroups[folderId].libraryId + var library = this.db.libraries.find(lib => lib.id === libraryId) + if (!library) { + Logger.error(`[Scanner] Library not found in files changed ${libraryId}`) + continue; + } + var folder = library.getFolderById(folderId) + if (!folder) { + Logger.error(`[Scanner] Folder is not in library in files changed "${folderId}", Library "${library.name}"`) + continue; + } + var relFilePaths = folderGroups[folderId].fileUpdates.map(fileUpdate => fileUpdate.relPath) + var fileUpdateBookGroup = groupFilesIntoAudiobookPaths(relFilePaths, true) + var folderScanResults = await this.scanFolderUpdates(library, folder, fileUpdateBookGroup) + Logger.debug(`[Scanner] Folder scan results`, folderScanResults) + } + } + + async scanFolderUpdates(library, folder, fileUpdateBookGroup) { + Logger.debug(`[Scanner] Scanning file update groups in folder "${folder.id}" of library "${library.name}"`) + + var bookGroupingResults = {} + for (const bookDir in fileUpdateBookGroup) { + var fullPath = Path.posix.join(folder.fullPath.replace(/\\/g, '/'), bookDir) + + // Check if book dir group is already an audiobook or in a subdir of an audiobook + var existingAudiobook = this.db.audiobooks.find(ab => fullPath.startsWith(ab.fullPath)) + if (existingAudiobook) { + + // Is the audiobook exactly - check if was deleted + if (existingAudiobook.fullPath === fullPath) { + var exists = await fs.pathExists(fullPath) + if (!exists) { + Logger.info(`[Scanner] Scanning file update group and audiobook was deleted "${existingAudiobook.title}" - marking as missing`) + existingAudiobook.setMissing() + await this.db.updateAudiobook(existingAudiobook) + this.emitter('audiobook_updated', existingAudiobook.toJSONExpanded()) + + bookGroupingResults[bookDir] = ScanResult.REMOVED + continue; + } + } + + // Scan audiobook for updates + Logger.debug(`[Scanner] Folder update for relative path "${bookDir}" is in audiobook "${existingAudiobook.title}" - scan for updates`) + bookGroupingResults[bookDir] = await this.scanAudiobook(folder, existingAudiobook) + continue; + } + + // Check if an audiobook is a subdirectory of this dir + var childAudiobook = this.db.audiobooks.find(ab => ab.fullPath.startsWith(fullPath)) + if (childAudiobook) { + Logger.warn(`[Scanner] Files were modified in a parent directory of an audiobook "${childAudiobook.title}" - ignoring`) + bookGroupingResults[bookDir] = ScanResult.NOTHING + continue; + } + + Logger.debug(`[Scanner] Folder update group must be a new book "${bookDir}" in library "${library.name}"`) + var newAudiobook = await this.scanPotentialNewAudiobook(folder, fullPath) + if (newAudiobook) { + await this.db.insertEntity('audiobook', newAudiobook) + this.emitter('audiobook_added', newAudiobook.toJSONExpanded()) + } + bookGroupingResults[bookDir] = newAudiobook ? ScanResult.ADDED : ScanResult.NOTHING + } + + return bookGroupingResults + } + + async scanPotentialNewAudiobook(folder, fullPath) { + var audiobookData = await getAudiobookFileData(folder, fullPath, this.db.serverSettings) + if (!audiobookData) return null + var serverSettings = this.db.serverSettings + return this.scanNewAudiobook(audiobookData, serverSettings.scannerPreferAudioMetadata, serverSettings.scannerPreferOpfMetadata, serverSettings.scannerFindCovers) + } + + async searchForCover(audiobook, libraryScan = null) { + var options = { + titleDistance: 2, + authorDistance: 2 + } + var results = await this.bookFinder.findCovers('google', audiobook.title, audiobook.authorFL, options) + if (results.length) { + if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Found best cover for "${audiobook.title}"`) + else Logger.debug(`[Scanner] Found best cover for "${audiobook.title}"`) + + // If the first cover result fails, attempt to download the second + for (let i = 0; i < results.length && i < 2; i++) { + + // Downloads and updates the book cover + var result = await this.coverController.downloadCoverFromUrl(audiobook, results[i]) + + if (result.error) { + Logger.error(`[Scanner] Failed to download cover from url "${results[i]}" | Attempt ${i + 1}`, result.error) + } else { + return true + } + } + } + return false + } } module.exports = Scanner \ No newline at end of file diff --git a/server/utils/constants.js b/server/utils/constants.js index 446a6778..d36ce925 100644 --- a/server/utils/constants.js +++ b/server/utils/constants.js @@ -6,14 +6,6 @@ module.exports.ScanResult = { UPTODATE: 4 } -module.exports.ScanStatus = { - NOTHING: 0, - ADDED: 1, - UPDATED: 2, - REMOVED: 3, - UPTODATE: 4 -} - module.exports.CoverDestination = { METADATA: 0, AUDIOBOOK: 1 diff --git a/server/utils/scandir.js b/server/utils/scandir.js index 481b8ffa..d7854e28 100644 --- a/server/utils/scandir.js +++ b/server/utils/scandir.js @@ -16,7 +16,7 @@ function isBookFile(path) { // TODO: Function needs to be re-done // Input: array of relative file paths // Output: map of files grouped into potential audiobook dirs -function groupFilesIntoAudiobookPaths(paths, useAllFileTypes = false) { +function groupFilesIntoAudiobookPaths(paths) { // Step 1: Clean path, Remove leading "/", Filter out files in root dir var pathsFiltered = paths.map(path => { return path.startsWith('/') ? path.slice(1) : path