diff --git a/client/components/tables/library/LibrariesTable.vue b/client/components/tables/library/LibrariesTable.vue index a9255e2d..598b12b7 100644 --- a/client/components/tables/library/LibrariesTable.vue +++ b/client/components/tables/library/LibrariesTable.vue @@ -11,10 +11,6 @@ {{ $strings.ButtonAddYourFirstLibrary }} -

- *{{ $strings.ButtonForceReScan }} {{ $strings.MessageForceReScanDescription }} -

-

**{{ $strings.ButtonMatchBooks }} {{ $strings.MessageMatchBooksDescription }}

diff --git a/client/components/tables/library/LibraryItem.vue b/client/components/tables/library/LibraryItem.vue index 6cec8867..b84dec44 100644 --- a/client/components/tables/library/LibraryItem.vue +++ b/client/components/tables/library/LibraryItem.vue @@ -71,11 +71,6 @@ export default { text: this.$strings.ButtonScan, action: 'scan', value: 'scan' - }, - { - text: this.$strings.ButtonForceReScan, - action: 'force-scan', - value: 'force-scan' } ] if (this.isBookLibrary) { @@ -137,26 +132,6 @@ export default { this.$toast.error(this.$strings.ToastLibraryScanFailedToStart) }) }, - forceScan() { - const payload = { - message: this.$strings.MessageConfirmForceReScan, - callback: (confirmed) => { - if (confirmed) { - this.$store - .dispatch('libraries/requestLibraryScan', { libraryId: this.library.id, force: 1 }) - .then(() => { - this.$toast.success(this.$strings.ToastLibraryScanStarted) - }) - .catch((error) => { - console.error('Failed to start scan', error) - this.$toast.error(this.$strings.ToastLibraryScanFailedToStart) - }) - } - }, - type: 'yesNo' - } - this.$store.commit('globals/setConfirmPrompt', payload) - }, deleteClick() { const payload = { message: this.$getString('MessageConfirmDeleteLibrary', [this.library.name]), diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 7a0d1710..c252dc8b 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -974,12 +974,9 @@ class LibraryController { Logger.error(`[LibraryController] Non-root user attempted to scan library`, req.user) return res.sendStatus(403) } - const options = { - forceRescan: req.query.force == 1 - } res.sendStatus(200) - await LibraryScanner.scan(req.library, options) + await LibraryScanner.scan(req.library) await Database.resetLibraryIssuesFilterData(req.library.id) Logger.info('[LibraryController] Scan complete') diff --git a/server/controllers/LibraryItemController.js b/server/controllers/LibraryItemController.js index 1520bf03..d396790b 100644 --- a/server/controllers/LibraryItemController.js +++ b/server/controllers/LibraryItemController.js @@ -9,6 +9,7 @@ const { reqSupportsWebp } = require('../utils/index') const { ScanResult } = require('../utils/constants') const { getAudioMimeTypeFromExtname } = require('../utils/fileUtils') const LibraryItemScanner = require('../scanner/LibraryItemScanner') +const AudioFileScanner = require('../scanner/AudioFileScanner') class LibraryItemController { constructor() { } @@ -555,7 +556,7 @@ class LibraryItemController { return res.sendStatus(404) } - const ffprobeData = await this.scanner.probeAudioFile(audioFile) + const ffprobeData = await AudioFileScanner.probeAudioFile(audioFile) res.json(ffprobeData) } diff --git a/server/objects/LibraryItem.js b/server/objects/LibraryItem.js index 1f8d6c47..e36a5a92 100644 --- a/server/objects/LibraryItem.js +++ b/server/objects/LibraryItem.js @@ -338,183 +338,6 @@ class LibraryItem { return hasUpdated } - // Data pulled from scandir during a scan, check it with current data - checkScanData(dataFound) { - let hasUpdated = false - - if (this.isMissing) { - // Item no longer missing - this.isMissing = false - hasUpdated = true - } - - if (dataFound.isFile !== this.isFile && dataFound.isFile !== undefined) { - Logger.info(`[LibraryItem] Check scan item isFile toggled from ${this.isFile} => ${dataFound.isFile}`) - this.isFile = dataFound.isFile - hasUpdated = true - } - - if (dataFound.ino !== this.ino) { - Logger.warn(`[LibraryItem] Check scan item changed inode "${this.ino}" -> "${dataFound.ino}"`) - this.ino = dataFound.ino - hasUpdated = true - } - - if (dataFound.folderId !== this.folderId) { - Logger.warn(`[LibraryItem] Check scan item changed folder ${this.folderId} -> ${dataFound.folderId}`) - this.folderId = dataFound.folderId - hasUpdated = true - } - - if (dataFound.path !== this.path) { - Logger.warn(`[LibraryItem] Check scan item changed path "${this.path}" -> "${dataFound.path}" (inode ${this.ino})`) - this.path = dataFound.path - this.relPath = dataFound.relPath - hasUpdated = true - } - - ['mtimeMs', 'ctimeMs', 'birthtimeMs'].forEach((key) => { - if (dataFound[key] != this[key]) { - this[key] = dataFound[key] || 0 - hasUpdated = true - } - }) - - const newLibraryFiles = [] - const existingLibraryFiles = [] - - dataFound.libraryFiles.forEach((lf) => { - const fileFoundCheck = this.checkFileFound(lf, true) - if (fileFoundCheck === null) { - newLibraryFiles.push(lf) - } else if (fileFoundCheck && lf.metadata.format !== 'abs' && lf.metadata.filename !== 'metadata.json') { // Ignore abs file updates - hasUpdated = true - existingLibraryFiles.push(lf) - } else { - existingLibraryFiles.push(lf) - } - }) - - const filesRemoved = [] - - // Remove files not found (inodes will all be up to date at this point) - this.libraryFiles = this.libraryFiles.filter(lf => { - if (!dataFound.libraryFiles.find(_lf => _lf.ino === lf.ino)) { - // Check if removing cover path - if (lf.metadata.path === this.media.coverPath) { - Logger.debug(`[LibraryItem] "${this.media.metadata.title}" check scan cover removed`) - this.media.updateCover('') - } - filesRemoved.push(lf.toJSON()) - this.media.removeFileWithInode(lf.ino) - return false - } - return true - }) - if (filesRemoved.length) { - if (this.media.mediaType === 'book') { - this.media.checkUpdateMissingTracks() - } - hasUpdated = true - } - - // Add library files to library item - if (newLibraryFiles.length) { - newLibraryFiles.forEach((lf) => this.libraryFiles.push(lf.clone())) - hasUpdated = true - } - - // Check if invalid - this.isInvalid = !this.media.hasMediaEntities - - // If cover path is in item folder, make sure libraryFile exists for it - if (this.media.coverPath && this.media.coverPath.startsWith(this.path)) { - const lf = this.libraryFiles.find(lf => lf.metadata.path === this.media.coverPath) - if (!lf) { - Logger.warn(`[LibraryItem] Invalid cover path - library file dne "${this.media.coverPath}"`) - this.media.updateCover('') - hasUpdated = true - } - } - - if (hasUpdated) { - this.setLastScan() - } - - return { - updated: hasUpdated, - newLibraryFiles, - filesRemoved, - existingLibraryFiles // Existing file data may get re-scanned if forceRescan is set - } - } - - // Set metadata from files - async syncFiles(preferOpfMetadata, librarySettings) { - let hasUpdated = false - - if (this.isBook) { - // Add/update ebook files (ebooks that were removed are removed in checkScanData) - if (librarySettings.audiobooksOnly) { - hasUpdated = this.media.ebookFile - if (hasUpdated) { - // If library was set to audiobooks only then set primary ebook as supplementary - Logger.info(`[LibraryItem] Library is audiobooks only so setting ebook "${this.media.ebookFile.metadata.filename}" as supplementary`) - } - this.setPrimaryEbook(null) - } else if (this.media.ebookFile) { - const matchingLibraryFile = this.libraryFiles.find(lf => lf.ino === this.media.ebookFile.ino) - if (matchingLibraryFile && this.media.ebookFile.updateFromLibraryFile(matchingLibraryFile)) { - hasUpdated = true - } - // Set any other ebook files as supplementary - const suppEbookLibraryFiles = this.libraryFiles.filter(lf => lf.isEBookFile && !lf.isSupplementary && this.media.ebookFile.ino !== lf.ino) - if (suppEbookLibraryFiles.length) { - for (const libraryFile of suppEbookLibraryFiles) { - libraryFile.isSupplementary = true - } - hasUpdated = true - } - } else { - const ebookLibraryFiles = this.libraryFiles.filter(lf => lf.isEBookFile && !lf.isSupplementary) - - // Prefer epub ebook then fallback to first other ebook file - const ebookLibraryFile = ebookLibraryFiles.find(lf => lf.metadata.format === 'epub') || ebookLibraryFiles[0] - if (ebookLibraryFile) { - this.setPrimaryEbook(ebookLibraryFile) - hasUpdated = true - } - } - } - - // Set cover image if not set - const imageFiles = this.libraryFiles.filter(lf => lf.fileType === 'image') - if (imageFiles.length && !this.media.coverPath) { - // attempt to find a file called cover. otherwise just fall back to the first image found - const coverMatch = imageFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path)) - if (coverMatch) { - this.media.coverPath = coverMatch.metadata.path - } else { - this.media.coverPath = imageFiles[0].metadata.path - } - Logger.info('[LibraryItem] Set media cover path', this.media.coverPath) - hasUpdated = true - } - - // Parse metadata files - const textMetadataFiles = this.libraryFiles.filter(lf => lf.fileType === 'metadata' || lf.fileType === 'text') - if (textMetadataFiles.length) { - if (await this.media.syncMetadataFiles(textMetadataFiles, preferOpfMetadata)) { - hasUpdated = true - } - } - - if (hasUpdated) { - this.updatedAt = Date.now() - } - return hasUpdated - } - searchQuery(query) { query = cleanStringForSearch(query) return this.media.searchQuery(query) @@ -556,19 +379,27 @@ class LibraryItem { return fs.writeFile(metadataFilePath, JSON.stringify(this.media.toJSONForMetadataFile(), null, 2)).then(async () => { // Add metadata.json to libraryFiles array if it is new let metadataLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) - if (storeMetadataWithItem && !metadataLibraryFile) { - metadataLibraryFile = new LibraryFile() - await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`) - this.libraryFiles.push(metadataLibraryFile) - } else if (storeMetadataWithItem) { - const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) - if (fileTimestamps) { - metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs - metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs - metadataLibraryFile.metadata.size = fileTimestamps.size - metadataLibraryFile.ino = fileTimestamps.ino + if (storeMetadataWithItem) { + if (!metadataLibraryFile) { + metadataLibraryFile = new LibraryFile() + await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`) + this.libraryFiles.push(metadataLibraryFile) + } else { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } + } + const libraryItemDirTimestamps = await getFileTimestampsWithIno(this.path) + if (libraryItemDirTimestamps) { + this.mtimeMs = libraryItemDirTimestamps.mtimeMs + this.ctimeMs = libraryItemDirTimestamps.ctimeMs } } + Logger.debug(`[LibraryItem] Success saving abmetadata to "${metadataFilePath}"`) return metadataLibraryFile @@ -593,17 +424,24 @@ class LibraryItem { } // Add metadata.abs to libraryFiles array if it is new let metadataLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) - if (storeMetadataWithItem && !metadataLibraryFile) { - metadataLibraryFile = new LibraryFile() - await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`) - this.libraryFiles.push(metadataLibraryFile) - } else if (storeMetadataWithItem) { - const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) - if (fileTimestamps) { - metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs - metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs - metadataLibraryFile.metadata.size = fileTimestamps.size - metadataLibraryFile.ino = fileTimestamps.ino + if (storeMetadataWithItem) { + if (!metadataLibraryFile) { + metadataLibraryFile = new LibraryFile() + await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`) + this.libraryFiles.push(metadataLibraryFile) + } else { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } + } + const libraryItemDirTimestamps = await getFileTimestampsWithIno(this.path) + if (libraryItemDirTimestamps) { + this.mtimeMs = libraryItemDirTimestamps.mtimeMs + this.ctimeMs = libraryItemDirTimestamps.ctimeMs } } diff --git a/server/objects/files/AudioFile.js b/server/objects/files/AudioFile.js index 7ffff19c..8a3c2a74 100644 --- a/server/objects/files/AudioFile.js +++ b/server/objects/files/AudioFile.js @@ -116,7 +116,7 @@ class AudioFile { return !this.invalid && !this.exclude } - // New scanner creates AudioFile from MediaFileScanner + // New scanner creates AudioFile from AudioFileScanner setDataFromProbe(libraryFile, probeData) { this.ino = libraryFile.ino || null diff --git a/server/scanner/AudioFileScanner.js b/server/scanner/AudioFileScanner.js index 75b157e9..05f0e430 100644 --- a/server/scanner/AudioFileScanner.js +++ b/server/scanner/AudioFileScanner.js @@ -153,12 +153,12 @@ class AudioFileScanner { const probeData = await prober.probe(libraryFile.metadata.path) if (probeData.error) { - Logger.error(`[MediaFileScanner] ${probeData.error} : "${libraryFile.metadata.path}"`) + Logger.error(`[AudioFileScanner] ${probeData.error} : "${libraryFile.metadata.path}"`) return null } if (!probeData.audioStream) { - Logger.error('[MediaFileScanner] Invalid audio file no audio stream') + Logger.error('[AudioFileScanner] Invalid audio file no audio stream') return null } @@ -195,5 +195,15 @@ class AudioFileScanner { return results } + + /** + * + * @param {AudioFile} audioFile + * @returns {object} + */ + probeAudioFile(audioFile) { + Logger.debug(`[AudioFileScanner] Running ffprobe for audio file at "${audioFile.metadata.path}"`) + return prober.rawProbe(audioFile.metadata.path) + } } module.exports = new AudioFileScanner() \ No newline at end of file diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js index b16c799c..ff3b773c 100644 --- a/server/scanner/BookScanner.js +++ b/server/scanner/BookScanner.js @@ -899,20 +899,31 @@ class BookScanner { return fsExtra.writeFile(metadataFilePath, JSON.stringify(jsonObject, null, 2)).then(async () => { // Add metadata.json to libraryFiles array if it is new let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) - if (storeMetadataWithItem && !metadataLibraryFile) { - const newLibraryFile = new LibraryFile() - await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`) - metadataLibraryFile = newLibraryFile.toJSON() - libraryItem.libraryFiles.push(metadataLibraryFile) - } else if (storeMetadataWithItem) { - const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) - if (fileTimestamps) { - metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs - metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs - metadataLibraryFile.metadata.size = fileTimestamps.size - metadataLibraryFile.ino = fileTimestamps.ino + if (storeMetadataWithItem) { + if (!metadataLibraryFile) { + const newLibraryFile = new LibraryFile() + await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`) + metadataLibraryFile = newLibraryFile.toJSON() + libraryItem.libraryFiles.push(metadataLibraryFile) + } else { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } + } + const libraryItemDirTimestamps = await getFileTimestampsWithIno(libraryItem.path) + if (libraryItemDirTimestamps) { + libraryItem.mtime = libraryItemDirTimestamps.mtimeMs + libraryItem.ctime = libraryItemDirTimestamps.ctimeMs + let size = 0 + libraryItem.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0)) + libraryItem.size = size } } + libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`) return metadataLibraryFile @@ -935,18 +946,28 @@ class BookScanner { } // Add metadata.abs to libraryFiles array if it is new let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) - if (storeMetadataWithItem && !metadataLibraryFile) { - const newLibraryFile = new LibraryFile() - await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`) - metadataLibraryFile = newLibraryFile.toJSON() - libraryItem.libraryFiles.push(metadataLibraryFile) - } else if (storeMetadataWithItem) { - const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) - if (fileTimestamps) { - metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs - metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs - metadataLibraryFile.metadata.size = fileTimestamps.size - metadataLibraryFile.ino = fileTimestamps.ino + if (storeMetadataWithItem) { + if (!metadataLibraryFile) { + const newLibraryFile = new LibraryFile() + await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`) + metadataLibraryFile = newLibraryFile.toJSON() + libraryItem.libraryFiles.push(metadataLibraryFile) + } else { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } + } + const libraryItemDirTimestamps = await getFileTimestampsWithIno(libraryItem.path) + if (libraryItemDirTimestamps) { + libraryItem.mtime = libraryItemDirTimestamps.mtimeMs + libraryItem.ctime = libraryItemDirTimestamps.ctimeMs + let size = 0 + libraryItem.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0)) + libraryItem.size = size } } diff --git a/server/scanner/LibraryScan.js b/server/scanner/LibraryScan.js index 5ec1c09b..432ff3cc 100644 --- a/server/scanner/LibraryScan.js +++ b/server/scanner/LibraryScan.js @@ -17,8 +17,6 @@ class LibraryScan { this.library = null this.verbose = false - this.scanOptions = null - this.startedAt = null this.finishedAt = null this.elapsed = null @@ -40,12 +38,6 @@ class LibraryScan { get libraryMediaType() { return this.library.mediaType } get folders() { return this.library.folders } - get _scanOptions() { return this.scanOptions || {} } - get forceRescan() { return !!this._scanOptions.forceRescan } - get preferAudioMetadata() { return !!this._scanOptions.preferAudioMetadata } - get preferOpfMetadata() { return !!this._scanOptions.preferOpfMetadata } - get preferOverdriveMediaMarker() { return !!this._scanOptions.preferOverdriveMediaMarker } - get findCovers() { return !!this._scanOptions.findCovers } get timestamp() { return (new Date()).toISOString() } @@ -80,7 +72,6 @@ class LibraryScan { id: this.id, type: this.type, library: this.library.toJSON(), - scanOptions: this.scanOptions ? this.scanOptions.toJSON() : null, startedAt: this.startedAt, finishedAt: this.finishedAt, elapsed: this.elapsed, @@ -90,13 +81,11 @@ class LibraryScan { } } - setData(library, scanOptions, type = 'scan') { + setData(library, type = 'scan') { this.id = uuidv4() this.type = type this.library = new Library(library.toJSON()) // clone library - this.scanOptions = scanOptions - this.startedAt = Date.now() } diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index bfbec4a9..bbff5198 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -10,7 +10,6 @@ const scanUtils = require('../utils/scandir') const { LogLevel, ScanResult } = require('../utils/constants') const libraryFilters = require('../utils/queries/libraryFilters') const LibraryItemScanner = require('./LibraryItemScanner') -const ScanOptions = require('./ScanOptions') const LibraryScan = require('./LibraryScan') const LibraryItemScanData = require('./LibraryItemScanData') @@ -58,11 +57,8 @@ class LibraryScanner { return } - const scanOptions = new ScanOptions() - scanOptions.setData(options, Database.serverSettings) - const libraryScan = new LibraryScan() - libraryScan.setData(library, scanOptions) + libraryScan.setData(library) libraryScan.verbose = true this.librariesScanning.push(libraryScan.getScanEmitData) @@ -471,7 +467,7 @@ class LibraryScanner { }) if (existingLibraryItem) { Logger.debug(`[LibraryScanner] scanFolderUpdates: Library item found by inode value=${dirIno}. "${existingLibraryItem.relPath} => ${itemDir}"`) - // Update library item paths for scan and all library item paths will get updated in LibraryItem.checkScanData + // Update library item paths for scan existingLibraryItem.path = fullPath existingLibraryItem.relPath = itemDir } diff --git a/server/scanner/MediaFileScanner.js b/server/scanner/MediaFileScanner.js deleted file mode 100644 index aa7255bf..00000000 --- a/server/scanner/MediaFileScanner.js +++ /dev/null @@ -1,342 +0,0 @@ -const Path = require('path') - -const AudioFile = require('../objects/files/AudioFile') -const VideoFile = require('../objects/files/VideoFile') - -const prober = require('../utils/prober') -const Logger = require('../Logger') -const { LogLevel } = require('../utils/constants') - -class MediaFileScanner { - constructor() { } - - /** - * Get track and disc number from audio filename - * @param {{title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} mediaMetadataFromScan - * @param {import('../objects/files/LibraryFile')} audioLibraryFile - * @returns {{trackNumber:number, discNumber:number}} - */ - getTrackAndDiscNumberFromFilename(mediaMetadataFromScan, audioLibraryFile) { - const { title, author, series, publishedYear } = mediaMetadataFromScan - const { filename, path } = audioLibraryFile.metadata - let partbasename = Path.basename(filename, Path.extname(filename)) - - // Remove title, author, series, and publishedYear from filename if there - if (title) partbasename = partbasename.replace(title, '') - if (author) partbasename = partbasename.replace(author, '') - if (series) partbasename = partbasename.replace(series, '') - if (publishedYear) partbasename = partbasename.replace(publishedYear) - - // Look for disc number - let discNumber = null - const discMatch = partbasename.match(/\b(disc|cd) ?(\d\d?)\b/i) - if (discMatch && discMatch.length > 2 && discMatch[2]) { - if (!isNaN(discMatch[2])) { - discNumber = Number(discMatch[2]) - } - - // Remove disc number from filename - partbasename = partbasename.replace(/\b(disc|cd) ?(\d\d?)\b/i, '') - } - - // Look for disc number in folder path e.g. /Book Title/CD01/audiofile.mp3 - const pathdir = Path.dirname(path).split('/').pop() - if (pathdir && /^cd\d{1,3}$/i.test(pathdir)) { - const discFromFolder = Number(pathdir.replace(/cd/i, '')) - if (!isNaN(discFromFolder) && discFromFolder !== null) discNumber = discFromFolder - } - - const numbersinpath = partbasename.match(/\d{1,4}/g) - const trackNumber = numbersinpath && numbersinpath.length ? parseInt(numbersinpath[0]) : null - return { - trackNumber, - discNumber - } - } - - getAverageScanDurationMs(results) { - if (!results.length) return 0 - let total = 0 - for (let i = 0; i < results.length; i++) total += results[i].elapsed - return Math.floor(total / results.length) - } - - async scan(mediaType, libraryFile, mediaMetadataFromScan, verbose = false) { - const probeStart = Date.now() - - const probeData = await prober.probe(libraryFile.metadata.path, verbose) - - if (probeData.error) { - Logger.error(`[MediaFileScanner] ${probeData.error} : "${libraryFile.metadata.path}"`) - return null - } - - if (mediaType === 'video') { - if (!probeData.videoStream) { - Logger.error('[MediaFileScanner] Invalid video file no video stream') - return null - } - - const videoFile = new VideoFile() - videoFile.setDataFromProbe(libraryFile, probeData) - - return { - videoFile, - elapsed: Date.now() - probeStart - } - } else { - if (!probeData.audioStream) { - Logger.error('[MediaFileScanner] Invalid audio file no audio stream') - return null - } - - const audioFile = new AudioFile() - audioFile.trackNumFromMeta = probeData.audioMetaTags.trackNumber - audioFile.discNumFromMeta = probeData.audioMetaTags.discNumber - if (mediaType === 'book') { - const { trackNumber, discNumber } = this.getTrackAndDiscNumberFromFilename(mediaMetadataFromScan, libraryFile) - audioFile.trackNumFromFilename = trackNumber - audioFile.discNumFromFilename = discNumber - } - audioFile.setDataFromProbe(libraryFile, probeData) - - return { - audioFile, - elapsed: Date.now() - probeStart - } - } - } - - /** - * Returns array of { MediaFile, elapsed, averageScanDuration } from audio file scan objects - * @param {import('../objects/LibraryItem')} libraryItem - * @param {import('../objects/files/LibraryFile')[]} mediaLibraryFiles - * @returns {Promise} - */ - async executeMediaFileScans(libraryItem, mediaLibraryFiles) { - const mediaType = libraryItem.mediaType - - const scanStart = Date.now() - const mediaMetadata = libraryItem.media.metadata || null - const batchSize = 32 - const results = [] - for (let batch = 0; batch < mediaLibraryFiles.length; batch += batchSize) { - const proms = [] - for (let i = batch; i < Math.min(batch + batchSize, mediaLibraryFiles.length); i++) { - proms.push(this.scan(mediaType, mediaLibraryFiles[i], mediaMetadata)) - } - results.push(...await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr))) - } - - return { - audioFiles: results.filter(r => r.audioFile).map(r => r.audioFile), - videoFiles: results.filter(r => r.videoFile).map(r => r.videoFile), - elapsed: Date.now() - scanStart, - averageScanDuration: this.getAverageScanDurationMs(results) - } - } - - isSequential(nums) { - if (!nums || !nums.length) return false - if (nums.length === 1) return true - let prev = nums[0] - for (let i = 1; i < nums.length; i++) { - if (nums[i] - prev > 1) return false - prev = nums[i] - } - return true - } - - removeDupes(nums) { - if (!nums || !nums.length) return [] - if (nums.length === 1) return nums - - var nodupes = [nums[0]] - nums.forEach((num) => { - if (num > nodupes[nodupes.length - 1]) nodupes.push(num) - }) - return nodupes - } - - runSmartTrackOrder(libraryItem, audioFiles) { - var discsFromFilename = [] - var tracksFromFilename = [] - var discsFromMeta = [] - var tracksFromMeta = [] - - audioFiles.forEach((af) => { - if (af.discNumFromFilename !== null) discsFromFilename.push(af.discNumFromFilename) - if (af.discNumFromMeta !== null) discsFromMeta.push(af.discNumFromMeta) - if (af.trackNumFromFilename !== null) tracksFromFilename.push(af.trackNumFromFilename) - if (af.trackNumFromMeta !== null) tracksFromMeta.push(af.trackNumFromMeta) - }) - discsFromFilename.sort((a, b) => a - b) - discsFromMeta.sort((a, b) => a - b) - tracksFromFilename.sort((a, b) => a - b) - tracksFromMeta.sort((a, b) => a - b) - - var discKey = null - if (discsFromMeta.length === audioFiles.length && this.isSequential(discsFromMeta)) { - discKey = 'discNumFromMeta' - } else if (discsFromFilename.length === audioFiles.length && this.isSequential(discsFromFilename)) { - discKey = 'discNumFromFilename' - } - - var trackKey = null - tracksFromFilename = this.removeDupes(tracksFromFilename) - tracksFromMeta = this.removeDupes(tracksFromMeta) - if (tracksFromFilename.length > tracksFromMeta.length) { - trackKey = 'trackNumFromFilename' - } else { - trackKey = 'trackNumFromMeta' - } - - if (discKey !== null) { - Logger.debug(`[MediaFileScanner] Smart track order for "${libraryItem.media.metadata.title}" using disc key ${discKey} and track key ${trackKey}`) - audioFiles.sort((a, b) => { - let Dx = a[discKey] - b[discKey] - if (Dx === 0) Dx = a[trackKey] - b[trackKey] - return Dx - }) - } else { - Logger.debug(`[MediaFileScanner] Smart track order for "${libraryItem.media.metadata.title}" using track key ${trackKey}`) - audioFiles.sort((a, b) => a[trackKey] - b[trackKey]) - } - - for (let i = 0; i < audioFiles.length; i++) { - audioFiles[i].index = i + 1 - var existingAF = libraryItem.media.findFileWithInode(audioFiles[i].ino) - if (existingAF) { - if (existingAF.updateFromScan) existingAF.updateFromScan(audioFiles[i]) - } else { - libraryItem.media.addAudioFile(audioFiles[i]) - } - } - } - - /** - * Scans media files for a library item and adds them as audio tracks and sets library item metadata - * @param {import('../objects/files/LibraryFile')[]} mediaLibraryFiles - * @param {import('../objects/LibraryItem')} libraryItem - * @param {import('./LibraryScan')} [libraryScan=null] - Optional when doing a library scan to use LibraryScan config/logs - * @return {Promise} True if any updates were made - */ - async scanMediaFiles(mediaLibraryFiles, libraryItem, libraryScan = null) { - const preferAudioMetadata = libraryScan ? !!libraryScan.preferAudioMetadata : !!global.ServerSettings.scannerPreferAudioMetadata - - let hasUpdated = false - - const mediaScanResult = await this.executeMediaFileScans(libraryItem, mediaLibraryFiles) - - if (libraryItem.mediaType === 'video') { - if (mediaScanResult.videoFiles.length) { - // TODO: Check for updates etc - hasUpdated = true - libraryItem.media.setVideoFile(mediaScanResult.videoFiles[0]) - } - } else if (mediaScanResult.audioFiles.length) { - if (libraryScan) { - libraryScan.addLog(LogLevel.DEBUG, `Library Item "${libraryItem.path}" Media file scan took ${mediaScanResult.elapsed}ms for ${mediaScanResult.audioFiles.length} with average time of ${mediaScanResult.averageScanDuration}ms per MB`) - } - Logger.debug(`Library Item "${libraryItem.path}" Media file scan took ${mediaScanResult.elapsed}ms with ${mediaScanResult.audioFiles.length} audio files averaging ${mediaScanResult.averageScanDuration}ms per MB`) - - const newAudioFiles = mediaScanResult.audioFiles.filter(af => { - return !libraryItem.media.findFileWithInode(af.ino) - }) - - // Book: Adding audio files to book media - if (libraryItem.mediaType === 'book') { - const mediaScanFileInodes = mediaScanResult.audioFiles.map(af => af.ino) - // Filter for existing valid track audio files not included in the audio files scanned - const existingAudioFiles = libraryItem.media.audioFiles.filter(af => af.isValidTrack && !mediaScanFileInodes.includes(af.ino)) - - if (newAudioFiles.length) { - // Single Track Audiobooks - if (mediaScanFileInodes.length + existingAudioFiles.length === 1) { - const af = mediaScanResult.audioFiles[0] - af.index = 1 - libraryItem.media.addAudioFile(af) - hasUpdated = true - } else { - const allAudioFiles = existingAudioFiles.concat(mediaScanResult.audioFiles) - this.runSmartTrackOrder(libraryItem, allAudioFiles) - hasUpdated = true - } - } else { - // Only update metadata not index - mediaScanResult.audioFiles.forEach((af) => { - const existingAF = libraryItem.media.findFileWithInode(af.ino) - if (existingAF) { - af.index = existingAF.index - if (existingAF.updateFromScan && existingAF.updateFromScan(af)) { - hasUpdated = true - } - } - }) - } - - // Set book details from audio file ID3 tags, optional prefer - if (libraryItem.media.setMetadataFromAudioFile(preferAudioMetadata)) { - hasUpdated = true - } - - if (hasUpdated) { - libraryItem.media.rebuildTracks() - } - } else if (libraryItem.mediaType === 'podcast') { // Podcast Media Type - const existingAudioFiles = mediaScanResult.audioFiles.filter(af => libraryItem.media.findFileWithInode(af.ino)) - - if (newAudioFiles.length) { - let newIndex = Math.max(...libraryItem.media.episodes.filter(ep => ep.index == null || isNaN(ep.index)).map(ep => Number(ep.index))) + 1 - newAudioFiles.forEach((newAudioFile) => { - libraryItem.media.addNewEpisodeFromAudioFile(newAudioFile, newIndex++) - }) - hasUpdated = true - } - - // Update audio file metadata for audio files already there - existingAudioFiles.forEach((af) => { - const podcastEpisode = libraryItem.media.findEpisodeWithInode(af.ino) - af.index = 1 - if (podcastEpisode?.audioFile.updateFromScan(af)) { - hasUpdated = true - - podcastEpisode.setDataFromAudioMetaTags(podcastEpisode.audioFile.metaTags, false) - } - }) - - if (libraryItem.media.setMetadataFromAudioFile(preferAudioMetadata)) { - hasUpdated = true - } - } else if (libraryItem.mediaType === 'music') { // Music - // Only one audio file in library item - if (newAudioFiles.length) { // New audio file - libraryItem.media.setAudioFile(newAudioFiles[0]) - hasUpdated = true - } else if (libraryItem.media.audioFile && libraryItem.media.audioFile.updateFromScan(mediaScanResult.audioFiles[0])) { - hasUpdated = true - console.log('Updated from scan') - } - - if (libraryItem.media.setMetadataFromAudioFile()) { - hasUpdated = true - } - - // If the audio track has no title meta tag then use the audio file name - if (!libraryItem.media.metadata.title && libraryItem.media.audioFile) { - const audioFileName = libraryItem.media.audioFile.metadata.filename - libraryItem.media.metadata.title = Path.basename(audioFileName, Path.extname(audioFileName)) - hasUpdated = true - } - } - } - - return hasUpdated - } - - probeAudioFile(audioFile) { - Logger.debug(`[MediaFileScanner] Running ffprobe for audio file at "${audioFile.metadata.path}"`) - return prober.rawProbe(audioFile.metadata.path) - } -} -module.exports = new MediaFileScanner() \ No newline at end of file diff --git a/server/scanner/PodcastScanner.js b/server/scanner/PodcastScanner.js index 15236263..4a80d40a 100644 --- a/server/scanner/PodcastScanner.js +++ b/server/scanner/PodcastScanner.js @@ -490,20 +490,31 @@ class PodcastScanner { return fsExtra.writeFile(metadataFilePath, JSON.stringify(jsonObject, null, 2)).then(async () => { // Add metadata.json to libraryFiles array if it is new let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) - if (storeMetadataWithItem && !metadataLibraryFile) { - const newLibraryFile = new LibraryFile() - await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`) - metadataLibraryFile = newLibraryFile.toJSON() - libraryItem.libraryFiles.push(metadataLibraryFile) - } else if (storeMetadataWithItem) { - const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) - if (fileTimestamps) { - metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs - metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs - metadataLibraryFile.metadata.size = fileTimestamps.size - metadataLibraryFile.ino = fileTimestamps.ino + if (storeMetadataWithItem) { + if (!metadataLibraryFile) { + const newLibraryFile = new LibraryFile() + await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`) + metadataLibraryFile = newLibraryFile.toJSON() + libraryItem.libraryFiles.push(metadataLibraryFile) + } else { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } + } + const libraryItemDirTimestamps = await getFileTimestampsWithIno(libraryItem.path) + if (libraryItemDirTimestamps) { + libraryItem.mtime = libraryItemDirTimestamps.mtimeMs + libraryItem.ctime = libraryItemDirTimestamps.ctimeMs + let size = 0 + libraryItem.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0)) + libraryItem.size = size } } + libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`) return metadataLibraryFile @@ -526,18 +537,28 @@ class PodcastScanner { } // Add metadata.abs to libraryFiles array if it is new let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) - if (storeMetadataWithItem && !metadataLibraryFile) { - const newLibraryFile = new LibraryFile() - await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`) - metadataLibraryFile = newLibraryFile.toJSON() - libraryItem.libraryFiles.push(metadataLibraryFile) - } else if (storeMetadataWithItem) { - const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) - if (fileTimestamps) { - metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs - metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs - metadataLibraryFile.metadata.size = fileTimestamps.size - metadataLibraryFile.ino = fileTimestamps.ino + if (storeMetadataWithItem) { + if (!metadataLibraryFile) { + const newLibraryFile = new LibraryFile() + await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`) + metadataLibraryFile = newLibraryFile.toJSON() + libraryItem.libraryFiles.push(metadataLibraryFile) + } else { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } + } + const libraryItemDirTimestamps = await getFileTimestampsWithIno(libraryItem.path) + if (libraryItemDirTimestamps) { + libraryItem.mtime = libraryItemDirTimestamps.mtimeMs + libraryItem.ctime = libraryItemDirTimestamps.ctimeMs + let size = 0 + libraryItem.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0)) + libraryItem.size = size } } diff --git a/server/scanner/ScanOptions.js b/server/scanner/ScanOptions.js deleted file mode 100644 index 873371bb..00000000 --- a/server/scanner/ScanOptions.js +++ /dev/null @@ -1,40 +0,0 @@ -class ScanOptions { - constructor() { - this.forceRescan = false - - // Server settings - this.parseSubtitles = false - this.findCovers = false - this.storeCoverWithItem = false - this.preferAudioMetadata = false - this.preferOpfMetadata = false - this.preferMatchedMetadata = false - this.preferOverdriveMediaMarker = false - } - - toJSON() { - return { - forceRescan: this.forceRescan, - parseSubtitles: this.parseSubtitles, - findCovers: this.findCovers, - storeCoverWithItem: this.storeCoverWithItem, - preferAudioMetadata: this.preferAudioMetadata, - preferOpfMetadata: this.preferOpfMetadata, - preferMatchedMetadata: this.preferMatchedMetadata, - preferOverdriveMediaMarker: this.preferOverdriveMediaMarker - } - } - - setData(options, serverSettings) { - this.forceRescan = !!options.forceRescan - - this.parseSubtitles = !!serverSettings.scannerParseSubtitle - this.findCovers = !!serverSettings.scannerFindCovers - this.storeCoverWithItem = serverSettings.storeCoverWithItem - this.preferAudioMetadata = serverSettings.scannerPreferAudioMetadata - this.preferOpfMetadata = serverSettings.scannerPreferOpfMetadata - this.scannerPreferMatchedMetadata = serverSettings.scannerPreferMatchedMetadata - this.preferOverdriveMediaMarker = serverSettings.scannerPreferOverdriveMediaMarker - } -} -module.exports = ScanOptions \ No newline at end of file diff --git a/server/scanner/Scanner.js b/server/scanner/Scanner.js index 091c4e48..74702307 100644 --- a/server/scanner/Scanner.js +++ b/server/scanner/Scanner.js @@ -6,7 +6,6 @@ const Database = require('../Database') const { LogLevel } = require('../utils/constants') const { findMatchingEpisodesInFeed, getPodcastFeed } = require('../utils/podcastUtils') -const MediaFileScanner = require('./MediaFileScanner') const BookFinder = require('../finders/BookFinder') const PodcastFinder = require('../finders/PodcastFinder') const LibraryScan = require('./LibraryScan') @@ -374,9 +373,5 @@ class Scanner { this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id) SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData) } - - probeAudioFile(audioFile) { - return MediaFileScanner.probeAudioFile(audioFile) - } } module.exports = Scanner