diff --git a/server/Db.js b/server/Db.js index 95122b09..b25cd54d 100644 --- a/server/Db.js +++ b/server/Db.js @@ -201,6 +201,20 @@ class Db { }) } + insertEntities(entityName, entities) { + var entityDb = this.getEntityDb(entityName) + return entityDb.insert(entities).then((results) => { + Logger.debug(`[DB] Inserted ${results.inserted} ${entityName}`) + + var arrayKey = this.getEntityArrayKey(entityName) + this[arrayKey] = this[arrayKey].concat(entities) + return true + }).catch((error) => { + Logger.error(`[DB] Failed to insert ${entityName}`, error) + return false + }) + } + insertEntity(entityName, entity) { var entityDb = this.getEntityDb(entityName) return entityDb.insert([entity]).then((results) => { @@ -215,6 +229,26 @@ class Db { }) } + updateEntities(entityName, entities) { + var entityDb = this.getEntityDb(entityName) + + var entityIds = entities.map(ent => ent.id) + return entityDb.update((record) => entityIds.includes(record.id), (record) => { + return entities.find(ent => ent.id === record.id) + }).then((results) => { + Logger.debug(`[DB] Updated ${entityName}: ${results.updated}`) + var arrayKey = this.getEntityArrayKey(entityName) + this[arrayKey] = this[arrayKey].map(e => { + if (entityIds.includes(e.id)) return entities.find(_e => _e.id === e.id) + return e + }) + return true + }).catch((error) => { + Logger.error(`[DB] Update ${entityName} Failed: ${error}`) + return false + }) + } + updateEntity(entityName, entity) { var entityDb = this.getEntityDb(entityName) @@ -224,7 +258,7 @@ class Db { } return entityDb.update((record) => record.id === entity.id, () => jsonEntity).then((results) => { - Logger.debug(`[DB] Updated entity ${entityName}: ${results.updated}`) + Logger.debug(`[DB] Updated ${entityName}: ${results.updated}`) var arrayKey = this.getEntityArrayKey(entityName) this[arrayKey] = this[arrayKey].map(e => { return e.id === entity.id ? entity : e diff --git a/server/Scanner.js b/server/Scanner.js index 8f3d4b16..60e2c015 100644 --- a/server/Scanner.js +++ b/server/Scanner.js @@ -6,8 +6,7 @@ const Logger = require('./Logger') const { version } = require('../package.json') const audioFileScanner = require('./utils/audioFileScanner') const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('./utils/scandir') -const { comparePaths, getIno, getId } = require('./utils/index') -const { secondsToTimestamp } = require('./utils/fileUtils') +const { comparePaths, getIno, getId, secondsToTimestamp } = require('./utils/index') const { ScanResult, CoverDestination } = require('./utils/constants') const BookFinder = require('./BookFinder') @@ -440,7 +439,6 @@ class Scanner { var scanPayload = { id: libraryId, name: library.name, - scanType: 'library', folders: library.folders.length } this.emitter('scan_start', scanPayload) @@ -489,7 +487,7 @@ class Scanner { Logger.info(`[Scanner] Canceling scan ${libraryId}`) delete this.cancelLibraryScan[libraryId] this.librariesScanning = this.librariesScanning.filter(l => l.id !== libraryId) - this.emitter('scan_complete', { id: libraryId, name: library.name, scanType: 'library', results: null }) + this.emitter('scan_complete', { id: libraryId, name: library.name, results: null }) return null } @@ -516,7 +514,7 @@ class Scanner { Logger.info(`[Scanner] Canceling scan ${libraryId}`) delete this.cancelLibraryScan[libraryId] this.librariesScanning = this.librariesScanning.filter(l => l.id !== libraryId) - this.emitter('scan_complete', { id: libraryId, name: library.name, scanType: 'library', results: scanResults }) + this.emitter('scan_complete', { id: libraryId, name: library.name, results: scanResults }) return } } @@ -532,7 +530,6 @@ class Scanner { this.emitter('scan_progress', { id: libraryId, name: library.name, - scanType: 'library', progress: { total: audiobookDataFound.length, done: i + 1, @@ -548,7 +545,7 @@ class Scanner { const scanElapsed = Math.floor((Date.now() - scanStart) / 1000) Logger.info(`[Scanned] Finished | ${scanResults.added} added | ${scanResults.updated} updated | ${scanResults.removed} removed | ${scanResults.missing} missing | elapsed: ${secondsToTimestamp(scanElapsed)}`) this.librariesScanning = this.librariesScanning.filter(l => l.id !== libraryId) - this.emitter('scan_complete', { id: libraryId, name: library.name, scanType: 'library', results: scanResults }) + this.emitter('scan_complete', { id: libraryId, name: library.name, results: scanResults }) } async scanAudiobookById(audiobookId) { diff --git a/server/Server.js b/server/Server.js index ad68efaa..7a6c6710 100644 --- a/server/Server.js +++ b/server/Server.js @@ -11,13 +11,14 @@ const { version } = require('../package.json') // Utils const { ScanResult } = require('./utils/constants') const filePerms = require('./utils/filePerms') -const { secondsToTimestamp } = require('./utils/fileUtils') +const { secondsToTimestamp } = require('./utils/index') const Logger = require('./Logger') // Classes const Auth = require('./Auth') const Watcher = require('./Watcher') const Scanner = require('./Scanner') +const Scanner2 = require('./scanner/Scanner') const Db = require('./Db') const BackupManager = require('./BackupManager') const LogManager = require('./LogManager') @@ -49,6 +50,8 @@ class Server { this.watcher = new Watcher(this.AudiobookPath) this.coverController = new CoverController(this.db, this.MetadataPath, this.AudiobookPath) this.scanner = new Scanner(this.AudiobookPath, this.MetadataPath, this.db, this.coverController, this.emitter.bind(this)) + this.scanner2 = new Scanner2(this.AudiobookPath, this.MetadataPath, this.db, this.coverController, this.emitter.bind(this)) + this.streamManager = new StreamManager(this.db, this.MetadataPath, this.emitter.bind(this), this.clientEmitter.bind(this)) this.rssFeeds = new RssFeeds(this.Port, this.db) this.downloadManager = new DownloadManager(this.db, this.MetadataPath, this.AudiobookPath, this.emitter.bind(this)) @@ -314,7 +317,8 @@ class Server { async scan(libraryId, forceAudioFileScan = false) { Logger.info('[Server] Starting Scan') - await this.scanner.scan(libraryId, forceAudioFileScan) + // await this.scanner2.scan(libraryId) + await this.scanner(libraryId, forceAudioFileScan) Logger.info('[Server] Scan complete') } diff --git a/server/objects/AudioFile.js b/server/objects/AudioFile.js index 4b43837c..da7be888 100644 --- a/server/objects/AudioFile.js +++ b/server/objects/AudioFile.js @@ -1,3 +1,5 @@ +const { isNullOrNaN } = require('../utils/index') + const Logger = require('../Logger') const AudioFileMetadata = require('./AudioFileMetadata') @@ -154,7 +156,7 @@ class AudioFile { } // New scanner creates AudioFile from AudioFileScanner - setData2(fileData, probeData) { + setDataFromProbe(fileData, probeData) { this.index = fileData.index || null this.ino = fileData.ino || null this.filename = fileData.filename @@ -162,6 +164,42 @@ class AudioFile { this.path = fileData.path this.fullPath = fileData.fullPath this.addedAt = Date.now() + + this.trackNumFromMeta = fileData.trackNumFromMeta || null + this.trackNumFromFilename = fileData.trackNumFromFilename || null + this.cdNumFromFilename = fileData.cdNumFromFilename || null + + this.format = probeData.format + this.duration = probeData.duration + this.size = probeData.size + this.bitRate = probeData.bitRate || null + this.language = probeData.language + this.codec = probeData.codec || null + this.timeBase = probeData.timeBase + this.channels = probeData.channels + this.channelLayout = probeData.channelLayout + this.chapters = probeData.chapters || [] + this.metadata = probeData.audioFileMetadata + } + + validateTrackIndex(isSingleTrack) { + var numFromMeta = isNullOrNaN(this.trackNumFromMeta) ? null : Number(this.trackNumFromMeta) + var numFromFilename = isNullOrNaN(this.trackNumFromFilename) ? null : Number(this.trackNumFromFilename) + + if (isSingleTrack) { // Single audio track audiobook only use metadata tag and default to 1 + return numFromMeta ? numFromMeta : 1 + } + if (numFromMeta !== null) return numFromMeta + if (numFromFilename !== null) return numFromFilename + + this.invalid = true + this.error = 'Failed to get track number' + return null + } + + setDuplicateTrackNumber(num) { + this.invalid = true + this.error = 'Duplicate track number "' + num + '"' } syncChapters(updatedChapters) { diff --git a/server/objects/Audiobook.js b/server/objects/Audiobook.js index a2fa4526..9ffbe455 100644 --- a/server/objects/Audiobook.js +++ b/server/objects/Audiobook.js @@ -1,7 +1,7 @@ const Path = require('path') const fs = require('fs-extra') -const { bytesPretty, elapsedPretty, readTextFile } = require('../utils/fileUtils') -const { comparePaths, getIno, getId } = require('../utils/index') +const { bytesPretty, readTextFile } = require('../utils/fileUtils') +const { comparePaths, getIno, getId, elapsedPretty } = require('../utils/index') const { parseOpfMetadataXML } = require('../utils/parseOpfMetadata') const { extractCoverArt } = require('../utils/ffmpegHelpers') const nfoGenerator = require('../utils/nfoGenerator') @@ -128,6 +128,8 @@ class Audiobook { get _otherFiles() { return this.otherFiles || [] } get _tracks() { return this.tracks || [] } + get audioFilesToInclude() { return this._audioFiles.filter(af => !af.exclude) } + get ebooks() { return this.otherFiles.filter(file => file.filetype === 'ebook') } @@ -346,6 +348,11 @@ class Audiobook { this.scanVersion = version } + setMissing() { + this.isMissing = true + this.lastUpdate = Date.now() + } + setBook(data) { // Use first image file as cover if (this.otherFiles && this.otherFiles.length) { @@ -353,7 +360,6 @@ class Audiobook { if (imageFile) { data.coverFullPath = imageFile.fullPath var relImagePath = imageFile.path.replace(this.path, '') - console.log('SET BOOK PATH', imageFile.path, 'REPLACE', this.path, 'RESULT', relImagePath) data.cover = Path.posix.join(`/s/book/${this.id}`, relImagePath) } } @@ -383,10 +389,15 @@ class Audiobook { } addAudioFile(audioFileData) { - var audioFile = new AudioFile() - audioFile.setData(audioFileData) - this.audioFiles.push(audioFile) - return audioFile + if (audioFileData instanceof AudioFile) { + this.audioFiles.push(audioFileData) + return audioFileData + } else { + var audioFile = new AudioFile() + audioFile.setData(audioFileData) + this.audioFiles.push(audioFile) + return audioFile + } } addOtherFile(fileData) { @@ -426,6 +437,10 @@ class Audiobook { return this.book.updateCover(cover, coverFullPath) } + checkHasTrackNum(trackNum) { + return this.tracks.find(t => t.index === trackNum) + } + updateAudioTracks(orderedFileData) { var index = 1 this.audioFiles = orderedFileData.map((fileData) => { @@ -444,8 +459,12 @@ class Audiobook { return audioFile }) - this.audioFiles.sort((a, b) => a.index - b.index) + this.rebuildTracks() + } + // After audio files have been added/removed/updated this method sets tracks + rebuildTracks() { + this.audioFiles.sort((a, b) => a.index - b.index) this.tracks = [] this.missingParts = [] this.audioFiles.forEach((file) => { @@ -570,7 +589,6 @@ class Audiobook { } }) - var imageFiles = this.otherFiles.filter(f => f.filetype === 'image') // OLD Path Check if cover was a local image and that it still exists @@ -866,7 +884,7 @@ class Audiobook { return hasUpdated } - checkShouldScan(dataFound) { + checkScanData(dataFound) { var hasUpdated = false if (dataFound.ino !== this.ino) { diff --git a/server/objects/ServerSettings.js b/server/objects/ServerSettings.js index 2d1f5e60..70e04122 100644 --- a/server/objects/ServerSettings.js +++ b/server/objects/ServerSettings.js @@ -12,6 +12,8 @@ class ServerSettings { // Scanner this.scannerParseSubtitle = false this.scannerFindCovers = false + this.scannerPreferAudioMetadata = false + this.scannerPreferOpfMetadata = false // Metadata this.coverDestination = CoverDestination.METADATA diff --git a/server/objects/Stream.js b/server/objects/Stream.js index 2f4fa4d8..9be1ff0e 100644 --- a/server/objects/Stream.js +++ b/server/objects/Stream.js @@ -3,8 +3,7 @@ const EventEmitter = require('events') const Path = require('path') const fs = require('fs-extra') const Logger = require('../Logger') -const { getId } = require('../utils/index') -const { secondsToTimestamp } = require('../utils/fileUtils') +const { getId, secondsToTimestamp } = require('../utils/index') const { writeConcatFile } = require('../utils/ffmpegHelpers') const hlsPlaylistGenerator = require('../utils/hlsPlaylistGenerator') diff --git a/server/scanner/AudioFileScanner.js b/server/scanner/AudioFileScanner.js index 83accbe8..e73cbcfc 100644 --- a/server/scanner/AudioFileScanner.js +++ b/server/scanner/AudioFileScanner.js @@ -1,22 +1,106 @@ +const Path = require('path') + const AudioFile = require('../objects/AudioFile') -const AudioProbeData = require('./AudioProbeData') const prober = require('../utils/prober') const Logger = require('../Logger') +const { msToTimestamp } = require('../utils') class AudioFileScanner { constructor() { } - async scan(audioFileData, verbose = false) { + getTrackNumberFromMeta(scanData) { + return !isNaN(scanData.trackNumber) && scanData.trackNumber !== null ? Math.trunc(Number(scanData.trackNumber)) : null + } + + getTrackNumberFromFilename(bookScanData, filename) { + const { title, author, series, publishYear } = bookScanData + var partbasename = Path.basename(filename, Path.extname(filename)) + + // Remove title, author, series, and publishYear from filename if there + if (title) partbasename = partbasename.replace(title, '') + if (author) partbasename = partbasename.replace(author, '') + if (series) partbasename = partbasename.replace(series, '') + if (publishYear) partbasename = partbasename.replace(publishYear) + + // Remove eg. "disc 1" from path + partbasename = partbasename.replace(/\bdisc \d\d?\b/i, '') + + // Remove "cd01" or "cd 01" from path + partbasename = partbasename.replace(/\bcd ?\d\d?\b/i, '') + + var numbersinpath = partbasename.match(/\d{1,4}/g) + if (!numbersinpath) return null + + var number = numbersinpath.length ? parseInt(numbersinpath[0]) : null + return number + } + + getCdNumberFromFilename(bookScanData, filename) { + const { title, author, series, publishYear } = bookScanData + var partbasename = Path.basename(filename, Path.extname(filename)) + + // Remove title, author, series, and publishYear from filename if there + if (title) partbasename = partbasename.replace(title, '') + if (author) partbasename = partbasename.replace(author, '') + if (series) partbasename = partbasename.replace(series, '') + if (publishYear) partbasename = partbasename.replace(publishYear) + + var cdNumber = null + + var cdmatch = partbasename.match(/\b(disc|cd) ?(\d\d?)\b/i) + if (cdmatch && cdmatch.length > 2 && cdmatch[2]) { + if (!isNaN(cdmatch[2])) { + cdNumber = Number(cdmatch[2]) + } + } + + return cdNumber + } + + getAverageScanDurationMs(results) { + if (!results.length) return 0 + var total = 0 + for (let i = 0; i < results.length; i++) total += results[i].elapsed + return Math.floor(total / results.length) + } + + async scan(audioFileData, bookScanData, verbose = false) { + var probeStart = Date.now() + // Logger.debug(`[AudioFileScanner] Start Probe ${audioFileData.fullPath}`) var probeData = await prober.probe2(audioFileData.fullPath, verbose) if (probeData.error) { Logger.error(`[AudioFileScanner] ${probeData.error} : "${audioFileData.fullPath}"`) return null } + // Logger.debug(`[AudioFileScanner] Finished Probe ${audioFileData.fullPath} elapsed ${msToTimestamp(Date.now() - probeStart, true)}`) var audioFile = new AudioFile() - // TODO: Build audio file - return audioFile + audioFileData.trackNumFromMeta = this.getTrackNumberFromMeta(probeData) + audioFileData.trackNumFromFilename = this.getTrackNumberFromFilename(bookScanData, audioFileData.filename) + audioFileData.cdNumFromFilename = this.getCdNumberFromFilename(bookScanData, audioFileData.filename) + audioFile.setDataFromProbe(audioFileData, probeData) + return { + audioFile, + elapsed: Date.now() - probeStart + } + } + + + // Returns array of { AudioFile, elapsed } from audio file scan objects + async scanAudioFiles(audioFileDataArray, bookScanData) { + var proms = [] + for (let i = 0; i < audioFileDataArray.length; i++) { + var prom = this.scan(audioFileDataArray[i], bookScanData) + proms.push(prom) + } + var scanStart = Date.now() + var results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr)) + return { + audioFiles: results.map(r => r.audioFile), + elapsed: Date.now() - scanStart, + averageScanDuration: this.getAverageScanDurationMs(results) + } } } module.exports = new AudioFileScanner() \ No newline at end of file diff --git a/server/scanner/AudioProbeData.js b/server/scanner/AudioProbeData.js index 5f27fe26..758cb47a 100644 --- a/server/scanner/AudioProbeData.js +++ b/server/scanner/AudioProbeData.js @@ -34,7 +34,7 @@ class AudioProbeData { } setData(data) { - var audioStream = getDefaultAudioStream(data.audio_streams) + var audioStream = this.getDefaultAudioStream(data.audio_streams) this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : false this.format = data.format diff --git a/server/scanner/LibraryScan.js b/server/scanner/LibraryScan.js index 4819fb38..5e32876b 100644 --- a/server/scanner/LibraryScan.js +++ b/server/scanner/LibraryScan.js @@ -1,6 +1,7 @@ const Folder = require('../objects/Folder') +const Constants = require('../utils/constants') -const { getId } = require('../utils/index') +const { getId, secondsToTimestamp } = require('../utils/index') class LibraryScan { constructor() { @@ -13,22 +14,49 @@ class LibraryScan { this.startedAt = null this.finishedAt = null + this.elapsed = null - this.folderScans = [] + this.status = Constants.ScanStatus.NOTHING + this.resultsMissing = 0 + this.resultsAdded = 0 + this.resultsUpdated = 0 } get _scanOptions() { return this.scanOptions || {} } get forceRescan() { return !!this._scanOptions.forceRescan } + get resultStats() { + return `${this.resultsAdded} Added | ${this.resultsUpdated} Updated | ${this.resultsMissing} Missing` + } + get elapsedTimestamp() { + return secondsToTimestamp(this.elapsed / 1000) + } + get getScanEmitData() { + return { + id: this.libraryId, + name: this.libraryName, + results: { + added: this.resultsAdded, + updated: this.resultsUpdated, + missing: this.resultsMissing + } + } + } + setData(library, scanOptions) { this.id = getId('lscan') this.libraryId = library.id this.libraryName = library.name - this.folders = library.folders.map(folder => Folder(folder.toJSON())) + this.folders = library.folders.map(folder => new Folder(folder.toJSON())) this.scanOptions = scanOptions this.startedAt = Date.now() } + + setComplete() { + this.finishedAt = Date.now() + this.elapsed = this.finishedAt - this.startedAt + } } module.exports = LibraryScan \ No newline at end of file diff --git a/server/scanner/ScanOptions.js b/server/scanner/ScanOptions.js index 5ca4d7cc..851cff01 100644 --- a/server/scanner/ScanOptions.js +++ b/server/scanner/ScanOptions.js @@ -4,33 +4,35 @@ class ScanOptions { constructor(options) { this.forceRescan = false - this.metadataPrecedence = [ - { - id: 'directory', - include: true - }, - { - id: 'reader-desc-txt', - include: true - }, - { - id: 'audio-file-metadata', - include: true - }, - { - id: 'metadata-opf', - include: true - }, - { - id: 'external-source', - include: false - } - ] + // this.metadataPrecedence = [ + // { + // id: 'directory', + // include: true + // }, + // { + // id: 'reader-desc-txt', + // include: true + // }, + // { + // id: 'audio-file-metadata', + // include: true + // }, + // { + // id: 'metadata-opf', + // include: true + // }, + // { + // id: 'external-source', + // include: false + // } + // ] // Server settings this.parseSubtitles = false this.findCovers = false this.coverDestination = CoverDestination.METADATA + this.preferAudioMetadata = false + this.preferOpfMetadata = false if (options) { this.construct(options) @@ -53,7 +55,9 @@ class ScanOptions { metadataPrecedence: this.metadataPrecedence, parseSubtitles: this.parseSubtitles, findCovers: this.findCovers, - coverDestination: this.coverDestination + coverDestination: this.coverDestination, + preferAudioMetadata: this.preferAudioMetadata, + preferOpfMetadata: this.preferOpfMetadata } } @@ -63,6 +67,8 @@ class ScanOptions { this.parseSubtitles = !!serverSettings.scannerParseSubtitle this.findCovers = !!serverSettings.scannerFindCovers this.coverDestination = serverSettings.coverDestination + this.preferAudioMetadata = serverSettings.scannerPreferAudioMetadata + this.preferOpfMetadata = serverSettings.scannerPreferOpfMetadata } } module.exports = ScanOptions \ No newline at end of file diff --git a/server/scanner/Scanner.js b/server/scanner/Scanner.js index 66497069..ddd4c21f 100644 --- a/server/scanner/Scanner.js +++ b/server/scanner/Scanner.js @@ -6,8 +6,7 @@ const Logger = require('../Logger') const { version } = require('../../package.json') const audioFileScanner = require('../utils/audioFileScanner') const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('../utils/scandir') -const { comparePaths, getIno, getId } = require('../utils/index') -const { secondsToTimestamp } = require('../utils/fileUtils') +const { comparePaths, getIno, getId, msToTimestamp } = require('../utils/index') const { ScanResult, CoverDestination } = require('../utils/constants') const AudioFileScanner = require('./AudioFileScanner') @@ -33,6 +32,20 @@ class Scanner { this.bookFinder = new BookFinder() } + getCoverDirectory(audiobook) { + if (this.db.serverSettings.coverDestination === CoverDestination.AUDIOBOOK) { + return { + fullPath: audiobook.fullPath, + relPath: '/s/book/' + audiobook.id + } + } else { + return { + fullPath: Path.posix.join(this.BookMetadataPath, audiobook.id), + relPath: Path.posix.join('/metadata', 'books', audiobook.id) + } + } + } + async scan(libraryId, options = {}) { if (this.librariesScanning.includes(libraryId)) { Logger.error(`[Scanner] Already scanning ${libraryId}`) @@ -53,14 +66,19 @@ class Scanner { var libraryScan = new LibraryScan() libraryScan.setData(library, scanOptions) + this.librariesScanning.push(libraryScan) + + this.emitter('scan_start', libraryScan.getScanEmitData) Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`) - var results = await this.scanLibrary(libraryScan) + await this.scanLibrary(libraryScan) - Logger.info(`[Scanner] Library scan ${libraryScan.id} complete`) + libraryScan.setComplete() + Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp}. ${libraryScan.resultStats}`) - return results + this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id) + this.emitter('scan_complete', libraryScan.getScanEmitData) } async scanLibrary(libraryScan) { @@ -77,9 +95,9 @@ class Scanner { var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryScan.libraryId) - const audiobooksToUpdate = [] - const audiobooksToRescan = [] - const newAudiobookData = [] + var audiobooksToUpdate = [] + var audiobookRescans = [] + var newAudiobookScans = [] // Check for existing & removed audiobooks for (let i = 0; i < audiobooksInLibrary.length; i++) { @@ -87,21 +105,20 @@ class Scanner { var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path)) if (!dataFound) { Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`) - audiobook.isMissing = true - audiobook.lastUpdate = Date.now() - scanResults.missing++ + audiobook.setMissing() audiobooksToUpdate.push(audiobook) } else { - var checkRes = audiobook.checkShouldRescan(dataFound) + var checkRes = audiobook.checkScanData(dataFound) if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) { // existing audiobook has new files checkRes.audiobook = audiobook - audiobooksToRescan.push(checkRes) + checkRes.bookScanData = dataFound + audiobookRescans.push(this.rescanAudiobook(checkRes, libraryScan)) + libraryScan.resultsMissing++ } else if (checkRes.updated) { audiobooksToUpdate.push(audiobook) + libraryScan.resultsUpdated++ } - - // Remove this abf audiobookDataFound = audiobookDataFound.filter(abf => abf.ino !== dataFound.ino) } } @@ -113,60 +130,108 @@ class Scanner { if (!hasEbook && !dataFound.audioFiles.length) { Logger.info(`[Scanner] Directory found "${audiobookDataFound.path}" has no ebook or audio files`) } else { - newAudiobookData.push(dataFound) + newAudiobookScans.push(this.scanNewAudiobook(dataFound, libraryScan)) } } - var rescans = [] - for (let i = 0; i < audiobooksToRescan.length; i++) { - var rescan = this.rescanAudiobook(audiobooksToRescan[i]) - rescans.push(rescan) + if (audiobookRescans.length) { + var updatedAudiobooks = (await Promise.all(audiobookRescans)).filter(ab => !!ab) + if (updatedAudiobooks.length) { + audiobooksToUpdate = audiobooksToUpdate.concat(updatedAudiobooks) + libraryScan.resultsUpdated += updatedAudiobooks.length + } } - var newscans = [] - for (let i = 0; i < newAudiobookData.length; i++) { - var newscan = this.scanNewAudiobook(newAudiobookData[i]) - newscans.push(newscan) + if (audiobooksToUpdate.length) { + Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" updating ${audiobooksToUpdate.length} books`) + await this.db.updateEntities('audiobook', audiobooksToUpdate) } - var rescanResults = await Promise.all(rescans) - - var newscanResults = await Promise.all(newscans) - - // TODO: Return report - return { - updates: 0, - additions: 0 + if (newAudiobookScans.length) { + var newAudiobooks = (await Promise.all(newAudiobookScans)).filter(ab => !!ab) + if (newAudiobooks.length) { + Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" inserting ${newAudiobooks.length} books`) + await this.db.insertEntities('audiobook', newAudiobooks) + libraryScan.resultsAdded = newAudiobooks.length + } } } - // Return scan result payload - async rescanAudiobook(audiobookCheckData) { - const { newAudioFileData, newOtherFileData, audiobook } = audiobookCheckData + async rescanAudiobook(audiobookCheckData, libraryScan) { + const { newAudioFileData, newOtherFileData, audiobook, bookScanData } = audiobookCheckData + Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`) + if (newAudioFileData.length) { - var newAudioFiles = await this.scanAudioFiles(newAudioFileData) - // TODO: Update audiobook tracks + var audioScanResult = await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData) + Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobook.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`) + if (audioScanResult.audioFiles.length) { + var totalAudioFilesToInclude = audiobook.audioFilesToInclude.length + audioScanResult.audioFiles.length + + // validate & add audio files to audiobook + for (let i = 0; i < audioScanResult.audioFiles.length; i++) { + var newAF = audioScanResult.audioFiles[i] + var trackIndex = newAF.validateTrackIndex(totalAudioFilesToInclude === 1) + if (trackIndex !== null) { + if (audiobook.checkHasTrackNum(trackIndex)) { + newAF.setDuplicateTrackNumber(trackIndex) + } else { + newAF.index = trackIndex + } + } + audiobook.addAudioFile(newAF) + } + + audiobook.rebuildTracks() + } } if (newOtherFileData.length) { - // TODO: Check other files - } - - return { - updated: true + await audiobook.syncOtherFiles(newOtherFileData, this.MetadataPath) } + return audiobook } - async scanNewAudiobook(audiobookData) { - // TODO: Return new audiobook - return null - } + async scanNewAudiobook(audiobookData, libraryScan) { + Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Scanning new "${audiobookData.path}"`) + var audiobook = new Audiobook() + audiobook.setData(audiobookData) - async scanAudioFiles(audioFileData) { - var proms = [] - for (let i = 0; i < audioFileData.length; i++) { - var prom = AudioFileScanner.scan(audioFileData[i]) - proms.push(prom) + if (audiobookData.audioFiles.length) { + var audioScanResult = await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData) + Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobookData.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`) + if (audioScanResult.audioFiles.length) { + // validate & add audio files to audiobook + for (let i = 0; i < audioScanResult.audioFiles.length; i++) { + var newAF = audioScanResult.audioFiles[i] + var trackIndex = newAF.validateTrackIndex(audioScanResult.audioFiles.length === 1) + if (trackIndex !== null) { + if (audiobook.checkHasTrackNum(trackIndex)) { + newAF.setDuplicateTrackNumber(trackIndex) + } else { + newAF.index = trackIndex + } + } + audiobook.addAudioFile(newAF) + } + audiobook.rebuildTracks() + } else if (!audiobook.ebooks.length) { + // Audiobook has no ebooks and no valid audio tracks do not continue + Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`) + return null + } } - return Promise.all(proms) + + // Look for desc.txt and reader.txt and update + await audiobook.saveDataFromTextFiles() + + // Extract embedded cover art if cover is not already in directory + if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) { + var outputCoverDirs = this.getCoverDirectory(audiobook) + var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath) + if (relativeDir) { + Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`) + } + } + + return audiobook } } module.exports = Scanner \ No newline at end of file diff --git a/server/utils/constants.js b/server/utils/constants.js index d36ce925..446a6778 100644 --- a/server/utils/constants.js +++ b/server/utils/constants.js @@ -6,6 +6,14 @@ module.exports.ScanResult = { UPTODATE: 4 } +module.exports.ScanStatus = { + NOTHING: 0, + ADDED: 1, + UPDATED: 2, + REMOVED: 3, + UPTODATE: 4 +} + module.exports.CoverDestination = { METADATA: 0, AUDIOBOOK: 1 diff --git a/server/utils/fileUtils.js b/server/utils/fileUtils.js index 3ba73bf4..2876a562 100644 --- a/server/utils/fileUtils.js +++ b/server/utils/fileUtils.js @@ -51,34 +51,6 @@ function bytesPretty(bytes, decimals = 0) { } module.exports.bytesPretty = bytesPretty -function elapsedPretty(seconds) { - var minutes = Math.floor(seconds / 60) - if (minutes < 70) { - return `${minutes} min` - } - var hours = Math.floor(minutes / 60) - minutes -= hours * 60 - if (!minutes) { - return `${hours} hr` - } - return `${hours} hr ${minutes} min` -} -module.exports.elapsedPretty = elapsedPretty - -function secondsToTimestamp(seconds) { - var _seconds = seconds - var _minutes = Math.floor(seconds / 60) - _seconds -= _minutes * 60 - var _hours = Math.floor(_minutes / 60) - _minutes -= _hours * 60 - _seconds = Math.floor(_seconds) - if (!_hours) { - return `${_minutes}:${_seconds.toString().padStart(2, '0')}` - } - return `${_hours}:${_minutes.toString().padStart(2, '0')}:${_seconds.toString().padStart(2, '0')}` -} -module.exports.secondsToTimestamp = secondsToTimestamp - function setFileOwner(path, uid, gid) { try { return fs.chown(path, uid, gid).then(() => true) diff --git a/server/utils/index.js b/server/utils/index.js index bdb10e69..79b6a93a 100644 --- a/server/utils/index.js +++ b/server/utils/index.js @@ -45,6 +45,10 @@ module.exports.getIno = (path) => { }) } +module.exports.isNullOrNaN = (num) => { + return num === null || isNaN(num) +} + const xmlToJSON = (xml) => { return new Promise((resolve, reject) => { parseString(xml, (err, results) => { @@ -63,4 +67,38 @@ module.exports.getId = (prepend = '') => { var _id = Math.random().toString(36).substring(2, 8) + Math.random().toString(36).substring(2, 8) + Math.random().toString(36).substring(2, 8) if (prepend) return prepend + '_' + _id return _id -} \ No newline at end of file +} + +function elapsedPretty(seconds) { + var minutes = Math.floor(seconds / 60) + if (minutes < 70) { + return `${minutes} min` + } + var hours = Math.floor(minutes / 60) + minutes -= hours * 60 + if (!minutes) { + return `${hours} hr` + } + return `${hours} hr ${minutes} min` +} +module.exports.elapsedPretty = elapsedPretty + +function secondsToTimestamp(seconds, includeMs = false) { + var _seconds = seconds + var _minutes = Math.floor(seconds / 60) + _seconds -= _minutes * 60 + var _hours = Math.floor(_minutes / 60) + _minutes -= _hours * 60 + + var ms = _seconds - Math.floor(seconds) + _seconds = Math.floor(_seconds) + + var msString = '.' + (includeMs ? ms.toFixed(3) : '0.0').split('.')[1] + if (!_hours) { + return `${_minutes}:${_seconds.toString().padStart(2, '0')}${msString}` + } + return `${_hours}:${_minutes.toString().padStart(2, '0')}:${_seconds.toString().padStart(2, '0')}${msString}` +} +module.exports.secondsToTimestamp = secondsToTimestamp + +module.exports.msToTimestamp = (ms, includeMs) => secondsToTimestamp(ms / 1000, includeMs) \ No newline at end of file