diff --git a/client/package.json b/client/package.json index 6a1d5577..ab48a468 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "audiobookshelf-client", - "version": "1.6.25", + "version": "1.6.26", "description": "Audiobook manager and player", "main": "index.js", "scripts": { diff --git a/package.json b/package.json index db17557e..9098af3d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "audiobookshelf", - "version": "1.6.25", + "version": "1.6.26", "description": "Self-hosted audiobook server for managing and playing audiobooks", "main": "index.js", "scripts": { diff --git a/server/Scanner.js b/server/Scanner.js index f0fd278e..8f3d4b16 100644 --- a/server/Scanner.js +++ b/server/Scanner.js @@ -454,7 +454,6 @@ class Scanner { var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryId) // TODO: This temporary fix from pre-release should be removed soon, "checkUpdateInos" - // TEMP - update ino for each audiobook if (audiobooksInLibrary.length) { for (let i = 0; i < audiobooksInLibrary.length; i++) { var ab = audiobooksInLibrary[i] @@ -463,7 +462,7 @@ class Scanner { if (shouldUpdateIno) { var filesWithMissingIno = ab.getFilesWithMissingIno() - Logger.debug(`\n\Updating inos for "${ab.title}"`) + Logger.debug(`\nUpdating inos for "${ab.title}"`) Logger.debug(`In Scan, Files with missing inode`, filesWithMissingIno) var hasUpdates = await ab.checkUpdateInos() @@ -504,7 +503,7 @@ class Scanner { // Check for removed audiobooks for (let i = 0; i < audiobooksInLibrary.length; i++) { var audiobook = audiobooksInLibrary[i] - var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino) + var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path)) if (!dataFound) { Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`) audiobook.isMissing = true diff --git a/server/objects/AudioFile.js b/server/objects/AudioFile.js index cf873e42..4b43837c 100644 --- a/server/objects/AudioFile.js +++ b/server/objects/AudioFile.js @@ -153,6 +153,17 @@ class AudioFile { this.metadata.setData(data) } + // New scanner creates AudioFile from AudioFileScanner + setData2(fileData, probeData) { + this.index = fileData.index || null + this.ino = fileData.ino || null + this.filename = fileData.filename + this.ext = fileData.ext + this.path = fileData.path + this.fullPath = fileData.fullPath + this.addedAt = Date.now() + } + syncChapters(updatedChapters) { if (this.chapters.length !== updatedChapters.length) { this.chapters = updatedChapters.map(ch => ({ ...ch })) diff --git a/server/objects/Audiobook.js b/server/objects/Audiobook.js index 58499e30..a2fa4526 100644 --- a/server/objects/Audiobook.js +++ b/server/objects/Audiobook.js @@ -823,5 +823,141 @@ class Audiobook { var audioFile = this.audioFiles[0] return this.book.setDetailsFromFileMetadata(audioFile.metadata) } + + // Returns null if file not found, true if file was updated, false if up to date + checkFileFound(fileFound, isAudioFile) { + var hasUpdated = false + + const arrayToCheck = isAudioFile ? this.audioFiles : this.otherFiles + + var existingFile = arrayToCheck.find(_af => _af.ino === fileFound.ino) + if (!existingFile) { + existingFile = arrayToCheck.find(_af => _af.path === fileFound.path) + if (existingFile) { + // file inode was updated + existingFile.ino = fileFound.ino + hasUpdated = true + } else { + // file not found + return null + } + } + + if (existingFile.filename !== fileFound.filename) { + existingFile.filename = fileFound.filename + existingFile.ext = fileFound.ext + hasUpdated = true + } + + if (existingFile.path !== fileFound.path) { + existingFile.path = fileFound.path + existingFile.fullPath = fileFound.fullPath + hasUpdated = true + } else if (existingFile.fullPath !== fileFound.fullPath) { + existingFile.fullPath = fileFound.fullPath + hasUpdated = true + } + + if (!isAudioFile && existingFile.filetype !== fileFound.filetype) { + existingFile.filetype = fileFound.filetype + hasUpdated = true + } + + return hasUpdated + } + + checkShouldScan(dataFound) { + var hasUpdated = false + + if (dataFound.ino !== this.ino) { + this.ino = dataFound.ino + hasUpdated = true + } + + if (dataFound.folderId !== this.folderId) { + Logger.warn(`[Audiobook] Check scan audiobook changed folder ${this.folderId} -> ${dataFound.folderId}`) + this.folderId = dataFound.folderId + hasUpdated = true + } + + if (dataFound.path !== this.path) { + Logger.warn(`[Audiobook] Check scan audiobook changed path "${this.path}" -> "${dataFound.path}"`) + this.path = dataFound.path + this.fullPath = dataFound.fullPath + hasUpdated = true + } else if (dataFound.fullPath !== this.fullPath) { + Logger.warn(`[Audiobook] Check scan audiobook changed fullpath "${this.fullPath}" -> "${dataFound.fullPath}"`) + this.fullPath = dataFound.fullPath + hasUpdated = true + } + + var newAudioFileData = [] + var newOtherFileData = [] + + dataFound.audioFiles.forEach((af) => { + var audioFileFoundCheck = this.checkFileFound(af, true) + if (audioFileFoundCheck === null) { + newAudioFileData.push(af) + } else if (audioFileFoundCheck === true) { + hasUpdated = true + } + }) + + dataFound.otherFiles.forEach((otherFileData) => { + var fileFoundCheck = this.checkFileFound(otherFileData, false) + if (fileFoundCheck === null) { + newOtherFileData.push(otherFileData) + } else if (fileFoundCheck === true) { + hasUpdated = true + } + }) + + const audioFilesRemoved = [] + const otherFilesRemoved = [] + + // inodes will all be up to date at this point + this.audioFiles = this.audioFiles.filter(af => { + if (!dataFound.audioFiles.find(_af => _af.ino === af.ino)) { + audioFilesRemoved.push(af.toJSON()) + return false + } + return true + }) + + // Remove all tracks that were associated with removed audio files + if (audioFilesRemoved.length) { + const audioFilesRemovedInodes = audioFilesRemoved.map(afr => afr.ino) + this.tracks = this.tracks.filter(t => !audioFilesRemovedInodes.includes(t.ino)) + this.checkUpdateMissingParts() + hasUpdated = true + } + + this.otherFiles = this.otherFiles.filter(otherFile => { + if (!dataFound.otherFiles.find(_otherFile => _otherFile.ino === otherFile.ino)) { + otherFilesRemoved.push(otherFile.toJSON()) + + // Check remove cover + if (otherFile.fullPath === this.book.coverFullPath) { + Logger.debug(`[Audiobook] "${this.title}" Check scan book cover removed`) + this.book.removeCover() + } + + return false + } + return true + }) + + if (otherFilesRemoved.length) { + hasUpdated = true + } + + return { + updated: hasUpdated, + newAudioFileData, + newOtherFileData, + audioFilesRemoved, + otherFilesRemoved + } + } } module.exports = Audiobook \ No newline at end of file diff --git a/server/scanner/AudioFileScanner.js b/server/scanner/AudioFileScanner.js new file mode 100644 index 00000000..83accbe8 --- /dev/null +++ b/server/scanner/AudioFileScanner.js @@ -0,0 +1,22 @@ +const AudioFile = require('../objects/AudioFile') +const AudioProbeData = require('./AudioProbeData') + +const prober = require('../utils/prober') +const Logger = require('../Logger') + +class AudioFileScanner { + constructor() { } + + async scan(audioFileData, verbose = false) { + var probeData = await prober.probe2(audioFileData.fullPath, verbose) + if (probeData.error) { + Logger.error(`[AudioFileScanner] ${probeData.error} : "${audioFileData.fullPath}"`) + return null + } + + var audioFile = new AudioFile() + // TODO: Build audio file + return audioFile + } +} +module.exports = new AudioFileScanner() \ No newline at end of file diff --git a/server/scanner/AudioProbeData.js b/server/scanner/AudioProbeData.js new file mode 100644 index 00000000..5f27fe26 --- /dev/null +++ b/server/scanner/AudioProbeData.js @@ -0,0 +1,74 @@ +const AudioFileMetadata = require('../objects/AudioFileMetadata') + +class AudioProbeData { + constructor() { + this.embeddedCoverArt = null + this.format = null + this.duration = null + this.size = null + this.bitRate = null + this.codec = null + this.timeBase = null + this.language = null + this.channelLayout = null + this.channels = null + this.sampleRate = null + this.chapters = [] + + this.audioFileMetadata = null + + this.trackNumber = null + this.trackTotal = null + } + + getDefaultAudioStream(audioStreams) { + if (audioStreams.length === 1) return audioStreams[0] + var defaultStream = audioStreams.find(a => a.is_default) + if (!defaultStream) return audioStreams[0] + return defaultStream + } + + getEmbeddedCoverArt(videoStream) { + const ImageCodecs = ['mjpeg', 'jpeg', 'png'] + return ImageCodecs.includes(videoStream.codec) ? videoStream.codec : null + } + + setData(data) { + var audioStream = getDefaultAudioStream(data.audio_streams) + + this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : false + this.format = data.format + this.duration = data.duration + this.size = data.size + this.bitRate = audioStream.bit_rate || data.bit_rate + this.codec = audioStream.codec + this.timeBase = audioStream.time_base + this.language = audioStream.language + this.channelLayout = audioStream.channel_layout + this.channels = audioStream.channels + this.sampleRate = audioStream.sample_rate + this.chapters = data.chapters || [] + + var metatags = {} + for (const key in data) { + if (data[key] && key.startsWith('file_tag')) { + metatags[key] = data[key] + } + } + + this.audioFileMetadata = new AudioFileMetadata() + this.audioFileMetadata.setData(metatags) + + // Track ID3 tag might be "3/10" or just "3" + if (this.audioFileMetadata.tagTrack) { + var trackParts = this.audioFileMetadata.tagTrack.split('/').map(part => Number(part)) + if (trackParts.length > 0) { + this.trackNumber = trackParts[0] + } + if (trackParts.length > 1) { + this.trackTotal = trackParts[1] + } + } + } +} +module.exports = AudioProbeData \ No newline at end of file diff --git a/server/scanner/LibraryScan.js b/server/scanner/LibraryScan.js new file mode 100644 index 00000000..4819fb38 --- /dev/null +++ b/server/scanner/LibraryScan.js @@ -0,0 +1,34 @@ +const Folder = require('../objects/Folder') + +const { getId } = require('../utils/index') + +class LibraryScan { + constructor() { + this.id = null + this.libraryId = null + this.libraryName = null + this.folders = null + + this.scanOptions = null + + this.startedAt = null + this.finishedAt = null + + this.folderScans = [] + } + + get _scanOptions() { return this.scanOptions || {} } + get forceRescan() { return !!this._scanOptions.forceRescan } + + setData(library, scanOptions) { + this.id = getId('lscan') + this.libraryId = library.id + this.libraryName = library.name + this.folders = library.folders.map(folder => Folder(folder.toJSON())) + + this.scanOptions = scanOptions + + this.startedAt = Date.now() + } +} +module.exports = LibraryScan \ No newline at end of file diff --git a/server/scanner/ScanOptions.js b/server/scanner/ScanOptions.js new file mode 100644 index 00000000..5ca4d7cc --- /dev/null +++ b/server/scanner/ScanOptions.js @@ -0,0 +1,68 @@ +const { CoverDestination } = require('../utils/constants') + +class ScanOptions { + constructor(options) { + this.forceRescan = false + + this.metadataPrecedence = [ + { + id: 'directory', + include: true + }, + { + id: 'reader-desc-txt', + include: true + }, + { + id: 'audio-file-metadata', + include: true + }, + { + id: 'metadata-opf', + include: true + }, + { + id: 'external-source', + include: false + } + ] + + // Server settings + this.parseSubtitles = false + this.findCovers = false + this.coverDestination = CoverDestination.METADATA + + if (options) { + this.construct(options) + } + } + + construct(options) { + for (const key in options) { + if (key === 'metadataPrecedence' && options[key].length) { + this.metadataPrecedence = [...options[key]] + } else if (this[key] !== undefined) { + this[key] = options[key] + } + } + } + + toJSON() { + return { + forceRescan: this.forceRescan, + metadataPrecedence: this.metadataPrecedence, + parseSubtitles: this.parseSubtitles, + findCovers: this.findCovers, + coverDestination: this.coverDestination + } + } + + setData(options, serverSettings) { + this.forceRescan = !!options.forceRescan + + this.parseSubtitles = !!serverSettings.scannerParseSubtitle + this.findCovers = !!serverSettings.scannerFindCovers + this.coverDestination = serverSettings.coverDestination + } +} +module.exports = ScanOptions \ No newline at end of file diff --git a/server/scanner/Scanner.js b/server/scanner/Scanner.js new file mode 100644 index 00000000..66497069 --- /dev/null +++ b/server/scanner/Scanner.js @@ -0,0 +1,172 @@ +const fs = require('fs-extra') +const Path = require('path') + +// Utils +const Logger = require('../Logger') +const { version } = require('../../package.json') +const audioFileScanner = require('../utils/audioFileScanner') +const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('../utils/scandir') +const { comparePaths, getIno, getId } = require('../utils/index') +const { secondsToTimestamp } = require('../utils/fileUtils') +const { ScanResult, CoverDestination } = require('../utils/constants') + +const AudioFileScanner = require('./AudioFileScanner') +const BookFinder = require('../BookFinder') +const Audiobook = require('../objects/Audiobook') +const LibraryScan = require('./LibraryScan') +const ScanOptions = require('./ScanOptions') + +class Scanner { + constructor(AUDIOBOOK_PATH, METADATA_PATH, db, coverController, emitter) { + this.AudiobookPath = AUDIOBOOK_PATH + this.MetadataPath = METADATA_PATH + this.BookMetadataPath = Path.posix.join(this.MetadataPath.replace(/\\/g, '/'), 'books') + + this.db = db + this.coverController = coverController + this.emitter = emitter + + this.cancelScan = false + this.cancelLibraryScan = {} + this.librariesScanning = [] + + this.bookFinder = new BookFinder() + } + + async scan(libraryId, options = {}) { + if (this.librariesScanning.includes(libraryId)) { + Logger.error(`[Scanner] Already scanning ${libraryId}`) + return + } + + var library = this.db.libraries.find(lib => lib.id === libraryId) + if (!library) { + Logger.error(`[Scanner] Library not found for scan ${libraryId}`) + return + } else if (!library.folders.length) { + Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`) + return + } + + var scanOptions = new ScanOptions() + scanOptions.setData(options, this.db.serverSettings) + + var libraryScan = new LibraryScan() + libraryScan.setData(library, scanOptions) + + Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`) + + var results = await this.scanLibrary(libraryScan) + + Logger.info(`[Scanner] Library scan ${libraryScan.id} complete`) + + return results + } + + async scanLibrary(libraryScan) { + var audiobookDataFound = [] + for (let i = 0; i < libraryScan.folders.length; i++) { + var folder = libraryScan.folders[i] + var abDataFoundInFolder = await scanRootDir(folder, this.db.serverSettings) + Logger.debug(`[Scanner] ${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`) + audiobookDataFound = audiobookDataFound.concat(abDataFoundInFolder) + } + + // Remove audiobooks with no inode + audiobookDataFound = audiobookDataFound.filter(abd => abd.ino) + + var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryScan.libraryId) + + const audiobooksToUpdate = [] + const audiobooksToRescan = [] + const newAudiobookData = [] + + // Check for existing & removed audiobooks + for (let i = 0; i < audiobooksInLibrary.length; i++) { + var audiobook = audiobooksInLibrary[i] + var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path)) + if (!dataFound) { + Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`) + audiobook.isMissing = true + audiobook.lastUpdate = Date.now() + scanResults.missing++ + audiobooksToUpdate.push(audiobook) + } else { + var checkRes = audiobook.checkShouldRescan(dataFound) + if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) { + // existing audiobook has new files + checkRes.audiobook = audiobook + audiobooksToRescan.push(checkRes) + } else if (checkRes.updated) { + audiobooksToUpdate.push(audiobook) + } + + // Remove this abf + audiobookDataFound = audiobookDataFound.filter(abf => abf.ino !== dataFound.ino) + } + } + + // Potential NEW Audiobooks + for (let i = 0; i < audiobookDataFound.length; i++) { + var dataFound = audiobookDataFound[i] + var hasEbook = dataFound.otherFiles.find(otherFile => otherFile.filetype === 'ebook') + if (!hasEbook && !dataFound.audioFiles.length) { + Logger.info(`[Scanner] Directory found "${audiobookDataFound.path}" has no ebook or audio files`) + } else { + newAudiobookData.push(dataFound) + } + } + + var rescans = [] + for (let i = 0; i < audiobooksToRescan.length; i++) { + var rescan = this.rescanAudiobook(audiobooksToRescan[i]) + rescans.push(rescan) + } + var newscans = [] + for (let i = 0; i < newAudiobookData.length; i++) { + var newscan = this.scanNewAudiobook(newAudiobookData[i]) + newscans.push(newscan) + } + + var rescanResults = await Promise.all(rescans) + + var newscanResults = await Promise.all(newscans) + + // TODO: Return report + return { + updates: 0, + additions: 0 + } + } + + // Return scan result payload + async rescanAudiobook(audiobookCheckData) { + const { newAudioFileData, newOtherFileData, audiobook } = audiobookCheckData + if (newAudioFileData.length) { + var newAudioFiles = await this.scanAudioFiles(newAudioFileData) + // TODO: Update audiobook tracks + } + if (newOtherFileData.length) { + // TODO: Check other files + } + + return { + updated: true + } + } + + async scanNewAudiobook(audiobookData) { + // TODO: Return new audiobook + return null + } + + async scanAudioFiles(audioFileData) { + var proms = [] + for (let i = 0; i < audioFileData.length; i++) { + var prom = AudioFileScanner.scan(audioFileData[i]) + proms.push(prom) + } + return Promise.all(proms) + } +} +module.exports = Scanner \ No newline at end of file diff --git a/server/utils/audioFileScanner.js b/server/utils/audioFileScanner.js index b7322de3..05d4d8e4 100644 --- a/server/utils/audioFileScanner.js +++ b/server/utils/audioFileScanner.js @@ -13,7 +13,7 @@ function getDefaultAudioStream(audioStreams) { async function scan(path, verbose = false) { Logger.debug(`Scanning path "${path}"`) - var probeData = await prober(path, verbose) + var probeData = await prober.probe(path, verbose) if (!probeData || !probeData.audio_streams || !probeData.audio_streams.length) { return { error: 'Invalid audio file' diff --git a/server/utils/prober.js b/server/utils/prober.js index eec75d13..7d53611b 100644 --- a/server/utils/prober.js +++ b/server/utils/prober.js @@ -1,5 +1,8 @@ var Ffmpeg = require('fluent-ffmpeg') const Path = require('path') + +const AudioProbeData = require('../scanner/AudioProbeData') + const Logger = require('../Logger') function tryGrabBitRate(stream, all_streams, total_bit_rate) { @@ -241,4 +244,31 @@ function probe(filepath, verbose = false) { }) }) } -module.exports = probe \ No newline at end of file +module.exports.probe = probe + +// Updated probe returns AudioProbeData object +function probe2(filepath, verbose = false) { + return new Promise((resolve) => { + Ffmpeg.ffprobe(filepath, ['-show_chapters'], (err, raw) => { + if (err) { + console.error(err) + var errorMsg = err ? err.message : null + resolve({ + error: errorMsg || 'Probe Error' + }) + } else { + var rawProbeData = parseProbeData(raw, verbose) + if (!rawProbeData || !rawProbeData.audio_streams.length) { + resolve({ + error: rawProbeData ? 'Invalid audio file: no audio streams found' : 'Probe Failed' + }) + } else { + var probeData = new AudioProbeData() + probeData.setData(rawProbeData) + resolve(probeData) + } + } + }) + }) +} +module.exports.probe2 = probe2 \ No newline at end of file