From d6cab8e591dd6276b02093727cd163e3d5e0c4fb Mon Sep 17 00:00:00 2001 From: Mark Cooper Date: Thu, 30 Sep 2021 18:52:32 -0500 Subject: [PATCH] logLevel as server setting, logger config page, re-scan audiobook option, fix embedded cover extraction, flac and mobi support, fix series bookshelf not wrapping --- Dockerfile | 1 - client/components/AudioPlayer.vue | 2 +- client/components/app/BookShelf.vue | 4 +- .../components/modals/edit-tabs/Details.vue | 26 +++- client/components/ui/Dropdown.vue | 72 ++++++++++ client/layouts/default.vue | 21 +-- client/middleware/routed.js | 9 +- client/package.json | 2 +- client/pages/config/index.vue | 10 +- client/pages/config/log.vue | 136 ++++++++++++++++++ client/store/logs.js | 31 ++++ package.json | 2 +- server/HlsController.js | 5 +- server/Logger.js | 97 ++++++++++--- server/Scanner.js | 60 ++++++-- server/Server.js | 29 +++- server/objects/AudioFile.js | 79 +++++++++- server/objects/AudioFileMetadata.js | 28 ++++ server/objects/AudioTrack.js | 34 ++--- server/objects/Audiobook.js | 21 ++- server/objects/ServerSettings.js | 13 +- server/objects/Stream.js | 32 +++-- server/utils/audioFileScanner.js | 51 ++++++- server/utils/constants.js | 10 ++ server/utils/ffmpegHelpers.js | 2 +- server/utils/hlsPlaylistGenerator.js | 15 +- server/utils/prober.js | 1 - server/utils/scandir.js | 4 +- 28 files changed, 684 insertions(+), 113 deletions(-) create mode 100644 client/components/ui/Dropdown.vue create mode 100644 client/pages/config/log.vue create mode 100644 client/store/logs.js diff --git a/Dockerfile b/Dockerfile index fa6d2479..8657f448 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,6 @@ RUN npm run generate ### STAGE 2: Build server ### FROM node:12-alpine ENV NODE_ENV=production -ENV LOG_LEVEL=INFO COPY --from=build /client/dist /client/dist COPY --from=ffmpeg / / COPY index.js index.js diff --git a/client/components/AudioPlayer.vue b/client/components/AudioPlayer.vue index ffbf1260..1d45fa8d 100644 --- a/client/components/AudioPlayer.vue +++ b/client/components/AudioPlayer.vue @@ -440,7 +440,7 @@ export default { }) this.hlsInstance.on(Hls.Events.ERROR, (e, data) => { - console.error('[HLS] Error', data.type, data.details) + console.error('[HLS] Error', data.type, data.details, data) if (data.details === Hls.ErrorDetails.BUFFER_STALLED_ERROR) { console.error('[HLS] BUFFER STALLED ERROR') } diff --git a/client/components/app/BookShelf.vue b/client/components/app/BookShelf.vue index d35e5d27..ed62546b 100644 --- a/client/components/app/BookShelf.vue +++ b/client/components/app/BookShelf.vue @@ -102,7 +102,8 @@ export default { return 16 * this.sizeMultiplier }, bookWidth() { - return this.bookCoverWidth + this.paddingX * 2 + var _width = this.bookCoverWidth + this.paddingX * 2 + return this.showGroups ? _width * 1.6 : _width }, isSelectionMode() { return this.$store.getters['getNumAudiobooksSelected'] @@ -161,6 +162,7 @@ export default { setBookshelfEntities() { this.wrapperClientWidth = this.$refs.wrapper.clientWidth var width = Math.max(0, this.wrapperClientWidth - this.rowPaddingX * 2) + var booksPerRow = Math.floor(width / this.bookWidth) var entities = this.entities diff --git a/client/components/modals/edit-tabs/Details.vue b/client/components/modals/edit-tabs/Details.vue index a2f03297..3448e829 100644 --- a/client/components/modals/edit-tabs/Details.vue +++ b/client/components/modals/edit-tabs/Details.vue @@ -56,10 +56,14 @@
Remove - + Save Metadata + + Re-Scan + +
Submit
@@ -93,7 +97,8 @@ export default { newTags: [], resettingProgress: false, isScrollable: false, - savingMetadata: false + savingMetadata: false, + rescanning: false } }, watch: { @@ -136,6 +141,23 @@ export default { } }, methods: { + audiobookScanComplete(result) { + this.rescanning = false + if (!result) { + this.$toast.error(`Re-Scan Failed for "${this.title}"`) + } else if (result === 'UPDATED') { + this.$toast.success(`Re-Scan complete audiobook was updated`) + } else if (result === 'UPTODATE') { + this.$toast.success(`Re-Scan complete audiobook was up to date`) + } else if (result === 'REMOVED') { + this.$toast.error(`Re-Scan complete audiobook was removed`) + } + }, + rescan() { + this.rescanning = true + this.$root.socket.once('audiobook_scan_complete', this.audiobookScanComplete) + this.$root.socket.emit('scan_audiobook', this.audiobookId) + }, saveMetadataComplete(result) { this.savingMetadata = false if (result.error) { diff --git a/client/components/ui/Dropdown.vue b/client/components/ui/Dropdown.vue new file mode 100644 index 00000000..737b055d --- /dev/null +++ b/client/components/ui/Dropdown.vue @@ -0,0 +1,72 @@ + + + \ No newline at end of file diff --git a/client/layouts/default.vue b/client/layouts/default.vue index 92c6e9fb..a552b4b7 100644 --- a/client/layouts/default.vue +++ b/client/layouts/default.vue @@ -127,21 +127,6 @@ export default { this.$store.commit('setScanProgress', progress) } }, - saveMetadataComplete(result) { - if (result.error) { - this.$toast.error(result.error) - } else if (result.audiobookId) { - var { savedPath } = result - if (!savedPath) { - this.$toast.error(`Failed to save metadata file (${result.audiobookId})`) - } else { - this.$toast.success(`Metadata file saved (${result.audiobookId})`) - } - } else { - var { success, failed } = result - this.$toast.success(`Metadata save complete\n${success} Succeeded\n${failed} Failed`) - } - }, userUpdated(user) { if (this.$store.state.user.user.id === user.id) { this.$store.commit('user/setUser', user) @@ -205,6 +190,9 @@ export default { download.status = this.$constants.DownloadStatus.EXPIRED this.$store.commit('downloads/addUpdateDownload', download) }, + logEvtReceived(payload) { + this.$store.commit('logs/logEvt', payload) + }, initializeSocket() { this.socket = this.$nuxtSocket({ name: process.env.NODE_ENV === 'development' ? 'dev' : 'prod', @@ -245,7 +233,6 @@ export default { this.socket.on('scan_start', this.scanStart) this.socket.on('scan_complete', this.scanComplete) this.socket.on('scan_progress', this.scanProgress) - // this.socket.on('save_metadata_complete', this.saveMetadataComplete) // Download Listeners this.socket.on('download_started', this.downloadStarted) @@ -253,6 +240,8 @@ export default { this.socket.on('download_failed', this.downloadFailed) this.socket.on('download_killed', this.downloadKilled) this.socket.on('download_expired', this.downloadExpired) + + this.socket.on('log', this.logEvtReceived) }, showUpdateToast(versionData) { var ignoreVersion = localStorage.getItem('ignoreVersion') diff --git a/client/middleware/routed.js b/client/middleware/routed.js index db2e916d..b4fbea81 100644 --- a/client/middleware/routed.js +++ b/client/middleware/routed.js @@ -6,8 +6,13 @@ export default function (context) { if (route.name === 'login' || from.name === 'login') return - if (route.name === 'config' || route.name === 'upload' || route.name === 'account' || route.name.startsWith('audiobook-id')) { - if (from.name !== route.name && from.name !== 'audiobook-id-edit' && from.name !== 'config' && from.name !== 'upload' && from.name !== 'account') { + if (!route.name) { + console.warn('No Route name', route) + return + } + + if (route.name.startsWith('config') || route.name === 'upload' || route.name === 'account' || route.name.startsWith('audiobook-id')) { + if (from.name !== route.name && from.name !== 'audiobook-id-edit' && from.name !== 'config' && from.name !== 'config-log' && from.name !== 'upload' && from.name !== 'account') { var _history = [...store.state.routeHistory] if (!_history.length || _history[_history.length - 1] !== from.fullPath) { _history.push(from.fullPath) diff --git a/client/package.json b/client/package.json index 58b36088..1de00cec 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "audiobookshelf-client", - "version": "1.2.9", + "version": "1.3.1", "description": "Audiobook manager and player", "main": "index.js", "scripts": { diff --git a/client/pages/config/index.vue b/client/pages/config/index.vue index 5c29c21b..a0bb7efa 100644 --- a/client/pages/config/index.vue +++ b/client/pages/config/index.vue @@ -34,8 +34,10 @@
+
-
+ +

Scanner

@@ -65,6 +67,8 @@
Reset All Audiobooks +
+ View Logger
@@ -134,6 +138,9 @@ export default { var payload = { scannerParseSubtitle: val } + this.updateServerSettings(payload) + }, + updateServerSettings(payload) { this.$store .dispatch('updateServerSettings', payload) .then((success) => { @@ -175,6 +182,7 @@ export default { .then(() => { this.isResettingAudiobooks = false this.$toast.success('Successfully reset audiobooks') + location.reload() }) .catch((error) => { console.error('failed to reset audiobooks', error) diff --git a/client/pages/config/log.vue b/client/pages/config/log.vue new file mode 100644 index 00000000..7e49d1d6 --- /dev/null +++ b/client/pages/config/log.vue @@ -0,0 +1,136 @@ + + + + + \ No newline at end of file diff --git a/client/store/logs.js b/client/store/logs.js new file mode 100644 index 00000000..4f0e7c03 --- /dev/null +++ b/client/store/logs.js @@ -0,0 +1,31 @@ +export const state = () => ({ + isListening: false, + logs: [] +}) + +export const getters = { + +} + +export const actions = { + setLogListener({ state, commit, dispatch }) { + dispatch('$nuxtSocket/emit', { + label: 'main', + evt: 'set_log_listener', + msg: 0 + }, { root: true }) + commit('setIsListening', true) + } +} + +export const mutations = { + setIsListening(state, val) { + state.isListening = val + }, + logEvt(state, payload) { + state.logs.push(payload) + if (state.logs.length > 500) { + state.logs = state.logs.slice(50) + } + } +} \ No newline at end of file diff --git a/package.json b/package.json index 6f46131f..36cc57c7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "audiobookshelf", - "version": "1.2.9", + "version": "1.3.1", "description": "Self-hosted audiobook server for managing and playing audiobooks", "main": "index.js", "scripts": { diff --git a/server/HlsController.js b/server/HlsController.js index b1bd08ef..e29c4cbc 100644 --- a/server/HlsController.js +++ b/server/HlsController.js @@ -21,7 +21,7 @@ class HlsController { } parseSegmentFilename(filename) { - var basename = Path.basename(filename, '.ts') + var basename = Path.basename(filename, Path.extname(filename)) var num_part = basename.split('-')[1] return Number(num_part) } @@ -41,7 +41,7 @@ class HlsController { Logger.warn('File path does not exist', fullFilePath) var fileExt = Path.extname(req.params.file) - if (fileExt === '.ts') { + if (fileExt === '.ts' || fileExt === '.m4s') { var segNum = this.parseSegmentFilename(req.params.file) var stream = this.streamManager.getStream(streamId) if (!stream) { @@ -66,6 +66,7 @@ class HlsController { } } } + // Logger.info('Sending file', fullFilePath) res.sendFile(fullFilePath) } diff --git a/server/Logger.js b/server/Logger.js index 0ca5f0f1..01ae43d3 100644 --- a/server/Logger.js +++ b/server/Logger.js @@ -1,55 +1,110 @@ -const LOG_LEVEL = { - TRACE: 0, - DEBUG: 1, - INFO: 2, - WARN: 3, - ERROR: 4, - FATAL: 5 -} +const { LogLevel } = require('./utils/constants') class Logger { constructor() { - let env_log_level = process.env.LOG_LEVEL || 'TRACE' - this.LogLevel = LOG_LEVEL[env_log_level] || LOG_LEVEL.TRACE - this.info(`Log Level: ${this.LogLevel}`) + this.logLevel = process.env.NODE_ENV === 'production' ? LogLevel.INFO : LogLevel.TRACE + this.socketListeners = [] } get timestamp() { return (new Date()).toISOString() } + get levelString() { + for (const key in LogLevel) { + if (LogLevel[key] === this.logLevel) { + return key + } + } + return 'UNKNOWN' + } + + getLogLevelString(level) { + for (const key in LogLevel) { + if (LogLevel[key] === level) { + return key + } + } + return 'UNKNOWN' + } + + addSocketListener(socket, level) { + var index = this.socketListeners.findIndex(s => s.id === socket.id) + if (index >= 0) { + this.socketListeners.splice(index, 1, { + id: socket.id, + socket, + level + }) + } else { + this.socketListeners.push({ + id: socket.id, + socket, + level + }) + } + } + + removeSocketListener(socketId) { + this.socketListeners = this.socketListeners.filter(s => s.id !== socketId) + } + + logToSockets(level, args) { + this.socketListeners.forEach((socketListener) => { + if (socketListener.level <= level) { + socketListener.socket.emit('log', { + timestamp: this.timestamp, + message: args.join(' '), + levelName: this.getLogLevelString(level), + level + }) + } + }) + } + + setLogLevel(level) { + this.logLevel = level + this.debug(`Set Log Level to ${this.levelString}`) + } + trace(...args) { - if (this.LogLevel > LOG_LEVEL.TRACE) return + if (this.logLevel > LogLevel.TRACE) return console.trace(`[${this.timestamp}] TRACE:`, ...args) + this.logToSockets(LogLevel.TRACE, args) } debug(...args) { - if (this.LogLevel > LOG_LEVEL.DEBUG) return + if (this.logLevel > LogLevel.DEBUG) return console.debug(`[${this.timestamp}] DEBUG:`, ...args) + this.logToSockets(LogLevel.DEBUG, args) } info(...args) { - if (this.LogLevel > LOG_LEVEL.INFO) return + if (this.logLevel > LogLevel.INFO) return console.info(`[${this.timestamp}] INFO:`, ...args) - } - - note(...args) { - if (this.LogLevel > LOG_LEVEL.INFO) return - console.log(`[${this.timestamp}] NOTE:`, ...args) + this.logToSockets(LogLevel.INFO, args) } warn(...args) { - if (this.LogLevel > LOG_LEVEL.WARN) return + if (this.logLevel > LogLevel.WARN) return console.warn(`[${this.timestamp}] WARN:`, ...args) + this.logToSockets(LogLevel.WARN, args) } error(...args) { - if (this.LogLevel > LOG_LEVEL.ERROR) return + if (this.logLevel > LogLevel.ERROR) return console.error(`[${this.timestamp}] ERROR:`, ...args) + this.logToSockets(LogLevel.ERROR, args) } fatal(...args) { console.error(`[${this.timestamp}] FATAL:`, ...args) + this.logToSockets(LogLevel.FATAL, args) + } + + note(...args) { + console.log(`[${this.timestamp}] NOTE:`, ...args) + this.logToSockets(LogLevel.NOTE, args) } } module.exports = new Logger() \ No newline at end of file diff --git a/server/Scanner.js b/server/Scanner.js index beefa003..8eae3bb6 100644 --- a/server/Scanner.js +++ b/server/Scanner.js @@ -60,11 +60,19 @@ class Scanner { return audiobookDataAudioFiles.filter(abdFile => !!abdFile.ino) } - async scanAudiobookData(audiobookData) { + async scanAudiobookData(audiobookData, forceAudioFileScan = false) { var existingAudiobook = this.audiobooks.find(a => a.ino === audiobookData.ino) // Logger.debug(`[Scanner] Scanning "${audiobookData.title}" (${audiobookData.ino}) - ${!!existingAudiobook ? 'Exists' : 'New'}`) if (existingAudiobook) { + + // TEMP: Check if is older audiobook and needs force rescan + if (!forceAudioFileScan && existingAudiobook.checkNeedsAudioFileRescan()) { + Logger.info(`[Scanner] Re-Scanning all audio files for "${existingAudiobook.title}" (last scan <= 1.3.0)`) + forceAudioFileScan = true + } + + // REMOVE: No valid audio files // TODO: Label as incomplete, do not actually delete if (!audiobookData.audioFiles.length) { @@ -94,7 +102,6 @@ class Scanner { removedAudioTracks.forEach((at) => existingAudiobook.removeAudioTrack(at)) } - // Check for new audio files and sync existing audio files var newAudioFiles = [] var hasUpdatedAudioFiles = false @@ -113,13 +120,35 @@ class Scanner { } } }) + + // Rescan audio file metadata + if (forceAudioFileScan) { + Logger.info(`[Scanner] Rescanning ${existingAudiobook.audioFiles.length} audio files for "${existingAudiobook.title}"`) + var numAudioFilesUpdated = await audioFileScanner.rescanAudioFiles(existingAudiobook) + if (numAudioFilesUpdated > 0) { + Logger.info(`[Scanner] Rescan complete, ${numAudioFilesUpdated} audio files were updated for "${existingAudiobook.title}"`) + hasUpdatedAudioFiles = true + + // Use embedded cover art if audiobook has no cover + if (existingAudiobook.hasEmbeddedCoverArt && !existingAudiobook.cover) { + var outputCoverDirs = this.getCoverDirectory(existingAudiobook) + var relativeDir = await existingAudiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath) + if (relativeDir) { + Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`) + } + } + } else { + Logger.info(`[Scanner] Rescan complete, audio files were up to date for "${existingAudiobook.title}"`) + } + } + + // Scan and add new audio files found and set tracks if (newAudioFiles.length) { Logger.info(`[Scanner] ${newAudioFiles.length} new audio files were found for audiobook "${existingAudiobook.title}"`) - // Scan new audio files found - sets tracks await audioFileScanner.scanAudioFiles(existingAudiobook, newAudioFiles) } - // REMOVE: No valid audio tracks + // If after a scan no valid audio tracks remain // TODO: Label as incomplete, do not actually delete if (!existingAudiobook.tracks.length) { Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`) @@ -131,12 +160,14 @@ class Scanner { var hasUpdates = removedAudioFiles.length || removedAudioTracks.length || newAudioFiles.length || hasUpdatedAudioFiles + // Check that audio tracks are in sequential order with no gaps if (existingAudiobook.checkUpdateMissingParts()) { Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`) hasUpdates = true } - var otherFilesUpdated = await existingAudiobook.syncOtherFiles(audiobookData.otherFiles) + // Sync other files (all files that are not audio files) + var otherFilesUpdated = await existingAudiobook.syncOtherFiles(audiobookData.otherFiles, forceAudioFileScan) if (otherFilesUpdated) { hasUpdates = true } @@ -202,7 +233,7 @@ class Scanner { return ScanResult.ADDED } - async scan() { + async scan(forceAudioFileScan = false) { // TODO: This temporary fix from pre-release should be removed soon, "checkUpdateInos" // TEMP - update ino for each audiobook if (this.audiobooks.length) { @@ -258,8 +289,7 @@ class Scanner { // Check for new and updated audiobooks for (let i = 0; i < audiobookDataFound.length; i++) { - var audiobookData = audiobookDataFound[i] - var result = await this.scanAudiobookData(audiobookData) + var result = await this.scanAudiobookData(audiobookDataFound[i], forceAudioFileScan) if (result === ScanResult.ADDED) scanResults.added++ if (result === ScanResult.REMOVED) scanResults.removed++ if (result === ScanResult.UPDATED) scanResults.updated++ @@ -283,14 +313,24 @@ class Scanner { return scanResults } - async scanAudiobook(audiobookPath) { + async scanAudiobookById(audiobookId) { + const audiobook = this.db.audiobooks.find(ab => ab.id === audiobookId) + if (!audiobook) { + Logger.error(`[Scanner] Scan audiobook by id not found ${audiobookId}`) + return ScanResult.NOTHING + } + Logger.info(`[Scanner] Scanning Audiobook "${audiobook.title}"`) + return this.scanAudiobook(audiobook.fullPath, true) + } + + async scanAudiobook(audiobookPath, forceAudioFileScan = false) { Logger.debug('[Scanner] scanAudiobook', audiobookPath) var audiobookData = await getAudiobookFileData(this.AudiobookPath, audiobookPath, this.db.serverSettings) if (!audiobookData) { return ScanResult.NOTHING } audiobookData.ino = await getIno(audiobookData.fullPath) - return this.scanAudiobookData(audiobookData) + return this.scanAudiobookData(audiobookData, forceAudioFileScan) } // Files were modified in this directory, check it out diff --git a/server/Server.js b/server/Server.js index 245fa868..64f2bb83 100644 --- a/server/Server.js +++ b/server/Server.js @@ -6,6 +6,8 @@ const fs = require('fs-extra') const fileUpload = require('express-fileupload') const rateLimit = require('express-rate-limit') +const { ScanResult } = require('./utils/constants') + const Auth = require('./Auth') const Watcher = require('./Watcher') const Scanner = require('./Scanner') @@ -82,20 +84,31 @@ class Server { async filesChanged(files) { Logger.info('[Server]', files.length, 'Files Changed') var result = await this.scanner.filesChanged(files) - Logger.info('[Server] Files changed result', result) + Logger.debug('[Server] Files changed result', result) } - async scan() { + async scan(forceAudioFileScan = false) { Logger.info('[Server] Starting Scan') this.isScanning = true this.isInitialized = true this.emitter('scan_start', 'files') - var results = await this.scanner.scan() + var results = await this.scanner.scan(forceAudioFileScan) this.isScanning = false this.emitter('scan_complete', { scanType: 'files', results }) Logger.info('[Server] Scan complete') } + async scanAudiobook(socket, audiobookId) { + var result = await this.scanner.scanAudiobookById(audiobookId) + var scanResultName = '' + for (const key in ScanResult) { + if (ScanResult[key] === result) { + scanResultName = key + } + } + socket.emit('audiobook_scan_complete', scanResultName) + } + async scanCovers() { Logger.info('[Server] Start cover scan') this.isScanningCovers = true @@ -287,6 +300,7 @@ class Server { socket.on('scan', this.scan.bind(this)) socket.on('scan_covers', this.scanCovers.bind(this)) socket.on('cancel_scan', this.cancelScan.bind(this)) + socket.on('scan_audiobook', (audiobookId) => this.scanAudiobook(socket, audiobookId)) socket.on('save_metadata', (audiobookId) => this.saveMetadata(socket, audiobookId)) // Streaming @@ -300,11 +314,15 @@ class Server { socket.on('download', (payload) => this.downloadManager.downloadSocketRequest(socket, payload)) socket.on('remove_download', (downloadId) => this.downloadManager.removeSocketRequest(socket, downloadId)) + socket.on('set_log_listener', (level) => Logger.addSocketListener(socket, level)) + socket.on('test', () => { socket.emit('test_received', socket.id) }) socket.on('disconnect', () => { + Logger.removeSocketListener(socket.id) + var _client = this.clients[socket.id] if (!_client) { Logger.warn('[SOCKET] Socket disconnect, no client ' + socket.id) @@ -368,6 +386,11 @@ class Server { stream: client.stream || null } client.socket.emit('init', initialPayload) + + // Setup log listener for root user + if (user.type === 'root') { + Logger.addSocketListener(socket, this.db.serverSettings.logLevel || 0) + } } async stop() { diff --git a/server/objects/AudioFile.js b/server/objects/AudioFile.js index 5faba3f0..16a773d3 100644 --- a/server/objects/AudioFile.js +++ b/server/objects/AudioFile.js @@ -1,3 +1,4 @@ +const Logger = require('../Logger') const AudioFileMetadata = require('./AudioFileMetadata') class AudioFile { @@ -33,6 +34,9 @@ class AudioFile { this.exclude = false this.error = null + // TEMP: For forcing rescan + this.isOldAudioFile = false + if (data) { this.construct(data) } @@ -58,6 +62,7 @@ class AudioFile { size: this.size, bitRate: this.bitRate, language: this.language, + codec: this.codec, timeBase: this.timeBase, channels: this.channels, channelLayout: this.channelLayout, @@ -88,7 +93,7 @@ class AudioFile { this.size = data.size this.bitRate = data.bitRate this.language = data.language - this.codec = data.codec + this.codec = data.codec || null this.timeBase = data.timeBase this.channels = data.channels this.channelLayout = data.channelLayout @@ -98,15 +103,11 @@ class AudioFile { // Old version of AudioFile used `tagAlbum` etc. var isOldVersion = Object.keys(data).find(key => key.startsWith('tag')) if (isOldVersion) { + this.isOldAudioFile = true this.metadata = new AudioFileMetadata(data) } else { this.metadata = new AudioFileMetadata(data.metadata || {}) } - // this.tagAlbum = data.tagAlbum - // this.tagArtist = data.tagArtist - // this.tagGenre = data.tagGenre - // this.tagTitle = data.tagTitle - // this.tagTrack = data.tagTrack } setData(data) { @@ -131,7 +132,7 @@ class AudioFile { this.size = data.size this.bitRate = data.bit_rate || null this.language = data.language - this.codec = data.codec + this.codec = data.codec || null this.timeBase = data.time_base this.channels = data.channels this.channelLayout = data.channel_layout @@ -142,10 +143,74 @@ class AudioFile { this.metadata.setData(data) } + syncChapters(updatedChapters) { + if (this.chapters.length !== updatedChapters.length) { + this.chapters = updatedChapters.map(ch => ({ ...ch })) + return true + } else if (updatedChapters.length === 0) { + if (this.chapters.length > 0) { + this.chapters = [] + return true + } + return false + } + + var hasUpdates = false + for (let i = 0; i < updatedChapters.length; i++) { + if (JSON.stringify(updatedChapters[i]) !== JSON.stringify(this.chapters[i])) { + hasUpdates = true + } + } + if (hasUpdates) { + this.chapters = updatedChapters.map(ch => ({ ...ch })) + } + return hasUpdates + } + + // Called from audioFileScanner.js with scanData + updateMetadata(data) { + if (!this.metadata) this.metadata = new AudioFileMetadata() + + var dataMap = { + format: data.format, + duration: data.duration, + size: data.size, + bitRate: data.bit_rate || null, + language: data.language, + codec: data.codec || null, + timeBase: data.time_base, + channels: data.channels, + channelLayout: data.channel_layout, + chapters: data.chapters || [], + embeddedCoverArt: data.embedded_cover_art || null + } + + var hasUpdates = false + for (const key in dataMap) { + if (key === 'chapters') { + var chaptersUpdated = this.syncChapters(dataMap.chapters) + if (chaptersUpdated) { + hasUpdates = true + } + } else if (dataMap[key] !== this[key]) { + // Logger.debug(`[AudioFile] "${key}" from ${this[key]} => ${dataMap[key]}`) + this[key] = dataMap[key] + hasUpdates = true + } + } + + if (this.metadata.updateData(data)) { + hasUpdates = true + } + + return hasUpdates + } + clone() { return new AudioFile(this.toJSON()) } + // If the file or parent directory was renamed it is synced here syncFile(newFile) { var hasUpdates = false var keysToSync = ['path', 'fullPath', 'ext', 'filename'] diff --git a/server/objects/AudioFileMetadata.js b/server/objects/AudioFileMetadata.js index 13d1c74e..615578f9 100644 --- a/server/objects/AudioFileMetadata.js +++ b/server/objects/AudioFileMetadata.js @@ -65,5 +65,33 @@ class AudioFileMetadata { this.tagEncoder = payload.file_tag_encoder || null this.tagEncodedBy = payload.file_tag_encodedby || null } + + updateData(payload) { + const dataMap = { + tagAlbum: payload.file_tag_album || null, + tagArtist: payload.file_tag_artist || null, + tagGenre: payload.file_tag_genre || null, + tagTitle: payload.file_tag_title || null, + tagTrack: payload.file_tag_track || null, + tagSubtitle: payload.file_tag_subtitle || null, + tagAlbumArtist: payload.file_tag_albumartist || null, + tagDate: payload.file_tag_date || null, + tagComposer: payload.file_tag_composer || null, + tagPublisher: payload.file_tag_publisher || null, + tagComment: payload.file_tag_comment || null, + tagDescription: payload.file_tag_description || null, + tagEncoder: payload.file_tag_encoder || null, + tagEncodedBy: payload.file_tag_encodedby || null + } + + var hasUpdates = false + for (const key in dataMap) { + if (dataMap[key] !== this[key]) { + this[key] = dataMap[key] + hasUpdates = true + } + } + return hasUpdates + } } module.exports = AudioFileMetadata \ No newline at end of file diff --git a/server/objects/AudioTrack.js b/server/objects/AudioTrack.js index 704212e7..c6306dee 100644 --- a/server/objects/AudioTrack.js +++ b/server/objects/AudioTrack.js @@ -20,13 +20,6 @@ class AudioTrack { this.channels = null this.channelLayout = null - // Storing tags in audio track is unnecessary, tags are stored on audio file - // this.tagAlbum = null - // this.tagArtist = null - // this.tagGenre = null - // this.tagTitle = null - // this.tagTrack = null - if (audioTrack) { this.construct(audioTrack) } @@ -50,12 +43,6 @@ class AudioTrack { this.timeBase = audioTrack.timeBase this.channels = audioTrack.channels this.channelLayout = audioTrack.channelLayout - - // this.tagAlbum = audioTrack.tagAlbum - // this.tagArtist = audioTrack.tagArtist - // this.tagGenre = audioTrack.tagGenre - // this.tagTitle = audioTrack.tagTitle - // this.tagTrack = audioTrack.tagTrack } get name() { @@ -78,11 +65,6 @@ class AudioTrack { timeBase: this.timeBase, channels: this.channels, channelLayout: this.channelLayout, - // tagAlbum: this.tagAlbum, - // tagArtist: this.tagArtist, - // tagGenre: this.tagGenre, - // tagTitle: this.tagTitle, - // tagTrack: this.tagTrack } } @@ -104,12 +86,18 @@ class AudioTrack { this.timeBase = probeData.timeBase this.channels = probeData.channels this.channelLayout = probeData.channelLayout + } - // this.tagAlbum = probeData.file_tag_album || null - // this.tagArtist = probeData.file_tag_artist || null - // this.tagGenre = probeData.file_tag_genre || null - // this.tagTitle = probeData.file_tag_title || null - // this.tagTrack = probeData.file_tag_track || null + syncMetadata(audioFile) { + var hasUpdates = false + var keysToSync = ['format', 'duration', 'size', 'bitRate', 'language', 'codec', 'timeBase', 'channels', 'channelLayout'] + keysToSync.forEach((key) => { + if (audioFile[key] !== undefined && audioFile[key] !== this[key]) { + hasUpdates = true + this[key] = audioFile[key] + } + }) + return hasUpdates } syncFile(newFile) { diff --git a/server/objects/Audiobook.js b/server/objects/Audiobook.js index a2f29645..26b23642 100644 --- a/server/objects/Audiobook.js +++ b/server/objects/Audiobook.js @@ -205,15 +205,22 @@ class Audiobook { // this function checks all files and sets the inode async checkUpdateInos() { var hasUpdates = false + + // Audiobook folder needs inode if (!this.ino) { this.ino = await getIno(this.fullPath) hasUpdates = true } + + // Check audio files have an inode for (let i = 0; i < this.audioFiles.length; i++) { var af = this.audioFiles[i] var at = this.tracks.find(t => t.ino === af.ino) if (!at) { at = this.tracks.find(t => comparePaths(t.path, af.path)) + if (!at && !af.exclude) { + Logger.warn(`[Audiobook] No matching track for audio file "${af.filename}"`) + } } if (!af.ino || af.ino === this.ino) { af.ino = await getIno(af.fullPath) @@ -229,6 +236,7 @@ class Audiobook { hasUpdates = true } } + for (let i = 0; i < this.tracks.length; i++) { var at = this.tracks[i] if (!at.ino) { @@ -252,6 +260,7 @@ class Audiobook { } } } + for (let i = 0; i < this.otherFiles.length; i++) { var file = this.otherFiles[i] if (!file.ino || file.ino === this.ino) { @@ -267,6 +276,11 @@ class Audiobook { return hasUpdates } + // Scans in v1.3.0 or lower will need to rescan audiofiles to pickup metadata and embedded cover + checkNeedsAudioFileRescan() { + return !!(this.audioFiles || []).find(af => af.isOldAudioFile || af.codec === null) + } + setData(data) { this.id = (Math.trunc(Math.random() * 1000) + Date.now()).toString(36) this.ino = data.ino || null @@ -409,19 +423,22 @@ class Audiobook { } // On scan check other files found with other files saved - async syncOtherFiles(newOtherFiles) { + async syncOtherFiles(newOtherFiles, forceRescan = false) { var hasUpdates = false var currOtherFileNum = this.otherFiles.length + var alreadyHadDescTxt = this.otherFiles.find(of => of.filename === 'desc.txt') + var newOtherFilePaths = newOtherFiles.map(f => f.path) this.otherFiles = this.otherFiles.filter(f => newOtherFilePaths.includes(f.path)) // Some files are not there anymore and filtered out if (currOtherFileNum !== this.otherFiles.length) hasUpdates = true + // If desc.txt is new or forcing rescan then read it and update description if empty var descriptionTxt = newOtherFiles.find(file => file.filename === 'desc.txt') - if (descriptionTxt) { + if (descriptionTxt && (!alreadyHadDescTxt || forceRescan)) { var newDescription = await readTextFile(descriptionTxt.fullPath) if (newDescription) { Logger.debug(`[Audiobook] Sync Other File desc.txt: ${newDescription}`) diff --git a/server/objects/ServerSettings.js b/server/objects/ServerSettings.js index 469c6156..ea89c1e1 100644 --- a/server/objects/ServerSettings.js +++ b/server/objects/ServerSettings.js @@ -1,4 +1,5 @@ const { CoverDestination } = require('../utils/constants') +const Logger = require('../Logger') class ServerSettings { constructor(settings) { @@ -11,6 +12,7 @@ class ServerSettings { this.saveMetadataFile = false this.rateLimitLoginRequests = 10 this.rateLimitLoginWindow = 10 * 60 * 1000 // 10 Minutes + this.logLevel = Logger.logLevel if (settings) { this.construct(settings) @@ -25,6 +27,11 @@ class ServerSettings { this.saveMetadataFile = !!settings.saveMetadataFile this.rateLimitLoginRequests = !isNaN(settings.rateLimitLoginRequests) ? Number(settings.rateLimitLoginRequests) : 10 this.rateLimitLoginWindow = !isNaN(settings.rateLimitLoginWindow) ? Number(settings.rateLimitLoginWindow) : 10 * 60 * 1000 // 10 Minutes + this.logLevel = settings.logLevel || Logger.logLevel + + if (this.logLevel !== Logger.logLevel) { + Logger.setLogLevel(this.logLevel) + } } toJSON() { @@ -36,7 +43,8 @@ class ServerSettings { coverDestination: this.coverDestination, saveMetadataFile: !!this.saveMetadataFile, rateLimitLoginRequests: this.rateLimitLoginRequests, - rateLimitLoginWindow: this.rateLimitLoginWindow + rateLimitLoginWindow: this.rateLimitLoginWindow, + logLevel: this.logLevel } } @@ -44,6 +52,9 @@ class ServerSettings { var hasUpdates = false for (const key in payload) { if (this[key] !== payload[key]) { + if (key === 'logLevel') { + Logger.setLogLevel(payload[key]) + } this[key] = payload[key] hasUpdates = true } diff --git a/server/objects/Stream.js b/server/objects/Stream.js index e54e9ebb..cfed5de7 100644 --- a/server/objects/Stream.js +++ b/server/objects/Stream.js @@ -16,7 +16,6 @@ class Stream extends EventEmitter { this.audiobook = audiobook this.segmentLength = 6 - this.segmentBasename = 'output-%d.ts' this.streamPath = Path.join(streamPath, this.id) this.concatFilesPath = Path.join(this.streamPath, 'files.txt') this.playlistPath = Path.join(this.streamPath, 'output.m3u8') @@ -51,6 +50,16 @@ class Stream extends EventEmitter { return this.audiobook.totalDuration } + get hlsSegmentType() { + var hasFlac = this.tracks.find(t => t.ext.toLowerCase() === '.flac') + return hasFlac ? 'fmp4' : 'mpegts' + } + + get segmentBasename() { + if (this.hlsSegmentType === 'fmp4') return 'output-%d.m4s' + return 'output-%d.ts' + } + get segmentStartNumber() { if (!this.startTime) return 0 return Math.floor(this.startTime / this.segmentLength) @@ -98,7 +107,7 @@ class Stream extends EventEmitter { var userAudiobook = clientUserAudiobooks[this.audiobookId] || null if (userAudiobook) { var timeRemaining = this.totalDuration - userAudiobook.currentTime - Logger.info('[STREAM] User has progress for audiobook', userAudiobook, `Time Remaining: ${timeRemaining}s`) + Logger.info('[STREAM] User has progress for audiobook', userAudiobook.progress, `Time Remaining: ${timeRemaining}s`) if (timeRemaining > 15) { this.startTime = userAudiobook.currentTime this.clientCurrentTime = this.startTime @@ -133,7 +142,7 @@ class Stream extends EventEmitter { async generatePlaylist() { fs.ensureDirSync(this.streamPath) - await hlsPlaylistGenerator(this.playlistPath, 'output', this.totalDuration, this.segmentLength) + await hlsPlaylistGenerator(this.playlistPath, 'output', this.totalDuration, this.segmentLength, this.hlsSegmentType) return this.clientPlaylistUri } @@ -142,7 +151,7 @@ class Stream extends EventEmitter { var files = await fs.readdir(this.streamPath) files.forEach((file) => { var extname = Path.extname(file) - if (extname === '.ts') { + if (extname === '.ts' || extname === '.m4s') { var basename = Path.basename(file, extname) var num_part = basename.split('-')[1] var part_num = Number(num_part) @@ -238,24 +247,31 @@ class Stream extends EventEmitter { } const logLevel = process.env.NODE_ENV === 'production' ? 'error' : 'warning' + const audioCodec = this.hlsSegmentType === 'fmp4' ? 'aac' : 'copy' this.ffmpeg.addOption([ `-loglevel ${logLevel}`, '-map 0:a', - '-c:a copy' + `-c:a ${audioCodec}` ]) - this.ffmpeg.addOption([ + const hlsOptions = [ '-f hls', "-copyts", "-avoid_negative_ts disabled", "-max_delay 5000000", "-max_muxing_queue_size 2048", `-hls_time 6`, - "-hls_segment_type mpegts", + `-hls_segment_type ${this.hlsSegmentType}`, `-start_number ${this.segmentStartNumber}`, "-hls_playlist_type vod", "-hls_list_size 0", "-hls_allow_cache 0" - ]) + ] + if (this.hlsSegmentType === 'fmp4') { + hlsOptions.push('-strict -2') + var fmp4InitFilename = Path.join(this.streamPath, 'init.mp4') + hlsOptions.push(`-hls_fmp4_init_filename ${fmp4InitFilename}`) + } + this.ffmpeg.addOption(hlsOptions) var segmentFilename = Path.join(this.streamPath, this.segmentBasename) this.ffmpeg.addOption(`-hls_segment_filename ${segmentFilename}`) this.ffmpeg.output(this.finalPlaylistPath) diff --git a/server/utils/audioFileScanner.js b/server/utils/audioFileScanner.js index 1ad77ec3..ba9434bd 100644 --- a/server/utils/audioFileScanner.js +++ b/server/utils/audioFileScanner.js @@ -83,6 +83,9 @@ function getTrackNumberFromFilename(title, author, series, publishYear, filename if (series) partbasename = partbasename.replace(series, '') if (publishYear) partbasename = partbasename.replace(publishYear) + // Remove eg. "disc 1" from path + partbasename = partbasename.replace(/ disc \d\d? /i, '') + var numbersinpath = partbasename.match(/\d+/g) if (!numbersinpath) return null @@ -95,9 +98,11 @@ async function scanAudioFiles(audiobook, newAudioFiles) { Logger.error('[AudioFileScanner] Scan Audio Files no new files', audiobook.title) return } + var tracks = [] var numDuplicateTracks = 0 var numInvalidTracks = 0 + for (let i = 0; i < newAudioFiles.length; i++) { var audioFile = newAudioFiles[i] var scanData = await scan(audioFile.fullPath) @@ -109,6 +114,7 @@ async function scanAudioFiles(audiobook, newAudioFiles) { var trackNumFromMeta = getTrackNumberFromMeta(scanData) var book = audiobook.book || {} + var trackNumFromFilename = getTrackNumberFromFilename(book.title, book.author, book.series, book.publishYear, audioFile.filename) var audioFileObj = { @@ -182,4 +188,47 @@ async function scanAudioFiles(audiobook, newAudioFiles) { audiobook.tracks.sort((a, b) => a.index - b.index) } } -module.exports.scanAudioFiles = scanAudioFiles \ No newline at end of file +module.exports.scanAudioFiles = scanAudioFiles + + +async function rescanAudioFiles(audiobook) { + + var audioFiles = audiobook.audioFiles + var updates = 0 + + for (let i = 0; i < audioFiles.length; i++) { + var audioFile = audioFiles[i] + var scanData = await scan(audioFile.fullPath) + if (!scanData || scanData.error) { + Logger.error('[AudioFileScanner] Scan failed for', audioFile.path) + // audiobook.invalidAudioFiles.push(parts[i]) + continue; + } + var hasUpdates = audioFile.updateMetadata(scanData) + if (hasUpdates) { + // Sync audio track with audio file + var matchingAudioTrack = audiobook.tracks.find(t => t.ino === audioFile.ino) + if (matchingAudioTrack) { + matchingAudioTrack.syncMetadata(audioFile) + } else if (!audioFile.exclude) { // If audio file is not excluded then it should have an audio track + + // Fallback to checking path + matchingAudioTrack = audiobook.tracks.find(t => t.path === audioFile.path) + if (matchingAudioTrack) { + Logger.warn(`[AudioFileScanner] Audio File mismatch ino with audio track "${audioFile.filename}"`) + matchingAudioTrack.ino = audioFile.ino + matchingAudioTrack.syncMetadata(audioFile) + } else { + Logger.error(`[AudioFileScanner] Audio File has no matching Track ${audioFile.filename} for "${audiobook.title}"`) + + // Exclude audio file to prevent further errors + // audioFile.exclude = true + } + } + updates++ + } + } + + return updates +} +module.exports.rescanAudioFiles = rescanAudioFiles \ No newline at end of file diff --git a/server/utils/constants.js b/server/utils/constants.js index 97fabd51..b01e2163 100644 --- a/server/utils/constants.js +++ b/server/utils/constants.js @@ -9,4 +9,14 @@ module.exports.ScanResult = { module.exports.CoverDestination = { METADATA: 0, AUDIOBOOK: 1 +} + +module.exports.LogLevel = { + TRACE: 0, + DEBUG: 1, + INFO: 2, + WARN: 3, + ERROR: 4, + FATAL: 5, + NOTE: 6 } \ No newline at end of file diff --git a/server/utils/ffmpegHelpers.js b/server/utils/ffmpegHelpers.js index 186ab5a2..9ba53364 100644 --- a/server/utils/ffmpegHelpers.js +++ b/server/utils/ffmpegHelpers.js @@ -75,7 +75,7 @@ async function extractCoverArt(filepath, outputpath) { return new Promise((resolve) => { var ffmpeg = Ffmpeg(filepath) - ffmpeg.addOption(['-map 0:v']) + ffmpeg.addOption(['-map 0:v', '-frames:v 1']) ffmpeg.output(outputpath) ffmpeg.on('start', (cmd) => { diff --git a/server/utils/hlsPlaylistGenerator.js b/server/utils/hlsPlaylistGenerator.js index bfe892ea..414e38b7 100644 --- a/server/utils/hlsPlaylistGenerator.js +++ b/server/utils/hlsPlaylistGenerator.js @@ -1,6 +1,8 @@ const fs = require('fs-extra') -function getPlaylistStr(segmentName, duration, segmentLength) { +function getPlaylistStr(segmentName, duration, segmentLength, hlsSegmentType) { + var ext = hlsSegmentType === 'fmp4' ? 'm4s' : 'ts' + var lines = [ '#EXTM3U', '#EXT-X-VERSION:3', @@ -9,22 +11,25 @@ function getPlaylistStr(segmentName, duration, segmentLength) { '#EXT-X-MEDIA-SEQUENCE:0', '#EXT-X-PLAYLIST-TYPE:VOD' ] + if (hlsSegmentType === 'fmp4') { + lines.push('#EXT-X-MAP:URI="init.mp4"') + } var numSegments = Math.floor(duration / segmentLength) var lastSegment = duration - (numSegments * segmentLength) for (let i = 0; i < numSegments; i++) { lines.push(`#EXTINF:6,`) - lines.push(`${segmentName}-${i}.ts`) + lines.push(`${segmentName}-${i}.${ext}`) } if (lastSegment > 0) { lines.push(`#EXTINF:${lastSegment},`) - lines.push(`${segmentName}-${numSegments}.ts`) + lines.push(`${segmentName}-${numSegments}.${ext}`) } lines.push('#EXT-X-ENDLIST') return lines.join('\n') } -function generatePlaylist(outputPath, segmentName, duration, segmentLength) { - var playlistStr = getPlaylistStr(segmentName, duration, segmentLength) +function generatePlaylist(outputPath, segmentName, duration, segmentLength, hlsSegmentType) { + var playlistStr = getPlaylistStr(segmentName, duration, segmentLength, hlsSegmentType) return fs.writeFile(outputPath, playlistStr) } module.exports = generatePlaylist \ No newline at end of file diff --git a/server/utils/prober.js b/server/utils/prober.js index da30a337..e8c2c4d3 100644 --- a/server/utils/prober.js +++ b/server/utils/prober.js @@ -137,7 +137,6 @@ function parseChapters(chapters) { function parseTags(format) { if (!format.tags) { - Logger.debug('No Tags') return {} } // Logger.debug('Tags', format.tags) diff --git a/server/utils/scandir.js b/server/utils/scandir.js index daca7e09..279fcf0a 100644 --- a/server/utils/scandir.js +++ b/server/utils/scandir.js @@ -3,10 +3,10 @@ const dir = require('node-dir') const Logger = require('../Logger') const { getIno } = require('./index') -const AUDIO_FORMATS = ['m4b', 'mp3', 'm4a'] +const AUDIO_FORMATS = ['m4b', 'mp3', 'm4a', 'flac'] const INFO_FORMATS = ['nfo'] const IMAGE_FORMATS = ['png', 'jpg', 'jpeg', 'webp'] -const EBOOK_FORMATS = ['epub', 'pdf'] +const EBOOK_FORMATS = ['epub', 'pdf', 'mobi'] function getPaths(path) { return new Promise((resolve) => {