From 04f92c33c250d71c002e9feb4cc50aebac79f3ba Mon Sep 17 00:00:00 2001 From: advplyr Date: Sun, 10 Oct 2021 16:36:21 -0500 Subject: [PATCH] Abort backup if it is getting too large #89, support for ebook only book folders #75 --- .dockerignore | 1 + .gitignore | 1 + client/components/app/Reader.vue | 34 +++++++-- client/components/cards/BookCard.vue | 38 ++++++++-- .../components/modals/edit-tabs/Download.vue | 10 ++- client/components/modals/edit-tabs/Tracks.vue | 74 ++++++++++--------- .../modals/libraries/LibraryItem.vue | 35 +++++++-- client/components/tables/LibrariesTable.vue | 23 +----- client/layouts/default.vue | 1 + client/package.json | 2 +- client/pages/audiobook/_id/index.vue | 54 ++++++++------ client/store/index.js | 9 +++ package-lock.json | 11 +-- package.json | 5 +- server/BackupManager.js | 24 ++++++ server/Scanner.js | 50 +++++++------ server/objects/Audiobook.js | 21 ++++-- server/utils/scandir.js | 14 ++-- 18 files changed, 258 insertions(+), 149 deletions(-) diff --git a/.dockerignore b/.dockerignore index 32d455d3..942b75ec 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ npm-debug.log /config /audiobooks /audiobooks2 +/media/ /metadata dev.js test/ diff --git a/.gitignore b/.gitignore index 6af8b796..90b61fd4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ node_modules/ /config/ /audiobooks/ /audiobooks2/ +/media/ /metadata/ test/ /client/.nuxt/ diff --git a/client/components/app/Reader.vue b/client/components/app/Reader.vue index fc0fce6a..245395fc 100644 --- a/client/components/app/Reader.vue +++ b/client/components/app/Reader.vue @@ -1,5 +1,5 @@ diff --git a/client/package.json b/client/package.json index 426ad94f..981602f1 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "audiobookshelf-client", - "version": "1.4.3", + "version": "1.4.4", "description": "Audiobook manager and player", "main": "index.js", "scripts": { diff --git a/client/pages/audiobook/_id/index.vue b/client/pages/audiobook/_id/index.vue index 073b3e37..437d1c62 100644 --- a/client/pages/audiobook/_id/index.vue +++ b/client/pages/audiobook/_id/index.vue @@ -57,7 +57,7 @@ -
+
Duration
@@ -65,7 +65,7 @@ {{ durationPretty }}
-
+
Size
@@ -73,16 +73,20 @@ {{ sizePretty }}
- -
+ + +
+ warning_amber +

Book has no audio tracks but has valid ebook files. The e-reader is experimental and can be turned on in config.

+
+
+ warning_amber +

Book has valid ebook files, but the experimental e-reader currently only supports epub files.

+
+

Your Progress: {{ Math.round(progressPercent * 100) }}%

{{ $elapsedPretty(userTimeRemaining) }} remaining

@@ -92,13 +96,13 @@
- + play_arrow {{ streaming ? 'Streaming' : 'Play' }} - + error - Missing + {{ isMissing ? 'Missing' : 'Incomplete' }} @@ -141,7 +145,7 @@
- + @@ -150,7 +154,7 @@ - + @@ -175,7 +179,6 @@ export default { }, data() { return { - showReader: false, isRead: false, resettingProgress: false, isProcessingReadUpdate: false @@ -230,6 +233,12 @@ export default { isMissing() { return this.audiobook.isMissing }, + isIncomplete() { + return this.audiobook.isIncomplete + }, + showPlayButton() { + return !this.isMissing && !this.isIncomplete && this.tracks.length + }, missingParts() { return this.audiobook.missingParts || [] }, @@ -313,16 +322,15 @@ export default { ebooks() { return this.audiobook.ebooks }, + showEpubAlert() { + return this.ebooks.length && !this.epubEbook && !this.tracks.length + }, + showExperimentalReadAlert() { + return !this.tracks.length && this.ebooks.length && !this.showExperimentalFeatures + }, epubEbook() { return this.audiobook.ebooks.find((eb) => eb.ext === '.epub') }, - epubPath() { - return this.epubEbook ? this.epubEbook.path : null - }, - epubUrl() { - if (!this.epubPath) return null - return `/ebook/${this.libraryId}/${this.folderId}/${this.epubPath}` - }, userToken() { return this.$store.getters['user/getToken'] }, @@ -365,7 +373,7 @@ export default { }, methods: { openEbook() { - this.showReader = true + this.$store.commit('showEReader', this.audiobook) }, toggleRead() { var updatePayload = { diff --git a/client/store/index.js b/client/store/index.js index 822e534c..83633b16 100644 --- a/client/store/index.js +++ b/client/store/index.js @@ -7,6 +7,7 @@ export const state = () => ({ streamAudiobook: null, editModalTab: 'details', showEditModal: false, + showEReader: false, selectedAudiobook: null, playOnLoad: false, developerMode: false, @@ -111,6 +112,14 @@ export const mutations = { setShowEditModal(state, val) { state.showEditModal = val }, + showEReader(state, audiobook) { + console.log('Show EReader', audiobook) + state.selectedAudiobook = audiobook + state.showEReader = true + }, + setShowEReader(state, val) { + state.showEReader = val + }, setDeveloperMode(state, val) { state.developerMode = val }, diff --git a/package-lock.json b/package-lock.json index d3da252f..5e5e530a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "audiobookshelf", - "version": "1.4.1", + "version": "1.4.3", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -411,15 +411,6 @@ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" }, - "cookie-parser": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.5.tgz", - "integrity": "sha512-f13bPUj/gG/5mDr+xLmSxxDsB9DQiTIfhJS/sqjrmfAWiAN+x2O4i/XguTL9yDZ+/IFDanJ+5x7hC4CXT9Tdzw==", - "requires": { - "cookie": "0.4.0", - "cookie-signature": "1.0.6" - } - }, "cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", diff --git a/package.json b/package.json index 91460660..25ebed41 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "audiobookshelf", - "version": "1.4.3", + "version": "1.4.4", "description": "Self-hosted audiobook server for managing and playing audiobooks", "main": "index.js", "scripts": { @@ -8,7 +8,7 @@ "start": "node index.js", "client": "cd client && npm install && npm run generate", "prod": "npm run client && npm install && node prod.js", - "build-win": "npm run build-prep && pkg -t node12-win-x64 -o ./dist/win/audiobookshelf .", + "build-win": "pkg -t node12-win-x64 -o ./dist/win/audiobookshelf .", "build-linux": "build/linuxpackager" }, "bin": "prod.js", @@ -26,7 +26,6 @@ "axios": "^0.21.1", "bcryptjs": "^2.4.3", "command-line-args": "^5.2.0", - "cookie-parser": "^1.4.5", "date-and-time": "^2.0.1", "epub": "^1.2.1", "express": "^4.17.1", diff --git a/server/BackupManager.js b/server/BackupManager.js index 9f3f3cbd..795a9b9c 100644 --- a/server/BackupManager.js +++ b/server/BackupManager.js @@ -24,6 +24,9 @@ class BackupManager { this.scheduleTask = null this.backups = [] + + // If backup exceeds this value it will be aborted + this.MaxBytesBeforeAbort = 1000000000 // ~ 1GB } get serverSettings() { @@ -191,6 +194,7 @@ class BackupManager { } async runBackup() { + // Check if Metadata Path is inside Config Path (otherwise there will be an infinite loop as the archiver tries to zip itself) Logger.info(`[BackupManager] Running Backup`) var metadataBooksPath = this.serverSettings.backupMetadataCovers ? Path.join(this.MetadataPath, 'books') : null @@ -233,6 +237,7 @@ class BackupManager { async removeBackup(backup) { try { + Logger.debug(`[BackupManager] Removing Backup "${backup.fullPath}"`) await fs.remove(backup.fullPath) this.backups = this.backups.filter(b => b.id !== backup.id) Logger.info(`[BackupManager] Backup "${backup.id}" Removed`) @@ -263,6 +268,15 @@ class BackupManager { Logger.debug('Data has been drained') }) + output.on('finish', () => { + Logger.debug('Write Stream Finished') + }) + + output.on('error', (err) => { + Logger.debug('Write Stream Error', err) + reject(err) + }) + // good practice to catch warnings (ie stat failures and other non-blocking errors) archive.on('warning', function (err) { if (err.code === 'ENOENT') { @@ -279,6 +293,16 @@ class BackupManager { Logger.error(`[BackupManager] Archiver error: ${err.message}`) reject(err) }) + archive.on('progress', ({ fs: fsobj }) => { + if (fsobj.processedBytes > this.MaxBytesBeforeAbort) { + Logger.error(`[BackupManager] Archiver is too large - aborting to prevent endless loop, Bytes Processed: ${fsobj.processedBytes}`) + archive.abort() + setTimeout(() => { + this.removeBackup(backup) + output.destroy('Backup too large') // Promise is reject in write stream error evt + }, 500) + } + }) // pipe archive data to the file archive.pipe(output) diff --git a/server/Scanner.js b/server/Scanner.js index ecb76208..5300c3f6 100644 --- a/server/Scanner.js +++ b/server/Scanner.js @@ -143,18 +143,21 @@ class Scanner { forceAudioFileScan = true } - // ino is now set for every file in scandir + // inode is required audiobookData.audioFiles = audiobookData.audioFiles.filter(af => af.ino) - // REMOVE: No valid audio files - // TODO: Label as incomplete, do not actually delete - if (!audiobookData.audioFiles.length) { - Logger.error(`[Scanner] "${existingAudiobook.title}" no valid audio files found - removing audiobook`) - - await this.db.removeEntity('audiobook', existingAudiobook.id) - this.emitter('audiobook_removed', existingAudiobook.toJSONMinified()) - - return ScanResult.REMOVED + // No valid ebook and audio files found, mark as incomplete + var ebookFiles = audiobookData.otherFiles.filter(f => f.filetype === 'ebook') + if (!audiobookData.audioFiles.length && !ebookFiles.length) { + Logger.error(`[Scanner] "${existingAudiobook.title}" no valid book files found - marking as incomplete`) + existingAudiobook.setLastScan(version) + existingAudiobook.isIncomplete = true + await this.db.updateAudiobook(existingAudiobook) + this.emitter('audiobook_updated', existingAudiobook.toJSONMinified()) + return ScanResult.UPDATED + } else if (existingAudiobook.isIncomplete) { // Was incomplete but now is not + Logger.info(`[Scanner] "${existingAudiobook.title}" was incomplete but now has book files`) + existingAudiobook.isIncomplete = false } // Check for audio files that were removed @@ -219,14 +222,15 @@ class Scanner { await audioFileScanner.scanAudioFiles(existingAudiobook, newAudioFiles) } - // If after a scan no valid audio tracks remain - // TODO: Label as incomplete, do not actually delete - if (!existingAudiobook.tracks.length) { - Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`) - - await this.db.removeEntity('audiobook', existingAudiobook.id) - this.emitter('audiobook_removed', existingAudiobook.toJSONMinified()) - return ScanResult.REMOVED + // After scanning audio files, some may no longer be valid + // so make sure the directory still has valid book files + if (!existingAudiobook.tracks.length && !ebookFiles.length) { + Logger.error(`[Scanner] "${existingAudiobook.title}" no valid book files found after update - marking as incomplete`) + existingAudiobook.setLastScan(version) + existingAudiobook.isIncomplete = true + await this.db.updateAudiobook(existingAudiobook) + this.emitter('audiobook_updated', existingAudiobook.toJSONMinified()) + return ScanResult.UPDATED } var hasUpdates = hasUpdatedIno || hasUpdatedLibraryOrFolder || removedAudioFiles.length || removedAudioTracks.length || newAudioFiles.length || hasUpdatedAudioFiles @@ -269,8 +273,9 @@ class Scanner { } async scanNewAudiobook(audiobookData) { - if (!audiobookData.audioFiles.length) { - Logger.error('[Scanner] No valid audio tracks for Audiobook', audiobookData.path) + var ebookFiles = audiobookData.otherFiles.map(f => f.filetype === 'ebook') + if (!audiobookData.audioFiles.length && !ebookFiles.length) { + Logger.error('[Scanner] No valid audio files and ebooks for Audiobook', audiobookData.path) return null } @@ -279,8 +284,9 @@ class Scanner { // Scan audio files and set tracks, pulls metadata await audioFileScanner.scanAudioFiles(audiobook, audiobookData.audioFiles) - if (!audiobook.tracks.length) { - Logger.warn('[Scanner] Invalid audiobook, no valid tracks', audiobook.title) + + if (!audiobook.tracks.length && !audiobook.ebooks.length) { + Logger.warn('[Scanner] Invalid audiobook, no valid audio tracks and ebook files', audiobook.title) return null } diff --git a/server/objects/Audiobook.js b/server/objects/Audiobook.js index d012194c..8de6b15a 100644 --- a/server/objects/Audiobook.js +++ b/server/objects/Audiobook.js @@ -37,6 +37,8 @@ class Audiobook { // Audiobook was scanned and not found this.isMissing = false + // Audiobook no longer has "book" files + this.isInvalid = false if (audiobook) { this.construct(audiobook) @@ -70,6 +72,7 @@ class Audiobook { } this.isMissing = !!audiobook.isMissing + this.isInvalid = !!audiobook.isInvalid } get title() { @@ -175,7 +178,8 @@ class Audiobook { audioFiles: this._audioFiles.map(audioFile => audioFile.toJSON()), otherFiles: this._otherFiles.map(otherFile => otherFile.toJSON()), chapters: this.chapters || [], - isMissing: !!this.isMissing + isMissing: !!this.isMissing, + isInvalid: !!this.isInvalid } } @@ -197,10 +201,12 @@ class Audiobook { hasMissingParts: this.missingParts ? this.missingParts.length : 0, hasInvalidParts: this.invalidParts ? this.invalidParts.length : 0, // numEbooks: this.ebooks.length, - numEbooks: this.hasEpub ? 1 : 0, + ebooks: this.ebooks.map(ebook => ebook.toJSON()), + numEbooks: this.hasEpub ? 1 : 0, // Only supporting epubs in the reader currently numTracks: this.tracks.length, chapters: this.chapters || [], - isMissing: !!this.isMissing + isMissing: !!this.isMissing, + isInvalid: !!this.isInvalid } } @@ -220,15 +226,16 @@ class Audiobook { sizePretty: this.sizePretty, missingParts: this.missingParts, invalidParts: this.invalidParts, - audioFiles: (this.audioFiles || []).map(audioFile => audioFile.toJSON()), - otherFiles: (this.otherFiles || []).map(otherFile => otherFile.toJSON()), - ebooks: (this.ebooks || []).map(ebook => ebook.toJSON()), + audioFiles: this._audioFiles.map(audioFile => audioFile.toJSON()), + otherFiles: this._otherFiles.map(otherFile => otherFile.toJSON()), + ebooks: this.ebooks.map(ebook => ebook.toJSON()), numEbooks: this.hasEpub ? 1 : 0, tags: this.tags, book: this.bookToJSON(), tracks: this.tracksToJSON(), chapters: this.chapters || [], - isMissing: !!this.isMissing + isMissing: !!this.isMissing, + isInvalid: !!this.isInvalid } } diff --git a/server/utils/scandir.js b/server/utils/scandir.js index 4fbeaa42..7a4f5276 100644 --- a/server/utils/scandir.js +++ b/server/utils/scandir.js @@ -16,11 +16,12 @@ function getPaths(path) { }) } -function isAudioFile(path) { +function isBookFile(path) { if (!path) return false var ext = Path.extname(path) if (!ext) return false - return globals.SupportedAudioTypes.includes(ext.slice(1).toLowerCase()) + var extclean = ext.slice(1).toLowerCase() + return globals.SupportedAudioTypes.includes(extclean) || globals.SupportedEbookTypes.includes(extclean) } // Input: array of relative file paths @@ -36,17 +37,18 @@ function groupFilesIntoAudiobookPaths(paths, useAllFileTypes = false) { return pathsA - pathsB }) - // Step 2.5: Seperate audio files and other files (optional) - var audioFilePaths = [] + // Step 2.5: Seperate audio/ebook files and other files (optional) + // - Directories without an audio or ebook file will not be included + var bookFilePaths = [] var otherFilePaths = [] pathsFiltered.forEach(path => { - if (isAudioFile(path) || useAllFileTypes) audioFilePaths.push(path) + if (isBookFile(path) || useAllFileTypes) bookFilePaths.push(path) else otherFilePaths.push(path) }) // Step 3: Group audio files in audiobooks var audiobookGroup = {} - audioFilePaths.forEach((path) => { + bookFilePaths.forEach((path) => { var dirparts = Path.dirname(path).split(Path.sep) var numparts = dirparts.length var _path = ''