From a1af672c7c8738eb9ba02f8d680e2159712b0096 Mon Sep 17 00:00:00 2001 From: mikiher Date: Fri, 15 Mar 2024 08:50:51 +0200 Subject: [PATCH 1/5] Add unit test workflow --- .github/workflows/unit-tests.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 00000000..4c88c16e --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,30 @@ +name: Unit Tests + +on: + workflow_dispatch: + inputs: + branch: + description: 'Branch name' + required: true + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name != 'workflow_dispatch' && github.ref_name || inputs.branch}} + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install dependencies + run: npm ci + + - name: Run tests + run: npm test From 0c612b48368f15e4a65e9ad27017dd375d5adc32 Mon Sep 17 00:00:00 2001 From: mikiher Date: Fri, 15 Mar 2024 09:51:40 +0200 Subject: [PATCH 2/5] Update unit test workflow to include push event --- .github/workflows/unit-tests.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 4c88c16e..24f6398c 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,22 +1,23 @@ -name: Unit Tests +name: Run Unit Tests on: workflow_dispatch: inputs: - branch: - description: 'Branch name' + ref: + description: 'Branch/Tag/SHA to test' required: true pull_request: + push: jobs: - build: + run-unit-tests: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ github.event_name != 'workflow_dispatch' && github.ref_name || inputs.branch}} + ref: ${{ github.event_name != 'workflow_dispatch' && github.ref_name || inputs.ref}} - name: Set up Node.js uses: actions/setup-node@v4 From 630ece82add1ed101e598ed67d3ee0938a39d7bd Mon Sep 17 00:00:00 2001 From: advplyr Date: Fri, 15 Mar 2024 14:35:09 -0500 Subject: [PATCH 3/5] Fix:Chapter modal scroll to current chapter --- client/components/modals/ChaptersModal.vue | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/components/modals/ChaptersModal.vue b/client/components/modals/ChaptersModal.vue index 2ace9891..4c005639 100644 --- a/client/components/modals/ChaptersModal.vue +++ b/client/components/modals/ChaptersModal.vue @@ -34,11 +34,6 @@ export default { data() { return {} }, - watch: { - value(newVal) { - this.$nextTick(this.scrollToChapter) - } - }, computed: { show: { get() { @@ -53,7 +48,7 @@ export default { return this.playbackRate }, currentChapterId() { - return this.currentChapter ? this.currentChapter.id : null + return this.currentChapter?.id || null }, currentChapterStart() { return (this.currentChapter?.start || 0) / this._playbackRate @@ -74,6 +69,11 @@ export default { } } } + }, + updated() { + if (this.value) { + this.$nextTick(this.scrollToChapter) + } } } From 88f9533b37eed2e5fe97f4c7d23598ca8c3cdcad Mon Sep 17 00:00:00 2001 From: advplyr Date: Fri, 15 Mar 2024 17:10:43 -0500 Subject: [PATCH 4/5] Fix:HLS.js retry fragments #2748 #2720 --- client/package-lock.json | 10 ++++---- client/package.json | 4 ++-- client/players/LocalAudioPlayer.js | 37 +++++++++++++++++++++++++----- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index a15762dc..9d513691 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -16,7 +16,7 @@ "cron-parser": "^4.7.1", "date-fns": "^2.25.0", "epubjs": "^0.3.88", - "hls.js": "^1.0.7", + "hls.js": "^1.5.7", "libarchive.js": "^1.3.0", "nuxt": "^2.17.3", "nuxt-socket-io": "^1.1.18", @@ -8627,9 +8627,9 @@ } }, "node_modules/hls.js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.5.1.tgz", - "integrity": "sha512-SsUSlpyjOGnwBhVrVEG6vRFPU2SAJ0gUqrFdGeo7YPbOC0vuWK0TDMyp7n3QiaBC/Wkic771uqPnnVdT8/x+3Q==" + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.5.7.tgz", + "integrity": "sha512-Hnyf7ojTBtXHeOW1/t6wCBJSiK1WpoKF9yg7juxldDx8u3iswrkPt2wbOA/1NiwU4j27DSIVoIEJRAhcdMef/A==" }, "node_modules/hmac-drbg": { "version": "1.0.1", @@ -16976,4 +16976,4 @@ } } } -} \ No newline at end of file +} diff --git a/client/package.json b/client/package.json index d9f11c99..55acee1b 100644 --- a/client/package.json +++ b/client/package.json @@ -21,7 +21,7 @@ "cron-parser": "^4.7.1", "date-fns": "^2.25.0", "epubjs": "^0.3.88", - "hls.js": "^1.0.7", + "hls.js": "^1.5.7", "libarchive.js": "^1.3.0", "nuxt": "^2.17.3", "nuxt-socket-io": "^1.1.18", @@ -36,4 +36,4 @@ "postcss": "^8.3.6", "tailwindcss": "^3.4.1" } -} \ No newline at end of file +} diff --git a/client/players/LocalAudioPlayer.js b/client/players/LocalAudioPlayer.js index 587a52c0..eb1484bb 100644 --- a/client/players/LocalAudioPlayer.js +++ b/client/players/LocalAudioPlayer.js @@ -139,11 +139,30 @@ export default class LocalAudioPlayer extends EventEmitter { } var hlsOptions = { - startPosition: this.startTime || -1 - // No longer needed because token is put in a query string - // xhrSetup: (xhr) => { - // xhr.setRequestHeader('Authorization', `Bearer ${this.token}`) - // } + startPosition: this.startTime || -1, + fragLoadPolicy: { + default: { + maxTimeToFirstByteMs: 10000, + maxLoadTimeMs: 120000, + timeoutRetry: { + maxNumRetry: 4, + retryDelayMs: 0, + maxRetryDelayMs: 0, + }, + errorRetry: { + maxNumRetry: 8, + retryDelayMs: 1000, + maxRetryDelayMs: 8000, + shouldRetry: (retryConfig, retryCount, isTimeout, httpStatus, retry) => { + if (httpStatus?.code === 404 && retryConfig?.maxNumRetry > retryCount) { + console.log(`[HLS] Server 404 for fragment retry ${retryCount} of ${retryConfig.maxNumRetry}`) + return true + } + return retry + } + }, + } + } } this.hlsInstance = new Hls(hlsOptions) @@ -156,9 +175,15 @@ export default class LocalAudioPlayer extends EventEmitter { }) this.hlsInstance.on(Hls.Events.ERROR, (e, data) => { - console.error('[HLS] Error', data.type, data.details, data) if (data.details === Hls.ErrorDetails.BUFFER_STALLED_ERROR) { console.error('[HLS] BUFFER STALLED ERROR') + } else if (data.details === Hls.ErrorDetails.FRAG_LOAD_ERROR) { + // Only show error if the fragment is not being retried + if (data.errorAction?.action !== 5) { + console.error('[HLS] FRAG LOAD ERROR', data) + } + } else { + console.error('[HLS] Error', data.type, data.details, data) } }) this.hlsInstance.on(Hls.Events.DESTROYING, () => { From a2b2a2d060e9c5392df21dbb17a7ac7b6eb2ad8c Mon Sep 17 00:00:00 2001 From: advplyr Date: Sat, 16 Mar 2024 15:12:33 -0500 Subject: [PATCH 5/5] Fix:Applying backup not properly overwriting existing sqlite file - Fixed resetting api cache on backup - Added loading indicator in backups table - Fixed apply backup api not responding with 200 http status code - Added additional logging and failsafes --- client/components/tables/BackupsTable.vue | 14 ++++-- server/Database.js | 1 - server/controllers/BackupController.js | 7 ++- server/managers/ApiCacheManager.js | 10 +++++ server/managers/BackupManager.js | 54 ++++++++++++++++++++++- 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/client/components/tables/BackupsTable.vue b/client/components/tables/BackupsTable.vue index 08692a4d..bd3d074a 100644 --- a/client/components/tables/BackupsTable.vue +++ b/client/components/tables/BackupsTable.vue @@ -1,5 +1,5 @@ @@ -64,6 +68,7 @@ export default { showConfirmApply: false, selectedBackup: null, isBackingUp: false, + isApplyingBackup: false, processing: false, backups: [] } @@ -85,19 +90,21 @@ export default { }, confirm() { this.showConfirmApply = false + this.isApplyingBackup = true this.$axios .$get(`/api/backups/${this.selectedBackup.id}/apply`) .then(() => { - this.isBackingUp = false location.replace('/config/backups?backup=1') }) .catch((error) => { - this.isBackingUp = false console.error('Failed to apply backup', error) const errorMsg = error.response.data || this.$strings.ToastBackupRestoreFailed this.$toast.error(errorMsg) }) + .finally(() => { + this.isApplyingBackup = false + }) }, deleteBackupClick(backup) { if (confirm(this.$getString('MessageConfirmDeleteBackup', [this.$formatDatetime(backup.createdAt, this.dateFormat, this.timeFormat)]))) { @@ -180,7 +187,6 @@ export default { this.loadBackups() if (this.$route.query.backup) { this.$toast.success('Backup applied successfully') - this.$router.replace('/config') } } } diff --git a/server/Database.js b/server/Database.js index e3fabe1f..64dc518e 100644 --- a/server/Database.js +++ b/server/Database.js @@ -217,7 +217,6 @@ class Database { async disconnect() { Logger.info(`[Database] Disconnecting sqlite db`) await this.sequelize.close() - this.sequelize = null } /** diff --git a/server/controllers/BackupController.js b/server/controllers/BackupController.js index 8104623e..bd6caa0b 100644 --- a/server/controllers/BackupController.js +++ b/server/controllers/BackupController.js @@ -49,8 +49,13 @@ class BackupController { res.sendFile(req.backup.fullPath) } + /** + * + * @param {import('express').Request} req + * @param {import('express').Response} res + */ apply(req, res) { - this.backupManager.requestApplyBackup(req.backup, res) + this.backupManager.requestApplyBackup(this.apiCacheManager, req.backup, res) } middleware(req, res, next) { diff --git a/server/managers/ApiCacheManager.js b/server/managers/ApiCacheManager.js index 1af069f3..bb99b8cb 100644 --- a/server/managers/ApiCacheManager.js +++ b/server/managers/ApiCacheManager.js @@ -22,6 +22,16 @@ class ApiCacheManager { this.cache.clear() } + /** + * Reset hooks and clear cache. Used when applying backups + */ + reset() { + Logger.info(`[ApiCacheManager] Resetting cache`) + + this.init() + this.cache.clear() + } + get middleware() { return (req, res, next) => { const key = { user: req.user.username, url: req.url } diff --git a/server/managers/BackupManager.js b/server/managers/BackupManager.js index ef8ed643..40a74f14 100644 --- a/server/managers/BackupManager.js +++ b/server/managers/BackupManager.js @@ -146,23 +146,73 @@ class BackupManager { } } - async requestApplyBackup(backup, res) { + /** + * + * @param {import('./ApiCacheManager')} apiCacheManager + * @param {Backup} backup + * @param {import('express').Response} res + */ + async requestApplyBackup(apiCacheManager, backup, res) { + Logger.info(`[BackupManager] Applying backup at "${backup.fullPath}"`) + const zip = new StreamZip.async({ file: backup.fullPath }) const entries = await zip.entries() + + // Ensure backup has an absdatabase.sqlite file if (!Object.keys(entries).includes('absdatabase.sqlite')) { Logger.error(`[BackupManager] Cannot apply old backup ${backup.fullPath}`) + await zip.close() return res.status(500).send('Invalid backup file. Does not include absdatabase.sqlite. This might be from an older Audiobookshelf server.') } await Database.disconnect() - await zip.extract('absdatabase.sqlite', global.ConfigPath) + const dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite') + const tempDbPath = Path.join(global.ConfigPath, 'absdatabase-temp.sqlite') + + // Extract backup sqlite file to temporary path + await zip.extract('absdatabase.sqlite', tempDbPath) + Logger.info(`[BackupManager] Extracted backup sqlite db to temp path ${tempDbPath}`) + + // Verify extract - Abandon backup if sqlite file did not extract + if (!await fs.pathExists(tempDbPath)) { + Logger.error(`[BackupManager] Sqlite file not found after extract - abandon backup apply and reconnect db`) + await zip.close() + await Database.reconnect() + return res.status(500).send('Failed to extract sqlite db from backup') + } + + // Attempt to remove existing db file + try { + await fs.remove(dbPath) + } catch (error) { + // Abandon backup and remove extracted sqlite file if unable to remove existing db file + Logger.error(`[BackupManager] Unable to overwrite existing db file - abandon backup apply and reconnect db`, error) + await fs.remove(tempDbPath) + await zip.close() + await Database.reconnect() + return res.status(500).send(`Failed to overwrite sqlite db: ${error?.message || 'Unknown Error'}`) + } + + // Rename temp db + await fs.move(tempDbPath, dbPath) + Logger.info(`[BackupManager] Saved backup sqlite file at "${dbPath}"`) + + // Extract /metadata/items and /metadata/authors folders await zip.extract('metadata-items/', this.ItemsMetadataPath) await zip.extract('metadata-authors/', this.AuthorsMetadataPath) + await zip.close() + // Reconnect db await Database.reconnect() + // Reset api cache, set hooks again + await apiCacheManager.reset() + + res.sendStatus(200) + + // Triggers browser refresh for all clients SocketAuthority.emitter('backup_applied') }