From 0123dacb292ba73bbc9f8bfe73d2e94e82c1fadb Mon Sep 17 00:00:00 2001 From: Vito0912 <86927734+Vito0912@users.noreply.github.com> Date: Mon, 17 Mar 2025 19:35:59 +0100 Subject: [PATCH 1/7] download multiple items --- server/controllers/LibraryController.js | 48 +++++++++++++++- server/routers/ApiRouter.js | 1 + server/utils/zipHelpers.js | 73 +++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 1 deletion(-) diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 3585dc51..f51b4974 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -23,6 +23,7 @@ const RssFeedManager = require('../managers/RssFeedManager') const libraryFilters = require('../utils/queries/libraryFilters') const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters') const authorFilters = require('../utils/queries/authorFilters') +const zipHelpers = require('../utils/zipHelpers') /** * @typedef RequestUserObject @@ -528,7 +529,7 @@ class LibraryController { Logger.error(`[LibraryController] Non-admin user "${req.user.username}" attempted to delete library`) return res.sendStatus(403) } - + // Remove library watcher Watcher.removeLibrary(req.library) @@ -1419,6 +1420,51 @@ class LibraryController { }) } + /** + * GET: /api/library/:id/download + * Downloads multiple library items + * + * @param {LibraryItemControllerRequest} req + * @param {Response} res + */ + async downloadMultiple(req, res) { + if (!req.user.canDownload) { + Logger.warn(`User "${req.user.username}" attempted to download without permission`) + return res.sendStatus(403) + } + + if(req.query.ids === undefined) { + res.status(400).send('Library not found') + } + + const itemIds = req.query.ids.split(',') + + const libraryItems = await Database.libraryItemModel.findAll({ + attributes: ['id', 'libraryId', 'path'], + where: { + id: itemIds, + libraryId: req.params.id, + mediaType: 'book' + } + }) + + Logger.info(`[LibraryItemController] User "${req.user.username}" requested download for items "${itemIds}"`) + + const filename = `LibraryItems-${Date.now()}.zip` + const libraryItemPaths = libraryItems.map((li) => li.path) + + console.log(libraryItemPaths) + + try { + await zipHelpers.zipDirectoriesPipe(libraryItemPaths, filename, res) + Logger.info(`[LibraryItemController] Downloaded item "${filename}" at "${libraryItemPaths}"`) + } catch (error) { + Logger.error(`[LibraryItemController] Download failed for item "${filename}" at "${libraryItemPaths}"`, error) + //LibraryItemController.handleDownloadError(error, res) + } + } + + /** * * @param {RequestWithUser} req diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index 6779d0af..78a5291d 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -94,6 +94,7 @@ class ApiRouter { this.router.post('/libraries/order', LibraryController.reorder.bind(this)) this.router.post('/libraries/:id/remove-metadata', LibraryController.middleware.bind(this), LibraryController.removeAllMetadataFiles.bind(this)) this.router.get('/libraries/:id/podcast-titles', LibraryController.middleware.bind(this), LibraryController.getPodcastTitles.bind(this)) + this.router.get('/libraries/:id/download', LibraryController.middleware.bind(this), LibraryController.downloadMultiple.bind(this)) // // Item Routes diff --git a/server/utils/zipHelpers.js b/server/utils/zipHelpers.js index 44b65296..1145f3a0 100644 --- a/server/utils/zipHelpers.js +++ b/server/utils/zipHelpers.js @@ -1,5 +1,6 @@ const Logger = require('../Logger') const archiver = require('../libs/archiver') +const { lstatSync } = require('node:fs') module.exports.zipDirectoryPipe = (path, filename, res) => { return new Promise((resolve, reject) => { @@ -50,3 +51,75 @@ module.exports.zipDirectoryPipe = (path, filename, res) => { archive.finalize() }) } + +/** + * Creates a zip archive containing multiple directories and streams it to the response. + * + * @param {string[]} paths - Array of directory paths to include in the zip archive. + * @param {string} filename - Name of the zip file to be sent as attachment. + * @param {object} res - Response object to pipe the archive data to. + * @returns {Promise} - Promise that resolves when the zip operation completes. + */ +module.exports.zipDirectoriesPipe = (paths, filename, res) => { + return new Promise((resolve, reject) => { + // create a file to stream archive data to + res.attachment(filename) + + const archive = archiver('zip', { + zlib: { level: 0 } // Sets the compression level. + }) + + // listen for all archive data to be written + // 'close' event is fired only when a file descriptor is involved + res.on('close', () => { + Logger.info(archive.pointer() + ' total bytes') + Logger.debug('archiver has been finalized and the output file descriptor has closed.') + resolve() + }) + + // This event is fired when the data source is drained no matter what was the data source. + // It is not part of this library but rather from the NodeJS Stream API. + // @see: https://nodejs.org/api/stream.html#stream_event_end + res.on('end', () => { + Logger.debug('Data has been drained') + }) + + // good practice to catch warnings (ie stat failures and other non-blocking errors) + archive.on('warning', function (err) { + if (err.code === 'ENOENT') { + // log warning + Logger.warn(`[DownloadManager] Archiver warning: ${err.message}`) + } else { + // throw error + Logger.error(`[DownloadManager] Archiver error: ${err.message}`) + // throw err + reject(err) + } + }) + archive.on('error', function (err) { + Logger.error(`[DownloadManager] Archiver error: ${err.message}`) + reject(err) + }) + + // pipe archive data to the file + archive.pipe(res) + + // Add each path as a directory in the zip + paths.forEach(path => { + + const paths = path.split('/') + + // Check if path is file or directory + if (lstatSync(path).isDirectory()) { + const dirName = path.split('/').pop() + + // Add the directory to the archive with its name as the root folder + archive.directory(path, dirName); + } else { + archive.file(path, { name: paths[paths.length - 1] }); + } + }); + + archive.finalize() + }) +} From 0a9a846a332480eeb08bb8f999960e6432c16b99 Mon Sep 17 00:00:00 2001 From: Vito0912 <86927734+Vito0912@users.noreply.github.com> Date: Mon, 17 Mar 2025 19:56:42 +0100 Subject: [PATCH 2/7] added download to frontend --- client/components/app/Appbar.vue | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/client/components/app/Appbar.vue b/client/components/app/Appbar.vue index bb452526..6fd2bc3e 100644 --- a/client/components/app/Appbar.vue +++ b/client/components/app/Appbar.vue @@ -180,6 +180,15 @@ export default { action: 'rescan' }) + // The limit of 50 is introduced because of the URL length. Each id has 36 chars, so 36 * 40 = 1440 + // + 40 , separators = 1480 chars + base path 280 chars = 1760 chars. This keeps the URL under 2000 chars even with longer domains + if(this.selectedMediaItems.length <= 40) { + options.push({ + text: this.$strings.LabelDownload, + action: 'download' + }) + } + return options } }, @@ -215,6 +224,8 @@ export default { this.batchAutoMatchClick() } else if (action === 'rescan') { this.batchRescan() + } else if (action === 'download') { + this.batchDownload() } }, async batchRescan() { @@ -241,6 +252,11 @@ export default { } this.$store.commit('globals/setConfirmPrompt', payload) }, + async batchDownload() { + const libraryItemIds = this.selectedMediaItems.map((i) => i.id) + console.log('Downloading library items', libraryItemIds) + this.$downloadFile(`/api/libraries/${this.$store.state.libraries.currentLibraryId}/download?token=${this.$store.getters['user/getToken']}&ids=${libraryItemIds.join(',')}`, null, true) + }, async playSelectedItems() { this.$store.commit('setProcessingBatch', true) From 9b79aab4d5b8604f8535d32340b965d65f665818 Mon Sep 17 00:00:00 2001 From: Vito0912 <86927734+Vito0912@users.noreply.github.com> Date: Mon, 17 Mar 2025 19:58:55 +0100 Subject: [PATCH 3/7] logging --- server/controllers/LibraryController.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index f51b4974..5ca80115 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -1452,15 +1452,24 @@ class LibraryController { const filename = `LibraryItems-${Date.now()}.zip` const libraryItemPaths = libraryItems.map((li) => li.path) - - console.log(libraryItemPaths) + try { await zipHelpers.zipDirectoriesPipe(libraryItemPaths, filename, res) Logger.info(`[LibraryItemController] Downloaded item "${filename}" at "${libraryItemPaths}"`) } catch (error) { Logger.error(`[LibraryItemController] Download failed for item "${filename}" at "${libraryItemPaths}"`, error) - //LibraryItemController.handleDownloadError(error, res) + LibraryController.handleDownloadError(error, res) + } + } + + static handleDownloadError(error, res) { + if (!res.headersSent) { + if (error.code === 'ENOENT') { + return res.status(404).send('File not found') + } else { + return res.status(500).send('Download failed') + } } } From 3c9966e84917196fac1578d5519662339effeb2a Mon Sep 17 00:00:00 2001 From: Vito0912 <86927734+Vito0912@users.noreply.github.com> Date: Mon, 17 Mar 2025 20:04:01 +0100 Subject: [PATCH 4/7] clean up --- server/controllers/LibraryController.js | 22 ++++++++-------------- server/utils/zipHelpers.js | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 5ca80115..673c5cf2 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -1433,8 +1433,8 @@ class LibraryController { return res.sendStatus(403) } - if(req.query.ids === undefined) { - res.status(400).send('Library not found') + if(req.query.ids === undefined || req.query.ids === '') { + res.status(400).send('Library items not found') } const itemIds = req.query.ids.split(',') @@ -1452,24 +1452,18 @@ class LibraryController { const filename = `LibraryItems-${Date.now()}.zip` const libraryItemPaths = libraryItems.map((li) => li.path) - + + if (!libraryItemPaths.length) { + Logger.warn(`[LibraryItemController] No library items found for ids "${itemIds}"`) + return res.status(404).send('Library items not found') + } try { await zipHelpers.zipDirectoriesPipe(libraryItemPaths, filename, res) Logger.info(`[LibraryItemController] Downloaded item "${filename}" at "${libraryItemPaths}"`) } catch (error) { Logger.error(`[LibraryItemController] Download failed for item "${filename}" at "${libraryItemPaths}"`, error) - LibraryController.handleDownloadError(error, res) - } - } - - static handleDownloadError(error, res) { - if (!res.headersSent) { - if (error.code === 'ENOENT') { - return res.status(404).send('File not found') - } else { - return res.status(500).send('Download failed') - } + zipHelpers.handleDownloadError(error, res) } } diff --git a/server/utils/zipHelpers.js b/server/utils/zipHelpers.js index 1145f3a0..6849df5d 100644 --- a/server/utils/zipHelpers.js +++ b/server/utils/zipHelpers.js @@ -123,3 +123,20 @@ module.exports.zipDirectoriesPipe = (paths, filename, res) => { archive.finalize() }) } + +/** + * Handles errors that occur during the download process. + * + * @param error + * @param res + * @returns {*} + */ +module.exports.handleDownloadError = (error, res) => { + if (!res.headersSent) { + if (error.code === 'ENOENT') { + return res.status(404).send('File not found') + } else { + return res.status(500).send('Download failed') + } + } +} From 1def32aa5053bb0a431920ffdcafb1aa29f7c1ab Mon Sep 17 00:00:00 2001 From: advplyr Date: Tue, 18 Mar 2025 17:03:43 -0500 Subject: [PATCH 5/7] Fix req.query check and response --- server/controllers/LibraryController.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 673c5cf2..1d2d5bfa 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -1424,7 +1424,7 @@ class LibraryController { * GET: /api/library/:id/download * Downloads multiple library items * - * @param {LibraryItemControllerRequest} req + * @param {LibraryControllerRequest} req * @param {Response} res */ async downloadMultiple(req, res) { @@ -1433,8 +1433,9 @@ class LibraryController { return res.sendStatus(403) } - if(req.query.ids === undefined || req.query.ids === '') { - res.status(400).send('Library items not found') + if (!req.query.ids || typeof req.query.ids !== 'string') { + res.status(400).send('Invalid request. ids must be a string') + return } const itemIds = req.query.ids.split(',') @@ -1467,7 +1468,6 @@ class LibraryController { } } - /** * * @param {RequestWithUser} req From ff36a9327cab943672d053f857443fd89f0bd88b Mon Sep 17 00:00:00 2001 From: advplyr Date: Tue, 18 Mar 2025 17:28:49 -0500 Subject: [PATCH 6/7] Fix multiple download for podcasts & cleanup --- server/controllers/LibraryController.js | 20 ++++++++--------- server/utils/zipHelpers.js | 29 ++++++++++--------------- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 1d2d5bfa..f9aeba3f 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -1441,29 +1441,27 @@ class LibraryController { const itemIds = req.query.ids.split(',') const libraryItems = await Database.libraryItemModel.findAll({ - attributes: ['id', 'libraryId', 'path'], + attributes: ['id', 'libraryId', 'path', 'isFile'], where: { - id: itemIds, - libraryId: req.params.id, - mediaType: 'book' + id: itemIds } }) - Logger.info(`[LibraryItemController] User "${req.user.username}" requested download for items "${itemIds}"`) + Logger.info(`[LibraryController] User "${req.user.username}" requested download for items "${itemIds}"`) const filename = `LibraryItems-${Date.now()}.zip` - const libraryItemPaths = libraryItems.map((li) => li.path) + const pathObjects = libraryItems.map((li) => ({ path: li.path, isFile: li.isFile })) - if (!libraryItemPaths.length) { - Logger.warn(`[LibraryItemController] No library items found for ids "${itemIds}"`) + if (!pathObjects.length) { + Logger.warn(`[LibraryController] No library items found for ids "${itemIds}"`) return res.status(404).send('Library items not found') } try { - await zipHelpers.zipDirectoriesPipe(libraryItemPaths, filename, res) - Logger.info(`[LibraryItemController] Downloaded item "${filename}" at "${libraryItemPaths}"`) + await zipHelpers.zipDirectoriesPipe(pathObjects, filename, res) + Logger.info(`[LibraryController] Downloaded ${pathObjects.length} items "${filename}"`) } catch (error) { - Logger.error(`[LibraryItemController] Download failed for item "${filename}" at "${libraryItemPaths}"`, error) + Logger.error(`[LibraryController] Download failed for items "${filename}" at ${pathObjects.map((po) => po.path).join(', ')}`, error) zipHelpers.handleDownloadError(error, res) } } diff --git a/server/utils/zipHelpers.js b/server/utils/zipHelpers.js index 6849df5d..421ca73c 100644 --- a/server/utils/zipHelpers.js +++ b/server/utils/zipHelpers.js @@ -1,6 +1,7 @@ +const Path = require('path') +const { Response } = require('express') const Logger = require('../Logger') const archiver = require('../libs/archiver') -const { lstatSync } = require('node:fs') module.exports.zipDirectoryPipe = (path, filename, res) => { return new Promise((resolve, reject) => { @@ -55,12 +56,12 @@ module.exports.zipDirectoryPipe = (path, filename, res) => { /** * Creates a zip archive containing multiple directories and streams it to the response. * - * @param {string[]} paths - Array of directory paths to include in the zip archive. + * @param {{ path: string, isFile: boolean }[]} pathObjects * @param {string} filename - Name of the zip file to be sent as attachment. - * @param {object} res - Response object to pipe the archive data to. + * @param {Response} res - Response object to pipe the archive data to. * @returns {Promise} - Promise that resolves when the zip operation completes. */ -module.exports.zipDirectoriesPipe = (paths, filename, res) => { +module.exports.zipDirectoriesPipe = (pathObjects, filename, res) => { return new Promise((resolve, reject) => { // create a file to stream archive data to res.attachment(filename) @@ -105,20 +106,14 @@ module.exports.zipDirectoriesPipe = (paths, filename, res) => { archive.pipe(res) // Add each path as a directory in the zip - paths.forEach(path => { - - const paths = path.split('/') - - // Check if path is file or directory - if (lstatSync(path).isDirectory()) { - const dirName = path.split('/').pop() - + pathObjects.forEach((pathObject) => { + if (!pathObject.isFile) { // Add the directory to the archive with its name as the root folder - archive.directory(path, dirName); + archive.directory(pathObject.path, Path.basename(pathObject.path)) } else { - archive.file(path, { name: paths[paths.length - 1] }); + archive.file(pathObject.path, { name: Path.basename(pathObject.path) }) } - }); + }) archive.finalize() }) @@ -127,8 +122,8 @@ module.exports.zipDirectoriesPipe = (paths, filename, res) => { /** * Handles errors that occur during the download process. * - * @param error - * @param res + * @param {*} error + * @param {Response} res * @returns {*} */ module.exports.handleDownloadError = (error, res) => { From 3b7db82bf0e2ff6679643589a8b4489655299cc3 Mon Sep 17 00:00:00 2001 From: advplyr Date: Tue, 18 Mar 2025 17:33:46 -0500 Subject: [PATCH 7/7] Update bulk download to not open in new tab --- client/components/app/Appbar.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/components/app/Appbar.vue b/client/components/app/Appbar.vue index 6fd2bc3e..d40794c3 100644 --- a/client/components/app/Appbar.vue +++ b/client/components/app/Appbar.vue @@ -182,7 +182,7 @@ export default { // The limit of 50 is introduced because of the URL length. Each id has 36 chars, so 36 * 40 = 1440 // + 40 , separators = 1480 chars + base path 280 chars = 1760 chars. This keeps the URL under 2000 chars even with longer domains - if(this.selectedMediaItems.length <= 40) { + if (this.selectedMediaItems.length <= 40) { options.push({ text: this.$strings.LabelDownload, action: 'download' @@ -255,7 +255,7 @@ export default { async batchDownload() { const libraryItemIds = this.selectedMediaItems.map((i) => i.id) console.log('Downloading library items', libraryItemIds) - this.$downloadFile(`/api/libraries/${this.$store.state.libraries.currentLibraryId}/download?token=${this.$store.getters['user/getToken']}&ids=${libraryItemIds.join(',')}`, null, true) + this.$downloadFile(`/api/libraries/${this.$store.state.libraries.currentLibraryId}/download?token=${this.$store.getters['user/getToken']}&ids=${libraryItemIds.join(',')}`) }, async playSelectedItems() { this.$store.commit('setProcessingBatch', true)