From 4e6b75d6506d94c19fad81ae62cbac0c3370fd1e Mon Sep 17 00:00:00 2001 From: jfrazx Date: Thu, 5 Oct 2023 13:48:55 -0700 Subject: [PATCH 1/6] fix; HTTP/429 when requesting authors information, resolves #1570 --- package-lock.json | 29 ++++++++- package.json | 3 +- server/providers/Audnexus.js | 117 ++++++++++++++++++++++++----------- 3 files changed, 111 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index 77948004..a1e7ccd6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "express": "^4.17.1", "graceful-fs": "^4.2.10", "htmlparser2": "^8.0.1", + "limiter": "^2.1.0", "node-tone": "^1.0.1", "nodemailer": "^6.9.2", "sequelize": "^6.32.1", @@ -1308,6 +1309,19 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "optional": true }, + "node_modules/just-performance": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.3.0.tgz", + "integrity": "sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q==" + }, + "node_modules/limiter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/limiter/-/limiter-2.1.0.tgz", + "integrity": "sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw==", + "dependencies": { + "just-performance": "4.3.0" + } + }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -3673,6 +3687,19 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "optional": true }, + "just-performance": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.3.0.tgz", + "integrity": "sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q==" + }, + "limiter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/limiter/-/limiter-2.1.0.tgz", + "integrity": "sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw==", + "requires": { + "just-performance": "4.3.0" + } + }, "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -4672,4 +4699,4 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } -} \ No newline at end of file +} diff --git a/package.json b/package.json index 4a00fa59..082006e8 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "express": "^4.17.1", "graceful-fs": "^4.2.10", "htmlparser2": "^8.0.1", + "limiter": "^2.1.0", "node-tone": "^1.0.1", "nodemailer": "^6.9.2", "sequelize": "^6.32.1", @@ -44,4 +45,4 @@ "devDependencies": { "nodemon": "^2.0.20" } -} \ No newline at end of file +} diff --git a/server/providers/Audnexus.js b/server/providers/Audnexus.js index b74d1d13..06433f5d 100644 --- a/server/providers/Audnexus.js +++ b/server/providers/Audnexus.js @@ -1,78 +1,123 @@ const axios = require('axios') const { levenshteinDistance } = require('../utils/index') const Logger = require('../Logger') +const { RateLimiter } = require('limiter'); class Audnexus { + static _instance = null; + constructor() { + // ensures Audnexus class is singleton + if (Audnexus._instance) { + return Audnexus._instance + } + this.baseUrl = 'https://api.audnex.us' + + // @see https://github.com/laxamentumtech/audnexus#-deployment- + this.limiter = new RateLimiter({ + tokensPerInterval: 100, + fireImmediately: true, + interval: 'minute', + }) + + Audnexus._instance = this } authorASINsRequest(name, region) { name = encodeURIComponent(name) const regionQuery = region ? `®ion=${region}` : '' const authorRequestUrl = `${this.baseUrl}/authors?name=${name}${regionQuery}` + Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - return axios.get(authorRequestUrl).then((res) => { - return res.data || [] - }).catch((error) => { - Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error) - return [] - }) + + return this._processRequest(() => axios.get(authorRequestUrl)) + .then((res) => res.data || []) + .catch((error) => { + Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error) + return [] + }) } authorRequest(asin, region) { asin = encodeURIComponent(asin) const regionQuery = region ? `?region=${region}` : '' const authorRequestUrl = `${this.baseUrl}/authors/${asin}${regionQuery}` + Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - return axios.get(authorRequestUrl).then((res) => { - return res.data - }).catch((error) => { - Logger.error(`[Audnexus] Author request failed for ${asin}`, error) - return null - }) + + return this._processRequest(() => axios.get(authorRequestUrl)) + .then((res) => res.data) + .catch((error) => { + Logger.error(`[Audnexus] Author request failed for ${asin}`, error) + return null + }) + } + + /** + * @description Process a request with a rate limiter + * + * @param {*} request + * @returns + */ + async _processRequest(request) { + const remainingTokens = await this.limiter.removeTokens(1) + Logger.info(`[Audnexus] Attempting request with ${remainingTokens} remaining tokens and ${this.limiter.tokensThisInterval} this interval`) + + if (remainingTokens >= 1) { + return request() + } + + // 100 tokens(requests) per minute give a refresh of ~1.67 per second, + // so a 10 second wait will yield ~16.7 additional tokens + Logger.info('[Audnexus] Sleeping for 10 seconds') + await new Promise(resolve => setTimeout(resolve, 10000)) + + return this._processRequest(request) } async findAuthorByASIN(asin, region) { const author = await this.authorRequest(asin, region) - if (!author) { - return null - } - return { - asin: author.asin, - description: author.description, - image: author.image || null, - name: author.name - } + + return author ? + { + asin: author.asin, + description: author.description, + image: author.image || null, + name: author.name + } : null } async findAuthorByName(name, region, maxLevenshtein = 3) { Logger.debug(`[Audnexus] Looking up author by name ${name}`) + const asins = await this.authorASINsRequest(name, region) const matchingAsin = asins.find(obj => levenshteinDistance(obj.name, name) <= maxLevenshtein) + if (!matchingAsin) { return null } + const author = await this.authorRequest(matchingAsin.asin) - if (!author) { - return null - } - return { - asin: author.asin, - description: author.description, - image: author.image || null, - name: author.name - } + return author ? + { + description: author.description, + image: author.image || null, + asin: author.asin, + name: author.name + } : null } getChaptersByASIN(asin, region) { Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`) - return axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`).then((res) => { - return res.data - }).catch((error) => { - Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error) - return null - }) + + return this._processRequest(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`)) + .then((res) => res.data) + .catch((error) => { + Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error) + return null + }) } } + module.exports = Audnexus \ No newline at end of file From 73c21242b4a70f0314bbfc62dd2841a38afc4bcb Mon Sep 17 00:00:00 2001 From: jfrazx Date: Mon, 22 Jan 2024 20:36:20 -0800 Subject: [PATCH 2/6] feat: utilize p-throttle instad of limiter --- package-lock.json | 26 +++++++++---------- package.json | 2 +- server/providers/Audnexus.js | 48 +++++++++++------------------------- 3 files changed, 28 insertions(+), 48 deletions(-) diff --git a/package-lock.json b/package-lock.json index 65316f65..30b40e28 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,11 +15,11 @@ "express-session": "^1.17.3", "graceful-fs": "^4.2.10", "htmlparser2": "^8.0.1", - "limiter": "^2.1.0", "lru-cache": "^10.0.3", "node-tone": "^1.0.1", "nodemailer": "^6.9.2", "openid-client": "^5.6.1", + "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", "sequelize": "^6.35.2", @@ -2841,11 +2841,6 @@ "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==", "dev": true }, - "node_modules/just-performance": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.3.0.tgz", - "integrity": "sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q==" - }, "node_modules/jwa": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", @@ -2865,14 +2860,6 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/limiter": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/limiter/-/limiter-2.1.0.tgz", - "integrity": "sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw==", - "dependencies": { - "just-performance": "4.3.0" - } - }, "node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -3977,6 +3964,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-throttle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-4.1.1.tgz", + "integrity": "sha512-TuU8Ato+pRTPJoDzYD4s7ocJYcNSEZRvlxoq3hcPI2kZDZ49IQ1Wkj7/gDJc3X7XiEAAvRGtDzdXJI0tC3IL1g==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", diff --git a/package.json b/package.json index 65dbdd1e..46624752 100644 --- a/package.json +++ b/package.json @@ -41,11 +41,11 @@ "express-session": "^1.17.3", "graceful-fs": "^4.2.10", "htmlparser2": "^8.0.1", - "limiter": "^2.1.0", "lru-cache": "^10.0.3", "node-tone": "^1.0.1", "nodemailer": "^6.9.2", "openid-client": "^5.6.1", + "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", "sequelize": "^6.35.2", diff --git a/server/providers/Audnexus.js b/server/providers/Audnexus.js index d9223374..14eab4a1 100644 --- a/server/providers/Audnexus.js +++ b/server/providers/Audnexus.js @@ -1,7 +1,7 @@ const axios = require('axios') const { levenshteinDistance } = require('../utils/index') const Logger = require('../Logger') -const { RateLimiter } = require('limiter') +const pThrottle = require('p-throttle') class Audnexus { static _instance = null @@ -14,11 +14,15 @@ class Audnexus { this.baseUrl = 'https://api.audnex.us' + // Rate limit is 100 requests per minute. // @see https://github.com/laxamentumtech/audnexus#-deployment- - this.limiter = new RateLimiter({ - tokensPerInterval: 100, - fireImmediately: true, - interval: 'minute', + this.limiter = pThrottle({ + // Setting the limit to 1 allows for a short pause between requests that is almost imperceptible to + // the end user. A larger limit will grab blocks faster and then wait for the alloted time(interval) before + // fetching another batch. + limit: 1, + strict: true, + interval: 300 }) Audnexus._instance = this @@ -31,8 +35,8 @@ class Audnexus { Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - return this._processRequest(() => axios.get(authorRequestUrl)) - .then((res) => res.data || []) + const throttle = this.limiter(() => axios.get(authorRequestUrl)) + return throttle().then((res) => res.data || []) .catch((error) => { Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error) return [] @@ -46,36 +50,14 @@ class Audnexus { Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - return this._processRequest(() => axios.get(authorRequestUrl)) - .then((res) => res.data) + const throttle = this.limiter(() => axios.get(authorRequestUrl)) + return throttle().then((res) => res.data) .catch((error) => { Logger.error(`[Audnexus] Author request failed for ${asin}`, error) return null }) } - /** - * @description Process a request with a rate limiter - * - * @param {*} request - * @returns - */ - async _processRequest(request) { - const remainingTokens = await this.limiter.removeTokens(1) - Logger.info(`[Audnexus] Attempting request with ${remainingTokens} remaining tokens and ${this.limiter.tokensThisInterval} this interval`) - - if (remainingTokens >= 1) { - return request() - } - - // 100 tokens(requests) per minute give a refresh of ~1.67 per second, - // so a 10 second wait will yield ~16.7 additional tokens - Logger.info('[Audnexus] Sleeping for 10 seconds') - await new Promise(resolve => setTimeout(resolve, 10000)) - - return this._processRequest(request) - } - async findAuthorByASIN(asin, region) { const author = await this.authorRequest(asin, region) @@ -111,8 +93,8 @@ class Audnexus { getChaptersByASIN(asin, region) { Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`) - return this._processRequest(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`)) - .then((res) => res.data) + const throttle = this.limiter(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`)) + return throttle().then((res) => res.data) .catch((error) => { Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error) return null From 70827727aa8edfff13e8acfd8fe5c486b7f254bd Mon Sep 17 00:00:00 2001 From: jfrazx Date: Mon, 22 Jan 2024 22:19:05 -0800 Subject: [PATCH 3/6] feat(429): retry 429 request errors --- server/providers/Audnexus.js | 48 ++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/server/providers/Audnexus.js b/server/providers/Audnexus.js index 14eab4a1..9e4dc6b7 100644 --- a/server/providers/Audnexus.js +++ b/server/providers/Audnexus.js @@ -1,7 +1,7 @@ const axios = require('axios') const { levenshteinDistance } = require('../utils/index') const Logger = require('../Logger') -const pThrottle = require('p-throttle') +const Throttle = require('p-throttle') class Audnexus { static _instance = null @@ -16,13 +16,13 @@ class Audnexus { // Rate limit is 100 requests per minute. // @see https://github.com/laxamentumtech/audnexus#-deployment- - this.limiter = pThrottle({ - // Setting the limit to 1 allows for a short pause between requests that is almost imperceptible to - // the end user. A larger limit will grab blocks faster and then wait for the alloted time(interval) before - // fetching another batch. + this.limiter = Throttle({ + // Setting the limit to 1 allows for a short pause between requests that is imperceptible to the end user. + // A larger limit will grab blocks faster and then wait for the alloted time(interval) before + // fetching another batch, but with a discernable pause from the user perspective. limit: 1, strict: true, - interval: 300 + interval: 150 }) Audnexus._instance = this @@ -35,8 +35,8 @@ class Audnexus { Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - const throttle = this.limiter(() => axios.get(authorRequestUrl)) - return throttle().then((res) => res.data || []) + return this._processRequest(this.limiter(() => axios.get(authorRequestUrl))) + .then((res) => res.data || []) .catch((error) => { Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error) return [] @@ -50,8 +50,8 @@ class Audnexus { Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - const throttle = this.limiter(() => axios.get(authorRequestUrl)) - return throttle().then((res) => res.data) + return this._processRequest(this.limiter(() => axios.get(authorRequestUrl))) + .then((res) => res.data) .catch((error) => { Logger.error(`[Audnexus] Author request failed for ${asin}`, error) return null @@ -93,13 +93,35 @@ class Audnexus { getChaptersByASIN(asin, region) { Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`) - const throttle = this.limiter(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`)) - return throttle().then((res) => res.data) + return this._processRequest(this.limiter(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`))) + .then((res) => res.data) .catch((error) => { Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error) return null }) } + + /** + * Internal method to process requests and retry if rate limit is exceeded. + */ + async _processRequest(request) { + try { + const response = await request() + return response + } catch (error) { + if (error.response?.status === 429) { + const retryAfter = parseInt(error.response.headers?.['retry-after'], 10) || 5 + + Logger.warn(`[Audnexus] Rate limit exceeded. Retrying in ${retryAfter} seconds.`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + + return this._processRequest(request) + } + + throw error + } + } } -module.exports = Audnexus \ No newline at end of file +module.exports = Audnexus + From ee501f70ed161055ca66e0a8769b15ba0da5cd69 Mon Sep 17 00:00:00 2001 From: advplyr Date: Sun, 9 Jun 2024 12:51:28 -0500 Subject: [PATCH 4/6] Auto-formatting --- server/providers/Audnexus.js | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/server/providers/Audnexus.js b/server/providers/Audnexus.js index 1756bfd8..e34402e4 100644 --- a/server/providers/Audnexus.js +++ b/server/providers/Audnexus.js @@ -73,8 +73,8 @@ class Audnexus { Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) return this._processRequest(this.limiter(() => axios.get(authorRequestUrl))) - .then(res => res.data) - .catch(error => { + .then((res) => res.data) + .catch((error) => { Logger.error(`[Audnexus] Author request failed for ${asin}`, error) return null }) @@ -89,13 +89,14 @@ class Audnexus { async findAuthorByASIN(asin, region) { const author = await this.authorRequest(asin, region) - return author ? - { - asin: author.asin, - description: author.description, - image: author.image || null, - name: author.name - } : null + return author + ? { + asin: author.asin, + description: author.description, + image: author.image || null, + name: author.name + } + : null } /** @@ -138,8 +139,8 @@ class Audnexus { Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`) return this._processRequest(this.limiter(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`))) - .then(res => res.data) - .catch(error => { + .then((res) => res.data) + .catch((error) => { Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error) return null }) @@ -156,7 +157,7 @@ class Audnexus { const retryAfter = parseInt(error.response.headers?.['retry-after'], 10) || 5 Logger.warn(`[Audnexus] Rate limit exceeded. Retrying in ${retryAfter} seconds.`) - await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000)) return this._processRequest(request) } From a018374d26d2ec4a7e4bbebc03eda64df27d6e62 Mon Sep 17 00:00:00 2001 From: advplyr Date: Sun, 9 Jun 2024 13:43:03 -0500 Subject: [PATCH 5/6] Update:Validate ASIN for author, chapter and match requests --- server/controllers/AuthorController.js | 4 +-- server/controllers/SearchController.js | 18 ++++++---- server/providers/Audible.js | 15 ++------ server/providers/Audnexus.js | 31 ++++++++++++---- server/utils/index.js | 49 +++++++++++++++++--------- 5 files changed, 72 insertions(+), 45 deletions(-) diff --git a/server/controllers/AuthorController.js b/server/controllers/AuthorController.js index 52f2aa01..79f48dbd 100644 --- a/server/controllers/AuthorController.js +++ b/server/controllers/AuthorController.js @@ -9,7 +9,7 @@ const CacheManager = require('../managers/CacheManager') const CoverManager = require('../managers/CoverManager') const AuthorFinder = require('../finders/AuthorFinder') -const { reqSupportsWebp } = require('../utils/index') +const { reqSupportsWebp, isValidASIN } = require('../utils/index') const naturalSort = createNewSortInstance({ comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare @@ -252,7 +252,7 @@ class AuthorController { async match(req, res) { let authorData = null const region = req.body.region || 'us' - if (req.body.asin) { + if (req.body.asin && isValidASIN(req.body.asin.toUpperCase?.())) { authorData = await AuthorFinder.findAuthorByASIN(req.body.asin, region) } else { authorData = await AuthorFinder.findAuthorByName(req.body.q, region) diff --git a/server/controllers/SearchController.js b/server/controllers/SearchController.js index 213d23e1..b0aebb31 100644 --- a/server/controllers/SearchController.js +++ b/server/controllers/SearchController.js @@ -1,12 +1,13 @@ -const Logger = require("../Logger") +const Logger = require('../Logger') const BookFinder = require('../finders/BookFinder') const PodcastFinder = require('../finders/PodcastFinder') const AuthorFinder = require('../finders/AuthorFinder') const MusicFinder = require('../finders/MusicFinder') -const Database = require("../Database") +const Database = require('../Database') +const { isValidASIN } = require('../utils') class SearchController { - constructor() { } + constructor() {} async findBooks(req, res) { const id = req.query.id @@ -37,9 +38,9 @@ class SearchController { /** * Find podcast RSS feeds given a term - * - * @param {import('express').Request} req - * @param {import('express').Response} res + * + * @param {import('express').Request} req + * @param {import('express').Response} res */ async findPodcasts(req, res) { const term = req.query.term @@ -63,6 +64,9 @@ class SearchController { async findChapters(req, res) { const asin = req.query.asin + if (!isValidASIN(asin.toUpperCase())) { + return res.json({ error: 'Invalid ASIN' }) + } const region = (req.query.region || 'us').toLowerCase() const chapterData = await BookFinder.findChapters(asin, region) if (!chapterData) { @@ -78,4 +82,4 @@ class SearchController { }) } } -module.exports = new SearchController() \ No newline at end of file +module.exports = new SearchController() diff --git a/server/providers/Audible.js b/server/providers/Audible.js index 96c6cccb..76225613 100644 --- a/server/providers/Audible.js +++ b/server/providers/Audible.js @@ -1,6 +1,7 @@ const axios = require('axios').default const htmlSanitizer = require('../utils/htmlSanitizer') const Logger = require('../Logger') +const { isValidASIN } = require('../utils/index') class Audible { #responseTimeout = 30000 @@ -81,16 +82,6 @@ class Audible { } } - /** - * Test if a search title matches an ASIN. Supports lowercase letters - * - * @param {string} title - * @returns {boolean} - */ - isProbablyAsin(title) { - return /^[0-9A-Za-z]{10}$/.test(title) - } - /** * * @param {string} asin @@ -137,11 +128,11 @@ class Audible { if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout let items - if (asin) { + if (asin && isValidASIN(asin.toUpperCase())) { items = [await this.asinSearch(asin, region, timeout)] } - if (!items && this.isProbablyAsin(title)) { + if (!items && isValidASIN(title.toUpperCase())) { items = [await this.asinSearch(title, region, timeout)] } diff --git a/server/providers/Audnexus.js b/server/providers/Audnexus.js index e34402e4..60762ede 100644 --- a/server/providers/Audnexus.js +++ b/server/providers/Audnexus.js @@ -1,7 +1,8 @@ const axios = require('axios').default -const { levenshteinDistance } = require('../utils/index') -const Logger = require('../Logger') const Throttle = require('p-throttle') +const Logger = require('../Logger') +const { levenshteinDistance } = require('../utils/index') +const { isValidASIN } = require('../utils/index') /** * @typedef AuthorSearchObj @@ -66,13 +67,19 @@ class Audnexus { * @returns {Promise} */ authorRequest(asin, region) { - asin = encodeURIComponent(asin) - const regionQuery = region ? `?region=${region}` : '' - const authorRequestUrl = `${this.baseUrl}/authors/${asin}${regionQuery}` + if (!isValidASIN(asin?.toUpperCase?.())) { + Logger.error(`[Audnexus] Invalid ASIN ${asin}`) + return null + } + + asin = encodeURIComponent(asin.toUpperCase()) + + const authorRequestUrl = new URL(`${this.baseUrl}/authors/${asin}`) + if (region) authorRequestUrl.searchParams.set('region', region) Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - return this._processRequest(this.limiter(() => axios.get(authorRequestUrl))) + return this._processRequest(this.limiter(() => axios.get(authorRequestUrl.toString()))) .then((res) => res.data) .catch((error) => { Logger.error(`[Audnexus] Author request failed for ${asin}`, error) @@ -135,10 +142,20 @@ class Audnexus { } } + /** + * + * @param {string} asin + * @param {string} region + * @returns {Promise} + */ getChaptersByASIN(asin, region) { Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`) - return this._processRequest(this.limiter(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`))) + asin = encodeURIComponent(asin.toUpperCase()) + const chaptersRequestUrl = new URL(`${this.baseUrl}/books/${asin}/chapters`) + if (region) chaptersRequestUrl.searchParams.set('region', region) + + return this._processRequest(this.limiter(() => axios.get(chaptersRequestUrl.toString()))) .then((res) => res.data) .catch((error) => { Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error) diff --git a/server/utils/index.js b/server/utils/index.js index 6a89621b..14f297c1 100644 --- a/server/utils/index.js +++ b/server/utils/index.js @@ -1,7 +1,7 @@ const Path = require('path') const uuid = require('uuid') const Logger = require('../Logger') -const { parseString } = require("xml2js") +const { parseString } = require('xml2js') const areEquivalent = require('./areEquivalent') const levenshteinDistance = (str1, str2, caseSensitive = false) => { @@ -11,8 +11,9 @@ const levenshteinDistance = (str1, str2, caseSensitive = false) => { str1 = str1.toLowerCase() str2 = str2.toLowerCase() } - const track = Array(str2.length + 1).fill(null).map(() => - Array(str1.length + 1).fill(null)) + const track = Array(str2.length + 1) + .fill(null) + .map(() => Array(str1.length + 1).fill(null)) for (let i = 0; i <= str1.length; i += 1) { track[0][i] = i } @@ -25,7 +26,7 @@ const levenshteinDistance = (str1, str2, caseSensitive = false) => { track[j][i] = Math.min( track[j][i - 1] + 1, // deletion track[j - 1][i] + 1, // insertion - track[j - 1][i - 1] + indicator, // substitution + track[j - 1][i - 1] + indicator // substitution ) } } @@ -138,7 +139,10 @@ module.exports.toNumber = (val, fallback = 0) => { module.exports.cleanStringForSearch = (str) => { if (!str) return '' // Remove ' . ` " , - return str.toLowerCase().replace(/[\'\.\`\",]/g, '').trim() + return str + .toLowerCase() + .replace(/[\'\.\`\",]/g, '') + .trim() } const getTitleParts = (title) => { @@ -156,7 +160,7 @@ const getTitleParts = (title) => { /** * Remove sortingPrefixes from title * @example "The Good Book" => "Good Book" - * @param {string} title + * @param {string} title * @returns {string} */ module.exports.getTitleIgnorePrefix = (title) => { @@ -164,9 +168,9 @@ module.exports.getTitleIgnorePrefix = (title) => { } /** - * Put sorting prefix at the end of title + * Put sorting prefix at the end of title * @example "The Good Book" => "Good Book, The" - * @param {string} title + * @param {string} title * @returns {string} */ module.exports.getTitlePrefixAtEnd = (title) => { @@ -178,8 +182,8 @@ module.exports.getTitlePrefixAtEnd = (title) => { * to lower case for only ascii characters * used to handle sqlite that doesnt support unicode lower * @see https://github.com/advplyr/audiobookshelf/issues/2187 - * - * @param {string} str + * + * @param {string} str * @returns {string} */ module.exports.asciiOnlyToLowerCase = (str) => { @@ -200,8 +204,8 @@ module.exports.asciiOnlyToLowerCase = (str) => { /** * Escape string used in RegExp * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping - * - * @param {string} str + * + * @param {string} str * @returns {string} */ module.exports.escapeRegExp = (str) => { @@ -211,8 +215,8 @@ module.exports.escapeRegExp = (str) => { /** * Validate url string with URL class - * - * @param {string} rawUrl + * + * @param {string} rawUrl * @returns {string} null if invalid */ module.exports.validateUrl = (rawUrl) => { @@ -227,11 +231,22 @@ module.exports.validateUrl = (rawUrl) => { /** * Check if a string is a valid UUID - * - * @param {string} str + * + * @param {string} str * @returns {boolean} */ module.exports.isUUID = (str) => { if (!str || typeof str !== 'string') return false return uuid.validate(str) -} \ No newline at end of file +} + +/** + * Check if a string is a valid ASIN + * + * @param {string} str + * @returns {boolean} + */ +module.exports.isValidASIN = (str) => { + if (!str || typeof str !== 'string') return false + return /^[A-Z0-9]{10}$/.test(str) +} From c2897f819d8195ffe3af58c279f9d355d7f69f90 Mon Sep 17 00:00:00 2001 From: advplyr Date: Sun, 9 Jun 2024 13:55:53 -0500 Subject: [PATCH 6/6] Update:findEpisode API endpoint validate title search param is a string --- server/controllers/PodcastController.js | 56 ++++++++++++++----------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/server/controllers/PodcastController.js b/server/controllers/PodcastController.js index e476efd5..11985486 100644 --- a/server/controllers/PodcastController.js +++ b/server/controllers/PodcastController.js @@ -14,7 +14,6 @@ const CoverManager = require('../managers/CoverManager') const LibraryItem = require('../objects/LibraryItem') class PodcastController { - async create(req, res) { if (!req.user.isAdminOrUp) { Logger.error(`[PodcastController] Non-admin user "${req.user.username}" attempted to create podcast`) @@ -28,7 +27,7 @@ class PodcastController { return res.status(404).send('Library not found') } - const folder = library.folders.find(fold => fold.id === payload.folderId) + const folder = library.folders.find((fold) => fold.id === payload.folderId) if (!folder) { Logger.error(`[PodcastController] Create: Folder not found "${payload.folderId}"`) return res.status(404).send('Folder not found') @@ -37,20 +36,24 @@ class PodcastController { const podcastPath = filePathToPOSIX(payload.path) // Check if a library item with this podcast folder exists already - const existingLibraryItem = (await Database.libraryItemModel.count({ - where: { - path: podcastPath - } - })) > 0 + const existingLibraryItem = + (await Database.libraryItemModel.count({ + where: { + path: podcastPath + } + })) > 0 if (existingLibraryItem) { Logger.error(`[PodcastController] Podcast already exists at path "${podcastPath}"`) return res.status(400).send('Podcast already exists') } - const success = await fs.ensureDir(podcastPath).then(() => true).catch((error) => { - Logger.error(`[PodcastController] Failed to ensure podcast dir "${podcastPath}"`, error) - return false - }) + const success = await fs + .ensureDir(podcastPath) + .then(() => true) + .catch((error) => { + Logger.error(`[PodcastController] Failed to ensure podcast dir "${podcastPath}"`, error) + return false + }) if (!success) return res.status(400).send('Invalid podcast path') const libraryItemFolderStats = await getFileTimestampsWithIno(podcastPath) @@ -105,12 +108,12 @@ class PodcastController { /** * POST: /api/podcasts/feed - * + * * @typedef getPodcastFeedReqBody * @property {string} rssFeed - * - * @param {import('express').Request<{}, {}, getPodcastFeedReqBody, {}} req - * @param {import('express').Response} res + * + * @param {import('express').Request<{}, {}, getPodcastFeedReqBody, {}} req + * @param {import('express').Response} res */ async getPodcastFeed(req, res) { if (!req.user.isAdminOrUp) { @@ -178,7 +181,7 @@ class PodcastController { var downloadsInQueue = this.podcastManager.getEpisodeDownloadsInQueue(libraryItem.id) res.json({ - downloads: downloadsInQueue.map(d => d.toJSONForClient()) + downloads: downloadsInQueue.map((d) => d.toJSONForClient()) }) } @@ -189,8 +192,8 @@ class PodcastController { return res.status(500).send('Podcast does not have an RSS feed URL') } - var searchTitle = req.query.title - if (!searchTitle) { + const searchTitle = req.query.title + if (!searchTitle || typeof searchTitle !== 'string') { return res.sendStatus(500) } const episodes = await findMatchingEpisodes(rssFeedUrl, searchTitle) @@ -254,7 +257,7 @@ class PodcastController { const episodeId = req.params.episodeId const libraryItem = req.libraryItem - const episode = libraryItem.media.episodes.find(ep => ep.id === episodeId) + const episode = libraryItem.media.episodes.find((ep) => ep.id === episodeId) if (!episode) { Logger.error(`[PodcastController] getEpisode episode ${episodeId} not found for item ${libraryItem.id}`) return res.sendStatus(404) @@ -269,7 +272,7 @@ class PodcastController { const libraryItem = req.libraryItem const hardDelete = req.query.hard === '1' - const episode = libraryItem.media.episodes.find(ep => ep.id === episodeId) + const episode = libraryItem.media.episodes.find((ep) => ep.id === episodeId) if (!episode) { Logger.error(`[PodcastController] removeEpisode episode ${episodeId} not found for item ${libraryItem.id}`) return res.sendStatus(404) @@ -278,11 +281,14 @@ class PodcastController { if (hardDelete) { const audioFile = episode.audioFile // TODO: this will trigger the watcher. should maybe handle this gracefully - await fs.remove(audioFile.metadata.path).then(() => { - Logger.info(`[PodcastController] Hard deleted episode file at "${audioFile.metadata.path}"`) - }).catch((error) => { - Logger.error(`[PodcastController] Failed to hard delete episode file at "${audioFile.metadata.path}"`, error) - }) + await fs + .remove(audioFile.metadata.path) + .then(() => { + Logger.info(`[PodcastController] Hard deleted episode file at "${audioFile.metadata.path}"`) + }) + .catch((error) => { + Logger.error(`[PodcastController] Failed to hard delete episode file at "${audioFile.metadata.path}"`, error) + }) } // Remove episode from Podcast and library file