From 4e6b75d6506d94c19fad81ae62cbac0c3370fd1e Mon Sep 17 00:00:00 2001 From: jfrazx Date: Thu, 5 Oct 2023 13:48:55 -0700 Subject: [PATCH] fix; HTTP/429 when requesting authors information, resolves #1570 --- package-lock.json | 29 ++++++++- package.json | 3 +- server/providers/Audnexus.js | 117 ++++++++++++++++++++++++----------- 3 files changed, 111 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index 77948004..a1e7ccd6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "express": "^4.17.1", "graceful-fs": "^4.2.10", "htmlparser2": "^8.0.1", + "limiter": "^2.1.0", "node-tone": "^1.0.1", "nodemailer": "^6.9.2", "sequelize": "^6.32.1", @@ -1308,6 +1309,19 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "optional": true }, + "node_modules/just-performance": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.3.0.tgz", + "integrity": "sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q==" + }, + "node_modules/limiter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/limiter/-/limiter-2.1.0.tgz", + "integrity": "sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw==", + "dependencies": { + "just-performance": "4.3.0" + } + }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -3673,6 +3687,19 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "optional": true }, + "just-performance": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.3.0.tgz", + "integrity": "sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q==" + }, + "limiter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/limiter/-/limiter-2.1.0.tgz", + "integrity": "sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw==", + "requires": { + "just-performance": "4.3.0" + } + }, "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -4672,4 +4699,4 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } -} \ No newline at end of file +} diff --git a/package.json b/package.json index 4a00fa59..082006e8 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "express": "^4.17.1", "graceful-fs": "^4.2.10", "htmlparser2": "^8.0.1", + "limiter": "^2.1.0", "node-tone": "^1.0.1", "nodemailer": "^6.9.2", "sequelize": "^6.32.1", @@ -44,4 +45,4 @@ "devDependencies": { "nodemon": "^2.0.20" } -} \ No newline at end of file +} diff --git a/server/providers/Audnexus.js b/server/providers/Audnexus.js index b74d1d13..06433f5d 100644 --- a/server/providers/Audnexus.js +++ b/server/providers/Audnexus.js @@ -1,78 +1,123 @@ const axios = require('axios') const { levenshteinDistance } = require('../utils/index') const Logger = require('../Logger') +const { RateLimiter } = require('limiter'); class Audnexus { + static _instance = null; + constructor() { + // ensures Audnexus class is singleton + if (Audnexus._instance) { + return Audnexus._instance + } + this.baseUrl = 'https://api.audnex.us' + + // @see https://github.com/laxamentumtech/audnexus#-deployment- + this.limiter = new RateLimiter({ + tokensPerInterval: 100, + fireImmediately: true, + interval: 'minute', + }) + + Audnexus._instance = this } authorASINsRequest(name, region) { name = encodeURIComponent(name) const regionQuery = region ? `®ion=${region}` : '' const authorRequestUrl = `${this.baseUrl}/authors?name=${name}${regionQuery}` + Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - return axios.get(authorRequestUrl).then((res) => { - return res.data || [] - }).catch((error) => { - Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error) - return [] - }) + + return this._processRequest(() => axios.get(authorRequestUrl)) + .then((res) => res.data || []) + .catch((error) => { + Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error) + return [] + }) } authorRequest(asin, region) { asin = encodeURIComponent(asin) const regionQuery = region ? `?region=${region}` : '' const authorRequestUrl = `${this.baseUrl}/authors/${asin}${regionQuery}` + Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - return axios.get(authorRequestUrl).then((res) => { - return res.data - }).catch((error) => { - Logger.error(`[Audnexus] Author request failed for ${asin}`, error) - return null - }) + + return this._processRequest(() => axios.get(authorRequestUrl)) + .then((res) => res.data) + .catch((error) => { + Logger.error(`[Audnexus] Author request failed for ${asin}`, error) + return null + }) + } + + /** + * @description Process a request with a rate limiter + * + * @param {*} request + * @returns + */ + async _processRequest(request) { + const remainingTokens = await this.limiter.removeTokens(1) + Logger.info(`[Audnexus] Attempting request with ${remainingTokens} remaining tokens and ${this.limiter.tokensThisInterval} this interval`) + + if (remainingTokens >= 1) { + return request() + } + + // 100 tokens(requests) per minute give a refresh of ~1.67 per second, + // so a 10 second wait will yield ~16.7 additional tokens + Logger.info('[Audnexus] Sleeping for 10 seconds') + await new Promise(resolve => setTimeout(resolve, 10000)) + + return this._processRequest(request) } async findAuthorByASIN(asin, region) { const author = await this.authorRequest(asin, region) - if (!author) { - return null - } - return { - asin: author.asin, - description: author.description, - image: author.image || null, - name: author.name - } + + return author ? + { + asin: author.asin, + description: author.description, + image: author.image || null, + name: author.name + } : null } async findAuthorByName(name, region, maxLevenshtein = 3) { Logger.debug(`[Audnexus] Looking up author by name ${name}`) + const asins = await this.authorASINsRequest(name, region) const matchingAsin = asins.find(obj => levenshteinDistance(obj.name, name) <= maxLevenshtein) + if (!matchingAsin) { return null } + const author = await this.authorRequest(matchingAsin.asin) - if (!author) { - return null - } - return { - asin: author.asin, - description: author.description, - image: author.image || null, - name: author.name - } + return author ? + { + description: author.description, + image: author.image || null, + asin: author.asin, + name: author.name + } : null } getChaptersByASIN(asin, region) { Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`) - return axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`).then((res) => { - return res.data - }).catch((error) => { - Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error) - return null - }) + + return this._processRequest(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`)) + .then((res) => res.data) + .catch((error) => { + Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error) + return null + }) } } + module.exports = Audnexus \ No newline at end of file