Merge pull request #2188 from jfrazx/fix/match-authors-429

fix: HTTP/429 when requesting authors information, resolves #1570

package.json

@@ -45,6 +45,7 @@
     "node-tone": "^1.0.1",
     "nodemailer": "^6.9.13",
     "openid-client": "^5.6.1",
+    "p-throttle": "^4.1.1",
     "passport": "^0.6.0",
     "passport-jwt": "^4.0.1",
     "sequelize": "^6.35.2",

server/controllers/AuthorController.js

@@ -9,7 +9,7 @@ const CacheManager = require('../managers/CacheManager')
 const CoverManager = require('../managers/CoverManager')
 const AuthorFinder = require('../finders/AuthorFinder')
 
-const { reqSupportsWebp } = require('../utils/index')
+const { reqSupportsWebp, isValidASIN } = require('../utils/index')
 
 const naturalSort = createNewSortInstance({
   comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare
@@ -252,7 +252,7 @@ class AuthorController {
   async match(req, res) {
     let authorData = null
     const region = req.body.region || 'us'
-    if (req.body.asin) {
+    if (req.body.asin && isValidASIN(req.body.asin.toUpperCase?.())) {
       authorData = await AuthorFinder.findAuthorByASIN(req.body.asin, region)
     } else {
       authorData = await AuthorFinder.findAuthorByName(req.body.q, region)

server/controllers/PodcastController.js

@@ -14,7 +14,6 @@ const CoverManager = require('../managers/CoverManager')
 const LibraryItem = require('../objects/LibraryItem')
 
 class PodcastController {
-
   async create(req, res) {
     if (!req.user.isAdminOrUp) {
       Logger.error(`[PodcastController] Non-admin user "${req.user.username}" attempted to create podcast`)
@@ -28,7 +27,7 @@ class PodcastController {
       return res.status(404).send('Library not found')
     }
 
-    const folder = library.folders.find(fold => fold.id === payload.folderId)
+    const folder = library.folders.find((fold) => fold.id === payload.folderId)
     if (!folder) {
       Logger.error(`[PodcastController] Create: Folder not found "${payload.folderId}"`)
       return res.status(404).send('Folder not found')
@@ -37,7 +36,8 @@ class PodcastController {
     const podcastPath = filePathToPOSIX(payload.path)
 
     // Check if a library item with this podcast folder exists already
-    const existingLibraryItem = (await Database.libraryItemModel.count({
+    const existingLibraryItem =
+      (await Database.libraryItemModel.count({
         where: {
           path: podcastPath
         }
@@ -47,7 +47,10 @@ class PodcastController {
       return res.status(400).send('Podcast already exists')
     }
 
-    const success = await fs.ensureDir(podcastPath).then(() => true).catch((error) => {
+    const success = await fs
+      .ensureDir(podcastPath)
+      .then(() => true)
+      .catch((error) => {
         Logger.error(`[PodcastController] Failed to ensure podcast dir "${podcastPath}"`, error)
         return false
       })
@@ -178,7 +181,7 @@ class PodcastController {
 
     var downloadsInQueue = this.podcastManager.getEpisodeDownloadsInQueue(libraryItem.id)
     res.json({
-      downloads: downloadsInQueue.map(d => d.toJSONForClient())
+      downloads: downloadsInQueue.map((d) => d.toJSONForClient())
     })
   }
 
@@ -189,8 +192,8 @@ class PodcastController {
       return res.status(500).send('Podcast does not have an RSS feed URL')
     }
 
-    var searchTitle = req.query.title
+    const searchTitle = req.query.title
-    if (!searchTitle) {
+    if (!searchTitle || typeof searchTitle !== 'string') {
       return res.sendStatus(500)
     }
     const episodes = await findMatchingEpisodes(rssFeedUrl, searchTitle)
@@ -254,7 +257,7 @@ class PodcastController {
     const episodeId = req.params.episodeId
     const libraryItem = req.libraryItem
 
-    const episode = libraryItem.media.episodes.find(ep => ep.id === episodeId)
+    const episode = libraryItem.media.episodes.find((ep) => ep.id === episodeId)
     if (!episode) {
       Logger.error(`[PodcastController] getEpisode episode ${episodeId} not found for item ${libraryItem.id}`)
       return res.sendStatus(404)
@@ -269,7 +272,7 @@ class PodcastController {
     const libraryItem = req.libraryItem
     const hardDelete = req.query.hard === '1'
 
-    const episode = libraryItem.media.episodes.find(ep => ep.id === episodeId)
+    const episode = libraryItem.media.episodes.find((ep) => ep.id === episodeId)
     if (!episode) {
       Logger.error(`[PodcastController] removeEpisode episode ${episodeId} not found for item ${libraryItem.id}`)
       return res.sendStatus(404)
@@ -278,9 +281,12 @@ class PodcastController {
     if (hardDelete) {
       const audioFile = episode.audioFile
       // TODO: this will trigger the watcher. should maybe handle this gracefully
-      await fs.remove(audioFile.metadata.path).then(() => {
+      await fs
+        .remove(audioFile.metadata.path)
+        .then(() => {
           Logger.info(`[PodcastController] Hard deleted episode file at "${audioFile.metadata.path}"`)
-      }).catch((error) => {
+        })
+        .catch((error) => {
           Logger.error(`[PodcastController] Failed to hard delete episode file at "${audioFile.metadata.path}"`, error)
         })
     }

server/controllers/SearchController.js

@@ -1,9 +1,10 @@
-const Logger = require("../Logger")
+const Logger = require('../Logger')
 const BookFinder = require('../finders/BookFinder')
 const PodcastFinder = require('../finders/PodcastFinder')
 const AuthorFinder = require('../finders/AuthorFinder')
 const MusicFinder = require('../finders/MusicFinder')
-const Database = require("../Database")
+const Database = require('../Database')
+const { isValidASIN } = require('../utils')
 
 class SearchController {
   constructor() {}
@@ -63,6 +64,9 @@ class SearchController {
 
   async findChapters(req, res) {
     const asin = req.query.asin
+    if (!isValidASIN(asin.toUpperCase())) {
+      return res.json({ error: 'Invalid ASIN' })
+    }
     const region = (req.query.region || 'us').toLowerCase()
     const chapterData = await BookFinder.findChapters(asin, region)
     if (!chapterData) {

server/providers/Audible.js

@@ -1,6 +1,7 @@
 const axios = require('axios').default
 const htmlSanitizer = require('../utils/htmlSanitizer')
 const Logger = require('../Logger')
+const { isValidASIN } = require('../utils/index')
 
 class Audible {
   #responseTimeout = 30000
@@ -81,16 +82,6 @@ class Audible {
     }
   }
 
-  /**
-   * Test if a search title matches an ASIN. Supports lowercase letters
-   *
-   * @param {string} title
-   * @returns {boolean}
-   */
-  isProbablyAsin(title) {
-    return /^[0-9A-Za-z]{10}$/.test(title)
-  }
-
   /**
    *
    * @param {string} asin
@@ -137,11 +128,11 @@ class Audible {
     if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout
 
     let items
-    if (asin) {
+    if (asin && isValidASIN(asin.toUpperCase())) {
       items = [await this.asinSearch(asin, region, timeout)]
     }
 
-    if (!items && this.isProbablyAsin(title)) {
+    if (!items && isValidASIN(title.toUpperCase())) {
       items = [await this.asinSearch(title, region, timeout)]
     }
 

server/providers/Audnexus.js

@@ -1,6 +1,8 @@
 const axios = require('axios').default
-const { levenshteinDistance } = require('../utils/index')
+const Throttle = require('p-throttle')
 const Logger = require('../Logger')
+const { levenshteinDistance } = require('../utils/index')
+const { isValidASIN } = require('../utils/index')
 
 /**
  * @typedef AuthorSearchObj
@@ -11,8 +13,28 @@ const Logger = require('../Logger')
  */
 
 class Audnexus {
+  static _instance = null
+
   constructor() {
+    // ensures Audnexus class is singleton
+    if (Audnexus._instance) {
+      return Audnexus._instance
+    }
+
     this.baseUrl = 'https://api.audnex.us'
+
+    // Rate limit is 100 requests per minute.
+    // @see https://github.com/laxamentumtech/audnexus#-deployment-
+    this.limiter = Throttle({
+      // Setting the limit to 1 allows for a short pause between requests that is imperceptible to the end user.
+      // A larger limit will grab blocks faster and then wait for the alloted time(interval) before
+      // fetching another batch, but with a discernable pause from the user perspective.
+      limit: 1,
+      strict: true,
+      interval: 150
+    })
+
+    Audnexus._instance = this
   }
 
   /**
@@ -24,14 +46,14 @@ class Audnexus {
   authorASINsRequest(name, region) {
     const searchParams = new URLSearchParams()
     searchParams.set('name', name)
+
     if (region) searchParams.set('region', region)
+
     const authorRequestUrl = `${this.baseUrl}/authors?${searchParams.toString()}`
     Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`)
-    return axios
+
-      .get(authorRequestUrl)
+    return this._processRequest(this.limiter(() => axios.get(authorRequestUrl)))
-      .then((res) => {
+      .then((res) => res.data || [])
-        return res.data || []
-      })
       .catch((error) => {
         Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error)
         return []
@@ -45,15 +67,20 @@ class Audnexus {
    * @returns {Promise<AuthorSearchObj>}
    */
   authorRequest(asin, region) {
-    asin = encodeURIComponent(asin)
+    if (!isValidASIN(asin?.toUpperCase?.())) {
-    const regionQuery = region ? `?region=${region}` : ''
+      Logger.error(`[Audnexus] Invalid ASIN ${asin}`)
-    const authorRequestUrl = `${this.baseUrl}/authors/${asin}${regionQuery}`
+      return null
+    }
+
+    asin = encodeURIComponent(asin.toUpperCase())
+
+    const authorRequestUrl = new URL(`${this.baseUrl}/authors/${asin}`)
+    if (region) authorRequestUrl.searchParams.set('region', region)
+
     Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`)
-    return axios
+
-      .get(authorRequestUrl)
+    return this._processRequest(this.limiter(() => axios.get(authorRequestUrl.toString())))
-      .then((res) => {
+      .then((res) => res.data)
-        return res.data
-      })
       .catch((error) => {
         Logger.error(`[Audnexus] Author request failed for ${asin}`, error)
         return null
@@ -68,15 +95,15 @@ class Audnexus {
    */
   async findAuthorByASIN(asin, region) {
     const author = await this.authorRequest(asin, region)
-    if (!author) {
+
-      return null
+    return author
-    }
+      ? {
-    return {
           asin: author.asin,
           description: author.description,
           image: author.image || null,
           name: author.name
         }
+      : null
   }
 
   /**
@@ -97,13 +124,16 @@ class Audnexus {
         closestMatch = authorAsinObj
       }
     })
+
     if (!closestMatch || closestMatch.levenshteinDistance > maxLevenshtein) {
       return null
     }
+
     const author = await this.authorRequest(closestMatch.asin)
     if (!author) {
       return null
     }
+
     return {
       asin: author.asin,
       description: author.description,
@@ -112,17 +142,46 @@ class Audnexus {
     }
   }
 
+  /**
+   *
+   * @param {string} asin
+   * @param {string} region
+   * @returns {Promise<Object>}
+   */
   getChaptersByASIN(asin, region) {
     Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`)
-    return axios
+
-      .get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`)
+    asin = encodeURIComponent(asin.toUpperCase())
-      .then((res) => {
+    const chaptersRequestUrl = new URL(`${this.baseUrl}/books/${asin}/chapters`)
-        return res.data
+    if (region) chaptersRequestUrl.searchParams.set('region', region)
-      })
+
+    return this._processRequest(this.limiter(() => axios.get(chaptersRequestUrl.toString())))
+      .then((res) => res.data)
       .catch((error) => {
         Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error)
         return null
       })
   }
+
+  /**
+   * Internal method to process requests and retry if rate limit is exceeded.
+   */
+  async _processRequest(request) {
+    try {
+      return await request()
+    } catch (error) {
+      if (error.response?.status === 429) {
+        const retryAfter = parseInt(error.response.headers?.['retry-after'], 10) || 5
+
+        Logger.warn(`[Audnexus] Rate limit exceeded. Retrying in ${retryAfter} seconds.`)
+        await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000))
+
+        return this._processRequest(request)
       }
+
+      throw error
+    }
+  }
+}
+
 module.exports = Audnexus

server/utils/index.js

@@ -1,7 +1,7 @@
 const Path = require('path')
 const uuid = require('uuid')
 const Logger = require('../Logger')
-const { parseString } = require("xml2js")
+const { parseString } = require('xml2js')
 const areEquivalent = require('./areEquivalent')
 
 const levenshteinDistance = (str1, str2, caseSensitive = false) => {
@@ -11,8 +11,9 @@ const levenshteinDistance = (str1, str2, caseSensitive = false) => {
     str1 = str1.toLowerCase()
     str2 = str2.toLowerCase()
   }
-  const track = Array(str2.length + 1).fill(null).map(() =>
+  const track = Array(str2.length + 1)
-    Array(str1.length + 1).fill(null))
+    .fill(null)
+    .map(() => Array(str1.length + 1).fill(null))
   for (let i = 0; i <= str1.length; i += 1) {
     track[0][i] = i
   }
@@ -25,7 +26,7 @@ const levenshteinDistance = (str1, str2, caseSensitive = false) => {
       track[j][i] = Math.min(
         track[j][i - 1] + 1, // deletion
         track[j - 1][i] + 1, // insertion
-        track[j - 1][i - 1] + indicator, // substitution
+        track[j - 1][i - 1] + indicator // substitution
       )
     }
   }
@@ -138,7 +139,10 @@ module.exports.toNumber = (val, fallback = 0) => {
 module.exports.cleanStringForSearch = (str) => {
   if (!str) return ''
   // Remove ' . ` " ,
-  return str.toLowerCase().replace(/[\'\.\`\",]/g, '').trim()
+  return str
+    .toLowerCase()
+    .replace(/[\'\.\`\",]/g, '')
+    .trim()
 }
 
 const getTitleParts = (title) => {
@@ -235,3 +239,14 @@ module.exports.isUUID = (str) => {
   if (!str || typeof str !== 'string') return false
   return uuid.validate(str)
 }
+
+/**
+ * Check if a string is a valid ASIN
+ *
+ * @param {string} str
+ * @returns {boolean}
+ */
+module.exports.isValidASIN = (str) => {
+  if (!str || typeof str !== 'string') return false
+  return /^[A-Z0-9]{10}$/.test(str)
+}