Merge pull request #2188 from jfrazx/fix/match-authors-429

fix: HTTP/429 when requesting authors information, resolves #1570

package.json

@@ -45,6 +45,7 @@
     "node-tone": "^1.0.1",
     "nodemailer": "^6.9.13",
     "openid-client": "^5.6.1",
+    "p-throttle": "^4.1.1",
     "passport": "^0.6.0",
     "passport-jwt": "^4.0.1",
     "sequelize": "^6.35.2",

server/controllers/AuthorController.js

@@ -9,7 +9,7 @@ const CacheManager = require('../managers/CacheManager')
 const CoverManager = require('../managers/CoverManager')
 const AuthorFinder = require('../finders/AuthorFinder')
 
-const { reqSupportsWebp } = require('../utils/index')
+const { reqSupportsWebp, isValidASIN } = require('../utils/index')
 
 const naturalSort = createNewSortInstance({
   comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare
@@ -252,7 +252,7 @@ class AuthorController {
   async match(req, res) {
     let authorData = null
     const region = req.body.region || 'us'
-    if (req.body.asin) {
+    if (req.body.asin && isValidASIN(req.body.asin.toUpperCase?.())) {
       authorData = await AuthorFinder.findAuthorByASIN(req.body.asin, region)
     } else {
       authorData = await AuthorFinder.findAuthorByName(req.body.q, region)

server/controllers/PodcastController.js

@@ -14,7 +14,6 @@ const CoverManager = require('../managers/CoverManager')
 const LibraryItem = require('../objects/LibraryItem')
 
 class PodcastController {
-
   async create(req, res) {
     if (!req.user.isAdminOrUp) {
       Logger.error(`[PodcastController] Non-admin user "${req.user.username}" attempted to create podcast`)
@@ -28,7 +27,7 @@ class PodcastController {
       return res.status(404).send('Library not found')
     }
 
-    const folder = library.folders.find(fold => fold.id === payload.folderId)
+    const folder = library.folders.find((fold) => fold.id === payload.folderId)
     if (!folder) {
       Logger.error(`[PodcastController] Create: Folder not found "${payload.folderId}"`)
       return res.status(404).send('Folder not found')
@@ -37,20 +36,24 @@ class PodcastController {
     const podcastPath = filePathToPOSIX(payload.path)
 
     // Check if a library item with this podcast folder exists already
-    const existingLibraryItem = (await Database.libraryItemModel.count({
-      where: {
-        path: podcastPath
-      }
-    })) > 0
+    const existingLibraryItem =
+      (await Database.libraryItemModel.count({
+        where: {
+          path: podcastPath
+        }
+      })) > 0
     if (existingLibraryItem) {
       Logger.error(`[PodcastController] Podcast already exists at path "${podcastPath}"`)
       return res.status(400).send('Podcast already exists')
     }
 
-    const success = await fs.ensureDir(podcastPath).then(() => true).catch((error) => {
-      Logger.error(`[PodcastController] Failed to ensure podcast dir "${podcastPath}"`, error)
-      return false
-    })
+    const success = await fs
+      .ensureDir(podcastPath)
+      .then(() => true)
+      .catch((error) => {
+        Logger.error(`[PodcastController] Failed to ensure podcast dir "${podcastPath}"`, error)
+        return false
+      })
     if (!success) return res.status(400).send('Invalid podcast path')
 
     const libraryItemFolderStats = await getFileTimestampsWithIno(podcastPath)
@@ -105,12 +108,12 @@ class PodcastController {
 
   /**
    * POST: /api/podcasts/feed
-   * 
+   *
    * @typedef getPodcastFeedReqBody
    * @property {string} rssFeed
-   * 
-   * @param {import('express').Request<{}, {}, getPodcastFeedReqBody, {}} req 
-   * @param {import('express').Response} res 
+   *
+   * @param {import('express').Request<{}, {}, getPodcastFeedReqBody, {}} req
+   * @param {import('express').Response} res
    */
   async getPodcastFeed(req, res) {
     if (!req.user.isAdminOrUp) {
@@ -178,7 +181,7 @@ class PodcastController {
 
     var downloadsInQueue = this.podcastManager.getEpisodeDownloadsInQueue(libraryItem.id)
     res.json({
-      downloads: downloadsInQueue.map(d => d.toJSONForClient())
+      downloads: downloadsInQueue.map((d) => d.toJSONForClient())
     })
   }
 
@@ -189,8 +192,8 @@ class PodcastController {
       return res.status(500).send('Podcast does not have an RSS feed URL')
     }
 
-    var searchTitle = req.query.title
-    if (!searchTitle) {
+    const searchTitle = req.query.title
+    if (!searchTitle || typeof searchTitle !== 'string') {
       return res.sendStatus(500)
     }
     const episodes = await findMatchingEpisodes(rssFeedUrl, searchTitle)
@@ -254,7 +257,7 @@ class PodcastController {
     const episodeId = req.params.episodeId
     const libraryItem = req.libraryItem
 
-    const episode = libraryItem.media.episodes.find(ep => ep.id === episodeId)
+    const episode = libraryItem.media.episodes.find((ep) => ep.id === episodeId)
     if (!episode) {
       Logger.error(`[PodcastController] getEpisode episode ${episodeId} not found for item ${libraryItem.id}`)
       return res.sendStatus(404)
@@ -269,7 +272,7 @@ class PodcastController {
     const libraryItem = req.libraryItem
     const hardDelete = req.query.hard === '1'
 
-    const episode = libraryItem.media.episodes.find(ep => ep.id === episodeId)
+    const episode = libraryItem.media.episodes.find((ep) => ep.id === episodeId)
     if (!episode) {
       Logger.error(`[PodcastController] removeEpisode episode ${episodeId} not found for item ${libraryItem.id}`)
       return res.sendStatus(404)
@@ -278,11 +281,14 @@ class PodcastController {
     if (hardDelete) {
       const audioFile = episode.audioFile
       // TODO: this will trigger the watcher. should maybe handle this gracefully
-      await fs.remove(audioFile.metadata.path).then(() => {
-        Logger.info(`[PodcastController] Hard deleted episode file at "${audioFile.metadata.path}"`)
-      }).catch((error) => {
-        Logger.error(`[PodcastController] Failed to hard delete episode file at "${audioFile.metadata.path}"`, error)
-      })
+      await fs
+        .remove(audioFile.metadata.path)
+        .then(() => {
+          Logger.info(`[PodcastController] Hard deleted episode file at "${audioFile.metadata.path}"`)
+        })
+        .catch((error) => {
+          Logger.error(`[PodcastController] Failed to hard delete episode file at "${audioFile.metadata.path}"`, error)
+        })
     }
 
     // Remove episode from Podcast and library file

server/controllers/SearchController.js

@@ -1,12 +1,13 @@
-const Logger = require("../Logger")
+const Logger = require('../Logger')
 const BookFinder = require('../finders/BookFinder')
 const PodcastFinder = require('../finders/PodcastFinder')
 const AuthorFinder = require('../finders/AuthorFinder')
 const MusicFinder = require('../finders/MusicFinder')
-const Database = require("../Database")
+const Database = require('../Database')
+const { isValidASIN } = require('../utils')
 
 class SearchController {
-  constructor() { }
+  constructor() {}
 
   async findBooks(req, res) {
     const id = req.query.id
@@ -37,9 +38,9 @@ class SearchController {
 
   /**
    * Find podcast RSS feeds given a term
-   * 
-   * @param {import('express').Request} req 
-   * @param {import('express').Response} res 
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
    */
   async findPodcasts(req, res) {
     const term = req.query.term
@@ -63,6 +64,9 @@ class SearchController {
 
   async findChapters(req, res) {
     const asin = req.query.asin
+    if (!isValidASIN(asin.toUpperCase())) {
+      return res.json({ error: 'Invalid ASIN' })
+    }
     const region = (req.query.region || 'us').toLowerCase()
     const chapterData = await BookFinder.findChapters(asin, region)
     if (!chapterData) {
@@ -78,4 +82,4 @@ class SearchController {
     })
   }
 }
-module.exports = new SearchController()
+module.exports = new SearchController()

server/providers/Audible.js

@@ -1,6 +1,7 @@
 const axios = require('axios').default
 const htmlSanitizer = require('../utils/htmlSanitizer')
 const Logger = require('../Logger')
+const { isValidASIN } = require('../utils/index')
 
 class Audible {
   #responseTimeout = 30000
@@ -81,16 +82,6 @@ class Audible {
     }
   }
 
-  /**
-   * Test if a search title matches an ASIN. Supports lowercase letters
-   *
-   * @param {string} title
-   * @returns {boolean}
-   */
-  isProbablyAsin(title) {
-    return /^[0-9A-Za-z]{10}$/.test(title)
-  }
-
   /**
    *
    * @param {string} asin
@@ -137,11 +128,11 @@ class Audible {
     if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout
 
     let items
-    if (asin) {
+    if (asin && isValidASIN(asin.toUpperCase())) {
       items = [await this.asinSearch(asin, region, timeout)]
     }
 
-    if (!items && this.isProbablyAsin(title)) {
+    if (!items && isValidASIN(title.toUpperCase())) {
       items = [await this.asinSearch(title, region, timeout)]
     }
 

server/providers/Audnexus.js

@@ -1,6 +1,8 @@
 const axios = require('axios').default
-const { levenshteinDistance } = require('../utils/index')
+const Throttle = require('p-throttle')
 const Logger = require('../Logger')
+const { levenshteinDistance } = require('../utils/index')
+const { isValidASIN } = require('../utils/index')
 
 /**
  * @typedef AuthorSearchObj
@@ -11,8 +13,28 @@ const Logger = require('../Logger')
  */
 
 class Audnexus {
+  static _instance = null
+
   constructor() {
+    // ensures Audnexus class is singleton
+    if (Audnexus._instance) {
+      return Audnexus._instance
+    }
+
     this.baseUrl = 'https://api.audnex.us'
+
+    // Rate limit is 100 requests per minute.
+    // @see https://github.com/laxamentumtech/audnexus#-deployment-
+    this.limiter = Throttle({
+      // Setting the limit to 1 allows for a short pause between requests that is imperceptible to the end user.
+      // A larger limit will grab blocks faster and then wait for the alloted time(interval) before
+      // fetching another batch, but with a discernable pause from the user perspective.
+      limit: 1,
+      strict: true,
+      interval: 150
+    })
+
+    Audnexus._instance = this
   }
 
   /**
@@ -24,14 +46,14 @@ class Audnexus {
   authorASINsRequest(name, region) {
     const searchParams = new URLSearchParams()
     searchParams.set('name', name)
+
     if (region) searchParams.set('region', region)
+
     const authorRequestUrl = `${this.baseUrl}/authors?${searchParams.toString()}`
     Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`)
-    return axios
-      .get(authorRequestUrl)
-      .then((res) => {
-        return res.data || []
-      })
+
+    return this._processRequest(this.limiter(() => axios.get(authorRequestUrl)))
+      .then((res) => res.data || [])
       .catch((error) => {
         Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error)
         return []
@@ -45,15 +67,20 @@ class Audnexus {
    * @returns {Promise<AuthorSearchObj>}
    */
   authorRequest(asin, region) {
-    asin = encodeURIComponent(asin)
-    const regionQuery = region ? `?region=${region}` : ''
-    const authorRequestUrl = `${this.baseUrl}/authors/${asin}${regionQuery}`
+    if (!isValidASIN(asin?.toUpperCase?.())) {
+      Logger.error(`[Audnexus] Invalid ASIN ${asin}`)
+      return null
+    }
+
+    asin = encodeURIComponent(asin.toUpperCase())
+
+    const authorRequestUrl = new URL(`${this.baseUrl}/authors/${asin}`)
+    if (region) authorRequestUrl.searchParams.set('region', region)
+
     Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`)
-    return axios
-      .get(authorRequestUrl)
-      .then((res) => {
-        return res.data
-      })
+
+    return this._processRequest(this.limiter(() => axios.get(authorRequestUrl.toString())))
+      .then((res) => res.data)
       .catch((error) => {
         Logger.error(`[Audnexus] Author request failed for ${asin}`, error)
         return null
@@ -68,15 +95,15 @@ class Audnexus {
    */
   async findAuthorByASIN(asin, region) {
     const author = await this.authorRequest(asin, region)
-    if (!author) {
-      return null
-    }
-    return {
-      asin: author.asin,
-      description: author.description,
-      image: author.image || null,
-      name: author.name
-    }
+
+    return author
+      ? {
+          asin: author.asin,
+          description: author.description,
+          image: author.image || null,
+          name: author.name
+        }
+      : null
   }
 
   /**
@@ -97,13 +124,16 @@ class Audnexus {
         closestMatch = authorAsinObj
       }
     })
+
     if (!closestMatch || closestMatch.levenshteinDistance > maxLevenshtein) {
       return null
     }
+
     const author = await this.authorRequest(closestMatch.asin)
     if (!author) {
       return null
     }
+
     return {
       asin: author.asin,
       description: author.description,
@@ -112,17 +142,46 @@ class Audnexus {
     }
   }
 
+  /**
+   *
+   * @param {string} asin
+   * @param {string} region
+   * @returns {Promise<Object>}
+   */
   getChaptersByASIN(asin, region) {
     Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`)
-    return axios
-      .get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`)
-      .then((res) => {
-        return res.data
-      })
+
+    asin = encodeURIComponent(asin.toUpperCase())
+    const chaptersRequestUrl = new URL(`${this.baseUrl}/books/${asin}/chapters`)
+    if (region) chaptersRequestUrl.searchParams.set('region', region)
+
+    return this._processRequest(this.limiter(() => axios.get(chaptersRequestUrl.toString())))
+      .then((res) => res.data)
       .catch((error) => {
         Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error)
         return null
       })
   }
+
+  /**
+   * Internal method to process requests and retry if rate limit is exceeded.
+   */
+  async _processRequest(request) {
+    try {
+      return await request()
+    } catch (error) {
+      if (error.response?.status === 429) {
+        const retryAfter = parseInt(error.response.headers?.['retry-after'], 10) || 5
+
+        Logger.warn(`[Audnexus] Rate limit exceeded. Retrying in ${retryAfter} seconds.`)
+        await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000))
+
+        return this._processRequest(request)
+      }
+
+      throw error
+    }
+  }
 }
+
 module.exports = Audnexus

server/utils/index.js

@@ -1,7 +1,7 @@
 const Path = require('path')
 const uuid = require('uuid')
 const Logger = require('../Logger')
-const { parseString } = require("xml2js")
+const { parseString } = require('xml2js')
 const areEquivalent = require('./areEquivalent')
 
 const levenshteinDistance = (str1, str2, caseSensitive = false) => {
@@ -11,8 +11,9 @@ const levenshteinDistance = (str1, str2, caseSensitive = false) => {
     str1 = str1.toLowerCase()
     str2 = str2.toLowerCase()
   }
-  const track = Array(str2.length + 1).fill(null).map(() =>
-    Array(str1.length + 1).fill(null))
+  const track = Array(str2.length + 1)
+    .fill(null)
+    .map(() => Array(str1.length + 1).fill(null))
   for (let i = 0; i <= str1.length; i += 1) {
     track[0][i] = i
   }
@@ -25,7 +26,7 @@ const levenshteinDistance = (str1, str2, caseSensitive = false) => {
       track[j][i] = Math.min(
         track[j][i - 1] + 1, // deletion
         track[j - 1][i] + 1, // insertion
-        track[j - 1][i - 1] + indicator, // substitution
+        track[j - 1][i - 1] + indicator // substitution
       )
     }
   }
@@ -138,7 +139,10 @@ module.exports.toNumber = (val, fallback = 0) => {
 module.exports.cleanStringForSearch = (str) => {
   if (!str) return ''
   // Remove ' . ` " ,
-  return str.toLowerCase().replace(/[\'\.\`\",]/g, '').trim()
+  return str
+    .toLowerCase()
+    .replace(/[\'\.\`\",]/g, '')
+    .trim()
 }
 
 const getTitleParts = (title) => {
@@ -156,7 +160,7 @@ const getTitleParts = (title) => {
 /**
  * Remove sortingPrefixes from title
  * @example "The Good Book" => "Good Book"
- * @param {string} title 
+ * @param {string} title
  * @returns {string}
  */
 module.exports.getTitleIgnorePrefix = (title) => {
@@ -164,9 +168,9 @@ module.exports.getTitleIgnorePrefix = (title) => {
 }
 
 /**
- * Put sorting prefix at the end of title 
+ * Put sorting prefix at the end of title
  * @example "The Good Book" => "Good Book, The"
- * @param {string} title 
+ * @param {string} title
  * @returns {string}
  */
 module.exports.getTitlePrefixAtEnd = (title) => {
@@ -178,8 +182,8 @@ module.exports.getTitlePrefixAtEnd = (title) => {
  * to lower case for only ascii characters
  * used to handle sqlite that doesnt support unicode lower
  * @see https://github.com/advplyr/audiobookshelf/issues/2187
- * 
- * @param {string} str 
+ *
+ * @param {string} str
  * @returns {string}
  */
 module.exports.asciiOnlyToLowerCase = (str) => {
@@ -200,8 +204,8 @@ module.exports.asciiOnlyToLowerCase = (str) => {
 /**
  * Escape string used in RegExp
  * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping
- * 
- * @param {string} str 
+ *
+ * @param {string} str
  * @returns {string}
  */
 module.exports.escapeRegExp = (str) => {
@@ -211,8 +215,8 @@ module.exports.escapeRegExp = (str) => {
 
 /**
  * Validate url string with URL class
- * 
- * @param {string} rawUrl 
+ *
+ * @param {string} rawUrl
  * @returns {string} null if invalid
  */
 module.exports.validateUrl = (rawUrl) => {
@@ -227,11 +231,22 @@ module.exports.validateUrl = (rawUrl) => {
 
 /**
  * Check if a string is a valid UUID
- * 
- * @param {string} str 
+ *
+ * @param {string} str
  * @returns {boolean}
  */
 module.exports.isUUID = (str) => {
   if (!str || typeof str !== 'string') return false
   return uuid.validate(str)
-}
+}
+
+/**
+ * Check if a string is a valid ASIN
+ *
+ * @param {string} str
+ * @returns {boolean}
+ */
+module.exports.isValidASIN = (str) => {
+  if (!str || typeof str !== 'string') return false
+  return /^[A-Z0-9]{10}$/.test(str)
+}