From 6fa49e0aab2309d6f64d08dcc18343f6d4c0694d Mon Sep 17 00:00:00 2001 From: advplyr Date: Sat, 25 May 2024 16:32:02 -0500 Subject: [PATCH] Fix:Add timeout to provider matching default to 30s #3000 --- server/finders/BookFinder.js | 217 +++++++++------- server/providers/Audible.js | 293 ++++++++++++---------- server/providers/AudiobookCovers.js | 36 ++- server/providers/Audnexus.js | 77 +++--- server/providers/CustomProviderAdapter.js | 166 ++++++------ server/providers/FantLab.js | 116 ++++++--- server/providers/GoogleBooks.js | 44 ++-- server/providers/OpenLibrary.js | 54 ++-- server/providers/iTunes.js | 74 ++++-- 9 files changed, 633 insertions(+), 444 deletions(-) diff --git a/server/finders/BookFinder.js b/server/finders/BookFinder.js index 9aa0a182..8aef4d11 100644 --- a/server/finders/BookFinder.js +++ b/server/finders/BookFinder.js @@ -10,6 +10,8 @@ const Logger = require('../Logger') const { levenshteinDistance, escapeRegExp } = require('../utils/index') class BookFinder { + #providerResponseTimeout = 30000 + constructor() { this.openLibrary = new OpenLibrary() this.googleBooks = new GoogleBooks() @@ -36,63 +38,75 @@ class BookFinder { filterSearchResults(books, title, author, maxTitleDistance, maxAuthorDistance) { var searchTitle = cleanTitleForCompares(title) var searchAuthor = cleanAuthorForCompares(author) - return books.map(b => { - b.cleanedTitle = cleanTitleForCompares(b.title) - b.titleDistance = levenshteinDistance(b.cleanedTitle, title) + return books + .map((b) => { + b.cleanedTitle = cleanTitleForCompares(b.title) + b.titleDistance = levenshteinDistance(b.cleanedTitle, title) - // Total length of search (title or both title & author) - b.totalPossibleDistance = b.title.length + // Total length of search (title or both title & author) + b.totalPossibleDistance = b.title.length - if (author) { - if (!b.author) { - b.authorDistance = author.length - } else { - b.totalPossibleDistance += b.author.length - b.cleanedAuthor = cleanAuthorForCompares(b.author) + if (author) { + if (!b.author) { + b.authorDistance = author.length + } else { + b.totalPossibleDistance += b.author.length + b.cleanedAuthor = cleanAuthorForCompares(b.author) - var cleanedAuthorDistance = levenshteinDistance(b.cleanedAuthor, searchAuthor) - var authorDistance = levenshteinDistance(b.author || '', author) + var cleanedAuthorDistance = levenshteinDistance(b.cleanedAuthor, searchAuthor) + var authorDistance = levenshteinDistance(b.author || '', author) - // Use best distance - b.authorDistance = Math.min(cleanedAuthorDistance, authorDistance) + // Use best distance + b.authorDistance = Math.min(cleanedAuthorDistance, authorDistance) - // Check book author contains searchAuthor - if (searchAuthor.length > 4 && b.cleanedAuthor.includes(searchAuthor)) b.includesAuthor = searchAuthor - else if (author.length > 4 && b.author.includes(author)) b.includesAuthor = author + // Check book author contains searchAuthor + if (searchAuthor.length > 4 && b.cleanedAuthor.includes(searchAuthor)) b.includesAuthor = searchAuthor + else if (author.length > 4 && b.author.includes(author)) b.includesAuthor = author + } } - } - b.totalDistance = b.titleDistance + (b.authorDistance || 0) + b.totalDistance = b.titleDistance + (b.authorDistance || 0) - // Check book title contains the searchTitle - if (searchTitle.length > 4 && b.cleanedTitle.includes(searchTitle)) b.includesTitle = searchTitle - else if (title.length > 4 && b.title.includes(title)) b.includesTitle = title + // Check book title contains the searchTitle + if (searchTitle.length > 4 && b.cleanedTitle.includes(searchTitle)) b.includesTitle = searchTitle + else if (title.length > 4 && b.title.includes(title)) b.includesTitle = title - return b - }).filter(b => { - if (b.includesTitle) { // If search title was found in result title then skip over leven distance check - if (this.verbose) Logger.debug(`Exact title was included in "${b.title}", Search: "${b.includesTitle}"`) - } else if (b.titleDistance > maxTitleDistance) { - if (this.verbose) Logger.debug(`Filtering out search result title distance = ${b.titleDistance}: "${b.cleanedTitle}"/"${searchTitle}"`) - return false - } - - if (author) { - if (b.includesAuthor) { // If search author was found in result author then skip over leven distance check - if (this.verbose) Logger.debug(`Exact author was included in "${b.author}", Search: "${b.includesAuthor}"`) - } else if (b.authorDistance > maxAuthorDistance) { - if (this.verbose) Logger.debug(`Filtering out search result "${b.author}", author distance = ${b.authorDistance}: "${b.author}"/"${author}"`) + return b + }) + .filter((b) => { + if (b.includesTitle) { + // If search title was found in result title then skip over leven distance check + if (this.verbose) Logger.debug(`Exact title was included in "${b.title}", Search: "${b.includesTitle}"`) + } else if (b.titleDistance > maxTitleDistance) { + if (this.verbose) Logger.debug(`Filtering out search result title distance = ${b.titleDistance}: "${b.cleanedTitle}"/"${searchTitle}"`) return false } - } - // If book total search length < 5 and was not exact match, then filter out - if (b.totalPossibleDistance < 5 && b.totalDistance > 0) return false - return true - }) + if (author) { + if (b.includesAuthor) { + // If search author was found in result author then skip over leven distance check + if (this.verbose) Logger.debug(`Exact author was included in "${b.author}", Search: "${b.includesAuthor}"`) + } else if (b.authorDistance > maxAuthorDistance) { + if (this.verbose) Logger.debug(`Filtering out search result "${b.author}", author distance = ${b.authorDistance}: "${b.author}"/"${author}"`) + return false + } + } + + // If book total search length < 5 and was not exact match, then filter out + if (b.totalPossibleDistance < 5 && b.totalDistance > 0) return false + return true + }) } + /** + * + * @param {string} title + * @param {string} author + * @param {number} maxTitleDistance + * @param {number} maxAuthorDistance + * @returns {Promise} + */ async getOpenLibResults(title, author, maxTitleDistance, maxAuthorDistance) { - var books = await this.openLibrary.searchTitle(title) + var books = await this.openLibrary.searchTitle(title, this.#providerResponseTimeout) if (this.verbose) Logger.debug(`OpenLib Book Search Results: ${books.length || 0}`) if (books.errorCode) { Logger.error(`OpenLib Search Error ${books.errorCode}`) @@ -109,8 +123,14 @@ class BookFinder { return booksFiltered } + /** + * + * @param {string} title + * @param {string} author + * @returns {Promise} + */ async getGoogleBooksResults(title, author) { - var books = await this.googleBooks.search(title, author) + var books = await this.googleBooks.search(title, author, this.#providerResponseTimeout) if (this.verbose) Logger.debug(`GoogleBooks Book Search Results: ${books.length || 0}`) if (books.errorCode) { Logger.error(`GoogleBooks Search Error ${books.errorCode}`) @@ -120,8 +140,14 @@ class BookFinder { return books } + /** + * + * @param {string} title + * @param {string} author + * @returns {Promise} + */ async getFantLabResults(title, author) { - var books = await this.fantLab.search(title, author) + var books = await this.fantLab.search(title, author, this.#providerResponseTimeout) if (this.verbose) Logger.debug(`FantLab Book Search Results: ${books.length || 0}`) if (books.errorCode) { Logger.error(`FantLab Search Error ${books.errorCode}`) @@ -131,41 +157,58 @@ class BookFinder { return books } + /** + * + * @param {string} search + * @returns {Promise} + */ async getAudiobookCoversResults(search) { - const covers = await this.audiobookCovers.search(search) + const covers = await this.audiobookCovers.search(search, this.#providerResponseTimeout) if (this.verbose) Logger.debug(`AudiobookCovers Search Results: ${covers.length || 0}`) return covers || [] } - async getiTunesAudiobooksResults(title, author) { - return this.iTunesApi.searchAudiobooks(title) + /** + * + * @param {string} title + * @returns {Promise} + */ + async getiTunesAudiobooksResults(title) { + return this.iTunesApi.searchAudiobooks(title, this.#providerResponseTimeout) } + /** + * + * @param {string} title + * @param {string} author + * @param {string} asin + * @param {string} provider + * @returns {Promise} + */ async getAudibleResults(title, author, asin, provider) { const region = provider.includes('.') ? provider.split('.').pop() : '' - const books = await this.audible.search(title, author, asin, region) + const books = await this.audible.search(title, author, asin, region, this.#providerResponseTimeout) if (this.verbose) Logger.debug(`Audible Book Search Results: ${books.length || 0}`) if (!books) return [] return books } /** - * - * @param {string} title + * + * @param {string} title * @param {string} author - * @param {string} isbn - * @param {string} providerSlug + * @param {string} isbn + * @param {string} providerSlug * @returns {Promise} */ async getCustomProviderResults(title, author, isbn, providerSlug) { - const books = await this.customProviderAdapter.search(title, author, isbn, providerSlug, 'book') + const books = await this.customProviderAdapter.search(title, author, isbn, providerSlug, 'book', this.#providerResponseTimeout) if (this.verbose) Logger.debug(`Custom provider '${providerSlug}' Search Results: ${books.length || 0}`) return books } static TitleCandidates = class { - constructor(cleanAuthor) { this.candidates = new Set() this.cleanAuthor = cleanAuthor @@ -179,13 +222,13 @@ class BookFinder { title = this.#removeAuthorFromTitle(title) const titleTransformers = [ - [/([,:;_]| by ).*/g, ''], // Remove subtitle - [/(^| )\d+k(bps)?( |$)/, ' '], // Remove bitrate + [/([,:;_]| by ).*/g, ''], // Remove subtitle + [/(^| )\d+k(bps)?( |$)/, ' '], // Remove bitrate [/ (2nd|3rd|\d+th)\s+ed(\.|ition)?/g, ''], // Remove edition - [/(^| |\.)(m4b|m4a|mp3)( |$)/g, ''], // Remove file-type - [/ a novel.*$/g, ''], // Remove "a novel" - [/(^| )(un)?abridged( |$)/g, ' '], // Remove "unabridged/abridged" - [/^\d+ | \d+$/g, ''], // Remove preceding/trailing numbers + [/(^| |\.)(m4b|m4a|mp3)( |$)/g, ''], // Remove file-type + [/ a novel.*$/g, ''], // Remove "a novel" + [/(^| )(un)?abridged( |$)/g, ' '], // Remove "unabridged/abridged" + [/^\d+ | \d+$/g, ''] // Remove preceding/trailing numbers ] // Main variant @@ -197,8 +240,7 @@ class BookFinder { let candidate = cleanTitle - for (const transformer of titleTransformers) - candidate = candidate.replace(transformer[0], transformer[1]).trim() + for (const transformer of titleTransformers) candidate = candidate.replace(transformer[0], transformer[1]).trim() if (candidate != cleanTitle) { if (candidate) { @@ -240,7 +282,7 @@ class BookFinder { #removeAuthorFromTitle(title) { if (!this.cleanAuthor) return title - const authorRe = new RegExp(`(^| | by |)${escapeRegExp(this.cleanAuthor)}(?= |$)`, "g") + const authorRe = new RegExp(`(^| | by |)${escapeRegExp(this.cleanAuthor)}(?= |$)`, 'g') const authorCleanedTitle = cleanAuthorForCompares(title) const authorCleanedTitleWithoutAuthor = authorCleanedTitle.replace(authorRe, '') if (authorCleanedTitleWithoutAuthor !== authorCleanedTitle) { @@ -297,7 +339,7 @@ class BookFinder { promises.push(this.validateAuthor(candidate)) } const results = [...new Set(await Promise.all(promises))] - filteredCandidates = results.filter(author => author) + filteredCandidates = results.filter((author) => author) // If no valid candidates were found, add back an aggresively cleaned author version if (!filteredCandidates.length && this.cleanAuthor) filteredCandidates.push(this.agressivelyCleanAuthor) // Always add an empty author candidate @@ -312,17 +354,16 @@ class BookFinder { } } - /** * Search for books including fuzzy searches - * + * * @param {Object} libraryItem - * @param {string} provider - * @param {string} title - * @param {string} author - * @param {string} isbn - * @param {string} asin - * @param {{titleDistance:number, authorDistance:number, maxFuzzySearches:number}} options + * @param {string} provider + * @param {string} title + * @param {string} author + * @param {string} isbn + * @param {string} asin + * @param {{titleDistance:number, authorDistance:number, maxFuzzySearches:number}} options * @returns {Promise} */ async search(libraryItem, provider, title, author, isbn, asin, options = {}) { @@ -337,8 +378,7 @@ class BookFinder { return this.getCustomProviderResults(title, author, isbn, provider) } - if (!title) - return books + if (!title) return books books = await this.runSearch(title, author, provider, asin, maxTitleDistance, maxAuthorDistance) @@ -353,17 +393,14 @@ class BookFinder { let authorCandidates = new BookFinder.AuthorCandidates(cleanAuthor, this.audnexus) // Remove underscores and parentheses with their contents, and replace with a separator - const cleanTitle = title.replace(/\[.*?\]|\(.*?\)|{.*?}|_/g, " - ") + const cleanTitle = title.replace(/\[.*?\]|\(.*?\)|{.*?}|_/g, ' - ') // Split title into hypen-separated parts const titleParts = cleanTitle.split(/ - | -|- /) - for (const titlePart of titleParts) - authorCandidates.add(titlePart) + for (const titlePart of titleParts) authorCandidates.add(titlePart) authorCandidates = await authorCandidates.getCandidates() - loop_author: - for (const authorCandidate of authorCandidates) { + loop_author: for (const authorCandidate of authorCandidates) { let titleCandidates = new BookFinder.TitleCandidates(authorCandidate) - for (const titlePart of titleParts) - titleCandidates.add(titlePart) + for (const titlePart of titleParts) titleCandidates.add(titlePart) titleCandidates = titleCandidates.getCandidates() for (const titleCandidate of titleCandidates) { if (titleCandidate == title && authorCandidate == author) continue // We already tried this @@ -393,10 +430,10 @@ class BookFinder { /** * Search for books - * - * @param {string} title - * @param {string} author - * @param {string} provider + * + * @param {string} title + * @param {string} author + * @param {string} provider * @param {string} asin only used for audible providers * @param {number} maxTitleDistance only used for openlibrary provider * @param {number} maxAuthorDistance only used for openlibrary provider @@ -412,7 +449,7 @@ class BookFinder { } else if (provider.startsWith('audible')) { books = await this.getAudibleResults(title, author, asin, provider) } else if (provider === 'itunes') { - books = await this.getiTunesAudiobooksResults(title, author) + books = await this.getiTunesAudiobooksResults(title) } else if (provider === 'openlibrary') { books = await this.getOpenLibResults(title, author, maxTitleDistance, maxAuthorDistance) } else if (provider === 'fantlab') { @@ -448,7 +485,7 @@ class BookFinder { covers.push(result.cover) } }) - return [...(new Set(covers))] + return [...new Set(covers)] } findChapters(asin, region) { @@ -468,7 +505,7 @@ function stripSubtitle(title) { function replaceAccentedChars(str) { try { - return str.normalize('NFD').replace(/[\u0300-\u036f]/g, "") + return str.normalize('NFD').replace(/[\u0300-\u036f]/g, '') } catch (error) { Logger.error('[BookFinder] str normalize error', error) return str @@ -483,7 +520,7 @@ function cleanTitleForCompares(title) { let stripped = stripSubtitle(title) // Remove text in paranthesis (i.e. "Ender's Game (Ender's Saga)" becomes "Ender's Game") - let cleaned = stripped.replace(/ *\([^)]*\) */g, "") + let cleaned = stripped.replace(/ *\([^)]*\) */g, '') // Remove single quotes (i.e. "Ender's Game" becomes "Enders Game") cleaned = cleaned.replace(/'/g, '') diff --git a/server/providers/Audible.js b/server/providers/Audible.js index 97fea620..96c6cccb 100644 --- a/server/providers/Audible.js +++ b/server/providers/Audible.js @@ -1,145 +1,176 @@ -const axios = require('axios') +const axios = require('axios').default const htmlSanitizer = require('../utils/htmlSanitizer') const Logger = require('../Logger') class Audible { - constructor() { - this.regionMap = { - 'us': '.com', - 'ca': '.ca', - 'uk': '.co.uk', - 'au': '.com.au', - 'fr': '.fr', - 'de': '.de', - 'jp': '.co.jp', - 'it': '.it', - 'in': '.in', - 'es': '.es' - } + #responseTimeout = 30000 + + constructor() { + this.regionMap = { + us: '.com', + ca: '.ca', + uk: '.co.uk', + au: '.com.au', + fr: '.fr', + de: '.de', + jp: '.co.jp', + it: '.it', + in: '.in', + es: '.es' + } + } + + /** + * Audible will sometimes send sequences with "Book 1" or "2, Dramatized Adaptation" + * @see https://github.com/advplyr/audiobookshelf/issues/2380 + * @see https://github.com/advplyr/audiobookshelf/issues/1339 + * + * @param {string} seriesName + * @param {string} sequence + * @returns {string} + */ + cleanSeriesSequence(seriesName, sequence) { + if (!sequence) return '' + // match any number with optional decimal (e.g, 1 or 1.5 or .5) + let numberFound = sequence.match(/\.\d+|\d+(?:\.\d+)?/) + let updatedSequence = numberFound ? numberFound[0] : sequence + if (sequence !== updatedSequence) { + Logger.debug(`[Audible] Series "${seriesName}" sequence was cleaned from "${sequence}" to "${updatedSequence}"`) + } + return updatedSequence + } + + cleanResult(item) { + const { title, subtitle, asin, authors, narrators, publisherName, summary, releaseDate, image, genres, seriesPrimary, seriesSecondary, language, runtimeLengthMin, formatType } = item + + const series = [] + if (seriesPrimary) { + series.push({ + series: seriesPrimary.name, + sequence: this.cleanSeriesSequence(seriesPrimary.name, seriesPrimary.position || '') + }) + } + if (seriesSecondary) { + series.push({ + series: seriesSecondary.name, + sequence: this.cleanSeriesSequence(seriesSecondary.name, seriesSecondary.position || '') + }) } - /** - * Audible will sometimes send sequences with "Book 1" or "2, Dramatized Adaptation" - * @see https://github.com/advplyr/audiobookshelf/issues/2380 - * @see https://github.com/advplyr/audiobookshelf/issues/1339 - * - * @param {string} seriesName - * @param {string} sequence - * @returns {string} - */ - cleanSeriesSequence(seriesName, sequence) { - if (!sequence) return '' - // match any number with optional decimal (e.g, 1 or 1.5 or .5) - let numberFound = sequence.match(/\.\d+|\d+(?:\.\d+)?/) - let updatedSequence = numberFound ? numberFound[0] : sequence - if (sequence !== updatedSequence) { - Logger.debug(`[Audible] Series "${seriesName}" sequence was cleaned from "${sequence}" to "${updatedSequence}"`) - } - return updatedSequence + const genresFiltered = genres ? genres.filter((g) => g.type == 'genre').map((g) => g.name) : [] + const tagsFiltered = genres ? genres.filter((g) => g.type == 'tag').map((g) => g.name) : [] + + return { + title, + subtitle: subtitle || null, + author: authors ? authors.map(({ name }) => name).join(', ') : null, + narrator: narrators ? narrators.map(({ name }) => name).join(', ') : null, + publisher: publisherName, + publishedYear: releaseDate ? releaseDate.split('-')[0] : null, + description: summary ? htmlSanitizer.stripAllTags(summary) : null, + cover: image, + asin, + genres: genresFiltered.length ? genresFiltered : null, + tags: tagsFiltered.length ? tagsFiltered.join(', ') : null, + series: series.length ? series : null, + language: language ? language.charAt(0).toUpperCase() + language.slice(1) : null, + duration: runtimeLengthMin && !isNaN(runtimeLengthMin) ? Number(runtimeLengthMin) : 0, + region: item.region || null, + rating: item.rating || null, + abridged: formatType === 'abridged' + } + } + + /** + * Test if a search title matches an ASIN. Supports lowercase letters + * + * @param {string} title + * @returns {boolean} + */ + isProbablyAsin(title) { + return /^[0-9A-Za-z]{10}$/.test(title) + } + + /** + * + * @param {string} asin + * @param {string} region + * @param {number} [timeout] response timeout in ms + * @returns {Promise} + */ + asinSearch(asin, region, timeout = this.#responseTimeout) { + if (!asin) return [] + if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout + + asin = encodeURIComponent(asin.toUpperCase()) + var regionQuery = region ? `?region=${region}` : '' + var url = `https://api.audnex.us/books/${asin}${regionQuery}` + Logger.debug(`[Audible] ASIN url: ${url}`) + return axios + .get(url, { + timeout + }) + .then((res) => { + if (!res || !res.data || !res.data.asin) return null + return res.data + }) + .catch((error) => { + Logger.error('[Audible] ASIN search error', error) + return [] + }) + } + + /** + * + * @param {string} title + * @param {string} author + * @param {string} asin + * @param {string} region + * @param {number} [timeout] response timeout in ms + * @returns {Promise} + */ + async search(title, author, asin, region, timeout = this.#responseTimeout) { + if (region && !this.regionMap[region]) { + Logger.error(`[Audible] search: Invalid region ${region}`) + region = '' + } + if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout + + let items + if (asin) { + items = [await this.asinSearch(asin, region, timeout)] } - cleanResult(item) { - const { title, subtitle, asin, authors, narrators, publisherName, summary, releaseDate, image, genres, seriesPrimary, seriesSecondary, language, runtimeLengthMin, formatType } = item - - const series = [] - if (seriesPrimary) { - series.push({ - series: seriesPrimary.name, - sequence: this.cleanSeriesSequence(seriesPrimary.name, seriesPrimary.position || '') - }) - } - if (seriesSecondary) { - series.push({ - series: seriesSecondary.name, - sequence: this.cleanSeriesSequence(seriesSecondary.name, seriesSecondary.position || '') - }) - } - - const genresFiltered = genres ? genres.filter(g => g.type == "genre").map(g => g.name) : [] - const tagsFiltered = genres ? genres.filter(g => g.type == "tag").map(g => g.name) : [] - - return { - title, - subtitle: subtitle || null, - author: authors ? authors.map(({ name }) => name).join(', ') : null, - narrator: narrators ? narrators.map(({ name }) => name).join(', ') : null, - publisher: publisherName, - publishedYear: releaseDate ? releaseDate.split('-')[0] : null, - description: summary ? htmlSanitizer.stripAllTags(summary) : null, - cover: image, - asin, - genres: genresFiltered.length ? genresFiltered : null, - tags: tagsFiltered.length ? tagsFiltered.join(', ') : null, - series: series.length ? series : null, - language: language ? language.charAt(0).toUpperCase() + language.slice(1) : null, - duration: runtimeLengthMin && !isNaN(runtimeLengthMin) ? Number(runtimeLengthMin) : 0, - region: item.region || null, - rating: item.rating || null, - abridged: formatType === 'abridged' - } + if (!items && this.isProbablyAsin(title)) { + items = [await this.asinSearch(title, region, timeout)] } - /** - * Test if a search title matches an ASIN. Supports lowercase letters - * - * @param {string} title - * @returns {boolean} - */ - isProbablyAsin(title) { - return /^[0-9A-Za-z]{10}$/.test(title) - } - - asinSearch(asin, region) { - if (!asin) return [] - asin = encodeURIComponent(asin.toUpperCase()) - var regionQuery = region ? `?region=${region}` : '' - var url = `https://api.audnex.us/books/${asin}${regionQuery}` - Logger.debug(`[Audible] ASIN url: ${url}`) - return axios.get(url).then((res) => { - if (!res || !res.data || !res.data.asin) return null - return res.data - }).catch(error => { - Logger.error('[Audible] ASIN search error', error) - return [] + if (!items) { + const queryObj = { + num_results: '10', + products_sort_by: 'Relevance', + title: title + } + if (author) queryObj.author = author + const queryString = new URLSearchParams(queryObj).toString() + const tld = region ? this.regionMap[region] : '.com' + const url = `https://api.audible${tld}/1.0/catalog/products?${queryString}` + Logger.debug(`[Audible] Search url: ${url}`) + items = await axios + .get(url, { + timeout + }) + .then((res) => { + if (!res?.data?.products) return null + return Promise.all(res.data.products.map((result) => this.asinSearch(result.asin, region, timeout))) + }) + .catch((error) => { + Logger.error('[Audible] query search error', error) + return [] }) } - - async search(title, author, asin, region) { - if (region && !this.regionMap[region]) { - Logger.error(`[Audible] search: Invalid region ${region}`) - region = '' - } - - let items - if (asin) { - items = [await this.asinSearch(asin, region)] - } - - if (!items && this.isProbablyAsin(title)) { - items = [await this.asinSearch(title, region)] - } - - if (!items) { - const queryObj = { - num_results: '10', - products_sort_by: 'Relevance', - title: title - } - if (author) queryObj.author = author - const queryString = (new URLSearchParams(queryObj)).toString() - const tld = region ? this.regionMap[region] : '.com' - const url = `https://api.audible${tld}/1.0/catalog/products?${queryString}` - Logger.debug(`[Audible] Search url: ${url}`) - items = await axios.get(url).then((res) => { - if (!res?.data?.products) return null - return Promise.all(res.data.products.map(result => this.asinSearch(result.asin, region))) - }).catch(error => { - Logger.error('[Audible] query search error', error) - return [] - }) - } - return items ? items.map(item => this.cleanResult(item)) : [] - } + return items?.map((item) => this.cleanResult(item)) || [] + } } -module.exports = Audible \ No newline at end of file +module.exports = Audible diff --git a/server/providers/AudiobookCovers.js b/server/providers/AudiobookCovers.js index 53ae0508..8e284fea 100644 --- a/server/providers/AudiobookCovers.js +++ b/server/providers/AudiobookCovers.js @@ -2,22 +2,32 @@ const axios = require('axios') const Logger = require('../Logger') class AudiobookCovers { - constructor() { } + #responseTimeout = 30000 + + constructor() {} + + /** + * + * @param {string} search + * @param {number} [timeout] + * @returns {Promise<{cover: string}[]>} + */ + async search(search, timeout = this.#responseTimeout) { + if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout - async search(search) { const url = `https://api.audiobookcovers.com/cover/bytext/` const params = new URLSearchParams([['q', search]]) - const items = await axios.get(url, { params }).then((res) => { - if (!res || !res.data) return [] - return res.data - }).catch(error => { - Logger.error('[AudiobookCovers] Cover search error', error) - return [] - }) - return items.map(item => ({ cover: item.versions.png.original })) + const items = await axios + .get(url, { + params, + timeout + }) + .then((res) => res?.data || []) + .catch((error) => { + Logger.error('[AudiobookCovers] Cover search error', error) + return [] + }) + return items.map((item) => ({ cover: item.versions.png.original })) } } - - - module.exports = AudiobookCovers diff --git a/server/providers/Audnexus.js b/server/providers/Audnexus.js index f5034bff..2a871e39 100644 --- a/server/providers/Audnexus.js +++ b/server/providers/Audnexus.js @@ -1,4 +1,4 @@ -const axios = require('axios') +const axios = require('axios').default const { levenshteinDistance } = require('../utils/index') const Logger = require('../Logger') @@ -16,10 +16,10 @@ class Audnexus { } /** - * - * @param {string} name - * @param {string} region - * @returns {Promise<{asin:string, name:string}[]>} + * + * @param {string} name + * @param {string} region + * @returns {Promise<{asin:string, name:string}[]>} */ authorASINsRequest(name, region) { const searchParams = new URLSearchParams() @@ -27,18 +27,21 @@ class Audnexus { if (region) searchParams.set('region', region) const authorRequestUrl = `${this.baseUrl}/authors?${searchParams.toString()}` Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - return axios.get(authorRequestUrl).then((res) => { - return res.data || [] - }).catch((error) => { - Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error) - return [] - }) + return axios + .get(authorRequestUrl) + .then((res) => { + return res.data || [] + }) + .catch((error) => { + Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error) + return [] + }) } /** - * - * @param {string} asin - * @param {string} region + * + * @param {string} asin + * @param {string} region * @returns {Promise} */ authorRequest(asin, region) { @@ -46,18 +49,21 @@ class Audnexus { const regionQuery = region ? `?region=${region}` : '' const authorRequestUrl = `${this.baseUrl}/authors/${asin}${regionQuery}` Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - return axios.get(authorRequestUrl).then((res) => { - return res.data - }).catch((error) => { - Logger.error(`[Audnexus] Author request failed for ${asin}`, error) - return null - }) + return axios + .get(authorRequestUrl) + .then((res) => { + return res.data + }) + .catch((error) => { + Logger.error(`[Audnexus] Author request failed for ${asin}`, error) + return null + }) } /** - * - * @param {string} asin - * @param {string} region + * + * @param {string} asin + * @param {string} region * @returns {Promise} */ async findAuthorByASIN(asin, region) { @@ -74,10 +80,10 @@ class Audnexus { } /** - * - * @param {string} name - * @param {string} region - * @param {number} maxLevenshtein + * + * @param {string} name + * @param {string} region + * @param {number} maxLevenshtein * @returns {Promise} */ async findAuthorByName(name, region, maxLevenshtein = 3) { @@ -108,12 +114,15 @@ class Audnexus { getChaptersByASIN(asin, region) { Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`) - return axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`).then((res) => { - return res.data - }).catch((error) => { - Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error) - return null - }) + return axios + .get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`) + .then((res) => { + return res.data + }) + .catch((error) => { + Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error) + return null + }) } } -module.exports = Audnexus \ No newline at end of file +module.exports = Audnexus diff --git a/server/providers/CustomProviderAdapter.js b/server/providers/CustomProviderAdapter.js index 28c1bc04..fe6537fd 100644 --- a/server/providers/CustomProviderAdapter.js +++ b/server/providers/CustomProviderAdapter.js @@ -1,97 +1,91 @@ +const axios = require('axios').default const Database = require('../Database') -const axios = require('axios') const Logger = require('../Logger') class CustomProviderAdapter { - constructor() { } + #responseTimeout = 30000 - /** - * - * @param {string} title - * @param {string} author - * @param {string} isbn - * @param {string} providerSlug - * @param {string} mediaType - * @returns {Promise} - */ - async search(title, author, isbn, providerSlug, mediaType) { - const providerId = providerSlug.split('custom-')[1] - const provider = await Database.customMetadataProviderModel.findByPk(providerId) + constructor() {} - if (!provider) { - throw new Error("Custom provider not found for the given id") - } + /** + * + * @param {string} title + * @param {string} author + * @param {string} isbn + * @param {string} providerSlug + * @param {string} mediaType + * @param {number} [timeout] response timeout in ms + * @returns {Promise} + */ + async search(title, author, isbn, providerSlug, mediaType, timeout = this.#responseTimeout) { + if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout - // Setup query params - const queryObj = { - mediaType, - query: title - } - if (author) { - queryObj.author = author - } - if (isbn) { - queryObj.isbn = isbn - } - const queryString = (new URLSearchParams(queryObj)).toString() + const providerId = providerSlug.split('custom-')[1] + const provider = await Database.customMetadataProviderModel.findByPk(providerId) - // Setup headers - const axiosOptions = {} - if (provider.authHeaderValue) { - axiosOptions.headers = { - 'Authorization': provider.authHeaderValue - } - } - - const matches = await axios.get(`${provider.url}/search?${queryString}`, axiosOptions).then((res) => { - if (!res?.data || !Array.isArray(res.data.matches)) return null - return res.data.matches - }).catch(error => { - Logger.error('[CustomMetadataProvider] Search error', error) - return [] - }) - - if (!matches) { - throw new Error("Custom provider returned malformed response") - } - - // re-map keys to throw out - return matches.map(({ - title, - subtitle, - author, - narrator, - publisher, - publishedYear, - description, - cover, - isbn, - asin, - genres, - tags, - series, - language, - duration - }) => { - return { - title, - subtitle, - author, - narrator, - publisher, - publishedYear, - description, - cover, - isbn, - asin, - genres, - tags: tags?.join(',') || null, - series: series?.length ? series : null, - language, - duration - } - }) + if (!provider) { + throw new Error('Custom provider not found for the given id') } + + // Setup query params + const queryObj = { + mediaType, + query: title + } + if (author) { + queryObj.author = author + } + if (isbn) { + queryObj.isbn = isbn + } + const queryString = new URLSearchParams(queryObj).toString() + + // Setup headers + const axiosOptions = { + timeout + } + if (provider.authHeaderValue) { + axiosOptions.headers = { + Authorization: provider.authHeaderValue + } + } + + const matches = await axios + .get(`${provider.url}/search?${queryString}`, axiosOptions) + .then((res) => { + if (!res?.data || !Array.isArray(res.data.matches)) return null + return res.data.matches + }) + .catch((error) => { + Logger.error('[CustomMetadataProvider] Search error', error) + return [] + }) + + if (!matches) { + throw new Error('Custom provider returned malformed response') + } + + // re-map keys to throw out + return matches.map(({ title, subtitle, author, narrator, publisher, publishedYear, description, cover, isbn, asin, genres, tags, series, language, duration }) => { + return { + title, + subtitle, + author, + narrator, + publisher, + publishedYear, + description, + cover, + isbn, + asin, + genres, + tags: tags?.join(',') || null, + series: series?.length ? series : null, + language, + duration + } + }) + } } -module.exports = CustomProviderAdapter \ No newline at end of file +module.exports = CustomProviderAdapter diff --git a/server/providers/FantLab.js b/server/providers/FantLab.js index 41328ca1..dd9f60cc 100644 --- a/server/providers/FantLab.js +++ b/server/providers/FantLab.js @@ -2,6 +2,7 @@ const axios = require('axios') const Logger = require('../Logger') class FantLab { + #responseTimeout = 30000 // 7 - other // 11 - essay // 12 - article @@ -22,28 +23,47 @@ class FantLab { _filterWorkType = [7, 11, 12, 22, 23, 24, 25, 26, 46, 47, 49, 51, 52, 55, 56, 57] _baseUrl = 'https://api.fantlab.ru' - constructor() { } + constructor() {} + + /** + * @param {string} title + * @param {string} author' + * @param {number} [timeout] response timeout in ms + * @returns {Promise} + **/ + async search(title, author, timeout = this.#responseTimeout) { + if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout - async search(title, author) { let searchString = encodeURIComponent(title) if (author) { searchString += encodeURIComponent(' ' + author) } const url = `${this._baseUrl}/search-works?q=${searchString}&page=1&onlymatches=1` Logger.debug(`[FantLab] Search url: ${url}`) - const items = await axios.get(url).then((res) => { - return res.data || [] - }).catch(error => { - Logger.error('[FantLab] search error', error) - return [] - }) + const items = await axios + .get(url, { + timeout + }) + .then((res) => { + return res.data || [] + }) + .catch((error) => { + Logger.error('[FantLab] search error', error) + return [] + }) - return Promise.all(items.map(async item => await this.getWork(item))).then(resArray => { - return resArray.filter(res => res) + return Promise.all(items.map(async (item) => await this.getWork(item, timeout))).then((resArray) => { + return resArray.filter((res) => res) }) } - async getWork(item) { + /** + * @param {Object} item + * @param {number} [timeout] response timeout in ms + * @returns {Promise} + **/ + async getWork(item, timeout = this.#responseTimeout) { + if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout const { work_id, work_type_id } = item if (this._filterWorkType.includes(work_type_id)) { @@ -51,23 +71,34 @@ class FantLab { } const url = `${this._baseUrl}/work/${work_id}/extended` - const bookData = await axios.get(url).then((resp) => { - return resp.data || null - }).catch((error) => { - Logger.error(`[FantLab] work info request for url "${url}" error`, error) - return null - }) + const bookData = await axios + .get(url, { + timeout + }) + .then((resp) => { + return resp.data || null + }) + .catch((error) => { + Logger.error(`[FantLab] work info request for url "${url}" error`, error) + return null + }) - return this.cleanBookData(bookData) + return this.cleanBookData(bookData, timeout) } - async cleanBookData(bookData) { + /** + * + * @param {Object} bookData + * @param {number} [timeout] + * @returns {Promise} + */ + async cleanBookData(bookData, timeout = this.#responseTimeout) { let { authors, work_name_alts, work_id, work_name, work_year, work_description, image, classificatory, editions_blocks } = bookData const subtitle = Array.isArray(work_name_alts) ? work_name_alts[0] : null - const authorNames = authors.map(au => (au.name || '').trim()).filter(au => au) + const authorNames = authors.map((au) => (au.name || '').trim()).filter((au) => au) - const imageAndIsbn = await this.tryGetCoverFromEditions(editions_blocks) + const imageAndIsbn = await this.tryGetCoverFromEditions(editions_blocks, timeout) const imageToUse = imageAndIsbn?.imageUrl || image @@ -88,7 +119,7 @@ class FantLab { tryGetGenres(classificatory) { if (!classificatory || !classificatory.genre_group) return [] - const genresGroup = classificatory.genre_group.find(group => group.genre_group_id == 1) // genres and subgenres + const genresGroup = classificatory.genre_group.find((group) => group.genre_group_id == 1) // genres and subgenres // genre_group_id=2 - General Characteristics // genre_group_id=3 - Arena @@ -108,10 +139,16 @@ class FantLab { tryGetSubGenres(rootGenre) { if (!rootGenre.genre || !rootGenre.genre.length) return [] - return rootGenre.genre.map(g => g.label).filter(g => g) + return rootGenre.genre.map((g) => g.label).filter((g) => g) } - async tryGetCoverFromEditions(editions) { + /** + * + * @param {Object} editions + * @param {number} [timeout] + * @returns {Promise<{imageUrl: string, isbn: string}> + */ + async tryGetCoverFromEditions(editions, timeout = this.#responseTimeout) { if (!editions) { return null } @@ -129,24 +166,37 @@ class FantLab { const isbn = lastEdition['isbn'] || null // get only from paper edition return { - imageUrl: await this.getCoverFromEdition(editionId), + imageUrl: await this.getCoverFromEdition(editionId, timeout), isbn } } - async getCoverFromEdition(editionId) { + /** + * + * @param {number} editionId + * @param {number} [timeout] + * @returns {Promise} + */ + async getCoverFromEdition(editionId, timeout = this.#responseTimeout) { if (!editionId) return null + if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout + const url = `${this._baseUrl}/edition/${editionId}` - const editionInfo = await axios.get(url).then((resp) => { - return resp.data || null - }).catch(error => { - Logger.error(`[FantLab] search cover from edition with url "${url}" error`, error) - return null - }) + const editionInfo = await axios + .get(url, { + timeout + }) + .then((resp) => { + return resp.data || null + }) + .catch((error) => { + Logger.error(`[FantLab] search cover from edition with url "${url}" error`, error) + return null + }) return editionInfo?.image || null } } -module.exports = FantLab \ No newline at end of file +module.exports = FantLab diff --git a/server/providers/GoogleBooks.js b/server/providers/GoogleBooks.js index 69cc392e..76a3dcea 100644 --- a/server/providers/GoogleBooks.js +++ b/server/providers/GoogleBooks.js @@ -2,12 +2,14 @@ const axios = require('axios') const Logger = require('../Logger') class GoogleBooks { - constructor() { } + #responseTimeout = 30000 + + constructor() {} extractIsbn(industryIdentifiers) { if (!industryIdentifiers || !industryIdentifiers.length) return null - var isbnObj = industryIdentifiers.find(i => i.type === 'ISBN_13') || industryIdentifiers.find(i => i.type === 'ISBN_10') + var isbnObj = industryIdentifiers.find((i) => i.type === 'ISBN_13') || industryIdentifiers.find((i) => i.type === 'ISBN_10') if (isbnObj && isbnObj.identifier) return isbnObj.identifier return null } @@ -38,24 +40,38 @@ class GoogleBooks { } } - async search(title, author) { + /** + * Search for a book by title and author + * @param {string} title + * @param {string} author + * @param {number} [timeout] response timeout in ms + * @returns {Promise} + **/ + async search(title, author, timeout = this.#responseTimeout) { + if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout + title = encodeURIComponent(title) - var queryString = `q=intitle:${title}` + let queryString = `q=intitle:${title}` if (author) { author = encodeURIComponent(author) queryString += `+inauthor:${author}` } - var url = `https://www.googleapis.com/books/v1/volumes?${queryString}` + const url = `https://www.googleapis.com/books/v1/volumes?${queryString}` Logger.debug(`[GoogleBooks] Search url: ${url}`) - var items = await axios.get(url).then((res) => { - if (!res || !res.data || !res.data.items) return [] - return res.data.items - }).catch(error => { - Logger.error('[GoogleBooks] Volume search error', error) - return [] - }) - return items.map(item => this.cleanResult(item)) + const items = await axios + .get(url, { + timeout + }) + .then((res) => { + if (!res || !res.data || !res.data.items) return [] + return res.data.items + }) + .catch((error) => { + Logger.error('[GoogleBooks] Volume search error', error) + return [] + }) + return items.map((item) => this.cleanResult(item)) } } -module.exports = GoogleBooks \ No newline at end of file +module.exports = GoogleBooks diff --git a/server/providers/OpenLibrary.js b/server/providers/OpenLibrary.js index e22a3ac0..453c919b 100644 --- a/server/providers/OpenLibrary.js +++ b/server/providers/OpenLibrary.js @@ -1,17 +1,31 @@ -var axios = require('axios') +const axios = require('axios').default class OpenLibrary { + #responseTimeout = 30000 + constructor() { this.baseUrl = 'https://openlibrary.org' } - get(uri) { - return axios.get(`${this.baseUrl}/${uri}`).then((res) => { - return res.data - }).catch((error) => { - console.error('Failed', error) - return false - }) + /** + * + * @param {string} uri + * @param {number} timeout + * @returns {Promise} + */ + get(uri, timeout = this.#responseTimeout) { + if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout + return axios + .get(`${this.baseUrl}/${uri}`, { + timeout + }) + .then((res) => { + return res.data + }) + .catch((error) => { + console.error('Failed', error) + return null + }) } async isbnLookup(isbn) { @@ -33,7 +47,7 @@ class OpenLibrary { } } if (!worksData.covers) worksData.covers = [] - var coverImages = worksData.covers.filter(c => c > 0).map(c => `https://covers.openlibrary.org/b/id/${c}-L.jpg`) + var coverImages = worksData.covers.filter((c) => c > 0).map((c) => `https://covers.openlibrary.org/b/id/${c}-L.jpg`) var description = null if (worksData.description) { if (typeof worksData.description === 'string') { @@ -73,27 +87,35 @@ class OpenLibrary { } async search(query) { - var queryString = Object.keys(query).map(key => key + '=' + query[key]).join('&') + var queryString = Object.keys(query) + .map((key) => key + '=' + query[key]) + .join('&') var lookupData = await this.get(`/search.json?${queryString}`) if (!lookupData) { return { errorCode: 404 } } - var searchDocs = await Promise.all(lookupData.docs.map(d => this.cleanSearchDoc(d))) + var searchDocs = await Promise.all(lookupData.docs.map((d) => this.cleanSearchDoc(d))) return searchDocs } - async searchTitle(title) { - title = encodeURIComponent(title); - var lookupData = await this.get(`/search.json?title=${title}`) + /** + * + * @param {string} title + * @param {number} timeout + * @returns {Promise} + */ + async searchTitle(title, timeout = this.#responseTimeout) { + title = encodeURIComponent(title) + var lookupData = await this.get(`/search.json?title=${title}`, timeout) if (!lookupData) { return { errorCode: 404 } } - var searchDocs = await Promise.all(lookupData.docs.map(d => this.cleanSearchDoc(d))) + var searchDocs = await Promise.all(lookupData.docs.map((d) => this.cleanSearchDoc(d))) return searchDocs } } -module.exports = OpenLibrary \ No newline at end of file +module.exports = OpenLibrary diff --git a/server/providers/iTunes.js b/server/providers/iTunes.js index 05a661b5..1ec051d1 100644 --- a/server/providers/iTunes.js +++ b/server/providers/iTunes.js @@ -28,19 +28,24 @@ const htmlSanitizer = require('../utils/htmlSanitizer') */ class iTunes { - constructor() { } + #responseTimeout = 30000 + + constructor() {} /** * @see https://developer.apple.com/library/archive/documentation/AudioVideo/Conceptual/iTuneSearchAPI/Searching.html - * - * @param {iTunesSearchParams} options + * + * @param {iTunesSearchParams} options + * @param {number} [timeout] response timeout in ms * @returns {Promise} */ - search(options) { + search(options, timeout = this.#responseTimeout) { if (!options.term) { Logger.error('[iTunes] Invalid search options - no term') return [] } + if (!timeout || isNaN(timeout)) timeout = this.#responseTimeout + const query = { term: options.term, media: options.media, @@ -49,12 +54,18 @@ class iTunes { limit: options.limit, country: options.country } - return axios.get('https://itunes.apple.com/search', { params: query }).then((response) => { - return response.data.results || [] - }).catch((error) => { - Logger.error(`[iTunes] search request error`, error) - return [] - }) + return axios + .get('https://itunes.apple.com/search', { + params: query, + timeout + }) + .then((response) => { + return response.data.results || [] + }) + .catch((error) => { + Logger.error(`[iTunes] search request error`, error) + return [] + }) } // Example cover art: https://is1-ssl.mzstatic.com/image/thumb/Music118/v4/cb/ea/73/cbea739b-ff3b-11c4-fb93-7889fbec7390/9781598874983_cover.jpg/100x100bb.jpg @@ -65,20 +76,22 @@ class iTunes { return data.artworkUrl600 } // Should already be sorted from small to large - var artworkSizes = Object.keys(data).filter(key => key.startsWith('artworkUrl')).map(key => { - return { - url: data[key], - size: Number(key.replace('artworkUrl', '')) - } - }) + var artworkSizes = Object.keys(data) + .filter((key) => key.startsWith('artworkUrl')) + .map((key) => { + return { + url: data[key], + size: Number(key.replace('artworkUrl', '')) + } + }) if (!artworkSizes.length) return null // Return next biggest size > 600 - var nextBestSize = artworkSizes.find(size => size.size > 600) + var nextBestSize = artworkSizes.find((size) => size.size > 600) if (nextBestSize) return nextBestSize.url // Find square artwork - var squareArtwork = artworkSizes.find(size => size.url.includes(`${size.size}x${size.size}bb`)) + var squareArtwork = artworkSizes.find((size) => size.url.includes(`${size.size}x${size.size}bb`)) // Square cover replace with 600x600bb if (squareArtwork) { @@ -106,15 +119,21 @@ class iTunes { } } - searchAudiobooks(term) { - return this.search({ term, entity: 'audiobook', media: 'audiobook' }).then((results) => { + /** + * + * @param {string} term + * @param {number} [timeout] response timeout in ms + * @returns {Promise} + */ + searchAudiobooks(term, timeout = this.#responseTimeout) { + return this.search({ term, entity: 'audiobook', media: 'audiobook' }, timeout).then((results) => { return results.map(this.cleanAudiobook.bind(this)) }) } /** - * - * @param {Object} data + * + * @param {Object} data * @returns {iTunesPodcastSearchResult} */ cleanPodcast(data) { @@ -136,13 +155,14 @@ class iTunes { } /** - * - * @param {string} term - * @param {{country:string}} options + * + * @param {string} term + * @param {{country:string}} options + * @param {number} [timeout] response timeout in ms * @returns {Promise} */ - searchPodcasts(term, options = {}) { - return this.search({ term, entity: 'podcast', media: 'podcast', ...options }).then((results) => { + searchPodcasts(term, options = {}, timeout = this.#responseTimeout) { + return this.search({ term, entity: 'podcast', media: 'podcast', ...options }, timeout).then((results) => { return results.map(this.cleanPodcast.bind(this)) }) }