From cf927f61a0b1714794cb38b026476a056ae59e25 Mon Sep 17 00:00:00 2001 From: Dmitry Naboychenko Date: Tue, 7 Feb 2023 00:25:18 +0300 Subject: [PATCH 1/4] Add FantLab.ru Book Finder --- client/store/scanners.js | 4 + server/finders/BookFinder.js | 20 ++++- server/providers/FantLab.js | 170 +++++++++++++++++++++++++++++++++++ 3 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 server/providers/FantLab.js diff --git a/client/store/scanners.js b/client/store/scanners.js index 8443baf9..0a339a03 100644 --- a/client/store/scanners.js +++ b/client/store/scanners.js @@ -52,6 +52,10 @@ export const state = () => ({ { text: 'Audible.es', value: 'audible.es' + }, + { + text: 'FantLab.ru', + value: 'fantlab' } ], podcastProviders: [ diff --git a/server/finders/BookFinder.js b/server/finders/BookFinder.js index 9c993782..a7d5b730 100644 --- a/server/finders/BookFinder.js +++ b/server/finders/BookFinder.js @@ -3,6 +3,7 @@ const GoogleBooks = require('../providers/GoogleBooks') const Audible = require('../providers/Audible') const iTunes = require('../providers/iTunes') const Audnexus = require('../providers/Audnexus') +const FantLab = require('../providers/FantLab') const Logger = require('../Logger') const { levenshteinDistance } = require('../utils/index') @@ -13,6 +14,7 @@ class BookFinder { this.audible = new Audible() this.iTunesApi = new iTunes() this.audnexus = new Audnexus() + this.fantLab = new FantLab() this.verbose = false } @@ -146,6 +148,17 @@ class BookFinder { return books } + async getFantLabResults(title, author) { + var books = await this.fantLab.search(title, author) + if (this.verbose) Logger.debug(`FantLab Book Search Results: ${books.length || 0}`) + if (books.errorCode) { + Logger.error(`FantLab Search Error ${books.errorCode}`) + return [] + } + + return books + } + async getiTunesAudiobooksResults(title, author) { return this.iTunesApi.searchAudiobooks(title) } @@ -172,7 +185,10 @@ class BookFinder { books = await this.getiTunesAudiobooksResults(title, author) } else if (provider === 'openlibrary') { books = await this.getOpenLibResults(title, author, maxTitleDistance, maxAuthorDistance) - } else { + } else if (provider === 'fantlab') { + books = await this.getFantLabResults(title, author) + } + else { books = await this.getGoogleBooksResults(title, author) } @@ -186,7 +202,7 @@ class BookFinder { return this.search(provider, cleanedTitle, cleanedAuthor, isbn, asin, options) } - if (["google", "audible", "itunes"].includes(provider)) return books + if (["google", "audible", "itunes", 'fantlab'].includes(provider)) return books return books.sort((a, b) => { return a.totalDistance - b.totalDistance diff --git a/server/providers/FantLab.js b/server/providers/FantLab.js new file mode 100644 index 00000000..50a74c87 --- /dev/null +++ b/server/providers/FantLab.js @@ -0,0 +1,170 @@ +const axios = require('axios') +const Logger = require('../Logger') + +class FantLab { + // 7 - other + // 11 - essay + // 12 - article + // 22 - disser + // 23 - monography + // 24 - study + // 25 - encyclopedy + // 26 - magazine + // 46 - sketch + // 47 - reportage + // 49 - excerpt + // 51 - interview + // 52 - review + // 55 - libretto + // 56 - anthology series + // 57 - newspaper + // types can get here https://api.fantlab.ru/config.json + _filterWorkType = [7, 11, 12, 22, 23, 24, 25, 26, 46, 47, 49, 51, 52, 55, 56, 57]; + _baseUrl = 'https://api.fantlab.ru' + + constructor() { } + + async search(title, author) { + var searchString = encodeURIComponent(title) + if (author) { + searchString += encodeURIComponent(' ' + author) + } + var url = `${this._baseUrl}/search-works?q=${searchString}&page=1&onlymatches=1` + Logger.debug(`[FantLab] Search url: ${url}`) + var items = await axios.get(url).then((res) => { + return res.data || [] + }).catch(error => { + Logger.error('[FantLab] search error', error) + return [] + }) + + return Promise.all(items.map(async item => await this.getWork(item))) + } + + async getWork(item) { + var { work_id, work_type_id } = item + + if (this._filterWorkType.includes(work_type_id)) { + return { title: null } + } + + var url = `${this._baseUrl}/work/${work_id}/extended` + var bookData = await axios.get(url).then((resp) => { + return resp.data || null + }).catch((error) => { + Logger.error(`[FantLab] work info reques error`, error) + return null + }) + + return await this.cleanBookData(bookData) + } + + async cleanBookData(bookData) { + var { authors, work_name_alts, work_id, work_name, work_year, work_description, image, classificatory, editions_blocks } = bookData; + + var subtitle = Array.isArray(work_name_alts) ? work_name_alts[0] : null + var auth = authors.map(function (author) { + return author.name + }); + + var genres = classificatory ? this.tryGetGenres(classificatory) : [] + + var imageAndIsbn = await this.tryGetCoverFromEditions(editions_blocks) + + if (imageAndIsbn) { + var { imageUrl, isbn } = imageAndIsbn + + if (imageUrl) { + image = imageUrl + } + } + + var cover = 'https://fantlab.ru' + image + + return { + id: work_id, + title: work_name, + subtitle: subtitle || null, + author: auth ? auth.join(', ') : null, + publisher: null, + publishedYear: work_year, + description: work_description, + cover: image ? cover : null, + genres: genres, + isbn: isbn + } + } + + tryGetGenres(classificatory) { + var { genre_group } = classificatory; + if (!genre_group) { + return [] + } + var genresGroup = genre_group.find(group => group.genre_group_id = 1) // genres and subgenres + + // genre_group_id=2 - General Characteristics + // genre_group_id=3 - Arena + // genre_group_id=4 - Duration of action + // genre_group_id=6 - Story moves + // genre_group_id=7 - Story linearity + // genre_group_id=5 - Recommended age of the reader + + if (!genresGroup) return [] + + var { genre } = genresGroup; + var rootGenre = genre[0]; + + var { label } = rootGenre + + return [label].concat(this.tryGetSubGenres(rootGenre)) + } + + tryGetSubGenres(rootGenre) { + var { genre } = rootGenre + return genre ? genre.map(genreObj => genreObj.label) : [] + } + + async tryGetCoverFromEditions(editions) { + + if (!editions) { + return null + } + + var bookEditions = editions['30'] // try get audiobooks first + if (!bookEditions) { + bookEditions = editions['10'] // paper editions in ru lang + } + + if (!bookEditions) { + return null + } + + var { list } = bookEditions + + var lastEdition = list[list.length - 1] + + var editionId = lastEdition['edition_id'] + var isbn = lastEdition['isbn'] // get only from paper edition + + return { + imageUrl: await this.getCoverFromEdition(editionId), + isbn: isbn + } + } + + async getCoverFromEdition(editionId) { + var url = `${this._baseUrl}/edition/${editionId}` + + var editionInfo = await axios.get(url).then((resp) => { + return resp.data || null + }).catch(error => { + Logger.error('[FantLab] search cover from edition error', error) + return null + }) + + return editionInfo ? editionInfo['image'] : null + } + +} + +module.exports = FantLab \ No newline at end of file From b9307143bd208b4c279ff81edfc10f26c15d9221 Mon Sep 17 00:00:00 2001 From: Dmitry Naboychenko Date: Wed, 8 Feb 2023 22:32:27 +0300 Subject: [PATCH 2/4] FantLab match provider fixes after code review --- server/providers/FantLab.js | 60 +++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/server/providers/FantLab.js b/server/providers/FantLab.js index 50a74c87..0cfd9f21 100644 --- a/server/providers/FantLab.js +++ b/server/providers/FantLab.js @@ -19,57 +19,59 @@ class FantLab { // 56 - anthology series // 57 - newspaper // types can get here https://api.fantlab.ru/config.json - _filterWorkType = [7, 11, 12, 22, 23, 24, 25, 26, 46, 47, 49, 51, 52, 55, 56, 57]; + _filterWorkType = [7, 11, 12, 22, 23, 24, 25, 26, 46, 47, 49, 51, 52, 55, 56, 57] _baseUrl = 'https://api.fantlab.ru' constructor() { } async search(title, author) { - var searchString = encodeURIComponent(title) + let searchString = encodeURIComponent(title) if (author) { searchString += encodeURIComponent(' ' + author) } - var url = `${this._baseUrl}/search-works?q=${searchString}&page=1&onlymatches=1` + const url = `${this._baseUrl}/search-works?q=${searchString}&page=1&onlymatches=1` Logger.debug(`[FantLab] Search url: ${url}`) - var items = await axios.get(url).then((res) => { + const items = await axios.get(url).then((res) => { return res.data || [] }).catch(error => { Logger.error('[FantLab] search error', error) return [] }) - return Promise.all(items.map(async item => await this.getWork(item))) + return Promise.all(items.map(async item => await this.getWork(item))).then(resArray => { + return resArray.filter(res => res != null) + }) } async getWork(item) { - var { work_id, work_type_id } = item + const { work_id, work_type_id } = item if (this._filterWorkType.includes(work_type_id)) { return { title: null } } - var url = `${this._baseUrl}/work/${work_id}/extended` - var bookData = await axios.get(url).then((resp) => { + const url = `${this._baseUrl}/work/${work_id}/extended` + const bookData = await axios.get(url).then((resp) => { return resp.data || null }).catch((error) => { Logger.error(`[FantLab] work info reques error`, error) return null }) - return await this.cleanBookData(bookData) + return this.cleanBookData(bookData) } async cleanBookData(bookData) { - var { authors, work_name_alts, work_id, work_name, work_year, work_description, image, classificatory, editions_blocks } = bookData; + let { authors, work_name_alts, work_id, work_name, work_year, work_description, image, classificatory, editions_blocks } = bookData - var subtitle = Array.isArray(work_name_alts) ? work_name_alts[0] : null - var auth = authors.map(function (author) { + const subtitle = Array.isArray(work_name_alts) ? work_name_alts[0] : null + const auth = authors.map(function (author) { return author.name - }); + }) - var genres = classificatory ? this.tryGetGenres(classificatory) : [] + const genres = classificatory ? this.tryGetGenres(classificatory) : [] - var imageAndIsbn = await this.tryGetCoverFromEditions(editions_blocks) + const imageAndIsbn = await this.tryGetCoverFromEditions(editions_blocks) if (imageAndIsbn) { var { imageUrl, isbn } = imageAndIsbn @@ -79,7 +81,7 @@ class FantLab { } } - var cover = 'https://fantlab.ru' + image + const cover = 'https://fantlab.ru' + image return { id: work_id, @@ -96,11 +98,11 @@ class FantLab { } tryGetGenres(classificatory) { - var { genre_group } = classificatory; + const { genre_group } = classificatory if (!genre_group) { return [] } - var genresGroup = genre_group.find(group => group.genre_group_id = 1) // genres and subgenres + const genresGroup = genre_group.find(group => group.genre_group_id == 1) // genres and subgenres // genre_group_id=2 - General Characteristics // genre_group_id=3 - Arena @@ -111,16 +113,16 @@ class FantLab { if (!genresGroup) return [] - var { genre } = genresGroup; - var rootGenre = genre[0]; + const { genre } = genresGroup + const rootGenre = genre[0] - var { label } = rootGenre + const { label } = rootGenre return [label].concat(this.tryGetSubGenres(rootGenre)) } tryGetSubGenres(rootGenre) { - var { genre } = rootGenre + const { genre } = rootGenre return genre ? genre.map(genreObj => genreObj.label) : [] } @@ -130,7 +132,7 @@ class FantLab { return null } - var bookEditions = editions['30'] // try get audiobooks first + let bookEditions = editions['30'] // try get audiobooks first if (!bookEditions) { bookEditions = editions['10'] // paper editions in ru lang } @@ -139,12 +141,12 @@ class FantLab { return null } - var { list } = bookEditions + const { list } = bookEditions - var lastEdition = list[list.length - 1] + const lastEdition = list[list.length - 1] - var editionId = lastEdition['edition_id'] - var isbn = lastEdition['isbn'] // get only from paper edition + const editionId = lastEdition['edition_id'] + const isbn = lastEdition['isbn'] // get only from paper edition return { imageUrl: await this.getCoverFromEdition(editionId), @@ -153,9 +155,9 @@ class FantLab { } async getCoverFromEdition(editionId) { - var url = `${this._baseUrl}/edition/${editionId}` + const url = `${this._baseUrl}/edition/${editionId}` - var editionInfo = await axios.get(url).then((resp) => { + const editionInfo = await axios.get(url).then((resp) => { return resp.data || null }).catch(error => { Logger.error('[FantLab] search cover from edition error', error) From 371cd3b2e5025d8c6dcec826d635e5085b124bf5 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 9 Feb 2023 23:09:44 +0300 Subject: [PATCH 3/4] Update server/providers/FantLab.js Co-authored-by: advplyr <67830747+advplyr@users.noreply.github.com> --- server/providers/FantLab.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/providers/FantLab.js b/server/providers/FantLab.js index 0cfd9f21..eb30c599 100644 --- a/server/providers/FantLab.js +++ b/server/providers/FantLab.js @@ -47,7 +47,7 @@ class FantLab { const { work_id, work_type_id } = item if (this._filterWorkType.includes(work_type_id)) { - return { title: null } + return null } const url = `${this._baseUrl}/work/${work_id}/extended` From f35c96e1186916e1aa78e4f1fc5620662d0a7b18 Mon Sep 17 00:00:00 2001 From: advplyr Date: Sat, 11 Feb 2023 14:25:25 -0600 Subject: [PATCH 4/4] FantLab minor refactor --- server/providers/FantLab.js | 70 +++++++++++++------------------------ 1 file changed, 25 insertions(+), 45 deletions(-) diff --git a/server/providers/FantLab.js b/server/providers/FantLab.js index eb30c599..41328ca1 100644 --- a/server/providers/FantLab.js +++ b/server/providers/FantLab.js @@ -39,7 +39,7 @@ class FantLab { }) return Promise.all(items.map(async item => await this.getWork(item))).then(resArray => { - return resArray.filter(res => res != null) + return resArray.filter(res => res) }) } @@ -54,7 +54,7 @@ class FantLab { const bookData = await axios.get(url).then((resp) => { return resp.data || null }).catch((error) => { - Logger.error(`[FantLab] work info reques error`, error) + Logger.error(`[FantLab] work info request for url "${url}" error`, error) return null }) @@ -65,44 +65,30 @@ class FantLab { let { authors, work_name_alts, work_id, work_name, work_year, work_description, image, classificatory, editions_blocks } = bookData const subtitle = Array.isArray(work_name_alts) ? work_name_alts[0] : null - const auth = authors.map(function (author) { - return author.name - }) - - const genres = classificatory ? this.tryGetGenres(classificatory) : [] + const authorNames = authors.map(au => (au.name || '').trim()).filter(au => au) const imageAndIsbn = await this.tryGetCoverFromEditions(editions_blocks) - if (imageAndIsbn) { - var { imageUrl, isbn } = imageAndIsbn - - if (imageUrl) { - image = imageUrl - } - } - - const cover = 'https://fantlab.ru' + image + const imageToUse = imageAndIsbn?.imageUrl || image return { id: work_id, title: work_name, subtitle: subtitle || null, - author: auth ? auth.join(', ') : null, + author: authorNames.length ? authorNames.join(', ') : null, publisher: null, publishedYear: work_year, description: work_description, - cover: image ? cover : null, - genres: genres, - isbn: isbn + cover: imageToUse ? `https://fantlab.ru${imageToUse}` : null, + genres: this.tryGetGenres(classificatory), + isbn: imageAndIsbn?.isbn || null } } tryGetGenres(classificatory) { - const { genre_group } = classificatory - if (!genre_group) { - return [] - } - const genresGroup = genre_group.find(group => group.genre_group_id == 1) // genres and subgenres + if (!classificatory || !classificatory.genre_group) return [] + + const genresGroup = classificatory.genre_group.find(group => group.genre_group_id == 1) // genres and subgenres // genre_group_id=2 - General Characteristics // genre_group_id=3 - Arena @@ -111,10 +97,9 @@ class FantLab { // genre_group_id=7 - Story linearity // genre_group_id=5 - Recommended age of the reader - if (!genresGroup) return [] + if (!genresGroup || !genresGroup.genre || !genresGroup.genre.length) return [] - const { genre } = genresGroup - const rootGenre = genre[0] + const rootGenre = genresGroup.genre[0] const { label } = rootGenre @@ -122,51 +107,46 @@ class FantLab { } tryGetSubGenres(rootGenre) { - const { genre } = rootGenre - return genre ? genre.map(genreObj => genreObj.label) : [] + if (!rootGenre.genre || !rootGenre.genre.length) return [] + return rootGenre.genre.map(g => g.label).filter(g => g) } async tryGetCoverFromEditions(editions) { - if (!editions) { return null } - let bookEditions = editions['30'] // try get audiobooks first - if (!bookEditions) { - bookEditions = editions['10'] // paper editions in ru lang - } - - if (!bookEditions) { + // 30 = audio, 10 = paper + // Prefer audio if available + const bookEditions = editions['30'] || editions['10'] + if (!bookEditions || !bookEditions.list || !bookEditions.list.length) { return null } - const { list } = bookEditions - - const lastEdition = list[list.length - 1] + const lastEdition = bookEditions.list.pop() const editionId = lastEdition['edition_id'] - const isbn = lastEdition['isbn'] // get only from paper edition + const isbn = lastEdition['isbn'] || null // get only from paper edition return { imageUrl: await this.getCoverFromEdition(editionId), - isbn: isbn + isbn } } async getCoverFromEdition(editionId) { + if (!editionId) return null const url = `${this._baseUrl}/edition/${editionId}` const editionInfo = await axios.get(url).then((resp) => { return resp.data || null }).catch(error => { - Logger.error('[FantLab] search cover from edition error', error) + Logger.error(`[FantLab] search cover from edition with url "${url}" error`, error) return null }) - return editionInfo ? editionInfo['image'] : null + return editionInfo?.image || null } - } module.exports = FantLab \ No newline at end of file