From 8027c4a06f01f3a13d43cf041af012a4f64d62f5 Mon Sep 17 00:00:00 2001 From: Barnabas Ratki Date: Wed, 3 Jan 2024 01:36:56 +0100 Subject: [PATCH] Added support for custom metadata providers WiP but already open to feedback --- client/components/app/ConfigSideNav.vue | 5 + .../modals/AddCustomMetadataProviderModal.vue | 104 ++++++++++++ .../tables/CustomMetadataProviderTable.vue | 150 ++++++++++++++++++ client/layouts/default.vue | 8 + .../config/custom-metadata-providers.vue | 45 ++++++ client/store/scanners.js | 31 +++- client/strings/en-us.json | 3 + server/Database.js | 45 ++++++ server/controllers/LibraryController.js | 10 ++ server/controllers/MiscController.js | 90 +++++++++++ server/finders/BookFinder.js | 17 +- server/models/CustomMetadataProvider.js | 58 +++++++ server/providers/CustomProviderAdapter.js | 76 +++++++++ server/routers/ApiRouter.js | 4 + 14 files changed, 642 insertions(+), 4 deletions(-) create mode 100644 client/components/modals/AddCustomMetadataProviderModal.vue create mode 100644 client/components/tables/CustomMetadataProviderTable.vue create mode 100644 client/pages/config/custom-metadata-providers.vue create mode 100644 server/models/CustomMetadataProvider.js create mode 100644 server/providers/CustomProviderAdapter.js diff --git a/client/components/app/ConfigSideNav.vue b/client/components/app/ConfigSideNav.vue index c2db0725..e253d1ae 100644 --- a/client/components/app/ConfigSideNav.vue +++ b/client/components/app/ConfigSideNav.vue @@ -109,6 +109,11 @@ export default { id: 'config-authentication', title: this.$strings.HeaderAuthentication, path: '/config/authentication' + }, + { + id: 'config-custom-metadata-providers', + title: this.$strings.HeaderCustomMetadataProviders, + path: '/config/custom-metadata-providers' } ] diff --git a/client/components/modals/AddCustomMetadataProviderModal.vue b/client/components/modals/AddCustomMetadataProviderModal.vue new file mode 100644 index 00000000..1b9f930c --- /dev/null +++ b/client/components/modals/AddCustomMetadataProviderModal.vue @@ -0,0 +1,104 @@ + + + diff --git a/client/components/tables/CustomMetadataProviderTable.vue b/client/components/tables/CustomMetadataProviderTable.vue new file mode 100644 index 00000000..8104cede --- /dev/null +++ b/client/components/tables/CustomMetadataProviderTable.vue @@ -0,0 +1,150 @@ + + + + + diff --git a/client/layouts/default.vue b/client/layouts/default.vue index c3cc3484..1d33c44c 100644 --- a/client/layouts/default.vue +++ b/client/layouts/default.vue @@ -328,6 +328,9 @@ export default { this.$store.commit('libraries/setEReaderDevices', data.ereaderDevices) }, + customMetadataProvidersChanged() { + this.$store.dispatch('scanners/reFetchCustom') + }, initializeSocket() { this.socket = this.$nuxtSocket({ name: process.env.NODE_ENV === 'development' ? 'dev' : 'prod', @@ -406,6 +409,10 @@ export default { this.socket.on('batch_quickmatch_complete', this.batchQuickMatchComplete) this.socket.on('admin_message', this.adminMessageEvt) + + // Custom metadata provider Listeners + this.socket.on('custom_metadata_provider_added', this.customMetadataProvidersChanged) + this.socket.on('custom_metadata_provider_removed', this.customMetadataProvidersChanged) }, showUpdateToast(versionData) { var ignoreVersion = localStorage.getItem('ignoreVersion') @@ -541,6 +548,7 @@ export default { window.addEventListener('keydown', this.keyDown) this.$store.dispatch('libraries/load') + this.$store.dispatch('scanners/reFetchCustom') this.initLocalStorage() diff --git a/client/pages/config/custom-metadata-providers.vue b/client/pages/config/custom-metadata-providers.vue new file mode 100644 index 00000000..10fdb21b --- /dev/null +++ b/client/pages/config/custom-metadata-providers.vue @@ -0,0 +1,45 @@ + + + + + diff --git a/client/store/scanners.js b/client/store/scanners.js index ccdc1791..32878a6a 100644 --- a/client/store/scanners.js +++ b/client/store/scanners.js @@ -73,6 +73,33 @@ export const state = () => ({ export const getters = {} -export const actions = {} +export const actions = { + reFetchCustom({ dispatch, commit }) { + return this.$axios + .$get(`/api/custom-metadata-providers`) + .then((data) => { + const providers = data.providers -export const mutations = {} \ No newline at end of file + commit('setCustomProviders', providers) + return data + }) + .catch((error) => { + console.error('Failed', error) + return false + }) + }, +} + +export const mutations = { + setCustomProviders(state, providers) { + // clear previous values, and add new values to the end + state.providers = state.providers.filter((p) => !p.value.startsWith("custom-")); + state.providers = [ + ...state.providers, + ...providers.map((p) => {return { + text: p.name, + value: p.slug, + }}) + ] + }, +} \ No newline at end of file diff --git a/client/strings/en-us.json b/client/strings/en-us.json index f69175fd..9dfde095 100644 --- a/client/strings/en-us.json +++ b/client/strings/en-us.json @@ -96,6 +96,7 @@ "HeaderAudiobookTools": "Audiobook File Management Tools", "HeaderAudioTracks": "Audio Tracks", "HeaderAuthentication": "Authentication", + "HeaderCustomMetadataProviders": "Custom metadata providers", "HeaderBackups": "Backups", "HeaderChangePassword": "Change Password", "HeaderChapters": "Chapters", @@ -540,6 +541,8 @@ "LabelYourBookmarks": "Your Bookmarks", "LabelYourPlaylists": "Your Playlists", "LabelYourProgress": "Your Progress", + "LabelUrl": "URL", + "LabelApiKey": "API Key", "MessageAddToPlayerQueue": "Add to player queue", "MessageAppriseDescription": "To use this feature you will need to have an instance of Apprise API running or an api that will handle those same requests.
The Apprise API Url should be the full URL path to send the notification, e.g., if your API instance is served at http://192.168.1.1:8337 then you would put http://192.168.1.1:8337/notify.", "MessageBackupsDescription": "Backups include users, user progress, library item details, server settings, and images stored in /metadata/items & /metadata/authors. Backups do not include any files stored in your library folders.", diff --git a/server/Database.js b/server/Database.js index fd606bac..bbea7352 100644 --- a/server/Database.js +++ b/server/Database.js @@ -132,6 +132,11 @@ class Database { return this.models.playbackSession } + /** @type {typeof import('./models/CustomMetadataProvider')} */ + get customMetadataProviderModel() { + return this.models.customMetadataProvider + } + /** * Check if db file exists * @returns {boolean} @@ -245,6 +250,7 @@ class Database { require('./models/Feed').init(this.sequelize) require('./models/FeedEpisode').init(this.sequelize) require('./models/Setting').init(this.sequelize) + require('./models/CustomMetadataProvider').init(this.sequelize) return this.sequelize.sync({ force, alter: false }) } @@ -694,6 +700,45 @@ class Database { }) } + /** + * Returns true if a custom provider with the given slug exists + * @param {string} providerSlug + * @return {boolean} + */ + async doesCustomProviderExistBySlug(providerSlug) { + const id = providerSlug.split("custom-")[1] + + if (!id) { + return false + } + + return !!await this.customMetadataProviderModel.findByPk(id) + } + + /** + * Removes a custom metadata provider + * @param {string} id + */ + async removeCustomMetadataProviderById(id) { + // destroy metadta provider + await this.customMetadataProviderModel.destroy({ + where: { + id, + } + }) + + const slug = `custom-${id}`; + + // fallback libraries using it to google + await this.libraryModel.update({ + provider: "google", + }, { + where: { + provider: slug, + } + }); + } + /** * Clean invalid records in database * Series should have atleast one Book diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 70baff85..304ca4f0 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -51,6 +51,11 @@ class LibraryController { } } + // Validate that the custom provider exists if given any + if (newLibraryPayload.provider && newLibraryPayload.provider.startsWith("custom-")) { + await Database.doesCustomProviderExistBySlug(newLibraryPayload.provider) + } + const library = new Library() let currentLargestDisplayOrder = await Database.libraryModel.getMaxDisplayOrder() @@ -175,6 +180,11 @@ class LibraryController { } } + // Validate that the custom provider exists if given any + if (req.body.provider && req.body.provider.startsWith("custom-")) { + await Database.doesCustomProviderExistBySlug(req.body.provider) + } + const hasUpdates = library.update(req.body) // TODO: Should check if this is an update to folder paths or name only if (hasUpdates) { diff --git a/server/controllers/MiscController.js b/server/controllers/MiscController.js index c2272ee6..31f4587b 100644 --- a/server/controllers/MiscController.js +++ b/server/controllers/MiscController.js @@ -717,5 +717,95 @@ class MiscController { const stats = await adminStats.getStatsForYear(year) res.json(stats) } + + /** + * GET: /api/custom-metadata-providers + * + * @param {import('express').Request} req + * @param {import('express').Response} res + */ + async getCustomMetadataProviders(req, res) { + const providers = await Database.customMetadataProviderModel.findAll() + + res.json({ + providers: providers.map((p) => p.toUserJson()), + }) + } + + /** + * GET: /api/custom-metadata-providers/admin + * + * @param {import('express').Request} req + * @param {import('express').Response} res + */ + async getAdminCustomMetadataProviders(req, res) { + if (!req.user.isAdminOrUp) { + Logger.error(`[MiscController] Non-admin user "${req.user.username}" attempted to get admin custom metadata providers`) + return res.sendStatus(403) + } + + const providers = await Database.customMetadataProviderModel.findAll() + + res.json({ + providers, + }) + } + + /** + * PATCH: /api/custom-metadata-providers/admin + * + * @param {import('express').Request} req + * @param {import('express').Response} res + */ + async addCustomMetadataProviders(req, res) { + if (!req.user.isAdminOrUp) { + Logger.error(`[MiscController] Non-admin user "${req.user.username}" attempted to get admin custom metadata providers`) + return res.sendStatus(403) + } + + const { name, url, apiKey } = req.body; + + if (!name || !url || !apiKey) { + return res.status(500).send(`Invalid patch data`) + } + + const provider = await Database.customMetadataProviderModel.create({ + name, + url, + apiKey, + }) + + SocketAuthority.adminEmitter('custom_metadata_provider_added', provider) + + res.json({ + provider, + }) + } + + /** + * DELETE: /api/custom-metadata-providers/admin/:id + * + * @param {import('express').Request} req + * @param {import('express').Response} res + */ + async deleteCustomMetadataProviders(req, res) { + if (!req.user.isAdminOrUp) { + Logger.error(`[MiscController] Non-admin user "${req.user.username}" attempted to get admin custom metadata providers`) + return res.sendStatus(403) + } + + const { id } = req.params; + + if (!id) { + return res.status(500).send(`Invalid delete data`) + } + + const provider = await Database.customMetadataProviderModel.findByPk(id); + await Database.removeCustomMetadataProviderById(id); + + SocketAuthority.adminEmitter('custom_metadata_provider_removed', provider) + + res.json({}) + } } module.exports = new MiscController() diff --git a/server/finders/BookFinder.js b/server/finders/BookFinder.js index 466c8701..6c35a5fb 100644 --- a/server/finders/BookFinder.js +++ b/server/finders/BookFinder.js @@ -5,6 +5,7 @@ const iTunes = require('../providers/iTunes') const Audnexus = require('../providers/Audnexus') const FantLab = require('../providers/FantLab') const AudiobookCovers = require('../providers/AudiobookCovers') +const CustomProviderAdapter = require('../providers/CustomProviderAdapter') const Logger = require('../Logger') const { levenshteinDistance, escapeRegExp } = require('../utils/index') @@ -17,6 +18,7 @@ class BookFinder { this.audnexus = new Audnexus() this.fantLab = new FantLab() this.audiobookCovers = new AudiobookCovers() + this.customProviderAdapter = new CustomProviderAdapter() this.providers = ['google', 'itunes', 'openlibrary', 'fantlab', 'audiobookcovers', 'audible', 'audible.ca', 'audible.uk', 'audible.au', 'audible.fr', 'audible.de', 'audible.jp', 'audible.it', 'audible.in', 'audible.es'] @@ -147,6 +149,13 @@ class BookFinder { return books } + async getCustomProviderResults(title, author, providerSlug) { + const books = await this.customProviderAdapter.search(title, author, providerSlug) + if (this.verbose) Logger.debug(`Custom provider '${providerSlug}' Search Results: ${books.length || 0}`) + + return books + } + static TitleCandidates = class { constructor(cleanAuthor) { @@ -315,6 +324,11 @@ class BookFinder { const maxFuzzySearches = !isNaN(options.maxFuzzySearches) ? Number(options.maxFuzzySearches) : 5 let numFuzzySearches = 0 + // Custom providers are assumed to be correct + if (provider.startsWith("custom-")) { + return await this.getCustomProviderResults(title, author, provider) + } + if (!title) return books @@ -397,8 +411,7 @@ class BookFinder { books = await this.getFantLabResults(title, author) } else if (provider === 'audiobookcovers') { books = await this.getAudiobookCoversResults(title) - } - else { + } else { books = await this.getGoogleBooksResults(title, author) } return books diff --git a/server/models/CustomMetadataProvider.js b/server/models/CustomMetadataProvider.js new file mode 100644 index 00000000..4f2488d2 --- /dev/null +++ b/server/models/CustomMetadataProvider.js @@ -0,0 +1,58 @@ +const { DataTypes, Model, Sequelize } = require('sequelize') + +class CustomMetadataProvider extends Model { + constructor(values, options) { + super(values, options) + + /** @type {UUIDV4} */ + this.id + /** @type {string} */ + this.name + /** @type {string} */ + this.url + /** @type {string} */ + this.apiKey + } + + getSlug() { + return `custom-${this.id}` + } + + toUserJson() { + return { + name: this.name, + id: this.id, + slug: this.getSlug() + } + } + + static findByPk(id) { + this.findOne({ + where: { + id, + } + }) + } + + /** + * Initialize model + * @param {import('../Database').sequelize} sequelize + */ + static init(sequelize) { + super.init({ + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true + }, + name: DataTypes.STRING, + url: DataTypes.STRING, + apiKey: DataTypes.STRING + }, { + sequelize, + modelName: 'customMetadataProvider' + }) + } +} + +module.exports = CustomMetadataProvider \ No newline at end of file diff --git a/server/providers/CustomProviderAdapter.js b/server/providers/CustomProviderAdapter.js new file mode 100644 index 00000000..1bf5a5ee --- /dev/null +++ b/server/providers/CustomProviderAdapter.js @@ -0,0 +1,76 @@ +const Database = require('../Database') +const axios = require("axios"); +const Logger = require("../Logger"); + +class CustomProviderAdapter { + constructor() { + } + + async search(title, author, providerSlug) { + const providerId = providerSlug.split("custom-")[1] + + console.log(providerId) + const provider = await Database.customMetadataProviderModel.findOne({ + where: { + id: providerId, + } + }); + + if (!provider) { + throw new Error("Custom provider not found for the given id"); + } + + const matches = await axios.get(`${provider.url}/search?query=${encodeURIComponent(title)}${!!author ? `&author=${encodeURIComponent(author)}` : ""}`, { + headers: { + "Authorization": provider.apiKey, + }, + }).then((res) => { + if (!res || !res.data || !Array.isArray(res.data.matches)) return null + return res.data.matches + }).catch(error => { + Logger.error('[CustomMetadataProvider] Search error', error) + return [] + }) + + if (matches === null) { + throw new Error("Custom provider returned malformed response"); + } + + // re-map keys to throw out + return matches.map(({ + title, + subtitle, + author, + narrator, + publisher, + published_year, + description, + cover, + isbn, + asin, + genres, + tags, + language, + duration, + }) => { + return { + title, + subtitle, + author, + narrator, + publisher, + publishedYear: published_year, + description, + cover, + isbn, + asin, + genres, + tags: tags.join(","), + language, + duration, + } + }) + } +} + +module.exports = CustomProviderAdapter \ No newline at end of file diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index 3edce256..f78d4539 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -318,6 +318,10 @@ class ApiRouter { this.router.patch('/auth-settings', MiscController.updateAuthSettings.bind(this)) this.router.post('/watcher/update', MiscController.updateWatchedPath.bind(this)) this.router.get('/stats/year/:year', MiscController.getAdminStatsForYear.bind(this)) + this.router.get('/custom-metadata-providers', MiscController.getCustomMetadataProviders.bind(this)) + this.router.get('/custom-metadata-providers/admin', MiscController.getAdminCustomMetadataProviders.bind(this)) + this.router.patch('/custom-metadata-providers/admin', MiscController.addCustomMetadataProviders.bind(this)) + this.router.delete('/custom-metadata-providers/admin/:id', MiscController.deleteCustomMetadataProviders.bind(this)) } async getDirectories(dir, relpath, excludedDirs, level = 0) {