From 6814adffcc6a935d74ba28eb74ef75d695d26de4 Mon Sep 17 00:00:00 2001 From: advplyr Date: Mon, 17 Jul 2023 16:48:46 -0500 Subject: [PATCH] Update:Only load feeds when needed --- Dockerfile | 3 +- server/Database.js | 4 -- server/controllers/CollectionController.js | 6 +-- server/controllers/LibraryController.js | 28 ++++++------ server/controllers/LibraryItemController.js | 6 +-- server/controllers/RSSFeedController.js | 6 +-- server/controllers/SeriesController.js | 2 +- server/managers/RssFeedManager.js | 43 +++++++++++++------ server/models/Feed.js | 47 +++++++++++++++++++++ server/utils/libraryHelpers.js | 35 ++++++++------- 10 files changed, 125 insertions(+), 55 deletions(-) diff --git a/Dockerfile b/Dockerfile index 64e70242..0526494d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,6 @@ FROM sandreas/tone:v0.1.5 AS tone FROM node:16-alpine ENV NODE_ENV=production -ENV NODE_OPTIONS=--max-old-space-size=8192 RUN apk update && \ apk add --no-cache --update \ @@ -30,6 +29,8 @@ RUN npm ci --only=production RUN apk del make python3 g++ +ENV NODE_OPTIONS=--max-old-space-size=8192 + EXPOSE 80 HEALTHCHECK \ --interval=30s \ diff --git a/server/Database.js b/server/Database.js index 3d53a282..a432c394 100644 --- a/server/Database.js +++ b/server/Database.js @@ -23,7 +23,6 @@ class Database { this.playlists = [] this.authors = [] this.series = [] - this.feeds = [] this.serverSettings = null this.notificationSettings = null @@ -147,7 +146,6 @@ class Database { this.playlists = await this.models.playlist.getOldPlaylists() this.authors = await this.models.author.getOldAuthors() this.series = await this.models.series.getAllOldSeries() - this.feeds = await this.models.feed.getOldFeeds() Logger.info(`[Database] Db data loaded in ${Date.now() - startTime}ms`) @@ -408,7 +406,6 @@ class Database { async createFeed(oldFeed) { if (!this.sequelize) return false await this.models.feed.fullCreateFromOld(oldFeed) - this.feeds.push(oldFeed) } updateFeed(oldFeed) { @@ -419,7 +416,6 @@ class Database { async removeFeed(feedId) { if (!this.sequelize) return false await this.models.feed.removeById(feedId) - this.feeds = this.feeds.filter(f => f.id !== feedId) } updateSeries(oldSeries) { diff --git a/server/controllers/CollectionController.js b/server/controllers/CollectionController.js index f4702d16..4665869e 100644 --- a/server/controllers/CollectionController.js +++ b/server/controllers/CollectionController.js @@ -26,14 +26,14 @@ class CollectionController { }) } - findOne(req, res) { + async findOne(req, res) { const includeEntities = (req.query.include || '').split(',') const collectionExpanded = req.collection.toJSONExpanded(Database.libraryItems) if (includeEntities.includes('rssfeed')) { - const feedData = this.rssFeedManager.findFeedForEntityId(collectionExpanded.id) - collectionExpanded.rssFeed = feedData ? feedData.toJSONMinified() : null + const feedData = await this.rssFeedManager.findFeedForEntityId(collectionExpanded.id) + collectionExpanded.rssFeed = feedData?.toJSONMinified() || null } res.json(collectionExpanded) diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index b0271f59..893bb5c7 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -179,7 +179,7 @@ class LibraryController { // api/libraries/:id/items // TODO: Optimize this method, items are iterated through several times but can be combined - getLibraryItems(req, res) { + async getLibraryItems(req, res) { let libraryItems = req.libraryItems const include = (req.query.include || '').split(',').map(v => v.trim().toLowerCase()).filter(v => !!v) @@ -203,7 +203,7 @@ class LibraryController { // Step 1 - Filter the retrieved library items let filterSeries = null if (payload.filterBy) { - libraryItems = libraryHelpers.getFilteredLibraryItems(libraryItems, payload.filterBy, req.user, Database.feeds) + libraryItems = await libraryHelpers.getFilteredLibraryItems(libraryItems, payload.filterBy, req.user) payload.total = libraryItems.length // Determining if we are filtering titles by a series, and if so, which series @@ -319,7 +319,7 @@ class LibraryController { } // Step 4 - Transform the items to pass to the client side - payload.results = libraryItems.map(li => { + payload.results = await Promise.all(libraryItems.map(async li => { const json = payload.minified ? li.toJSONMinified() : li.toJSON() if (li.collapsedSeries) { @@ -356,7 +356,7 @@ class LibraryController { } else { // add rssFeed object if "include=rssfeed" was put in query string (only for non-collapsed series) if (include.includes('rssfeed')) { - const feedData = this.rssFeedManager.findFeedForEntityId(json.id) + const feedData = await this.rssFeedManager.findFeedForEntityId(json.id) json.rssFeed = feedData ? feedData.toJSONMinified() : null } @@ -372,7 +372,7 @@ class LibraryController { } return json - }) + })) res.json(payload) } @@ -449,11 +449,11 @@ class LibraryController { // add rssFeed when "include=rssfeed" is in query string if (include.includes('rssfeed')) { - series = series.map((se) => { - const feedData = this.rssFeedManager.findFeedForEntityId(se.id) + series = await Promise.all(series.map(async (se) => { + const feedData = await this.rssFeedManager.findFeedForEntityId(se.id) se.rssFeed = feedData?.toJSONMinified() || null return se - }) + })) } payload.results = series @@ -489,7 +489,7 @@ class LibraryController { } if (include.includes('rssfeed')) { - const feedObj = this.rssFeedManager.findFeedForEntityId(seriesJson.id) + const feedObj = await this.rssFeedManager.findFeedForEntityId(seriesJson.id) seriesJson.rssFeed = feedObj?.toJSONMinified() || null } @@ -514,19 +514,21 @@ class LibraryController { include: include.join(',') } - let collections = Database.collections.filter(c => c.libraryId === req.library.id).map(c => { + let collections = await Promise.all(Database.collections.filter(c => c.libraryId === req.library.id).map(async c => { const expanded = c.toJSONExpanded(libraryItems, payload.minified) // If all books restricted to user in this collection then hide this collection if (!expanded.books.length && c.books.length) return null if (include.includes('rssfeed')) { - const feedData = this.rssFeedManager.findFeedForEntityId(c.id) + const feedData = await this.rssFeedManager.findFeedForEntityId(c.id) expanded.rssFeed = feedData?.toJSONMinified() || null } return expanded - }).filter(c => !!c) + })) + + collections = collections.filter(c => !!c) payload.total = collections.length @@ -595,7 +597,7 @@ class LibraryController { const limitPerShelf = req.query.limit && !isNaN(req.query.limit) ? Number(req.query.limit) || 10 : 10 const include = (req.query.include || '').split(',').map(v => v.trim().toLowerCase()).filter(v => !!v) - const categories = libraryHelpers.buildPersonalizedShelves(this, req.user, req.libraryItems, req.library, limitPerShelf, include) + const categories = await libraryHelpers.buildPersonalizedShelves(this, req.user, req.libraryItems, req.library, limitPerShelf, include) res.json(categories) } diff --git a/server/controllers/LibraryItemController.js b/server/controllers/LibraryItemController.js index dceed76e..4e10baa9 100644 --- a/server/controllers/LibraryItemController.js +++ b/server/controllers/LibraryItemController.js @@ -13,7 +13,7 @@ class LibraryItemController { constructor() { } // Example expand with authors: api/items/:id?expanded=1&include=authors - findOne(req, res) { + async findOne(req, res) { const includeEntities = (req.query.include || '').split(',') if (req.query.expanded == 1) { var item = req.libraryItem.toJSONExpanded() @@ -25,8 +25,8 @@ class LibraryItemController { } if (includeEntities.includes('rssfeed')) { - const feedData = this.rssFeedManager.findFeedForEntityId(item.id) - item.rssFeed = feedData ? feedData.toJSONMinified() : null + const feedData = await this.rssFeedManager.findFeedForEntityId(item.id) + item.rssFeed = feedData?.toJSONMinified() || null } if (item.mediaType == 'book') { diff --git a/server/controllers/RSSFeedController.js b/server/controllers/RSSFeedController.js index 02f24580..a6e6abc8 100644 --- a/server/controllers/RSSFeedController.js +++ b/server/controllers/RSSFeedController.js @@ -30,7 +30,7 @@ class RSSFeedController { } // Check that this slug is not being used for another feed (slug will also be the Feed id) - if (this.rssFeedManager.findFeedBySlug(options.slug)) { + if (await this.rssFeedManager.findFeedBySlug(options.slug)) { Logger.error(`[RSSFeedController] Cannot open RSS feed because slug "${options.slug}" is already in use`) return res.status(400).send('Slug already in use') } @@ -55,7 +55,7 @@ class RSSFeedController { } // Check that this slug is not being used for another feed (slug will also be the Feed id) - if (this.rssFeedManager.findFeedBySlug(options.slug)) { + if (await this.rssFeedManager.findFeedBySlug(options.slug)) { Logger.error(`[RSSFeedController] Cannot open RSS feed because slug "${options.slug}" is already in use`) return res.status(400).send('Slug already in use') } @@ -89,7 +89,7 @@ class RSSFeedController { } // Check that this slug is not being used for another feed (slug will also be the Feed id) - if (this.rssFeedManager.findFeedBySlug(options.slug)) { + if (await this.rssFeedManager.findFeedBySlug(options.slug)) { Logger.error(`[RSSFeedController] Cannot open RSS feed because slug "${options.slug}" is already in use`) return res.status(400).send('Slug already in use') } diff --git a/server/controllers/SeriesController.js b/server/controllers/SeriesController.js index 77135fd1..041c0716 100644 --- a/server/controllers/SeriesController.js +++ b/server/controllers/SeriesController.js @@ -35,7 +35,7 @@ class SeriesController { } if (include.includes('rssfeed')) { - const feedObj = this.rssFeedManager.findFeedForEntityId(seriesJson.id) + const feedObj = await this.rssFeedManager.findFeedForEntityId(seriesJson.id) seriesJson.rssFeed = feedObj?.toJSONMinified() || null } diff --git a/server/managers/RssFeedManager.js b/server/managers/RssFeedManager.js index bb52057e..d3d303be 100644 --- a/server/managers/RssFeedManager.js +++ b/server/managers/RssFeedManager.js @@ -35,8 +35,12 @@ class RssFeedManager { return true } + /** + * Validate all feeds and remove invalid + */ async init() { - for (const feed of Database.feeds) { + const feeds = await Database.models.feed.getOldFeeds() + for (const feed of feeds) { // Remove invalid feeds if (!this.validateFeedEntity(feed)) { await Database.removeFeed(feed.id) @@ -44,20 +48,35 @@ class RssFeedManager { } } + /** + * Find open feed for an entity (e.g. collection id, playlist id, library item id) + * @param {string} entityId + * @returns {Promise} oldFeed + */ findFeedForEntityId(entityId) { - return Database.feeds.find(feed => feed.entityId === entityId) + return Database.models.feed.findOneOld({ entityId }) } + /** + * Find open feed for a slug + * @param {string} slug + * @returns {Promise} oldFeed + */ findFeedBySlug(slug) { - return Database.feeds.find(feed => feed.slug === slug) + return Database.models.feed.findOneOld({ slug }) } + /** + * Find open feed for a slug + * @param {string} slug + * @returns {Promise} oldFeed + */ findFeed(id) { - return Database.feeds.find(feed => feed.id === id) + return Database.models.feed.findByPkOld(id) } async getFeed(req, res) { - const feed = this.findFeedBySlug(req.params.slug) + const feed = await this.findFeedBySlug(req.params.slug) if (!feed) { Logger.warn(`[RssFeedManager] Feed not found ${req.params.slug}`) res.sendStatus(404) @@ -134,8 +153,8 @@ class RssFeedManager { res.send(xml) } - getFeedItem(req, res) { - const feed = this.findFeedBySlug(req.params.slug) + async getFeedItem(req, res) { + const feed = await this.findFeedBySlug(req.params.slug) if (!feed) { Logger.debug(`[RssFeedManager] Feed not found ${req.params.slug}`) res.sendStatus(404) @@ -150,8 +169,8 @@ class RssFeedManager { res.sendFile(episodePath) } - getFeedCover(req, res) { - const feed = this.findFeedBySlug(req.params.slug) + async getFeedCover(req, res) { + const feed = await this.findFeedBySlug(req.params.slug) if (!feed) { Logger.debug(`[RssFeedManager] Feed not found ${req.params.slug}`) res.sendStatus(404) @@ -225,7 +244,7 @@ class RssFeedManager { } async closeRssFeed(req, res) { - const feed = this.findFeed(req.params.id) + const feed = await this.findFeed(req.params.id) if (!feed) { Logger.error(`[RssFeedManager] RSS feed not found with id "${req.params.id}"`) return res.sendStatus(404) @@ -234,8 +253,8 @@ class RssFeedManager { res.sendStatus(200) } - closeFeedForEntityId(entityId) { - const feed = this.findFeedForEntityId(entityId) + async closeFeedForEntityId(entityId) { + const feed = await this.findFeedForEntityId(entityId) if (!feed) return return this.handleCloseFeed(feed) } diff --git a/server/models/Feed.js b/server/models/Feed.js index 5c4f50f8..c9b2226e 100644 --- a/server/models/Feed.js +++ b/server/models/Feed.js @@ -56,6 +56,53 @@ module.exports = (sequelize) => { }) } + /** + * Find all library item ids that have an open feed (used in library filter) + * @returns {Promise>} array of library item ids + */ + static async findAllLibraryItemIds() { + const feeds = await this.findAll({ + attributes: ['entityId'], + where: { + entityType: 'libraryItem' + } + }) + return feeds.map(f => f.entityId).filter(f => f) || [] + } + + /** + * Find feed where and return oldFeed + * @param {object} where sequelize where object + * @returns {Promise} oldFeed + */ + static async findOneOld(where) { + if (!where) return null + const feedExpanded = await this.findOne({ + where, + include: { + model: sequelize.models.feedEpisode + } + }) + if (!feedExpanded) return null + return this.getOldFeed(feedExpanded) + } + + /** + * Find feed and return oldFeed + * @param {string} id + * @returns {Promise} oldFeed + */ + static async findByPkOld(id) { + if (!id) return null + const feedExpanded = await this.findByPk(id, { + include: { + model: sequelize.models.feedEpisode + } + }) + if (!feedExpanded) return null + return this.getOldFeed(feedExpanded) + } + static async fullCreateFromOld(oldFeed) { const feedObj = this.getFromOld(oldFeed) const newFeed = await this.create(feedObj) diff --git a/server/utils/libraryHelpers.js b/server/utils/libraryHelpers.js index aec74982..890e54b5 100644 --- a/server/utils/libraryHelpers.js +++ b/server/utils/libraryHelpers.js @@ -11,7 +11,7 @@ module.exports = { return Buffer.from(decodeURIComponent(text), 'base64').toString() }, - getFilteredLibraryItems(libraryItems, filterBy, user, feedsArray) { + async getFilteredLibraryItems(libraryItems, filterBy, user) { let filtered = libraryItems const searchGroups = ['genres', 'tags', 'series', 'authors', 'progress', 'narrators', 'publishers', 'missing', 'languages', 'tracks', 'ebooks'] @@ -71,7 +71,9 @@ module.exports = { } else if (filterBy === 'issues') { filtered = filtered.filter(li => li.hasIssues) } else if (filterBy === 'feed-open') { - filtered = filtered.filter(li => feedsArray.some(feed => feed.entityId === li.id)) + const libraryItemIdsWithFeed = await Database.models.feed.findAllLibraryItemIds() + filtered = filtered.filter(li => libraryItemIdsWithFeed.includes(li.id)) + // filtered = filtered.filter(li => feedsArray.some(feed => feed.entityId === li.id)) } else if (filterBy === 'abridged') { filtered = filtered.filter(li => !!li.media.metadata?.abridged) } else if (filterBy === 'ebook') { @@ -356,7 +358,7 @@ module.exports = { return filteredLibraryItems }, - buildPersonalizedShelves(ctx, user, libraryItems, library, maxEntitiesPerShelf, include) { + async buildPersonalizedShelves(ctx, user, libraryItems, library, maxEntitiesPerShelf, include) { const mediaType = library.mediaType const isPodcastLibrary = mediaType === 'podcast' const includeRssFeed = include.includes('rssfeed') @@ -846,27 +848,30 @@ module.exports = { const categoriesWithItems = Object.values(categoryMap).filter(cat => cat.items.length) - return categoriesWithItems.map(cat => { - const shelf = shelves.find(s => s.id === cat.id) - shelf.entities = cat.items + const finalShelves = [] + for (const categoryWithItems of categoriesWithItems) { + const shelf = shelves.find(s => s.id === categoryWithItems.id) + shelf.entities = categoryWithItems.items // Add rssFeed to entities if query string "include=rssfeed" was on request if (includeRssFeed) { if (shelf.type === 'book' || shelf.type === 'podcast') { - shelf.entities = shelf.entities.map((item) => { - item.rssFeed = ctx.rssFeedManager.findFeedForEntityId(item.id)?.toJSONMinified() || null + shelf.entities = await Promise.all(shelf.entities.map(async (item) => { + const feed = await ctx.rssFeedManager.findFeedForEntityId(item.id) + item.rssFeed = feed?.toJSONMinified() || null return item - }) + })) } else if (shelf.type === 'series') { - shelf.entities = shelf.entities.map((series) => { - series.rssFeed = ctx.rssFeedManager.findFeedForEntityId(series.id)?.toJSONMinified() || null + shelf.entities = await Promise.all(shelf.entities.map(async (series) => { + const feed = await ctx.rssFeedManager.findFeedForEntityId(series.id) + series.rssFeed = feed?.toJSONMinified() || null return series - }) + })) } } - - return shelf - }) + finalShelves.push(shelf) + } + return finalShelves }, groupMusicLibraryItemsIntoAlbums(libraryItems) {