Update:Only load feeds when needed

Dockerfile

@@ -10,7 +10,6 @@ FROM sandreas/tone:v0.1.5 AS tone
 FROM node:16-alpine
 
 ENV NODE_ENV=production
-ENV NODE_OPTIONS=--max-old-space-size=8192
 
 RUN apk update && \
     apk add --no-cache --update \
@@ -30,6 +29,8 @@ RUN npm ci --only=production
 
 RUN apk del make python3 g++
 
+ENV NODE_OPTIONS=--max-old-space-size=8192
+
 EXPOSE 80
 HEALTHCHECK \
     --interval=30s \

server/Database.js

@@ -23,7 +23,6 @@ class Database {
     this.playlists = []
     this.authors = []
     this.series = []
-    this.feeds = []
 
     this.serverSettings = null
     this.notificationSettings = null
@@ -147,7 +146,6 @@ class Database {
     this.playlists = await this.models.playlist.getOldPlaylists()
     this.authors = await this.models.author.getOldAuthors()
     this.series = await this.models.series.getAllOldSeries()
-    this.feeds = await this.models.feed.getOldFeeds()
 
     Logger.info(`[Database] Db data loaded in ${Date.now() - startTime}ms`)
 
@@ -408,7 +406,6 @@ class Database {
   async createFeed(oldFeed) {
     if (!this.sequelize) return false
     await this.models.feed.fullCreateFromOld(oldFeed)
-    this.feeds.push(oldFeed)
   }
 
   updateFeed(oldFeed) {
@@ -419,7 +416,6 @@ class Database {
   async removeFeed(feedId) {
     if (!this.sequelize) return false
     await this.models.feed.removeById(feedId)
-    this.feeds = this.feeds.filter(f => f.id !== feedId)
   }
 
   updateSeries(oldSeries) {

server/controllers/CollectionController.js

@@ -26,14 +26,14 @@ class CollectionController {
     })
   }
 
-  findOne(req, res) {
+  async findOne(req, res) {
     const includeEntities = (req.query.include || '').split(',')
 
     const collectionExpanded = req.collection.toJSONExpanded(Database.libraryItems)
 
     if (includeEntities.includes('rssfeed')) {
-      const feedData = this.rssFeedManager.findFeedForEntityId(collectionExpanded.id)
+      const feedData = await this.rssFeedManager.findFeedForEntityId(collectionExpanded.id)
-      collectionExpanded.rssFeed = feedData ? feedData.toJSONMinified() : null
+      collectionExpanded.rssFeed = feedData?.toJSONMinified() || null
     }
 
     res.json(collectionExpanded)

server/controllers/LibraryController.js

@@ -179,7 +179,7 @@ class LibraryController {
 
   // api/libraries/:id/items
   // TODO: Optimize this method, items are iterated through several times but can be combined
-  getLibraryItems(req, res) {
+  async getLibraryItems(req, res) {
     let libraryItems = req.libraryItems
 
     const include = (req.query.include || '').split(',').map(v => v.trim().toLowerCase()).filter(v => !!v)
@@ -203,7 +203,7 @@ class LibraryController {
     // Step 1 - Filter the retrieved library items
     let filterSeries = null
     if (payload.filterBy) {
-      libraryItems = libraryHelpers.getFilteredLibraryItems(libraryItems, payload.filterBy, req.user, Database.feeds)
+      libraryItems = await libraryHelpers.getFilteredLibraryItems(libraryItems, payload.filterBy, req.user)
       payload.total = libraryItems.length
 
       // Determining if we are filtering titles by a series, and if so, which series
@@ -319,7 +319,7 @@ class LibraryController {
     }
 
     // Step 4 - Transform the items to pass to the client side
-    payload.results = libraryItems.map(li => {
+    payload.results = await Promise.all(libraryItems.map(async li => {
       const json = payload.minified ? li.toJSONMinified() : li.toJSON()
 
       if (li.collapsedSeries) {
@@ -356,7 +356,7 @@ class LibraryController {
       } else {
         // add rssFeed object if "include=rssfeed" was put in query string (only for non-collapsed series)
         if (include.includes('rssfeed')) {
-          const feedData = this.rssFeedManager.findFeedForEntityId(json.id)
+          const feedData = await this.rssFeedManager.findFeedForEntityId(json.id)
           json.rssFeed = feedData ? feedData.toJSONMinified() : null
         }
 
@@ -372,7 +372,7 @@ class LibraryController {
       }
 
       return json
-    })
+    }))
 
     res.json(payload)
   }
@@ -449,11 +449,11 @@ class LibraryController {
 
     // add rssFeed when "include=rssfeed" is in query string
     if (include.includes('rssfeed')) {
-      series = series.map((se) => {
+      series = await Promise.all(series.map(async (se) => {
-        const feedData = this.rssFeedManager.findFeedForEntityId(se.id)
+        const feedData = await this.rssFeedManager.findFeedForEntityId(se.id)
         se.rssFeed = feedData?.toJSONMinified() || null
         return se
-      })
+      }))
     }
 
     payload.results = series
@@ -489,7 +489,7 @@ class LibraryController {
     }
 
     if (include.includes('rssfeed')) {
-      const feedObj = this.rssFeedManager.findFeedForEntityId(seriesJson.id)
+      const feedObj = await this.rssFeedManager.findFeedForEntityId(seriesJson.id)
       seriesJson.rssFeed = feedObj?.toJSONMinified() || null
     }
 
@@ -514,19 +514,21 @@ class LibraryController {
       include: include.join(',')
     }
 
-    let collections = Database.collections.filter(c => c.libraryId === req.library.id).map(c => {
+    let collections = await Promise.all(Database.collections.filter(c => c.libraryId === req.library.id).map(async c => {
       const expanded = c.toJSONExpanded(libraryItems, payload.minified)
 
       // If all books restricted to user in this collection then hide this collection
       if (!expanded.books.length && c.books.length) return null
 
       if (include.includes('rssfeed')) {
-        const feedData = this.rssFeedManager.findFeedForEntityId(c.id)
+        const feedData = await this.rssFeedManager.findFeedForEntityId(c.id)
         expanded.rssFeed = feedData?.toJSONMinified() || null
       }
 
       return expanded
-    }).filter(c => !!c)
+    }))
+
+    collections = collections.filter(c => !!c)
 
     payload.total = collections.length
 
@@ -595,7 +597,7 @@ class LibraryController {
     const limitPerShelf = req.query.limit && !isNaN(req.query.limit) ? Number(req.query.limit) || 10 : 10
     const include = (req.query.include || '').split(',').map(v => v.trim().toLowerCase()).filter(v => !!v)
 
-    const categories = libraryHelpers.buildPersonalizedShelves(this, req.user, req.libraryItems, req.library, limitPerShelf, include)
+    const categories = await libraryHelpers.buildPersonalizedShelves(this, req.user, req.libraryItems, req.library, limitPerShelf, include)
     res.json(categories)
   }
 

server/controllers/LibraryItemController.js

@@ -13,7 +13,7 @@ class LibraryItemController {
   constructor() { }
 
   // Example expand with authors: api/items/:id?expanded=1&include=authors
-  findOne(req, res) {
+  async findOne(req, res) {
     const includeEntities = (req.query.include || '').split(',')
     if (req.query.expanded == 1) {
       var item = req.libraryItem.toJSONExpanded()
@@ -25,8 +25,8 @@ class LibraryItemController {
       }
 
       if (includeEntities.includes('rssfeed')) {
-        const feedData = this.rssFeedManager.findFeedForEntityId(item.id)
+        const feedData = await this.rssFeedManager.findFeedForEntityId(item.id)
-        item.rssFeed = feedData ? feedData.toJSONMinified() : null
+        item.rssFeed = feedData?.toJSONMinified() || null
       }
 
       if (item.mediaType == 'book') {

server/controllers/RSSFeedController.js

@@ -30,7 +30,7 @@ class RSSFeedController {
     }
 
     // Check that this slug is not being used for another feed (slug will also be the Feed id)
-    if (this.rssFeedManager.findFeedBySlug(options.slug)) {
+    if (await this.rssFeedManager.findFeedBySlug(options.slug)) {
       Logger.error(`[RSSFeedController] Cannot open RSS feed because slug "${options.slug}" is already in use`)
       return res.status(400).send('Slug already in use')
     }
@@ -55,7 +55,7 @@ class RSSFeedController {
     }
 
     // Check that this slug is not being used for another feed (slug will also be the Feed id)
-    if (this.rssFeedManager.findFeedBySlug(options.slug)) {
+    if (await this.rssFeedManager.findFeedBySlug(options.slug)) {
       Logger.error(`[RSSFeedController] Cannot open RSS feed because slug "${options.slug}" is already in use`)
       return res.status(400).send('Slug already in use')
     }
@@ -89,7 +89,7 @@ class RSSFeedController {
     }
 
     // Check that this slug is not being used for another feed (slug will also be the Feed id)
-    if (this.rssFeedManager.findFeedBySlug(options.slug)) {
+    if (await this.rssFeedManager.findFeedBySlug(options.slug)) {
       Logger.error(`[RSSFeedController] Cannot open RSS feed because slug "${options.slug}" is already in use`)
       return res.status(400).send('Slug already in use')
     }

server/controllers/SeriesController.js

@@ -35,7 +35,7 @@ class SeriesController {
     }
 
     if (include.includes('rssfeed')) {
-      const feedObj = this.rssFeedManager.findFeedForEntityId(seriesJson.id)
+      const feedObj = await this.rssFeedManager.findFeedForEntityId(seriesJson.id)
       seriesJson.rssFeed = feedObj?.toJSONMinified() || null
     }
 

server/managers/RssFeedManager.js

@@ -35,8 +35,12 @@ class RssFeedManager {
     return true
   }
 
+  /**
+   * Validate all feeds and remove invalid
+   */
   async init() {
-    for (const feed of Database.feeds) {
+    const feeds = await Database.models.feed.getOldFeeds()
+    for (const feed of feeds) {
       // Remove invalid feeds
       if (!this.validateFeedEntity(feed)) {
         await Database.removeFeed(feed.id)
@@ -44,20 +48,35 @@ class RssFeedManager {
     }
   }
 
+  /**
+   * Find open feed for an entity (e.g. collection id, playlist id, library item id)
+   * @param {string} entityId 
+   * @returns {Promise<objects.Feed>} oldFeed
+   */
   findFeedForEntityId(entityId) {
-    return Database.feeds.find(feed => feed.entityId === entityId)
+    return Database.models.feed.findOneOld({ entityId })
   }
 
+  /**
+   * Find open feed for a slug
+   * @param {string} slug 
+   * @returns {Promise<objects.Feed>} oldFeed
+   */
   findFeedBySlug(slug) {
-    return Database.feeds.find(feed => feed.slug === slug)
+    return Database.models.feed.findOneOld({ slug })
   }
 
+  /**
+   * Find open feed for a slug
+   * @param {string} slug 
+   * @returns {Promise<objects.Feed>} oldFeed
+   */
   findFeed(id) {
-    return Database.feeds.find(feed => feed.id === id)
+    return Database.models.feed.findByPkOld(id)
   }
 
   async getFeed(req, res) {
-    const feed = this.findFeedBySlug(req.params.slug)
+    const feed = await this.findFeedBySlug(req.params.slug)
     if (!feed) {
       Logger.warn(`[RssFeedManager] Feed not found ${req.params.slug}`)
       res.sendStatus(404)
@@ -134,8 +153,8 @@ class RssFeedManager {
     res.send(xml)
   }
 
-  getFeedItem(req, res) {
+  async getFeedItem(req, res) {
-    const feed = this.findFeedBySlug(req.params.slug)
+    const feed = await this.findFeedBySlug(req.params.slug)
     if (!feed) {
       Logger.debug(`[RssFeedManager] Feed not found ${req.params.slug}`)
       res.sendStatus(404)
@@ -150,8 +169,8 @@ class RssFeedManager {
     res.sendFile(episodePath)
   }
 
-  getFeedCover(req, res) {
+  async getFeedCover(req, res) {
-    const feed = this.findFeedBySlug(req.params.slug)
+    const feed = await this.findFeedBySlug(req.params.slug)
     if (!feed) {
       Logger.debug(`[RssFeedManager] Feed not found ${req.params.slug}`)
       res.sendStatus(404)
@@ -225,7 +244,7 @@ class RssFeedManager {
   }
 
   async closeRssFeed(req, res) {
-    const feed = this.findFeed(req.params.id)
+    const feed = await this.findFeed(req.params.id)
     if (!feed) {
       Logger.error(`[RssFeedManager] RSS feed not found with id "${req.params.id}"`)
       return res.sendStatus(404)
@@ -234,8 +253,8 @@ class RssFeedManager {
     res.sendStatus(200)
   }
 
-  closeFeedForEntityId(entityId) {
+  async closeFeedForEntityId(entityId) {
-    const feed = this.findFeedForEntityId(entityId)
+    const feed = await this.findFeedForEntityId(entityId)
     if (!feed) return
     return this.handleCloseFeed(feed)
   }

server/models/Feed.js

@@ -56,6 +56,53 @@ module.exports = (sequelize) => {
       })
     }
 
+    /**
+     * Find all library item ids that have an open feed (used in library filter)
+     * @returns {Promise<Array<String>>} array of library item ids
+     */
+    static async findAllLibraryItemIds() {
+      const feeds = await this.findAll({
+        attributes: ['entityId'],
+        where: {
+          entityType: 'libraryItem'
+        }
+      })
+      return feeds.map(f => f.entityId).filter(f => f) || []
+    }
+
+    /**
+     * Find feed where and return oldFeed
+     * @param {object} where sequelize where object
+     * @returns {Promise<objects.Feed>} oldFeed
+     */
+    static async findOneOld(where) {
+      if (!where) return null
+      const feedExpanded = await this.findOne({
+        where,
+        include: {
+          model: sequelize.models.feedEpisode
+        }
+      })
+      if (!feedExpanded) return null
+      return this.getOldFeed(feedExpanded)
+    }
+
+    /**
+     * Find feed and return oldFeed
+     * @param {string} id
+     * @returns {Promise<objects.Feed>} oldFeed
+     */
+    static async findByPkOld(id) {
+      if (!id) return null
+      const feedExpanded = await this.findByPk(id, {
+        include: {
+          model: sequelize.models.feedEpisode
+        }
+      })
+      if (!feedExpanded) return null
+      return this.getOldFeed(feedExpanded)
+    }
+
     static async fullCreateFromOld(oldFeed) {
       const feedObj = this.getFromOld(oldFeed)
       const newFeed = await this.create(feedObj)

server/utils/libraryHelpers.js

@@ -11,7 +11,7 @@ module.exports = {
     return Buffer.from(decodeURIComponent(text), 'base64').toString()
   },
 
-  getFilteredLibraryItems(libraryItems, filterBy, user, feedsArray) {
+  async getFilteredLibraryItems(libraryItems, filterBy, user) {
     let filtered = libraryItems
 
     const searchGroups = ['genres', 'tags', 'series', 'authors', 'progress', 'narrators', 'publishers', 'missing', 'languages', 'tracks', 'ebooks']
@@ -71,7 +71,9 @@ module.exports = {
     } else if (filterBy === 'issues') {
       filtered = filtered.filter(li => li.hasIssues)
     } else if (filterBy === 'feed-open') {
-      filtered = filtered.filter(li => feedsArray.some(feed => feed.entityId === li.id))
+      const libraryItemIdsWithFeed = await Database.models.feed.findAllLibraryItemIds()
+      filtered = filtered.filter(li => libraryItemIdsWithFeed.includes(li.id))
+      // filtered = filtered.filter(li => feedsArray.some(feed => feed.entityId === li.id))
     } else if (filterBy === 'abridged') {
       filtered = filtered.filter(li => !!li.media.metadata?.abridged)
     } else if (filterBy === 'ebook') {
@@ -356,7 +358,7 @@ module.exports = {
     return filteredLibraryItems
   },
 
-  buildPersonalizedShelves(ctx, user, libraryItems, library, maxEntitiesPerShelf, include) {
+  async buildPersonalizedShelves(ctx, user, libraryItems, library, maxEntitiesPerShelf, include) {
     const mediaType = library.mediaType
     const isPodcastLibrary = mediaType === 'podcast'
     const includeRssFeed = include.includes('rssfeed')
@@ -846,27 +848,30 @@ module.exports = {
 
     const categoriesWithItems = Object.values(categoryMap).filter(cat => cat.items.length)
 
-    return categoriesWithItems.map(cat => {
+    const finalShelves = []
-      const shelf = shelves.find(s => s.id === cat.id)
+    for (const categoryWithItems of categoriesWithItems) {
-      shelf.entities = cat.items
+      const shelf = shelves.find(s => s.id === categoryWithItems.id)
+      shelf.entities = categoryWithItems.items
 
       // Add rssFeed to entities if query string "include=rssfeed" was on request
       if (includeRssFeed) {
         if (shelf.type === 'book' || shelf.type === 'podcast') {
-          shelf.entities = shelf.entities.map((item) => {
+          shelf.entities = await Promise.all(shelf.entities.map(async (item) => {
-            item.rssFeed = ctx.rssFeedManager.findFeedForEntityId(item.id)?.toJSONMinified() || null
+            const feed = await ctx.rssFeedManager.findFeedForEntityId(item.id)
+            item.rssFeed = feed?.toJSONMinified() || null
             return item
-          })
+          }))
         } else if (shelf.type === 'series') {
-          shelf.entities = shelf.entities.map((series) => {
+          shelf.entities = await Promise.all(shelf.entities.map(async (series) => {
-            series.rssFeed = ctx.rssFeedManager.findFeedForEntityId(series.id)?.toJSONMinified() || null
+            const feed = await ctx.rssFeedManager.findFeedForEntityId(series.id)
+            series.rssFeed = feed?.toJSONMinified() || null
             return series
-          })
+          }))
         }
       }
-
+      finalShelves.push(shelf)
-      return shelf
+    }
-    })
+    return finalShelves
   },
 
   groupMusicLibraryItemsIntoAlbums(libraryItems) {