Update database loading library items incrementally to reduce mem usage

Dockerfile

@@ -29,7 +29,7 @@ RUN npm ci --only=production
 
 RUN apk del make python3 g++
 
-ENV NODE_OPTIONS=--max-old-space-size=8192
+ENV NODE_OPTIONS=--max-old-space-size=4096
 
 EXPOSE 80
 HEALTHCHECK \

server/Database.js

@@ -119,6 +119,13 @@ class Database {
     return this.sequelize.sync({ force, alter: false })
   }
 
+  /**
+   * Checks if migration to sqlite db is necessary & runs migration.
+   * 
+   * Check if version was upgraded and run any version specific migrations.
+   * 
+   * Loads most of the data from the database. This is a temporary solution.
+   */
   async loadData() {
     if (this.isNew && await dbMigration.checkShouldMigrate()) {
       Logger.info(`[Database] New database was created and old database was detected - migrating old to new`)
@@ -139,15 +146,30 @@ class Database {
       await dbMigration.migrationPatch(this)
     }
 
-    this.libraryItems = await this.models.libraryItem.getAllOldLibraryItems()
-    this.users = await this.models.user.getOldUsers()
-    this.libraries = await this.models.library.getAllOldLibraries()
-    this.collections = await this.models.collection.getOldCollections()
-    this.playlists = await this.models.playlist.getOldPlaylists()
-    this.authors = await this.models.author.getOldAuthors()
-    this.series = await this.models.series.getAllOldSeries()
+    Logger.info(`[Database] Loading db data...`)
 
-    Logger.info(`[Database] Db data loaded in ${Date.now() - startTime}ms`)
+    this.libraryItems = await this.models.libraryItem.loadAllLibraryItems()
+    Logger.info(`[Database] Loaded ${this.libraryItems.length} library items`)
+
+    this.users = await this.models.user.getOldUsers()
+    Logger.info(`[Database] Loaded ${this.users.length} users`)
+
+    this.libraries = await this.models.library.getAllOldLibraries()
+    Logger.info(`[Database] Loaded ${this.libraries.length} libraries`)
+
+    this.collections = await this.models.collection.getOldCollections()
+    Logger.info(`[Database] Loaded ${this.collections.length} collections`)
+
+    this.playlists = await this.models.playlist.getOldPlaylists()
+    Logger.info(`[Database] Loaded ${this.playlists.length} playlists`)
+
+    this.authors = await this.models.author.getOldAuthors()
+    Logger.info(`[Database] Loaded ${this.authors.length} authors`)
+
+    this.series = await this.models.series.getAllOldSeries()
+    Logger.info(`[Database] Loaded ${this.series.length} series`)
+
+    Logger.info(`[Database] Db data loaded in ${((Date.now() - startTime) / 1000).toFixed(2)}s`)
 
     if (packageJson.version !== this.serverSettings.version) {
       Logger.info(`[Database] Server upgrade detected from ${this.serverSettings.version} to ${packageJson.version}`)

server/models/LibraryItem.js

@@ -5,6 +5,95 @@ const { areEquivalent } = require('../utils/index')
 
 module.exports = (sequelize) => {
   class LibraryItem extends Model {
+    /**
+     * Loads all podcast episodes, all library items in chunks of 500, then maps them to old library items
+     * @todo this is a temporary solution until we can use the sqlite without loading all the library items on init
+     * 
+     * @returns {Promise<objects.LibraryItem[]>} old library items
+     */
+    static async loadAllLibraryItems() {
+      let start = Date.now()
+      Logger.info(`[LibraryItem] Loading podcast episodes...`)
+      const podcastEpisodes = await sequelize.models.podcastEpisode.findAll()
+      Logger.info(`[LibraryItem] Finished loading ${podcastEpisodes.length} podcast episodes in ${((Date.now() - start) / 1000).toFixed(2)}s`)
+
+      start = Date.now()
+      Logger.info(`[LibraryItem] Loading library items...`)
+      let libraryItems = await this.getAllOldLibraryItemsIncremental()
+      Logger.info(`[LibraryItem] Finished loading ${libraryItems.length} library items in ${((Date.now() - start) / 1000).toFixed(2)}s`)
+
+      // Map LibraryItem to old library item
+      libraryItems = libraryItems.map(li => {
+        if (li.mediaType === 'podcast') {
+          li.media.podcastEpisodes = podcastEpisodes.filter(pe => pe.podcastId === li.media.id)
+        }
+        return this.getOldLibraryItem(li)
+      })
+
+      return libraryItems
+    }
+
+    /**
+     * Loads all LibraryItem in batches of 500
+     * @todo temporary solution
+     * 
+     * @param {Model<LibraryItem>[]} libraryItems 
+     * @param {number} offset 
+     * @returns {Promise<Model<LibraryItem>[]>}
+     */
+    static async getAllOldLibraryItemsIncremental(libraryItems = [], offset = 0) {
+      const limit = 500
+      const rows = await this.getLibraryItemsIncrement(offset, limit)
+      libraryItems.push(...rows)
+      if (!rows.length || rows.length < limit) {
+        return libraryItems
+      }
+      Logger.info(`[LibraryItem] Loaded ${rows.length} library items. ${libraryItems.length} loaded so far.`)
+      return this.getAllOldLibraryItemsIncremental(libraryItems, offset + rows.length)
+    }
+
+    /**
+     * Gets library items partially expanded, not including podcast episodes
+     * @todo temporary solution
+     * 
+     * @param {number} offset
+     * @param {number} limit
+     * @returns {Promise<Model<LibraryItem>[]>} LibraryItem
+     */
+    static getLibraryItemsIncrement(offset, limit) {
+      return this.findAll({
+        include: [
+          {
+            model: sequelize.models.book,
+            include: [
+              {
+                model: sequelize.models.author,
+                through: {
+                  attributes: []
+                }
+              },
+              {
+                model: sequelize.models.series,
+                through: {
+                  attributes: ['sequence']
+                }
+              }
+            ]
+          },
+          {
+            model: sequelize.models.podcast
+          }
+        ],
+        offset,
+        limit
+      })
+    }
+
+    /**
+     * Currently unused because this is too slow and uses too much mem
+     * 
+     * @returns {Array<objects.LibraryItem>} old library items
+     */
     static async getAllOldLibraryItems() {
       let libraryItems = await this.findAll({
         include: [
@@ -38,6 +127,12 @@ module.exports = (sequelize) => {
       return libraryItems.map(ti => this.getOldLibraryItem(ti))
     }
 
+    /**
+     * Convert an expanded LibraryItem into an old library item
+     * 
+     * @param {Model<LibraryItem>} libraryItemExpanded 
+     * @returns {oldLibraryItem}
+     */
     static getOldLibraryItem(libraryItemExpanded) {
       let media = null
       if (libraryItemExpanded.mediaType === 'book') {