From 9a5ed64fae6d07b64490600910c63a265e0c3d5d Mon Sep 17 00:00:00 2001 From: advplyr Date: Wed, 19 Jul 2023 15:36:18 -0500 Subject: [PATCH] Update database loading library items incrementally to reduce mem usage --- Dockerfile | 2 +- server/Database.js | 38 ++++++++++++--- server/models/LibraryItem.js | 95 ++++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0526494d..2f9297da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN npm ci --only=production RUN apk del make python3 g++ -ENV NODE_OPTIONS=--max-old-space-size=8192 +ENV NODE_OPTIONS=--max-old-space-size=4096 EXPOSE 80 HEALTHCHECK \ diff --git a/server/Database.js b/server/Database.js index a432c394..8b2979d6 100644 --- a/server/Database.js +++ b/server/Database.js @@ -119,6 +119,13 @@ class Database { return this.sequelize.sync({ force, alter: false }) } + /** + * Checks if migration to sqlite db is necessary & runs migration. + * + * Check if version was upgraded and run any version specific migrations. + * + * Loads most of the data from the database. This is a temporary solution. + */ async loadData() { if (this.isNew && await dbMigration.checkShouldMigrate()) { Logger.info(`[Database] New database was created and old database was detected - migrating old to new`) @@ -139,15 +146,30 @@ class Database { await dbMigration.migrationPatch(this) } - this.libraryItems = await this.models.libraryItem.getAllOldLibraryItems() - this.users = await this.models.user.getOldUsers() - this.libraries = await this.models.library.getAllOldLibraries() - this.collections = await this.models.collection.getOldCollections() - this.playlists = await this.models.playlist.getOldPlaylists() - this.authors = await this.models.author.getOldAuthors() - this.series = await this.models.series.getAllOldSeries() + Logger.info(`[Database] Loading db data...`) - Logger.info(`[Database] Db data loaded in ${Date.now() - startTime}ms`) + this.libraryItems = await this.models.libraryItem.loadAllLibraryItems() + Logger.info(`[Database] Loaded ${this.libraryItems.length} library items`) + + this.users = await this.models.user.getOldUsers() + Logger.info(`[Database] Loaded ${this.users.length} users`) + + this.libraries = await this.models.library.getAllOldLibraries() + Logger.info(`[Database] Loaded ${this.libraries.length} libraries`) + + this.collections = await this.models.collection.getOldCollections() + Logger.info(`[Database] Loaded ${this.collections.length} collections`) + + this.playlists = await this.models.playlist.getOldPlaylists() + Logger.info(`[Database] Loaded ${this.playlists.length} playlists`) + + this.authors = await this.models.author.getOldAuthors() + Logger.info(`[Database] Loaded ${this.authors.length} authors`) + + this.series = await this.models.series.getAllOldSeries() + Logger.info(`[Database] Loaded ${this.series.length} series`) + + Logger.info(`[Database] Db data loaded in ${((Date.now() - startTime) / 1000).toFixed(2)}s`) if (packageJson.version !== this.serverSettings.version) { Logger.info(`[Database] Server upgrade detected from ${this.serverSettings.version} to ${packageJson.version}`) diff --git a/server/models/LibraryItem.js b/server/models/LibraryItem.js index 4e200737..2a4d5609 100644 --- a/server/models/LibraryItem.js +++ b/server/models/LibraryItem.js @@ -5,6 +5,95 @@ const { areEquivalent } = require('../utils/index') module.exports = (sequelize) => { class LibraryItem extends Model { + /** + * Loads all podcast episodes, all library items in chunks of 500, then maps them to old library items + * @todo this is a temporary solution until we can use the sqlite without loading all the library items on init + * + * @returns {Promise} old library items + */ + static async loadAllLibraryItems() { + let start = Date.now() + Logger.info(`[LibraryItem] Loading podcast episodes...`) + const podcastEpisodes = await sequelize.models.podcastEpisode.findAll() + Logger.info(`[LibraryItem] Finished loading ${podcastEpisodes.length} podcast episodes in ${((Date.now() - start) / 1000).toFixed(2)}s`) + + start = Date.now() + Logger.info(`[LibraryItem] Loading library items...`) + let libraryItems = await this.getAllOldLibraryItemsIncremental() + Logger.info(`[LibraryItem] Finished loading ${libraryItems.length} library items in ${((Date.now() - start) / 1000).toFixed(2)}s`) + + // Map LibraryItem to old library item + libraryItems = libraryItems.map(li => { + if (li.mediaType === 'podcast') { + li.media.podcastEpisodes = podcastEpisodes.filter(pe => pe.podcastId === li.media.id) + } + return this.getOldLibraryItem(li) + }) + + return libraryItems + } + + /** + * Loads all LibraryItem in batches of 500 + * @todo temporary solution + * + * @param {Model[]} libraryItems + * @param {number} offset + * @returns {Promise[]>} + */ + static async getAllOldLibraryItemsIncremental(libraryItems = [], offset = 0) { + const limit = 500 + const rows = await this.getLibraryItemsIncrement(offset, limit) + libraryItems.push(...rows) + if (!rows.length || rows.length < limit) { + return libraryItems + } + Logger.info(`[LibraryItem] Loaded ${rows.length} library items. ${libraryItems.length} loaded so far.`) + return this.getAllOldLibraryItemsIncremental(libraryItems, offset + rows.length) + } + + /** + * Gets library items partially expanded, not including podcast episodes + * @todo temporary solution + * + * @param {number} offset + * @param {number} limit + * @returns {Promise[]>} LibraryItem + */ + static getLibraryItemsIncrement(offset, limit) { + return this.findAll({ + include: [ + { + model: sequelize.models.book, + include: [ + { + model: sequelize.models.author, + through: { + attributes: [] + } + }, + { + model: sequelize.models.series, + through: { + attributes: ['sequence'] + } + } + ] + }, + { + model: sequelize.models.podcast + } + ], + offset, + limit + }) + } + + /** + * Currently unused because this is too slow and uses too much mem + * + * @returns {Array} old library items + */ static async getAllOldLibraryItems() { let libraryItems = await this.findAll({ include: [ @@ -38,6 +127,12 @@ module.exports = (sequelize) => { return libraryItems.map(ti => this.getOldLibraryItem(ti)) } + /** + * Convert an expanded LibraryItem into an old library item + * + * @param {Model} libraryItemExpanded + * @returns {oldLibraryItem} + */ static getOldLibraryItem(libraryItemExpanded) { let media = null if (libraryItemExpanded.mediaType === 'book') {