From 10011bd6a350f671c60efaea3e58bf05936cf57c Mon Sep 17 00:00:00 2001 From: advplyr Date: Tue, 5 Sep 2023 17:58:13 -0500 Subject: [PATCH] Add startup function to remove invalid records from DB --- server/Database.js | 56 ++++++++++++------- server/models/BookSeries.js | 8 ++- server/models/LibraryItem.js | 51 ----------------- .../utils/queries/libraryItemsBookFilters.js | 4 -- server/utils/queries/seriesFilters.js | 4 -- 5 files changed, 41 insertions(+), 82 deletions(-) diff --git a/server/Database.js b/server/Database.js index d8ef7df2..f87dc031 100644 --- a/server/Database.js +++ b/server/Database.js @@ -156,6 +156,7 @@ class Database { await this.buildModels(force) Logger.info(`[Database] Db initialized with models:`, Object.keys(this.sequelize.models).join(', ')) + await this.cleanDatabase() await this.loadData() } @@ -380,27 +381,6 @@ class Database { return this.models.libraryItem.fullUpdateFromOld(oldLibraryItem) } - async updateBulkLibraryItems(oldLibraryItems) { - if (!this.sequelize) return false - let updatesMade = 0 - for (const oldLibraryItem of oldLibraryItems) { - await oldLibraryItem.saveMetadata() - const hasUpdates = await this.models.libraryItem.fullUpdateFromOld(oldLibraryItem) - if (hasUpdates) { - updatesMade++ - } - } - return updatesMade - } - - async createBulkLibraryItems(oldLibraryItems) { - if (!this.sequelize) return false - for (const oldLibraryItem of oldLibraryItems) { - await oldLibraryItem.saveMetadata() - await this.models.libraryItem.fullCreateFromOld(oldLibraryItem) - } - } - async removeLibraryItem(libraryItemId) { if (!this.sequelize) return false await this.models.libraryItem.removeById(libraryItemId) @@ -675,6 +655,40 @@ class Database { } }) } + + /** + * Clean invalid records in database + * Series should have atleast one Book + * Book and Podcast must have an associated LibraryItem + */ + async cleanDatabase() { + // Remove invalid Podcast records + const podcastsWithNoLibraryItem = await this.podcastModel.findAll({ + where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM libraryItems li WHERE li.mediaId = podcast.id)`), 0) + }) + for (const podcast of podcastsWithNoLibraryItem) { + Logger.warn(`Found podcast "${podcast.title}" with no libraryItem - removing it`) + await podcast.destroy() + } + + // Remove invalid Book records + const booksWithNoLibraryItem = await this.bookModel.findAll({ + where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM libraryItems li WHERE li.mediaId = book.id)`), 0) + }) + for (const book of booksWithNoLibraryItem) { + Logger.warn(`Found book "${book.title}" with no libraryItem - removing it`) + await book.destroy() + } + + // Remove empty series + const emptySeries = await this.seriesModel.findAll({ + where: Sequelize.where(Sequelize.literal(`(SELECT count(*) FROM bookSeries bs WHERE bs.seriesId = series.id)`), 0) + }) + for (const series of emptySeries) { + Logger.warn(`Found series "${series.name}" with no books - removing it`) + await series.destroy() + } + } } module.exports = new Database() \ No newline at end of file diff --git a/server/models/BookSeries.js b/server/models/BookSeries.js index 680ad0c1..fe2a07a5 100644 --- a/server/models/BookSeries.js +++ b/server/models/BookSeries.js @@ -50,10 +50,14 @@ class BookSeries extends Model { book.belongsToMany(series, { through: BookSeries }) series.belongsToMany(book, { through: BookSeries }) - book.hasMany(BookSeries) + book.hasMany(BookSeries, { + onDelete: 'CASCADE' + }) BookSeries.belongsTo(book) - series.hasMany(BookSeries) + series.hasMany(BookSeries, { + onDelete: 'CASCADE' + }) BookSeries.belongsTo(series) } } diff --git a/server/models/LibraryItem.js b/server/models/LibraryItem.js index 77782939..d7670e8f 100644 --- a/server/models/LibraryItem.js +++ b/server/models/LibraryItem.js @@ -63,53 +63,6 @@ class LibraryItem extends Model { this.updatedAt } - /** - * Loads all podcast episodes, all library items in chunks of 500, then maps them to old library items - * @todo this is a temporary solution until we can use the sqlite without loading all the library items on init - * - * @returns {Promise} old library items - */ - static async loadAllLibraryItems() { - let start = Date.now() - Logger.info(`[LibraryItem] Loading podcast episodes...`) - const podcastEpisodes = await this.sequelize.models.podcastEpisode.findAll() - Logger.info(`[LibraryItem] Finished loading ${podcastEpisodes.length} podcast episodes in ${((Date.now() - start) / 1000).toFixed(2)}s`) - - start = Date.now() - Logger.info(`[LibraryItem] Loading library items...`) - let libraryItems = await this.getAllOldLibraryItemsIncremental() - Logger.info(`[LibraryItem] Finished loading ${libraryItems.length} library items in ${((Date.now() - start) / 1000).toFixed(2)}s`) - - // Map LibraryItem to old library item - libraryItems = libraryItems.map(li => { - if (li.mediaType === 'podcast') { - li.media.podcastEpisodes = podcastEpisodes.filter(pe => pe.podcastId === li.media.id) - } - return this.getOldLibraryItem(li) - }) - - return libraryItems - } - - /** - * Loads all LibraryItem in batches of 500 - * @todo temporary solution - * - * @param {Model[]} libraryItems - * @param {number} offset - * @returns {Promise[]>} - */ - static async getAllOldLibraryItemsIncremental(libraryItems = [], offset = 0) { - const limit = 500 - const rows = await this.getLibraryItemsIncrement(offset, limit) - libraryItems.push(...rows) - if (!rows.length || rows.length < limit) { - return libraryItems - } - Logger.info(`[LibraryItem] Loaded ${rows.length} library items. ${libraryItems.length} loaded so far.`) - return this.getAllOldLibraryItemsIncremental(libraryItems, offset + rows.length) - } - /** * Gets library items partially expanded, not including podcast episodes * @todo temporary solution @@ -120,10 +73,6 @@ class LibraryItem extends Model { */ static getLibraryItemsIncrement(offset, limit, where = null) { return this.findAll({ - benchmark: true, - logging: (sql, timeMs) => { - console.log(`[Query] Elapsed ${timeMs}ms.`) - }, where, include: [ { diff --git a/server/utils/queries/libraryItemsBookFilters.js b/server/utils/queries/libraryItemsBookFilters.js index e81ddf0c..a9eb057d 100644 --- a/server/utils/queries/libraryItemsBookFilters.js +++ b/server/utils/queries/libraryItemsBookFilters.js @@ -547,10 +547,6 @@ module.exports = { distinct: true, attributes: bookAttributes, replacements, - benchmark: true, - logging: (sql, timeMs) => { - console.log(`[Query] Elapsed ${timeMs}ms`) - }, include: [ { model: Database.libraryItemModel, diff --git a/server/utils/queries/seriesFilters.js b/server/utils/queries/seriesFilters.js index 22466637..69e4df06 100644 --- a/server/utils/queries/seriesFilters.js +++ b/server/utils/queries/seriesFilters.js @@ -141,10 +141,6 @@ module.exports = { offset, distinct: true, subQuery: false, - benchmark: true, - logging: (sql, timeMs) => { - console.log(`[Query] Series filter/sort. Elapsed ${timeMs}ms`) - }, attributes: seriesAttributes, replacements: userPermissionBookWhere.replacements, include: [