From 435b7fda7e8ddc3ef413a4e583ea09f91627d485 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Fri, 8 Nov 2024 09:09:18 -0700 Subject: [PATCH 1/3] Add: check for changes to library items --- server/utils/queries/libraryFilters.js | 54 ++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index 34c3fe54..f66df568 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -485,6 +485,60 @@ module.exports = { } } } else { + // To reduce the cold-start load time, first check if any library items, series, + // or authors have had an "updatedAt" timestamp since the last time the filter + // data was loaded. If so, we can skip loading all of the data. + // Because many items could change, just check the count of items. + const lastLoadedAt = cachedFilterData ? cachedFilterData.loadedAt : 0 + + const changedBooks = await Database.bookModel.count({ + include: { + model: Database.libraryItemModel, + attributes: [], + where: { + libraryId: libraryId, + updatedAt: { + [Sequelize.Op.gt]: new Date(lastLoadedAt) + } + } + }, + where: { + updatedAt: { + [Sequelize.Op.gt]: new Date(lastLoadedAt) + } + }, + limit: 1 + }) + + const changedSeries = await Database.seriesModel.count({ + where: { + libraryId: libraryId, + updatedAt: { + [Sequelize.Op.gt]: new Date(lastLoadedAt) + } + }, + limit: 1 + }) + + const changedAuthors = await Database.authorModel.count({ + where: { + libraryId: libraryId, + updatedAt: { + [Sequelize.Op.gt]: new Date(lastLoadedAt) + } + }, + limit: 1 + }) + + if (changedBooks + changedSeries + changedAuthors === 0) { + // If nothing has changed, update the cache to current time for 30 minute + // cache time and return the cached data + Logger.debug(`Filter data for ${libraryId} has not changed, returning cached data and updating cache time after ${((Date.now() - start) / 1000).toFixed(2)}s`) + Database.libraryFilterData[libraryId].loadedAt = Date.now() + return cachedFilterData + } + + // Something has changed in one of the tables, so reload all of the filter data for library const books = await Database.bookModel.findAll({ include: { model: Database.libraryItemModel, From e57d4cc54435dfa651a7de30a1a4b0a8fcd8d926 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Fri, 8 Nov 2024 09:33:34 -0700 Subject: [PATCH 2/3] Add: filter update check to podcast libraries --- server/utils/queries/libraryFilters.js | 41 ++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index f66df568..2be415e2 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -462,7 +462,42 @@ module.exports = { numIssues: 0 } + const lastLoadedAt = cachedFilterData ? cachedFilterData.loadedAt : 0 + if (mediaType === 'podcast') { + // To reduce the cold-start load time, first check if any podcasts + // have an "updatedAt" timestamp since the last time the filter + // data was loaded. If so, we can skip loading all of the data. + // Because many items could change, just check the count of items instead + // of actually loading the data twice + const changedPodcasts = await Database.podcastModel.count({ + include: { + model: Database.libraryItemModel, + attributes: [], + where: { + libraryId: libraryId, + updatedAt: { + [Sequelize.Op.gt]: new Date(lastLoadedAt) + } + } + }, + where: { + updatedAt: { + [Sequelize.Op.gt]: new Date(lastLoadedAt) + } + }, + limit: 1 + }) + + if (changedPodcasts === 0) { + // If nothing has changed, update the cache to current time for 30 minute + // cache time and return the cached data + Logger.debug(`Filter data for ${libraryId} has not changed, returning cached data and updating cache time after ${((Date.now() - start) / 1000).toFixed(2)}s`) + Database.libraryFilterData[libraryId].loadedAt = Date.now() + return cachedFilterData + } + + // Something has changed in the podcasts table, so reload all of the filter data for library const podcasts = await Database.podcastModel.findAll({ include: { model: Database.libraryItemModel, @@ -486,10 +521,10 @@ module.exports = { } } else { // To reduce the cold-start load time, first check if any library items, series, - // or authors have had an "updatedAt" timestamp since the last time the filter + // or authors have an "updatedAt" timestamp since the last time the filter // data was loaded. If so, we can skip loading all of the data. - // Because many items could change, just check the count of items. - const lastLoadedAt = cachedFilterData ? cachedFilterData.loadedAt : 0 + // Because many items could change, just check the count of items instead + // of actually loading the data twice const changedBooks = await Database.bookModel.count({ include: { From e8d8b67c0aa170f5b0fe4fe8c5996a00b49bbdc0 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Fri, 8 Nov 2024 10:49:12 -0700 Subject: [PATCH 3/3] Add: check for deleted items --- server/utils/queries/libraryFilters.js | 72 ++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index 2be415e2..64ad07ee 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -459,12 +459,29 @@ module.exports = { languages: new Set(), publishers: new Set(), publishedDecades: new Set(), + bookCount: 0, // How many books returned from database query + authorCount: 0, // How many authors returned from database query + seriesCount: 0, // How many series returned from database query + podcastCount: 0, // How many podcasts returned from database query numIssues: 0 } const lastLoadedAt = cachedFilterData ? cachedFilterData.loadedAt : 0 if (mediaType === 'podcast') { + // Check how many podcasts are in library to determine if we need to load all of the data + // This is done to handle the edge case of podcasts having been deleted and not having + // an updatedAt timestamp to trigger a reload of the filter data + const podcastCountFromDatabase = await Database.podcastModel.count({ + include: { + model: Database.libraryItemModel, + attributes: [], + where: { + libraryId: libraryId + } + } + }) + // To reduce the cold-start load time, first check if any podcasts // have an "updatedAt" timestamp since the last time the filter // data was loaded. If so, we can skip loading all of the data. @@ -490,11 +507,14 @@ module.exports = { }) if (changedPodcasts === 0) { - // If nothing has changed, update the cache to current time for 30 minute - // cache time and return the cached data - Logger.debug(`Filter data for ${libraryId} has not changed, returning cached data and updating cache time after ${((Date.now() - start) / 1000).toFixed(2)}s`) - Database.libraryFilterData[libraryId].loadedAt = Date.now() - return cachedFilterData + // If nothing has changed, check if the number of podcasts in + // library is still the same as prior check before updating cache creation time + + if (podcastCountFromDatabase === Database.libraryFilterData[libraryId].podcastCount) { + Logger.debug(`Filter data for ${libraryId} has not changed, returning cached data and updating cache time after ${((Date.now() - start) / 1000).toFixed(2)}s`) + Database.libraryFilterData[libraryId].loadedAt = Date.now() + return cachedFilterData + } } // Something has changed in the podcasts table, so reload all of the filter data for library @@ -519,7 +539,32 @@ module.exports = { data.languages.add(podcast.language) } } + + // Set podcast count for later comparison + data.podcastCount = podcastCountFromDatabase } else { + const bookCountFromDatabase = await Database.bookModel.count({ + include: { + model: Database.libraryItemModel, + attributes: [], + where: { + libraryId: libraryId + } + } + }) + + const seriesCountFromDatabase = await Database.seriesModel.count({ + where: { + libraryId: libraryId + } + }) + + const authorCountFromDatabase = await Database.authorModel.count({ + where: { + libraryId: libraryId + } + }) + // To reduce the cold-start load time, first check if any library items, series, // or authors have an "updatedAt" timestamp since the last time the filter // data was loaded. If so, we can skip loading all of the data. @@ -566,13 +611,20 @@ module.exports = { }) if (changedBooks + changedSeries + changedAuthors === 0) { - // If nothing has changed, update the cache to current time for 30 minute - // cache time and return the cached data - Logger.debug(`Filter data for ${libraryId} has not changed, returning cached data and updating cache time after ${((Date.now() - start) / 1000).toFixed(2)}s`) - Database.libraryFilterData[libraryId].loadedAt = Date.now() - return cachedFilterData + // If nothing has changed, check if the number of authors, series, and books + // matches the prior check before updating cache creation time + if (bookCountFromDatabase === Database.libraryFilterData[libraryId].bookCount && seriesCountFromDatabase === Database.libraryFilterData[libraryId].seriesCount && authorCountFromDatabase === Database.libraryFilterData[libraryId].authorCount) { + Logger.debug(`Filter data for ${libraryId} has not changed, returning cached data and updating cache time after ${((Date.now() - start) / 1000).toFixed(2)}s`) + Database.libraryFilterData[libraryId].loadedAt = Date.now() + return cachedFilterData + } } + // Store the counts for later comparison + data.bookCount = bookCountFromDatabase + data.seriesCount = seriesCountFromDatabase + data.authorCount = authorCountFromDatabase + // Something has changed in one of the tables, so reload all of the filter data for library const books = await Database.bookModel.findAll({ include: {