mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2025-04-25 01:16:40 +02:00
Merge pull request #3996 from mikiher/optimize-podcast-queries
Improve podcast library page query performance on title, titleIgnorePrefix, and addedAt sort orders
This commit is contained in:
advplyr
GitHub
commit
9f9bee2ddc
@ -191,6 +191,10 @@ class Database {
|
||||
Logger.info(`[Database] Db initialized with models:`, Object.keys(this.sequelize.models).join(', '))
|
||||
|
||||
await this.loadData()
|
||||
|
||||
Logger.info(`[Database] running ANALYZE`)
|
||||
await this.sequelize.query('ANALYZE')
|
||||
Logger.info(`[Database] ANALYZE completed`)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -107,7 +107,9 @@ class PodcastController {
|
||||
libraryFiles: [],
|
||||
extraData: {},
|
||||
libraryId: library.id,
|
||||
libraryFolderId: folder.id
|
||||
libraryFolderId: folder.id,
|
||||
title: podcast.title,
|
||||
titleIgnorePrefix: podcast.titleIgnorePrefix
|
||||
},
|
||||
{ transaction }
|
||||
)
|
||||
@ -498,6 +500,10 @@ class PodcastController {
|
||||
req.libraryItem.changed('libraryFiles', true)
|
||||
await req.libraryItem.save()
|
||||
|
||||
// update number of episodes
|
||||
req.libraryItem.media.numEpisodes = req.libraryItem.media.podcastEpisodes.length
|
||||
await req.libraryItem.media.save()
|
||||
|
||||
SocketAuthority.emitter('item_updated', req.libraryItem.toOldJSONExpanded())
|
||||
res.json(req.libraryItem.toOldJSON())
|
||||
}
|
||||
|
||||
@ -232,6 +232,11 @@ class PodcastManager {
|
||||
|
||||
await libraryItem.save()
|
||||
|
||||
if (libraryItem.media.numEpisodes !== libraryItem.media.podcastEpisodes.length) {
|
||||
libraryItem.media.numEpisodes = libraryItem.media.podcastEpisodes.length
|
||||
await libraryItem.media.save()
|
||||
}
|
||||
|
||||
SocketAuthority.emitter('item_updated', libraryItem.toOldJSONExpanded())
|
||||
const podcastEpisodeExpanded = podcastEpisode.toOldJSONExpanded(libraryItem.id)
|
||||
podcastEpisodeExpanded.libraryItem = libraryItem.toOldJSONExpanded()
|
||||
@ -622,7 +627,9 @@ class PodcastManager {
|
||||
libraryFiles: [],
|
||||
extraData: {},
|
||||
libraryId: folder.libraryId,
|
||||
libraryFolderId: folder.id
|
||||
libraryFolderId: folder.id,
|
||||
title: podcast.title,
|
||||
titleIgnorePrefix: podcast.titleIgnorePrefix
|
||||
},
|
||||
{ transaction }
|
||||
)
|
||||
|
||||
@ -14,3 +14,4 @@ Please add a record of every database migration that you create to this file. Th
|
||||
| v2.17.6 | v2.17.6-share-add-isdownloadable | Adds the isDownloadable column to the mediaItemShares table |
|
||||
| v2.17.7 | v2.17.7-add-indices | Adds indices to the libraryItems and books tables to reduce query times |
|
||||
| v2.19.1 | v2.19.1-copy-title-to-library-items | Copies title and titleIgnorePrefix to the libraryItems table, creates update triggers and indices |
|
||||
| v2.19.4 | v2.19.4-improve-podcast-queries | Adds numEpisodes to podcasts, adds podcastId to mediaProgresses, copies podcast title to libraryItems |
|
||||
|
||||
219
server/migrations/v2.19.4-improve-podcast-queries.js
Normal file
219
server/migrations/v2.19.4-improve-podcast-queries.js
Normal file
@ -0,0 +1,219 @@
|
||||
const util = require('util')
|
||||
|
||||
/**
|
||||
* @typedef MigrationContext
|
||||
* @property {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object.
|
||||
* @property {import('../Logger')} logger - a Logger object.
|
||||
*
|
||||
* @typedef MigrationOptions
|
||||
* @property {MigrationContext} context - an object containing the migration context.
|
||||
*/
|
||||
|
||||
const migrationVersion = '2.19.4'
|
||||
const migrationName = `${migrationVersion}-improve-podcast-queries`
|
||||
const loggerPrefix = `[${migrationVersion} migration]`
|
||||
|
||||
/**
|
||||
* This upward migration adds a numEpisodes column to the podcasts table and populates it.
|
||||
* It also adds a podcastId column to the mediaProgresses table and populates it.
|
||||
* It also copies the title and titleIgnorePrefix columns from the podcasts table to the libraryItems table,
|
||||
* and adds triggers to update them when the corresponding columns in the podcasts table are updated.
|
||||
*
|
||||
* @param {MigrationOptions} options - an object containing the migration context.
|
||||
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
|
||||
*/
|
||||
async function up({ context: { queryInterface, logger } }) {
|
||||
// Upwards migration script
|
||||
logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
|
||||
|
||||
// Add numEpisodes column to podcasts table
|
||||
await addColumn(queryInterface, logger, 'podcasts', 'numEpisodes', { type: queryInterface.sequelize.Sequelize.INTEGER, allowNull: false, defaultValue: 0 })
|
||||
|
||||
// Populate numEpisodes column with the number of episodes for each podcast
|
||||
await populateNumEpisodes(queryInterface, logger)
|
||||
|
||||
// Add podcastId column to mediaProgresses table
|
||||
await addColumn(queryInterface, logger, 'mediaProgresses', 'podcastId', { type: queryInterface.sequelize.Sequelize.UUID, allowNull: true })
|
||||
|
||||
// Populate podcastId column with the podcastId for each mediaProgress
|
||||
await populatePodcastId(queryInterface, logger)
|
||||
|
||||
// Copy title and titleIgnorePrefix columns from podcasts to libraryItems
|
||||
await copyColumn(queryInterface, logger, 'podcasts', 'title', 'id', 'libraryItems', 'title', 'mediaId')
|
||||
await copyColumn(queryInterface, logger, 'podcasts', 'titleIgnorePrefix', 'id', 'libraryItems', 'titleIgnorePrefix', 'mediaId')
|
||||
|
||||
// Add triggers to update title and titleIgnorePrefix in libraryItems
|
||||
await addTrigger(queryInterface, logger, 'podcasts', 'title', 'id', 'libraryItems', 'title', 'mediaId')
|
||||
await addTrigger(queryInterface, logger, 'podcasts', 'titleIgnorePrefix', 'id', 'libraryItems', 'titleIgnorePrefix', 'mediaId')
|
||||
|
||||
logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* This downward migration removes the triggers on the podcasts table,
|
||||
* the numEpisodes column from the podcasts table, and the podcastId column from the mediaProgresses table.
|
||||
*
|
||||
* @param {MigrationOptions} options - an object containing the migration context.
|
||||
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
|
||||
*/
|
||||
async function down({ context: { queryInterface, logger } }) {
|
||||
// Downward migration script
|
||||
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
|
||||
|
||||
// Remove triggers from libraryItems
|
||||
await removeTrigger(queryInterface, logger, 'podcasts', 'title', 'libraryItems', 'title')
|
||||
await removeTrigger(queryInterface, logger, 'podcasts', 'titleIgnorePrefix', 'libraryItems', 'titleIgnorePrefix')
|
||||
|
||||
// Remove numEpisodes column from podcasts table
|
||||
await removeColumn(queryInterface, logger, 'podcasts', 'numEpisodes')
|
||||
|
||||
// Remove podcastId column from mediaProgresses table
|
||||
await removeColumn(queryInterface, logger, 'mediaProgresses', 'podcastId')
|
||||
|
||||
logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
|
||||
}
|
||||
|
||||
async function populateNumEpisodes(queryInterface, logger) {
|
||||
logger.info(`${loggerPrefix} populating numEpisodes column in podcasts table`)
|
||||
await queryInterface.sequelize.query(`
|
||||
UPDATE podcasts
|
||||
SET numEpisodes = (SELECT COUNT(*) FROM podcastEpisodes WHERE podcastEpisodes.podcastId = podcasts.id)
|
||||
`)
|
||||
logger.info(`${loggerPrefix} populated numEpisodes column in podcasts table`)
|
||||
}
|
||||
|
||||
async function populatePodcastId(queryInterface, logger) {
|
||||
logger.info(`${loggerPrefix} populating podcastId column in mediaProgresses table`)
|
||||
// bulk update podcastId to the podcastId of the podcastEpisode if the mediaItemType is podcastEpisode
|
||||
await queryInterface.sequelize.query(`
|
||||
UPDATE mediaProgresses
|
||||
SET podcastId = (SELECT podcastId FROM podcastEpisodes WHERE podcastEpisodes.id = mediaProgresses.mediaItemId)
|
||||
WHERE mediaItemType = 'podcastEpisode'
|
||||
`)
|
||||
logger.info(`${loggerPrefix} populated podcastId column in mediaProgresses table`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to add a column to a table. If the column already exists, it logs a message and continues.
|
||||
*
|
||||
* @param {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object.
|
||||
* @param {import('../Logger')} logger - a Logger object.
|
||||
* @param {string} table - the name of the table to add the column to.
|
||||
* @param {string} column - the name of the column to add.
|
||||
* @param {Object} options - the options for the column.
|
||||
*/
|
||||
async function addColumn(queryInterface, logger, table, column, options) {
|
||||
logger.info(`${loggerPrefix} adding column "${column}" to table "${table}"`)
|
||||
const tableDescription = await queryInterface.describeTable(table)
|
||||
if (!tableDescription[column]) {
|
||||
await queryInterface.addColumn(table, column, options)
|
||||
logger.info(`${loggerPrefix} added column "${column}" to table "${table}"`)
|
||||
} else {
|
||||
logger.info(`${loggerPrefix} column "${column}" already exists in table "${table}"`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to remove a column from a table. If the column does not exist, it logs a message and continues.
|
||||
*
|
||||
* @param {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object.
|
||||
* @param {import('../Logger')} logger - a Logger object.
|
||||
* @param {string} table - the name of the table to remove the column from.
|
||||
* @param {string} column - the name of the column to remove.
|
||||
*/
|
||||
async function removeColumn(queryInterface, logger, table, column) {
|
||||
logger.info(`${loggerPrefix} removing column "${column}" from table "${table}"`)
|
||||
const tableDescription = await queryInterface.describeTable(table)
|
||||
if (tableDescription[column]) {
|
||||
await queryInterface.sequelize.query(`ALTER TABLE ${table} DROP COLUMN ${column}`)
|
||||
logger.info(`${loggerPrefix} removed column "${column}" from table "${table}"`)
|
||||
} else {
|
||||
logger.info(`${loggerPrefix} column "${column}" does not exist in table "${table}"`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to add a trigger to update a column in a target table when a column in a source table is updated.
|
||||
* If the trigger already exists, it drops it and creates a new one.
|
||||
* sourceIdColumn and targetIdColumn are used to match the source and target rows.
|
||||
*
|
||||
* @param {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object.
|
||||
* @param {import('../Logger')} logger - a Logger object.
|
||||
* @param {string} sourceTable - the name of the source table.
|
||||
* @param {string} sourceColumn - the name of the column to update.
|
||||
* @param {string} sourceIdColumn - the name of the id column of the source table.
|
||||
* @param {string} targetTable - the name of the target table.
|
||||
* @param {string} targetColumn - the name of the column to update.
|
||||
* @param {string} targetIdColumn - the name of the id column of the target table.
|
||||
*/
|
||||
async function addTrigger(queryInterface, logger, sourceTable, sourceColumn, sourceIdColumn, targetTable, targetColumn, targetIdColumn) {
|
||||
logger.info(`${loggerPrefix} adding trigger to update ${targetTable}.${targetColumn} when ${sourceTable}.${sourceColumn} is updated`)
|
||||
const triggerName = convertToSnakeCase(`update_${targetTable}_${targetColumn}_from_${sourceTable}_${sourceColumn}`)
|
||||
|
||||
await queryInterface.sequelize.query(`DROP TRIGGER IF EXISTS ${triggerName}`)
|
||||
|
||||
await queryInterface.sequelize.query(`
|
||||
CREATE TRIGGER ${triggerName}
|
||||
AFTER UPDATE OF ${sourceColumn} ON ${sourceTable}
|
||||
FOR EACH ROW
|
||||
BEGIN
|
||||
UPDATE ${targetTable}
|
||||
SET ${targetColumn} = NEW.${sourceColumn}
|
||||
WHERE ${targetTable}.${targetIdColumn} = NEW.${sourceIdColumn};
|
||||
END;
|
||||
`)
|
||||
logger.info(`${loggerPrefix} added trigger to update ${targetTable}.${targetColumn} when ${sourceTable}.${sourceColumn} is updated`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to remove an update trigger from a table.
|
||||
*
|
||||
* @param {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object.
|
||||
* @param {import('../Logger')} logger - a Logger object.
|
||||
* @param {string} sourceTable - the name of the source table.
|
||||
* @param {string} sourceColumn - the name of the column to update.
|
||||
* @param {string} targetTable - the name of the target table.
|
||||
* @param {string} targetColumn - the name of the column to update.
|
||||
*/
|
||||
async function removeTrigger(queryInterface, logger, sourceTable, sourceColumn, targetTable, targetColumn) {
|
||||
logger.info(`${loggerPrefix} removing trigger to update ${targetTable}.${targetColumn}`)
|
||||
const triggerName = convertToSnakeCase(`update_${targetTable}_${targetColumn}_from_${sourceTable}_${sourceColumn}`)
|
||||
await queryInterface.sequelize.query(`DROP TRIGGER IF EXISTS ${triggerName}`)
|
||||
logger.info(`${loggerPrefix} removed trigger to update ${targetTable}.${targetColumn}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to copy a column from a source table to a target table.
|
||||
* sourceIdColumn and targetIdColumn are used to match the source and target rows.
|
||||
*
|
||||
* @param {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object.
|
||||
* @param {import('../Logger')} logger - a Logger object.
|
||||
* @param {string} sourceTable - the name of the source table.
|
||||
* @param {string} sourceColumn - the name of the column to copy.
|
||||
* @param {string} sourceIdColumn - the name of the id column of the source table.
|
||||
* @param {string} targetTable - the name of the target table.
|
||||
* @param {string} targetColumn - the name of the column to copy to.
|
||||
* @param {string} targetIdColumn - the name of the id column of the target table.
|
||||
*/
|
||||
async function copyColumn(queryInterface, logger, sourceTable, sourceColumn, sourceIdColumn, targetTable, targetColumn, targetIdColumn) {
|
||||
logger.info(`${loggerPrefix} copying column "${sourceColumn}" from table "${sourceTable}" to table "${targetTable}"`)
|
||||
await queryInterface.sequelize.query(`
|
||||
UPDATE ${targetTable}
|
||||
SET ${targetColumn} = ${sourceTable}.${sourceColumn}
|
||||
FROM ${sourceTable}
|
||||
WHERE ${targetTable}.${targetIdColumn} = ${sourceTable}.${sourceIdColumn}
|
||||
`)
|
||||
logger.info(`${loggerPrefix} copied column "${sourceColumn}" from table "${sourceTable}" to table "${targetTable}"`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to convert a string to snake case, e.g. "titleIgnorePrefix" -> "title_ignore_prefix"
|
||||
*
|
||||
* @param {string} str - the string to convert to snake case.
|
||||
* @returns {string} - the string in snake case.
|
||||
*/
|
||||
function convertToSnakeCase(str) {
|
||||
return str.replace(/([A-Z])/g, '_$1').toLowerCase()
|
||||
}
|
||||
|
||||
module.exports = { up, down }
|
||||
@ -34,6 +34,8 @@ class MediaProgress extends Model {
|
||||
this.updatedAt
|
||||
/** @type {Date} */
|
||||
this.createdAt
|
||||
/** @type {UUIDV4} */
|
||||
this.podcastId
|
||||
}
|
||||
|
||||
static removeById(mediaProgressId) {
|
||||
@ -69,7 +71,8 @@ class MediaProgress extends Model {
|
||||
ebookLocation: DataTypes.STRING,
|
||||
ebookProgress: DataTypes.FLOAT,
|
||||
finishedAt: DataTypes.DATE,
|
||||
extraData: DataTypes.JSON
|
||||
extraData: DataTypes.JSON,
|
||||
podcastId: DataTypes.UUID
|
||||
},
|
||||
{
|
||||
sequelize,
|
||||
@ -123,6 +126,16 @@ class MediaProgress extends Model {
|
||||
}
|
||||
})
|
||||
|
||||
// make sure to call the afterDestroy hook for each instance
|
||||
MediaProgress.addHook('beforeBulkDestroy', (options) => {
|
||||
options.individualHooks = true
|
||||
})
|
||||
|
||||
// update the potentially cached user after destroying the media progress
|
||||
MediaProgress.addHook('afterDestroy', (instance) => {
|
||||
user.mediaProgressRemoved(instance)
|
||||
})
|
||||
|
||||
user.hasMany(MediaProgress, {
|
||||
onDelete: 'CASCADE'
|
||||
})
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
const { DataTypes, Model } = require('sequelize')
|
||||
const { getTitlePrefixAtEnd, getTitleIgnorePrefix } = require('../utils')
|
||||
const Logger = require('../Logger')
|
||||
const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters')
|
||||
|
||||
/**
|
||||
* @typedef PodcastExpandedProperties
|
||||
@ -61,6 +62,8 @@ class Podcast extends Model {
|
||||
this.createdAt
|
||||
/** @type {Date} */
|
||||
this.updatedAt
|
||||
/** @type {number} */
|
||||
this.numEpisodes
|
||||
|
||||
/** @type {import('./PodcastEpisode')[]} */
|
||||
this.podcastEpisodes
|
||||
@ -138,13 +141,22 @@ class Podcast extends Model {
|
||||
maxNewEpisodesToDownload: DataTypes.INTEGER,
|
||||
coverPath: DataTypes.STRING,
|
||||
tags: DataTypes.JSON,
|
||||
genres: DataTypes.JSON
|
||||
genres: DataTypes.JSON,
|
||||
numEpisodes: DataTypes.INTEGER
|
||||
},
|
||||
{
|
||||
sequelize,
|
||||
modelName: 'podcast'
|
||||
}
|
||||
)
|
||||
|
||||
Podcast.addHook('afterDestroy', async (instance) => {
|
||||
libraryItemsPodcastFilters.clearCountCache('podcast', 'afterDestroy')
|
||||
})
|
||||
|
||||
Podcast.addHook('afterCreate', async (instance) => {
|
||||
libraryItemsPodcastFilters.clearCountCache('podcast', 'afterCreate')
|
||||
})
|
||||
}
|
||||
|
||||
get hasMediaFiles() {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
const { DataTypes, Model } = require('sequelize')
|
||||
|
||||
const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters')
|
||||
/**
|
||||
* @typedef ChapterObject
|
||||
* @property {number} id
|
||||
@ -132,6 +132,14 @@ class PodcastEpisode extends Model {
|
||||
onDelete: 'CASCADE'
|
||||
})
|
||||
PodcastEpisode.belongsTo(podcast)
|
||||
|
||||
PodcastEpisode.addHook('afterDestroy', async (instance) => {
|
||||
libraryItemsPodcastFilters.clearCountCache('podcastEpisode', 'afterDestroy')
|
||||
})
|
||||
|
||||
PodcastEpisode.addHook('afterCreate', async (instance) => {
|
||||
libraryItemsPodcastFilters.clearCountCache('podcastEpisode', 'afterCreate')
|
||||
})
|
||||
}
|
||||
|
||||
get size() {
|
||||
|
||||
@ -404,6 +404,14 @@ class User extends Model {
|
||||
return count > 0
|
||||
}
|
||||
|
||||
static mediaProgressRemoved(mediaProgress) {
|
||||
const cachedUser = userCache.getById(mediaProgress.userId)
|
||||
if (cachedUser) {
|
||||
Logger.debug(`[User] mediaProgressRemoved: ${mediaProgress.id} from user ${cachedUser.id}`)
|
||||
cachedUser.mediaProgresses = cachedUser.mediaProgresses.filter((mp) => mp.id !== mediaProgress.id)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize model
|
||||
* @param {import('../Database').sequelize} sequelize
|
||||
@ -626,6 +634,7 @@ class User extends Model {
|
||||
/** @type {import('./MediaProgress')|null} */
|
||||
let mediaProgress = null
|
||||
let mediaItemId = null
|
||||
let podcastId = null
|
||||
if (progressPayload.episodeId) {
|
||||
const podcastEpisode = await this.sequelize.models.podcastEpisode.findByPk(progressPayload.episodeId, {
|
||||
attributes: ['id', 'podcastId'],
|
||||
@ -654,6 +663,7 @@ class User extends Model {
|
||||
}
|
||||
mediaItemId = podcastEpisode.id
|
||||
mediaProgress = podcastEpisode.mediaProgresses?.[0]
|
||||
podcastId = podcastEpisode.podcastId
|
||||
} else {
|
||||
const libraryItem = await this.sequelize.models.libraryItem.findByPk(progressPayload.libraryItemId, {
|
||||
attributes: ['id', 'mediaId', 'mediaType'],
|
||||
@ -686,6 +696,7 @@ class User extends Model {
|
||||
const newMediaProgressPayload = {
|
||||
userId: this.id,
|
||||
mediaItemId,
|
||||
podcastId,
|
||||
mediaItemType: progressPayload.episodeId ? 'podcastEpisode' : 'book',
|
||||
duration: isNullOrNaN(progressPayload.duration) ? 0 : Number(progressPayload.duration),
|
||||
currentTime: isNullOrNaN(progressPayload.currentTime) ? 0 : Number(progressPayload.currentTime),
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
const uuidv4 = require("uuid").v4
|
||||
const uuidv4 = require('uuid').v4
|
||||
const Path = require('path')
|
||||
const { LogLevel } = require('../utils/constants')
|
||||
const { getTitleIgnorePrefix } = require('../utils/index')
|
||||
@ -8,9 +8,9 @@ const { filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtil
|
||||
const AudioFile = require('../objects/files/AudioFile')
|
||||
const CoverManager = require('../managers/CoverManager')
|
||||
const LibraryFile = require('../objects/files/LibraryFile')
|
||||
const fsExtra = require("../libs/fsExtra")
|
||||
const PodcastEpisode = require("../models/PodcastEpisode")
|
||||
const AbsMetadataFileScanner = require("./AbsMetadataFileScanner")
|
||||
const fsExtra = require('../libs/fsExtra')
|
||||
const PodcastEpisode = require('../models/PodcastEpisode')
|
||||
const AbsMetadataFileScanner = require('./AbsMetadataFileScanner')
|
||||
|
||||
/**
|
||||
* Metadata for podcasts pulled from files
|
||||
@ -32,13 +32,13 @@ const AbsMetadataFileScanner = require("./AbsMetadataFileScanner")
|
||||
*/
|
||||
|
||||
class PodcastScanner {
|
||||
constructor() { }
|
||||
constructor() {}
|
||||
|
||||
/**
|
||||
* @param {import('../models/LibraryItem')} existingLibraryItem
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
* @param {import('../models/LibraryItem')} existingLibraryItem
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @returns {Promise<{libraryItem:import('../models/LibraryItem'), wasUpdated:boolean}>}
|
||||
*/
|
||||
async rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
|
||||
@ -59,28 +59,34 @@ class PodcastScanner {
|
||||
|
||||
if (libraryItemData.hasAudioFileChanges || libraryItemData.audioLibraryFiles.length !== existingPodcastEpisodes.length) {
|
||||
// Filter out and destroy episodes that were removed
|
||||
existingPodcastEpisodes = await Promise.all(existingPodcastEpisodes.filter(async ep => {
|
||||
if (libraryItemData.checkAudioFileRemoved(ep.audioFile)) {
|
||||
libraryScan.addLog(LogLevel.INFO, `Podcast episode "${ep.title}" audio file was removed`)
|
||||
// TODO: Should clean up other data linked to this episode
|
||||
await ep.destroy()
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}))
|
||||
existingPodcastEpisodes = await Promise.all(
|
||||
existingPodcastEpisodes.filter(async (ep) => {
|
||||
if (libraryItemData.checkAudioFileRemoved(ep.audioFile)) {
|
||||
libraryScan.addLog(LogLevel.INFO, `Podcast episode "${ep.title}" audio file was removed`)
|
||||
// TODO: Should clean up other data linked to this episode
|
||||
await ep.destroy()
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
)
|
||||
|
||||
// Update audio files that were modified
|
||||
if (libraryItemData.audioLibraryFilesModified.length) {
|
||||
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified.map(lf => lf.new))
|
||||
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(
|
||||
existingLibraryItem.mediaType,
|
||||
libraryItemData,
|
||||
libraryItemData.audioLibraryFilesModified.map((lf) => lf.new)
|
||||
)
|
||||
|
||||
for (const podcastEpisode of existingPodcastEpisodes) {
|
||||
let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === podcastEpisode.audioFile.metadata.path)
|
||||
let matchedScannedAudioFile = scannedAudioFiles.find((saf) => saf.metadata.path === podcastEpisode.audioFile.metadata.path)
|
||||
if (!matchedScannedAudioFile) {
|
||||
matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.ino === podcastEpisode.audioFile.ino)
|
||||
matchedScannedAudioFile = scannedAudioFiles.find((saf) => saf.ino === podcastEpisode.audioFile.ino)
|
||||
}
|
||||
|
||||
if (matchedScannedAudioFile) {
|
||||
scannedAudioFiles = scannedAudioFiles.filter(saf => saf !== matchedScannedAudioFile)
|
||||
scannedAudioFiles = scannedAudioFiles.filter((saf) => saf !== matchedScannedAudioFile)
|
||||
const audioFile = new AudioFile(podcastEpisode.audioFile)
|
||||
audioFile.updateFromScan(matchedScannedAudioFile)
|
||||
podcastEpisode.audioFile = audioFile.toJSON()
|
||||
@ -131,15 +137,20 @@ class PodcastScanner {
|
||||
|
||||
let hasMediaChanges = false
|
||||
|
||||
if (existingPodcastEpisodes.length !== media.numEpisodes) {
|
||||
media.numEpisodes = existingPodcastEpisodes.length
|
||||
hasMediaChanges = true
|
||||
}
|
||||
|
||||
// Check if cover was removed
|
||||
if (media.coverPath && libraryItemData.imageLibraryFilesRemoved.some(lf => lf.metadata.path === media.coverPath)) {
|
||||
if (media.coverPath && libraryItemData.imageLibraryFilesRemoved.some((lf) => lf.metadata.path === media.coverPath)) {
|
||||
media.coverPath = null
|
||||
hasMediaChanges = true
|
||||
}
|
||||
|
||||
// Update cover if it was modified
|
||||
if (media.coverPath && libraryItemData.imageLibraryFilesModified.length) {
|
||||
let coverMatch = libraryItemData.imageLibraryFilesModified.find(iFile => iFile.old.metadata.path === media.coverPath)
|
||||
let coverMatch = libraryItemData.imageLibraryFilesModified.find((iFile) => iFile.old.metadata.path === media.coverPath)
|
||||
if (coverMatch) {
|
||||
const coverPath = coverMatch.new.metadata.path
|
||||
if (coverPath !== media.coverPath) {
|
||||
@ -154,7 +165,7 @@ class PodcastScanner {
|
||||
// Check if cover is not set and image files were found
|
||||
if (!media.coverPath && libraryItemData.imageLibraryFiles.length) {
|
||||
// Prefer using a cover image with the name "cover" otherwise use the first image
|
||||
const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path))
|
||||
const coverMatch = libraryItemData.imageLibraryFiles.find((iFile) => /\/cover\.[^.\/]*$/.test(iFile.metadata.path))
|
||||
media.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path
|
||||
hasMediaChanges = true
|
||||
}
|
||||
@ -167,7 +178,7 @@ class PodcastScanner {
|
||||
|
||||
if (key === 'genres') {
|
||||
const existingGenres = media.genres || []
|
||||
if (podcastMetadata.genres.some(g => !existingGenres.includes(g)) || existingGenres.some(g => !podcastMetadata.genres.includes(g))) {
|
||||
if (podcastMetadata.genres.some((g) => !existingGenres.includes(g)) || existingGenres.some((g) => !podcastMetadata.genres.includes(g))) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating podcast genres "${existingGenres.join(',')}" => "${podcastMetadata.genres.join(',')}" for podcast "${podcastMetadata.title}"`)
|
||||
media.genres = podcastMetadata.genres
|
||||
media.changed('genres', true)
|
||||
@ -175,7 +186,7 @@ class PodcastScanner {
|
||||
}
|
||||
} else if (key === 'tags') {
|
||||
const existingTags = media.tags || []
|
||||
if (podcastMetadata.tags.some(t => !existingTags.includes(t)) || existingTags.some(t => !podcastMetadata.tags.includes(t))) {
|
||||
if (podcastMetadata.tags.some((t) => !existingTags.includes(t)) || existingTags.some((t) => !podcastMetadata.tags.includes(t))) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating podcast tags "${existingTags.join(',')}" => "${podcastMetadata.tags.join(',')}" for podcast "${podcastMetadata.title}"`)
|
||||
media.tags = podcastMetadata.tags
|
||||
media.changed('tags', true)
|
||||
@ -190,7 +201,7 @@ class PodcastScanner {
|
||||
|
||||
// If no cover then extract cover from audio file if available
|
||||
if (!media.coverPath && existingPodcastEpisodes.length) {
|
||||
const audioFiles = existingPodcastEpisodes.map(ep => ep.audioFile)
|
||||
const audioFiles = existingPodcastEpisodes.map((ep) => ep.audioFile)
|
||||
const extractedCoverPath = await CoverManager.saveEmbeddedCoverArt(audioFiles, existingLibraryItem.id, existingLibraryItem.path)
|
||||
if (extractedCoverPath) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating podcast "${podcastMetadata.title}" extracted embedded cover art from audio file to path "${extractedCoverPath}"`)
|
||||
@ -222,10 +233,10 @@ class PodcastScanner {
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
*
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @returns {Promise<import('../models/LibraryItem')>}
|
||||
*/
|
||||
async scanNewPodcastLibraryItem(libraryItemData, librarySettings, libraryScan) {
|
||||
@ -267,7 +278,7 @@ class PodcastScanner {
|
||||
// Set cover image from library file
|
||||
if (libraryItemData.imageLibraryFiles.length) {
|
||||
// Prefer using a cover image with the name "cover" otherwise use the first image
|
||||
const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path))
|
||||
const coverMatch = libraryItemData.imageLibraryFiles.find((iFile) => /\/cover\.[^.\/]*$/.test(iFile.metadata.path))
|
||||
podcastMetadata.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path
|
||||
}
|
||||
|
||||
@ -283,7 +294,8 @@ class PodcastScanner {
|
||||
lastEpisodeCheck: 0,
|
||||
maxEpisodesToKeep: 0,
|
||||
maxNewEpisodesToDownload: 3,
|
||||
podcastEpisodes: newPodcastEpisodes
|
||||
podcastEpisodes: newPodcastEpisodes,
|
||||
numEpisodes: newPodcastEpisodes.length
|
||||
}
|
||||
|
||||
const libraryItemObj = libraryItemData.libraryItemObject
|
||||
@ -291,6 +303,8 @@ class PodcastScanner {
|
||||
libraryItemObj.isMissing = false
|
||||
libraryItemObj.isInvalid = false
|
||||
libraryItemObj.extraData = {}
|
||||
libraryItemObj.title = podcastObject.title
|
||||
libraryItemObj.titleIgnorePrefix = getTitleIgnorePrefix(podcastObject.title)
|
||||
|
||||
// If cover was not found in folder then check embedded covers in audio files
|
||||
if (!podcastObject.coverPath && scannedAudioFiles.length) {
|
||||
@ -324,10 +338,10 @@ class PodcastScanner {
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
* @param {PodcastEpisode[]} podcastEpisodes Not the models for new podcasts
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @param {string} [existingLibraryItemId]
|
||||
* @returns {Promise<PodcastMetadataObject>}
|
||||
*/
|
||||
@ -364,8 +378,8 @@ class PodcastScanner {
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {import('../models/LibraryItem')} libraryItem
|
||||
*
|
||||
* @param {import('../models/LibraryItem')} libraryItem
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @returns {Promise}
|
||||
*/
|
||||
@ -399,41 +413,44 @@ class PodcastScanner {
|
||||
explicit: !!libraryItem.media.explicit,
|
||||
podcastType: libraryItem.media.podcastType
|
||||
}
|
||||
return fsExtra.writeFile(metadataFilePath, JSON.stringify(jsonObject, null, 2)).then(async () => {
|
||||
// Add metadata.json to libraryFiles array if it is new
|
||||
let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
|
||||
if (storeMetadataWithItem) {
|
||||
if (!metadataLibraryFile) {
|
||||
const newLibraryFile = new LibraryFile()
|
||||
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
|
||||
metadataLibraryFile = newLibraryFile.toJSON()
|
||||
libraryItem.libraryFiles.push(metadataLibraryFile)
|
||||
} else {
|
||||
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
|
||||
if (fileTimestamps) {
|
||||
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
|
||||
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
|
||||
metadataLibraryFile.metadata.size = fileTimestamps.size
|
||||
metadataLibraryFile.ino = fileTimestamps.ino
|
||||
return fsExtra
|
||||
.writeFile(metadataFilePath, JSON.stringify(jsonObject, null, 2))
|
||||
.then(async () => {
|
||||
// Add metadata.json to libraryFiles array if it is new
|
||||
let metadataLibraryFile = libraryItem.libraryFiles.find((lf) => lf.metadata.path === filePathToPOSIX(metadataFilePath))
|
||||
if (storeMetadataWithItem) {
|
||||
if (!metadataLibraryFile) {
|
||||
const newLibraryFile = new LibraryFile()
|
||||
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
|
||||
metadataLibraryFile = newLibraryFile.toJSON()
|
||||
libraryItem.libraryFiles.push(metadataLibraryFile)
|
||||
} else {
|
||||
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
|
||||
if (fileTimestamps) {
|
||||
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
|
||||
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
|
||||
metadataLibraryFile.metadata.size = fileTimestamps.size
|
||||
metadataLibraryFile.ino = fileTimestamps.ino
|
||||
}
|
||||
}
|
||||
const libraryItemDirTimestamps = await getFileTimestampsWithIno(libraryItem.path)
|
||||
if (libraryItemDirTimestamps) {
|
||||
libraryItem.mtime = libraryItemDirTimestamps.mtimeMs
|
||||
libraryItem.ctime = libraryItemDirTimestamps.ctimeMs
|
||||
let size = 0
|
||||
libraryItem.libraryFiles.forEach((lf) => (size += !isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0))
|
||||
libraryItem.size = size
|
||||
}
|
||||
}
|
||||
const libraryItemDirTimestamps = await getFileTimestampsWithIno(libraryItem.path)
|
||||
if (libraryItemDirTimestamps) {
|
||||
libraryItem.mtime = libraryItemDirTimestamps.mtimeMs
|
||||
libraryItem.ctime = libraryItemDirTimestamps.ctimeMs
|
||||
let size = 0
|
||||
libraryItem.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0))
|
||||
libraryItem.size = size
|
||||
}
|
||||
}
|
||||
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`)
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`)
|
||||
|
||||
return metadataLibraryFile
|
||||
}).catch((error) => {
|
||||
libraryScan.addLog(LogLevel.ERROR, `Failed to save json file at "${metadataFilePath}"`, error)
|
||||
return null
|
||||
})
|
||||
return metadataLibraryFile
|
||||
})
|
||||
.catch((error) => {
|
||||
libraryScan.addLog(LogLevel.ERROR, `Failed to save json file at "${metadataFilePath}"`, error)
|
||||
return null
|
||||
})
|
||||
}
|
||||
}
|
||||
module.exports = new PodcastScanner()
|
||||
module.exports = new PodcastScanner()
|
||||
|
||||
@ -4,6 +4,7 @@ const Database = require('../../Database')
|
||||
const libraryItemsBookFilters = require('./libraryItemsBookFilters')
|
||||
const libraryItemsPodcastFilters = require('./libraryItemsPodcastFilters')
|
||||
const { createNewSortInstance } = require('../../libs/fastSort')
|
||||
const { profile } = require('../../utils/profiler')
|
||||
const naturalSort = createNewSortInstance({
|
||||
comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare
|
||||
})
|
||||
@ -474,7 +475,8 @@ module.exports = {
|
||||
// Check how many podcasts are in library to determine if we need to load all of the data
|
||||
// This is done to handle the edge case of podcasts having been deleted and not having
|
||||
// an updatedAt timestamp to trigger a reload of the filter data
|
||||
const podcastCountFromDatabase = await Database.podcastModel.count({
|
||||
const podcastModelCount = process.env.QUERY_PROFILING ? profile(Database.podcastModel.count.bind(Database.podcastModel)) : Database.podcastModel.count.bind(Database.podcastModel)
|
||||
const podcastCountFromDatabase = await podcastModelCount({
|
||||
include: {
|
||||
model: Database.libraryItemModel,
|
||||
attributes: [],
|
||||
@ -489,7 +491,7 @@ module.exports = {
|
||||
// data was loaded. If so, we can skip loading all of the data.
|
||||
// Because many items could change, just check the count of items instead
|
||||
// of actually loading the data twice
|
||||
const changedPodcasts = await Database.podcastModel.count({
|
||||
const changedPodcasts = await podcastModelCount({
|
||||
include: {
|
||||
model: Database.libraryItemModel,
|
||||
attributes: [],
|
||||
@ -520,7 +522,8 @@ module.exports = {
|
||||
}
|
||||
|
||||
// Something has changed in the podcasts table, so reload all of the filter data for library
|
||||
const podcasts = await Database.podcastModel.findAll({
|
||||
const findAll = process.env.QUERY_PROFILING ? profile(Database.podcastModel.findAll.bind(Database.podcastModel)) : Database.podcastModel.findAll.bind(Database.podcastModel)
|
||||
const podcasts = await findAll({
|
||||
include: {
|
||||
model: Database.libraryItemModel,
|
||||
attributes: [],
|
||||
|
||||
@ -1,6 +1,10 @@
|
||||
const Sequelize = require('sequelize')
|
||||
const Database = require('../../Database')
|
||||
const Logger = require('../../Logger')
|
||||
const { profile } = require('../../utils/profiler')
|
||||
const stringifySequelizeQuery = require('../stringifySequelizeQuery')
|
||||
|
||||
const countCache = new Map()
|
||||
|
||||
module.exports = {
|
||||
/**
|
||||
@ -84,9 +88,9 @@ module.exports = {
|
||||
return [[Sequelize.literal(`\`podcast\`.\`author\` COLLATE NOCASE ${nullDir}`)]]
|
||||
} else if (sortBy === 'media.metadata.title') {
|
||||
if (global.ServerSettings.sortingIgnorePrefix) {
|
||||
return [[Sequelize.literal('`podcast`.`titleIgnorePrefix` COLLATE NOCASE'), dir]]
|
||||
return [[Sequelize.literal('`libraryItem`.`titleIgnorePrefix` COLLATE NOCASE'), dir]]
|
||||
} else {
|
||||
return [[Sequelize.literal('`podcast`.`title` COLLATE NOCASE'), dir]]
|
||||
return [[Sequelize.literal('`libraryItem`.`title` COLLATE NOCASE'), dir]]
|
||||
}
|
||||
} else if (sortBy === 'media.numTracks') {
|
||||
return [['numEpisodes', dir]]
|
||||
@ -96,6 +100,29 @@ module.exports = {
|
||||
return []
|
||||
},
|
||||
|
||||
clearCountCache(model, hook) {
|
||||
Logger.debug(`[LibraryItemsPodcastFilters] ${model}.${hook}: Clearing count cache`)
|
||||
countCache.clear()
|
||||
},
|
||||
|
||||
async findAndCountAll(findOptions, model, limit, offset) {
|
||||
const cacheKey = stringifySequelizeQuery(findOptions)
|
||||
if (!countCache.has(cacheKey)) {
|
||||
const count = await model.count(findOptions)
|
||||
countCache.set(cacheKey, count)
|
||||
}
|
||||
|
||||
findOptions.limit = limit
|
||||
findOptions.offset = offset
|
||||
|
||||
const rows = await model.findAll(findOptions)
|
||||
|
||||
return {
|
||||
rows,
|
||||
count: countCache.get(cacheKey)
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Get library items for podcast media type using filter and sort
|
||||
* @param {string} libraryId
|
||||
@ -120,7 +147,8 @@ module.exports = {
|
||||
if (includeRSSFeed) {
|
||||
libraryItemIncludes.push({
|
||||
model: Database.feedModel,
|
||||
required: filterGroup === 'feed-open'
|
||||
required: filterGroup === 'feed-open',
|
||||
separate: true
|
||||
})
|
||||
}
|
||||
if (filterGroup === 'issues') {
|
||||
@ -139,9 +167,6 @@ module.exports = {
|
||||
}
|
||||
|
||||
const podcastIncludes = []
|
||||
if (includeNumEpisodesIncomplete) {
|
||||
podcastIncludes.push([Sequelize.literal(`(SELECT count(*) FROM podcastEpisodes pe LEFT OUTER JOIN mediaProgresses mp ON mp.mediaItemId = pe.id AND mp.userId = :userId WHERE pe.podcastId = podcast.id AND (mp.isFinished = 0 OR mp.isFinished IS NULL))`), 'numEpisodesIncomplete'])
|
||||
}
|
||||
|
||||
let { mediaWhere, replacements } = this.getMediaGroupQuery(filterGroup, filterValue)
|
||||
replacements.userId = user.id
|
||||
@ -153,12 +178,12 @@ module.exports = {
|
||||
replacements = { ...replacements, ...userPermissionPodcastWhere.replacements }
|
||||
podcastWhere.push(...userPermissionPodcastWhere.podcastWhere)
|
||||
|
||||
const { rows: podcasts, count } = await Database.podcastModel.findAndCountAll({
|
||||
const findOptions = {
|
||||
where: podcastWhere,
|
||||
replacements,
|
||||
distinct: true,
|
||||
attributes: {
|
||||
include: [[Sequelize.literal(`(SELECT count(*) FROM podcastEpisodes pe WHERE pe.podcastId = podcast.id)`), 'numEpisodes'], ...podcastIncludes]
|
||||
include: [...podcastIncludes]
|
||||
},
|
||||
include: [
|
||||
{
|
||||
@ -169,10 +194,12 @@ module.exports = {
|
||||
}
|
||||
],
|
||||
order: this.getOrder(sortBy, sortDesc),
|
||||
subQuery: false,
|
||||
limit: limit || null,
|
||||
offset
|
||||
})
|
||||
subQuery: false
|
||||
}
|
||||
|
||||
const findAndCountAll = process.env.QUERY_PROFILING ? profile(this.findAndCountAll) : this.findAndCountAll
|
||||
|
||||
const { rows: podcasts, count } = await findAndCountAll(findOptions, Database.podcastModel, limit, offset)
|
||||
|
||||
const libraryItems = podcasts.map((podcastExpanded) => {
|
||||
const libraryItem = podcastExpanded.libraryItem
|
||||
@ -183,11 +210,15 @@ module.exports = {
|
||||
if (libraryItem.feeds?.length) {
|
||||
libraryItem.rssFeed = libraryItem.feeds[0]
|
||||
}
|
||||
if (podcast.dataValues.numEpisodesIncomplete) {
|
||||
libraryItem.numEpisodesIncomplete = podcast.dataValues.numEpisodesIncomplete
|
||||
}
|
||||
if (podcast.dataValues.numEpisodes) {
|
||||
podcast.numEpisodes = podcast.dataValues.numEpisodes
|
||||
|
||||
if (includeNumEpisodesIncomplete) {
|
||||
const numEpisodesComplete = user.mediaProgresses.reduce((acc, mp) => {
|
||||
if (mp.podcastId === podcast.id && mp.isFinished) {
|
||||
acc += 1
|
||||
}
|
||||
return acc
|
||||
}, 0)
|
||||
libraryItem.numEpisodesIncomplete = podcast.numEpisodes - numEpisodesComplete
|
||||
}
|
||||
|
||||
libraryItem.media = podcast
|
||||
@ -268,28 +299,31 @@ module.exports = {
|
||||
|
||||
const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(user)
|
||||
|
||||
const { rows: podcastEpisodes, count } = await Database.podcastEpisodeModel.findAndCountAll({
|
||||
const findOptions = {
|
||||
where: podcastEpisodeWhere,
|
||||
replacements: userPermissionPodcastWhere.replacements,
|
||||
include: [
|
||||
{
|
||||
model: Database.podcastModel,
|
||||
required: true,
|
||||
where: userPermissionPodcastWhere.podcastWhere,
|
||||
include: [
|
||||
{
|
||||
model: Database.libraryItemModel,
|
||||
required: true,
|
||||
where: libraryItemWhere
|
||||
}
|
||||
]
|
||||
},
|
||||
...podcastEpisodeIncludes
|
||||
],
|
||||
distinct: true,
|
||||
subQuery: false,
|
||||
order: podcastEpisodeOrder,
|
||||
limit,
|
||||
offset
|
||||
})
|
||||
order: podcastEpisodeOrder
|
||||
}
|
||||
|
||||
const findAndCountAll = process.env.QUERY_PROFILING ? profile(this.findAndCountAll) : this.findAndCountAll
|
||||
|
||||
const { rows: podcastEpisodes, count } = await findAndCountAll(findOptions, Database.podcastEpisodeModel, limit, offset)
|
||||
|
||||
const libraryItems = podcastEpisodes.map((ep) => {
|
||||
const libraryItem = ep.podcast.libraryItem
|
||||
|
||||
265
test/server/migrations/v2.19.4-improve-podcast-queries.test.js
Normal file
265
test/server/migrations/v2.19.4-improve-podcast-queries.test.js
Normal file
@ -0,0 +1,265 @@
|
||||
const chai = require('chai')
|
||||
const sinon = require('sinon')
|
||||
const { expect } = chai
|
||||
|
||||
const { DataTypes, Sequelize } = require('sequelize')
|
||||
const Logger = require('../../../server/Logger')
|
||||
|
||||
const { up, down } = require('../../../server/migrations/v2.19.4-improve-podcast-queries')
|
||||
|
||||
describe('Migration v2.19.4-improve-podcast-queries', () => {
|
||||
let sequelize
|
||||
let queryInterface
|
||||
let loggerInfoStub
|
||||
|
||||
beforeEach(async () => {
|
||||
sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
|
||||
queryInterface = sequelize.getQueryInterface()
|
||||
loggerInfoStub = sinon.stub(Logger, 'info')
|
||||
|
||||
await queryInterface.createTable('libraryItems', {
|
||||
id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true },
|
||||
mediaId: { type: DataTypes.INTEGER, allowNull: false },
|
||||
title: { type: DataTypes.STRING, allowNull: true },
|
||||
titleIgnorePrefix: { type: DataTypes.STRING, allowNull: true }
|
||||
})
|
||||
await queryInterface.createTable('podcasts', {
|
||||
id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true },
|
||||
title: { type: DataTypes.STRING, allowNull: false },
|
||||
titleIgnorePrefix: { type: DataTypes.STRING, allowNull: false }
|
||||
})
|
||||
|
||||
await queryInterface.createTable('podcastEpisodes', {
|
||||
id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true },
|
||||
podcastId: { type: DataTypes.INTEGER, allowNull: false, references: { model: 'podcasts', key: 'id', onDelete: 'CASCADE' } }
|
||||
})
|
||||
|
||||
await queryInterface.createTable('mediaProgresses', {
|
||||
id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true },
|
||||
userId: { type: DataTypes.INTEGER, allowNull: false },
|
||||
mediaItemId: { type: DataTypes.INTEGER, allowNull: false },
|
||||
mediaItemType: { type: DataTypes.STRING, allowNull: false },
|
||||
isFinished: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }
|
||||
})
|
||||
|
||||
await queryInterface.bulkInsert('libraryItems', [
|
||||
{ id: 1, mediaId: 1, title: null, titleIgnorePrefix: null },
|
||||
{ id: 2, mediaId: 2, title: null, titleIgnorePrefix: null }
|
||||
])
|
||||
|
||||
await queryInterface.bulkInsert('podcasts', [
|
||||
{ id: 1, title: 'The Podcast 1', titleIgnorePrefix: 'Podcast 1, The' },
|
||||
{ id: 2, title: 'The Podcast 2', titleIgnorePrefix: 'Podcast 2, The' }
|
||||
])
|
||||
|
||||
await queryInterface.bulkInsert('podcastEpisodes', [
|
||||
{ id: 1, podcastId: 1 },
|
||||
{ id: 2, podcastId: 1 },
|
||||
{ id: 3, podcastId: 2 }
|
||||
])
|
||||
|
||||
await queryInterface.bulkInsert('mediaProgresses', [
|
||||
{ id: 1, userId: 1, mediaItemId: 1, mediaItemType: 'podcastEpisode', isFinished: 1 },
|
||||
{ id: 2, userId: 1, mediaItemId: 2, mediaItemType: 'podcastEpisode', isFinished: 0 },
|
||||
{ id: 3, userId: 1, mediaItemId: 3, mediaItemType: 'podcastEpisode', isFinished: 1 },
|
||||
{ id: 4, userId: 2, mediaItemId: 1, mediaItemType: 'podcastEpisode', isFinished: 0 },
|
||||
{ id: 5, userId: 2, mediaItemId: 2, mediaItemType: 'podcastEpisode', isFinished: 1 },
|
||||
{ id: 6, userId: 2, mediaItemId: 3, mediaItemType: 'podcastEpisode', isFinished: 0 },
|
||||
{ id: 7, userId: 1, mediaItemId: 1, mediaItemType: 'book', isFinished: 1 },
|
||||
{ id: 8, userId: 1, mediaItemId: 2, mediaItemType: 'book', isFinished: 0 }
|
||||
])
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
sinon.restore()
|
||||
})
|
||||
|
||||
describe('up', () => {
|
||||
it('should add numEpisodes column to podcasts', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [podcasts] = await queryInterface.sequelize.query('SELECT * FROM podcasts')
|
||||
expect(podcasts).to.deep.equal([
|
||||
{ id: 1, numEpisodes: 2, title: 'The Podcast 1', titleIgnorePrefix: 'Podcast 1, The' },
|
||||
{ id: 2, numEpisodes: 1, title: 'The Podcast 2', titleIgnorePrefix: 'Podcast 2, The' }
|
||||
])
|
||||
|
||||
// Make sure podcastEpisodes are not affected due to ON DELETE CASCADE
|
||||
const [podcastEpisodes] = await queryInterface.sequelize.query('SELECT * FROM podcastEpisodes')
|
||||
expect(podcastEpisodes).to.deep.equal([
|
||||
{ id: 1, podcastId: 1 },
|
||||
{ id: 2, podcastId: 1 },
|
||||
{ id: 3, podcastId: 2 }
|
||||
])
|
||||
})
|
||||
|
||||
it('should add podcastId column to mediaProgresses', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [mediaProgresses] = await queryInterface.sequelize.query('SELECT * FROM mediaProgresses')
|
||||
expect(mediaProgresses).to.deep.equal([
|
||||
{ id: 1, userId: 1, mediaItemId: 1, mediaItemType: 'podcastEpisode', podcastId: 1, isFinished: 1 },
|
||||
{ id: 2, userId: 1, mediaItemId: 2, mediaItemType: 'podcastEpisode', podcastId: 1, isFinished: 0 },
|
||||
{ id: 3, userId: 1, mediaItemId: 3, mediaItemType: 'podcastEpisode', podcastId: 2, isFinished: 1 },
|
||||
{ id: 4, userId: 2, mediaItemId: 1, mediaItemType: 'podcastEpisode', podcastId: 1, isFinished: 0 },
|
||||
{ id: 5, userId: 2, mediaItemId: 2, mediaItemType: 'podcastEpisode', podcastId: 1, isFinished: 1 },
|
||||
{ id: 6, userId: 2, mediaItemId: 3, mediaItemType: 'podcastEpisode', podcastId: 2, isFinished: 0 },
|
||||
{ id: 7, userId: 1, mediaItemId: 1, mediaItemType: 'book', podcastId: null, isFinished: 1 },
|
||||
{ id: 8, userId: 1, mediaItemId: 2, mediaItemType: 'book', podcastId: null, isFinished: 0 }
|
||||
])
|
||||
})
|
||||
|
||||
it('should copy title and titleIgnorePrefix from podcasts to libraryItems', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [libraryItems] = await queryInterface.sequelize.query('SELECT * FROM libraryItems')
|
||||
expect(libraryItems).to.deep.equal([
|
||||
{ id: 1, mediaId: 1, title: 'The Podcast 1', titleIgnorePrefix: 'Podcast 1, The' },
|
||||
{ id: 2, mediaId: 2, title: 'The Podcast 2', titleIgnorePrefix: 'Podcast 2, The' }
|
||||
])
|
||||
})
|
||||
|
||||
it('should add trigger to update title in libraryItems', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [[{ count }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='trigger' AND name='update_library_items_title_from_podcasts_title'`)
|
||||
expect(count).to.equal(1)
|
||||
})
|
||||
|
||||
it('should add trigger to update titleIgnorePrefix in libraryItems', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [[{ count }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='trigger' AND name='update_library_items_title_ignore_prefix_from_podcasts_title_ignore_prefix'`)
|
||||
expect(count).to.equal(1)
|
||||
})
|
||||
|
||||
it('should be idempotent', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [podcasts] = await queryInterface.sequelize.query('SELECT * FROM podcasts')
|
||||
expect(podcasts).to.deep.equal([
|
||||
{ id: 1, numEpisodes: 2, title: 'The Podcast 1', titleIgnorePrefix: 'Podcast 1, The' },
|
||||
{ id: 2, numEpisodes: 1, title: 'The Podcast 2', titleIgnorePrefix: 'Podcast 2, The' }
|
||||
])
|
||||
|
||||
const [mediaProgresses] = await queryInterface.sequelize.query('SELECT * FROM mediaProgresses')
|
||||
expect(mediaProgresses).to.deep.equal([
|
||||
{ id: 1, userId: 1, mediaItemId: 1, mediaItemType: 'podcastEpisode', podcastId: 1, isFinished: 1 },
|
||||
{ id: 2, userId: 1, mediaItemId: 2, mediaItemType: 'podcastEpisode', podcastId: 1, isFinished: 0 },
|
||||
{ id: 3, userId: 1, mediaItemId: 3, mediaItemType: 'podcastEpisode', podcastId: 2, isFinished: 1 },
|
||||
{ id: 4, userId: 2, mediaItemId: 1, mediaItemType: 'podcastEpisode', podcastId: 1, isFinished: 0 },
|
||||
{ id: 5, userId: 2, mediaItemId: 2, mediaItemType: 'podcastEpisode', podcastId: 1, isFinished: 1 },
|
||||
{ id: 6, userId: 2, mediaItemId: 3, mediaItemType: 'podcastEpisode', podcastId: 2, isFinished: 0 },
|
||||
{ id: 7, userId: 1, mediaItemId: 1, mediaItemType: 'book', podcastId: null, isFinished: 1 },
|
||||
{ id: 8, userId: 1, mediaItemId: 2, mediaItemType: 'book', podcastId: null, isFinished: 0 }
|
||||
])
|
||||
|
||||
const [libraryItems] = await queryInterface.sequelize.query('SELECT * FROM libraryItems')
|
||||
expect(libraryItems).to.deep.equal([
|
||||
{ id: 1, mediaId: 1, title: 'The Podcast 1', titleIgnorePrefix: 'Podcast 1, The' },
|
||||
{ id: 2, mediaId: 2, title: 'The Podcast 2', titleIgnorePrefix: 'Podcast 2, The' }
|
||||
])
|
||||
|
||||
const [[{ count: count1 }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='trigger' AND name='update_library_items_title_from_podcasts_title'`)
|
||||
expect(count1).to.equal(1)
|
||||
|
||||
const [[{ count: count2 }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='trigger' AND name='update_library_items_title_ignore_prefix_from_podcasts_title_ignore_prefix'`)
|
||||
expect(count2).to.equal(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('down', () => {
|
||||
it('should remove numEpisodes column from podcasts', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
try {
|
||||
await down({ context: { queryInterface, logger: Logger } })
|
||||
} catch (error) {
|
||||
console.log(error)
|
||||
}
|
||||
|
||||
const [podcasts] = await queryInterface.sequelize.query('SELECT * FROM podcasts')
|
||||
expect(podcasts).to.deep.equal([
|
||||
{ id: 1, title: 'The Podcast 1', titleIgnorePrefix: 'Podcast 1, The' },
|
||||
{ id: 2, title: 'The Podcast 2', titleIgnorePrefix: 'Podcast 2, The' }
|
||||
])
|
||||
|
||||
// Make sure podcastEpisodes are not affected due to ON DELETE CASCADE
|
||||
const [podcastEpisodes] = await queryInterface.sequelize.query('SELECT * FROM podcastEpisodes')
|
||||
expect(podcastEpisodes).to.deep.equal([
|
||||
{ id: 1, podcastId: 1 },
|
||||
{ id: 2, podcastId: 1 },
|
||||
{ id: 3, podcastId: 2 }
|
||||
])
|
||||
})
|
||||
|
||||
it('should remove podcastId column from mediaProgresses', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
await down({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [mediaProgresses] = await queryInterface.sequelize.query('SELECT * FROM mediaProgresses')
|
||||
expect(mediaProgresses).to.deep.equal([
|
||||
{ id: 1, userId: 1, mediaItemId: 1, mediaItemType: 'podcastEpisode', isFinished: 1 },
|
||||
{ id: 2, userId: 1, mediaItemId: 2, mediaItemType: 'podcastEpisode', isFinished: 0 },
|
||||
{ id: 3, userId: 1, mediaItemId: 3, mediaItemType: 'podcastEpisode', isFinished: 1 },
|
||||
{ id: 4, userId: 2, mediaItemId: 1, mediaItemType: 'podcastEpisode', isFinished: 0 },
|
||||
{ id: 5, userId: 2, mediaItemId: 2, mediaItemType: 'podcastEpisode', isFinished: 1 },
|
||||
{ id: 6, userId: 2, mediaItemId: 3, mediaItemType: 'podcastEpisode', isFinished: 0 },
|
||||
{ id: 7, userId: 1, mediaItemId: 1, mediaItemType: 'book', isFinished: 1 },
|
||||
{ id: 8, userId: 1, mediaItemId: 2, mediaItemType: 'book', isFinished: 0 }
|
||||
])
|
||||
})
|
||||
|
||||
it('should remove trigger to update title in libraryItems', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
await down({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [[{ count }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='trigger' AND name='update_library_items_title_from_podcasts_title'`)
|
||||
expect(count).to.equal(0)
|
||||
})
|
||||
|
||||
it('should remove trigger to update titleIgnorePrefix in libraryItems', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
await down({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [[{ count }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='trigger' AND name='update_library_items_title_ignore_prefix_from_podcasts_title_ignore_prefix'`)
|
||||
expect(count).to.equal(0)
|
||||
})
|
||||
|
||||
it('should be idempotent', async () => {
|
||||
await up({ context: { queryInterface, logger: Logger } })
|
||||
await down({ context: { queryInterface, logger: Logger } })
|
||||
await down({ context: { queryInterface, logger: Logger } })
|
||||
|
||||
const [podcasts] = await queryInterface.sequelize.query('SELECT * FROM podcasts')
|
||||
expect(podcasts).to.deep.equal([
|
||||
{ id: 1, title: 'The Podcast 1', titleIgnorePrefix: 'Podcast 1, The' },
|
||||
{ id: 2, title: 'The Podcast 2', titleIgnorePrefix: 'Podcast 2, The' }
|
||||
])
|
||||
|
||||
const [mediaProgresses] = await queryInterface.sequelize.query('SELECT * FROM mediaProgresses')
|
||||
expect(mediaProgresses).to.deep.equal([
|
||||
{ id: 1, userId: 1, mediaItemId: 1, mediaItemType: 'podcastEpisode', isFinished: 1 },
|
||||
{ id: 2, userId: 1, mediaItemId: 2, mediaItemType: 'podcastEpisode', isFinished: 0 },
|
||||
{ id: 3, userId: 1, mediaItemId: 3, mediaItemType: 'podcastEpisode', isFinished: 1 },
|
||||
{ id: 4, userId: 2, mediaItemId: 1, mediaItemType: 'podcastEpisode', isFinished: 0 },
|
||||
{ id: 5, userId: 2, mediaItemId: 2, mediaItemType: 'podcastEpisode', isFinished: 1 },
|
||||
{ id: 6, userId: 2, mediaItemId: 3, mediaItemType: 'podcastEpisode', isFinished: 0 },
|
||||
{ id: 7, userId: 1, mediaItemId: 1, mediaItemType: 'book', isFinished: 1 },
|
||||
{ id: 8, userId: 1, mediaItemId: 2, mediaItemType: 'book', isFinished: 0 }
|
||||
])
|
||||
|
||||
const [libraryItems] = await queryInterface.sequelize.query('SELECT * FROM libraryItems')
|
||||
expect(libraryItems).to.deep.equal([
|
||||
{ id: 1, mediaId: 1, title: 'The Podcast 1', titleIgnorePrefix: 'Podcast 1, The' },
|
||||
{ id: 2, mediaId: 2, title: 'The Podcast 2', titleIgnorePrefix: 'Podcast 2, The' }
|
||||
])
|
||||
|
||||
const [[{ count: count1 }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='trigger' AND name='update_library_items_title_from_podcasts_title'`)
|
||||
expect(count1).to.equal(0)
|
||||
|
||||
const [[{ count: count2 }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='trigger' AND name='update_library_items_title_ignore_prefix_from_podcasts_title_ignore_prefix'`)
|
||||
expect(count2).to.equal(0)
|
||||
})
|
||||
})
|
||||
})
|
||||
Loading…
Reference in New Issue
Block a user