Add feed migration and cleanup

This commit is contained in:
advplyr 2023-07-05 18:18:37 -05:00
parent a4b0f6c202
commit a0bc959850
12 changed files with 138 additions and 34 deletions

View File

@ -106,7 +106,7 @@ class Database {
require('./models/FeedEpisode')(this.sequelize) require('./models/FeedEpisode')(this.sequelize)
require('./models/Setting')(this.sequelize) require('./models/Setting')(this.sequelize)
return this.sequelize.sync({ force }) return this.sequelize.sync({ force, alter: false })
} }
async loadData(force = false) { async loadData(force = false) {
@ -354,12 +354,13 @@ class Database {
this.libraryItems = this.libraryItems.filter(li => li.id !== libraryItemId) this.libraryItems = this.libraryItems.filter(li => li.id !== libraryItemId)
} }
createFeed(oldFeed) { async createFeed(oldFeed) {
// TODO: Implement await this.models.feed.fullCreateFromOld(oldFeed)
this.feeds.push(oldFeed)
} }
updateFeed(oldFeed) { updateFeed(oldFeed) {
// TODO: Implement return this.models.feed.fullUpdateFromOld(oldFeed)
} }
async removeFeed(feedId) { async removeFeed(feedId) {

View File

@ -29,7 +29,7 @@ class RssFeedManager {
} }
} else if (feedObj.entityType === 'series') { } else if (feedObj.entityType === 'series') {
const series = Database.series.find(s => s.id === feedObj.entityId) const series = Database.series.find(s => s.id === feedObj.entityId)
const hasSeriesBook = Database.libraryItems.some(li => li.mediaType === 'book' && li.media.metadata.hasSeries(series.id) && li.media.tracks.length) const hasSeriesBook = series ? Database.libraryItems.some(li => li.mediaType === 'book' && li.media.metadata.hasSeries(series.id) && li.media.tracks.length) : false
if (!hasSeriesBook) { if (!hasSeriesBook) {
Logger.error(`[RssFeedManager] Removing feed "${feedObj.id}". Series "${feedObj.entityId}" not found or has no audio tracks`) Logger.error(`[RssFeedManager] Removing feed "${feedObj.id}". Series "${feedObj.entityId}" not found or has no audio tracks`)
return false return false
@ -42,16 +42,15 @@ class RssFeedManager {
} }
async init() { async init() {
const feedObjects = Database.feeds const feeds = Database.feeds
if (!feedObjects?.length) return if (!feeds?.length) return
for (const feedObj of feedObjects) { for (const feed of feeds) {
// Remove invalid feeds // Remove invalid feeds
if (!this.validateFeedEntity(feedObj)) { if (!this.validateFeedEntity(feed)) {
await Database.removeFeed(feedObj.id) await Database.removeFeed(feed.id)
} }
const feed = new Feed(feedObj)
this.feeds[feed.id] = feed this.feeds[feed.id] = feed
Logger.info(`[RssFeedManager] Opened rss feed ${feed.feedUrl}`) Logger.info(`[RssFeedManager] Opened rss feed ${feed.feedUrl}`)
} }

View File

@ -80,8 +80,6 @@ module.exports = (sequelize) => {
id: oldCollection.id, id: oldCollection.id,
name: oldCollection.name, name: oldCollection.name,
description: oldCollection.description, description: oldCollection.description,
createdAt: oldCollection.createdAt,
updatedAt: oldCollection.lastUpdate,
libraryId: oldCollection.libraryId libraryId: oldCollection.libraryId
} }
} }

View File

@ -1,5 +1,6 @@
const { DataTypes, Model } = require('sequelize') const { DataTypes, Model } = require('sequelize')
const oldFeed = require('../objects/Feed') const oldFeed = require('../objects/Feed')
const areEquivalent = require('../utils/areEquivalent')
/* /*
* Polymorphic association: https://sequelize.org/docs/v6/advanced-association-concepts/polymorphic-associations/ * Polymorphic association: https://sequelize.org/docs/v6/advanced-association-concepts/polymorphic-associations/
* Feeds can be created from LibraryItem, Collection, Playlist or Series * Feeds can be created from LibraryItem, Collection, Playlist or Series
@ -24,6 +25,7 @@ module.exports = (sequelize) => {
userId: feedExpanded.userId, userId: feedExpanded.userId,
entityType: feedExpanded.entityType, entityType: feedExpanded.entityType,
entityId: feedExpanded.entityId, entityId: feedExpanded.entityId,
entityUpdatedAt: feedExpanded.entityUpdatedAt?.valueOf() || null,
meta: { meta: {
title: feedExpanded.title, title: feedExpanded.title,
description: feedExpanded.description, description: feedExpanded.description,
@ -54,6 +56,92 @@ module.exports = (sequelize) => {
}) })
} }
static async fullCreateFromOld(oldFeed) {
const feedObj = this.getFromOld(oldFeed)
const newFeed = await this.create(feedObj)
if (oldFeed.episodes?.length) {
for (const oldFeedEpisode of oldFeed.episodes) {
const feedEpisode = sequelize.models.feedEpisode.getFromOld(oldFeedEpisode)
feedEpisode.feedId = newFeed.id
await sequelize.models.feedEpisode.create(feedEpisode)
}
}
}
static async fullUpdateFromOld(oldFeed) {
const oldFeedEpisodes = oldFeed.episodes || []
const feedObj = this.getFromOld(oldFeed)
const existingFeed = await this.findByPk(feedObj.id, {
include: sequelize.models.feedEpisode
})
if (!existingFeed) return false
let hasUpdates = false
for (const feedEpisode of existingFeed.feedEpisodes) {
const oldFeedEpisode = oldFeedEpisodes.find(ep => ep.id === feedEpisode.id)
// Episode removed
if (!oldFeedEpisode) {
feedEpisode.destroy()
} else {
let episodeHasUpdates = false
const oldFeedEpisodeCleaned = sequelize.models.feedEpisode.getFromOld(oldFeedEpisode)
for (const key in oldFeedEpisodeCleaned) {
if (!areEquivalent(oldFeedEpisodeCleaned[key], feedEpisode[key])) {
episodeHasUpdates = true
}
}
if (episodeHasUpdates) {
await feedEpisode.update(oldFeedEpisodeCleaned)
hasUpdates = true
}
}
}
let feedHasUpdates = false
for (const key in feedObj) {
let existingValue = existingFeed[key]
if (existingValue instanceof Date) existingValue = existingValue.valueOf()
if (!areEquivalent(existingValue, feedObj[key])) {
feedHasUpdates = true
}
}
if (feedHasUpdates) {
await existingFeed.update(feedObj)
hasUpdates = true
}
return hasUpdates
}
static getFromOld(oldFeed) {
const oldFeedMeta = oldFeed.meta || {}
return {
id: oldFeed.id,
slug: oldFeed.slug,
entityType: oldFeed.entityType,
entityId: oldFeed.entityId,
entityUpdatedAt: oldFeed.entityUpdatedAt,
serverAddress: oldFeed.serverAddress,
feedURL: oldFeed.feedUrl,
imageURL: oldFeedMeta.imageUrl,
siteURL: oldFeedMeta.link,
title: oldFeedMeta.title,
description: oldFeedMeta.description,
author: oldFeedMeta.author,
podcastType: oldFeedMeta.type || null,
language: oldFeedMeta.language || null,
ownerName: oldFeedMeta.ownerName || null,
ownerEmail: oldFeedMeta.ownerEmail || null,
explicit: !!oldFeedMeta.explicit,
preventIndexing: !!oldFeedMeta.preventIndexing,
userId: oldFeed.userId
}
}
getEntity(options) { getEntity(options) {
if (!this.entityType) return Promise.resolve(null) if (!this.entityType) return Promise.resolve(null)
const mixinMethodName = `get${sequelize.uppercaseFirst(this.entityType)}` const mixinMethodName = `get${sequelize.uppercaseFirst(this.entityType)}`

View File

@ -24,6 +24,26 @@ module.exports = (sequelize) => {
fullPath: this.filePath fullPath: this.filePath
} }
} }
static getFromOld(oldFeedEpisode) {
return {
id: oldFeedEpisode.id,
title: oldFeedEpisode.title,
author: oldFeedEpisode.author,
description: oldFeedEpisode.description,
siteURL: oldFeedEpisode.link,
enclosureURL: oldFeedEpisode.enclosure?.url || null,
enclosureType: oldFeedEpisode.enclosure?.type || null,
enclosureSize: oldFeedEpisode.enclosure?.size || null,
pubDate: oldFeedEpisode.pubDate,
season: oldFeedEpisode.season || null,
episode: oldFeedEpisode.episode || null,
episodeType: oldFeedEpisode.episodeType || null,
duration: oldFeedEpisode.duration,
filePath: oldFeedEpisode.fullPath,
explicit: !!oldFeedEpisode.explicit
}
}
} }
FeedEpisode.init({ FeedEpisode.init({

View File

@ -287,8 +287,6 @@ module.exports = (sequelize) => {
birthtime: oldLibraryItem.birthtimeMs, birthtime: oldLibraryItem.birthtimeMs,
lastScan: oldLibraryItem.lastScan, lastScan: oldLibraryItem.lastScan,
lastScanVersion: oldLibraryItem.scanVersion, lastScanVersion: oldLibraryItem.scanVersion,
createdAt: oldLibraryItem.addedAt,
updatedAt: oldLibraryItem.updatedAt,
libraryId: oldLibraryItem.libraryId, libraryId: oldLibraryItem.libraryId,
libraryFolderId: oldLibraryItem.folderId, libraryFolderId: oldLibraryItem.folderId,
libraryFiles: oldLibraryItem.libraryFiles?.map(lf => lf.toJSON()) || [] libraryFiles: oldLibraryItem.libraryFiles?.map(lf => lf.toJSON()) || []

View File

@ -104,8 +104,6 @@ module.exports = (sequelize) => {
id: oldPlaylist.id, id: oldPlaylist.id,
name: oldPlaylist.name, name: oldPlaylist.name,
description: oldPlaylist.description, description: oldPlaylist.description,
createdAt: oldPlaylist.createdAt,
updatedAt: oldPlaylist.lastUpdate,
userId: oldPlaylist.userId, userId: oldPlaylist.userId,
libraryId: oldPlaylist.libraryId libraryId: oldPlaylist.libraryId
} }

View File

@ -52,8 +52,6 @@ module.exports = (sequelize) => {
enclosureSize: oldEpisode.enclosure?.length || null, enclosureSize: oldEpisode.enclosure?.length || null,
enclosureType: oldEpisode.enclosure?.type || null, enclosureType: oldEpisode.enclosure?.type || null,
publishedAt: oldEpisode.publishedAt, publishedAt: oldEpisode.publishedAt,
createdAt: oldEpisode.addedAt,
updatedAt: oldEpisode.updatedAt,
podcastId: oldEpisode.podcastId, podcastId: oldEpisode.podcastId,
audioFile: oldEpisode.audioFile?.toJSON() || null, audioFile: oldEpisode.audioFile?.toJSON() || null,
chapters: oldEpisode.chapters chapters: oldEpisode.chapters
@ -88,7 +86,9 @@ module.exports = (sequelize) => {
}) })
const { podcast } = sequelize.models const { podcast } = sequelize.models
podcast.hasMany(PodcastEpisode) podcast.hasMany(PodcastEpisode, {
onDelete: 'CASCADE'
})
PodcastEpisode.belongsTo(podcast) PodcastEpisode.belongsTo(podcast)
return PodcastEpisode return PodcastEpisode

View File

@ -1,3 +1,4 @@
const uuidv4 = require("uuid").v4
const date = require('../libs/dateAndTime') const date = require('../libs/dateAndTime')
const { secondsToTimestamp } = require('../utils/index') const { secondsToTimestamp } = require('../utils/index')
@ -97,13 +98,11 @@ class FeedEpisode {
setFromAudiobookTrack(libraryItem, serverAddress, slug, audioTrack, meta, additionalOffset = null) { setFromAudiobookTrack(libraryItem, serverAddress, slug, audioTrack, meta, additionalOffset = null) {
// Example: <pubDate>Fri, 04 Feb 2015 00:00:00 GMT</pubDate> // Example: <pubDate>Fri, 04 Feb 2015 00:00:00 GMT</pubDate>
let timeOffset = isNaN(audioTrack.index) ? 0 : (Number(audioTrack.index) * 1000) // Offset pubdate to ensure correct order let timeOffset = isNaN(audioTrack.index) ? 0 : (Number(audioTrack.index) * 1000) // Offset pubdate to ensure correct order
let episodeId = String(audioTrack.index) let episodeId = uuidv4()
// Additional offset can be used for collections/series // Additional offset can be used for collections/series
if (additionalOffset !== null && !isNaN(additionalOffset)) { if (additionalOffset !== null && !isNaN(additionalOffset)) {
timeOffset += Number(additionalOffset) * 1000 timeOffset += Number(additionalOffset) * 1000
episodeId = String(additionalOffset) + '-' + episodeId
} }
// e.g. Track 1 will have a pub date before Track 2 // e.g. Track 1 will have a pub date before Track 2

View File

@ -153,8 +153,13 @@ class PodcastEpisode {
update(payload) { update(payload) {
let hasUpdates = false let hasUpdates = false
for (const key in this.toJSON()) { for (const key in this.toJSON()) {
if (payload[key] != undefined && !areEquivalent(payload[key], this[key])) { let newValue = payload[key]
this[key] = copyValue(payload[key]) if (newValue === "") newValue = null
let existingValue = this[key]
if (existingValue === "") existingValue = null
if (newValue != undefined && !areEquivalent(newValue, existingValue)) {
this[key] = copyValue(newValue)
hasUpdates = true hasUpdates = true
} }
} }

View File

@ -32,7 +32,7 @@ module.exports = function areEquivalent(value1, value2, numToString = false, sta
// Truthy check to handle value1=null, value2=Object // Truthy check to handle value1=null, value2=Object
if ((value1 && !value2) || (!value1 && value2)) { if ((value1 && !value2) || (!value1 && value2)) {
console.log('value1/value2 falsy mismatch', value1, value2) // console.log('value1/value2 falsy mismatch', value1, value2)
return false return false
} }
@ -40,7 +40,7 @@ module.exports = function areEquivalent(value1, value2, numToString = false, sta
// Ensure types match // Ensure types match
if (type1 !== typeof value2) { if (type1 !== typeof value2) {
console.log('type diff', type1, typeof value2) // console.log('type diff', type1, typeof value2)
return false return false
} }
@ -63,7 +63,7 @@ module.exports = function areEquivalent(value1, value2, numToString = false, sta
if (type1 === 'bigint' || type1 === 'boolean' || if (type1 === 'bigint' || type1 === 'boolean' ||
type1 === 'function' || type1 === 'string' || type1 === 'function' || type1 === 'string' ||
type1 === 'symbol') { type1 === 'symbol') {
console.log('no match for values', value1, value2) // console.log('no match for values', value1, value2)
return false return false
} }
@ -93,20 +93,17 @@ module.exports = function areEquivalent(value1, value2, numToString = false, sta
// Handle arrays // Handle arrays
if (Array.isArray(value1)) { if (Array.isArray(value1)) {
if (!Array.isArray(value2)) { if (!Array.isArray(value2)) {
console.log('value2 is not array but value1 is', value1, value2)
return false return false
} }
const length = value1.length const length = value1.length
if (length !== value2.length) { if (length !== value2.length) {
console.log('array length diff', length)
return false return false
} }
for (let i = 0; i < length; i++) { for (let i = 0; i < length; i++) {
if (!areEquivalent(value1[i], value2[i], numToString, stack)) { if (!areEquivalent(value1[i], value2[i], numToString, stack)) {
console.log('2 array items are not equiv', value1[i], value2[i])
return false return false
} }
} }
@ -121,7 +118,6 @@ module.exports = function areEquivalent(value1, value2, numToString = false, sta
const numKeys = keys1.length const numKeys = keys1.length
if (keys2.length !== numKeys) { if (keys2.length !== numKeys) {
console.log('Key length is diff', keys2.length, numKeys)
return false return false
} }
@ -139,7 +135,7 @@ module.exports = function areEquivalent(value1, value2, numToString = false, sta
// Ensure perfect match across all keys // Ensure perfect match across all keys
for (let i = 0; i < numKeys; i++) { for (let i = 0; i < numKeys; i++) {
if (keys1[i] !== keys2[i]) { if (keys1[i] !== keys2[i]) {
console.log('object key is not equiv', keys1[i], keys2[i]) // console.log('object key is not equiv', keys1[i], keys2[i])
return false return false
} }
} }
@ -147,7 +143,7 @@ module.exports = function areEquivalent(value1, value2, numToString = false, sta
// Ensure perfect match across all values // Ensure perfect match across all values
for (let i = 0; i < numKeys; i++) { for (let i = 0; i < numKeys; i++) {
if (!areEquivalent(value1[keys1[i]], value2[keys1[i]], numToString, stack)) { if (!areEquivalent(value1[keys1[i]], value2[keys1[i]], numToString, stack)) {
console.log('2 subobjects not equiv', keys1[i], value1[keys1[i]], value2[keys1[i]]) // console.log('2 subobjects not equiv', keys1[i], value1[keys1[i]], value2[keys1[i]])
return false return false
} }
} }

View File

@ -169,6 +169,8 @@ function migratePodcast(oldLibraryItem, LibraryItem) {
// //
const oldEpisodes = oldPodcast.episodes || [] const oldEpisodes = oldPodcast.episodes || []
for (const oldEpisode of oldEpisodes) { for (const oldEpisode of oldEpisodes) {
oldEpisode.audioFile.index = 1
const PodcastEpisode = { const PodcastEpisode = {
id: uuidv4(), id: uuidv4(),
index: oldEpisode.index, index: oldEpisode.index,