Merge pull request #3772 from advplyr/feed-episodes-upsert

Feed episode IDs changing on refresh & several other refresh issues

server/managers/RssFeedManager.js

@@ -98,11 +98,22 @@ class RssFeedManager {
             podcastId: feed.entity.mediaId
           },
           attributes: ['id', 'updatedAt'],
-          order: [['createdAt', 'DESC']]
+          order: [['updatedAt', 'DESC']]
         })
+
         if (mostRecentPodcastEpisode && mostRecentPodcastEpisode.updatedAt > newEntityUpdatedAt) {
           newEntityUpdatedAt = mostRecentPodcastEpisode.updatedAt
         }
+      } else {
+        const book = await Database.bookModel.findOne({
+          where: {
+            id: feed.entity.mediaId
+          },
+          attributes: ['id', 'updatedAt']
+        })
+        if (book && book.updatedAt > newEntityUpdatedAt) {
+          newEntityUpdatedAt = book.updatedAt
+        }
       }
 
       return newEntityUpdatedAt > feed.entityUpdatedAt
@@ -111,7 +122,7 @@ class RssFeedManager {
         attributes: ['id', 'updatedAt'],
         include: {
           model: Database.bookModel,
-          attributes: ['id'],
+          attributes: ['id', 'audioFiles', 'updatedAt'],
           through: {
             attributes: []
           },
@@ -122,13 +133,16 @@ class RssFeedManager {
         }
       })
 
+      const totalBookTracks = feed.entity.books.reduce((total, book) => total + book.includedAudioFiles.length, 0)
+      if (feed.feedEpisodes.length !== totalBookTracks) {
+        return true
+      }
+
       let newEntityUpdatedAt = feed.entity.updatedAt
 
       const mostRecentItemUpdatedAt = feed.entity.books.reduce((mostRecent, book) => {
-        if (book.libraryItem.updatedAt > mostRecent) {
+        let updatedAt = book.libraryItem.updatedAt > book.updatedAt ? book.libraryItem.updatedAt : book.updatedAt
-          return book.libraryItem.updatedAt
+        return updatedAt > mostRecent ? updatedAt : mostRecent
-        }
-        return mostRecent
       }, 0)
 
       if (mostRecentItemUpdatedAt > newEntityUpdatedAt) {
@@ -151,6 +165,9 @@ class RssFeedManager {
     let feed = await Database.feedModel.findOne({
       where: {
         slug: req.params.slug
+      },
+      include: {
+        model: Database.feedEpisodeModel
       }
     })
     if (!feed) {
@@ -163,8 +180,6 @@ class RssFeedManager {
     if (feedRequiresUpdate) {
       Logger.info(`[RssFeedManager] Feed "${feed.title}" requires update - updating feed`)
       feed = await feed.updateFeedForEntity()
-    } else {
-      feed.feedEpisodes = await feed.getFeedEpisodes()
     }
 
     const xml = feed.buildXml(req.originalHostPrefix)

server/models/Feed.js

@@ -107,6 +107,9 @@ class Feed extends Model {
       entityUpdatedAt = libraryItem.media.podcastEpisodes.reduce((mostRecent, episode) => {
         return episode.updatedAt > mostRecent ? episode.updatedAt : mostRecent
       }, entityUpdatedAt)
+    } else if (libraryItem.media.updatedAt > entityUpdatedAt) {
+      // Book feeds will use Book.updatedAt if more recent
+      entityUpdatedAt = libraryItem.media.updatedAt
     }
 
     const feedObj = {
@@ -187,7 +190,8 @@ class Feed extends Model {
     const booksWithTracks = collectionExpanded.books.filter((book) => book.includedAudioFiles.length)
 
     const entityUpdatedAt = booksWithTracks.reduce((mostRecent, book) => {
-      return book.libraryItem.updatedAt > mostRecent ? book.libraryItem.updatedAt : mostRecent
+      const updatedAt = book.libraryItem.updatedAt > book.updatedAt ? book.libraryItem.updatedAt : book.updatedAt
+      return updatedAt > mostRecent ? updatedAt : mostRecent
     }, collectionExpanded.updatedAt)
 
     const firstBookWithCover = booksWithTracks.find((book) => book.coverPath)
@@ -275,7 +279,8 @@ class Feed extends Model {
   static getFeedObjForSeries(userId, seriesExpanded, slug, serverAddress, feedOptions = null) {
     const booksWithTracks = seriesExpanded.books.filter((book) => book.includedAudioFiles.length)
     const entityUpdatedAt = booksWithTracks.reduce((mostRecent, book) => {
-      return book.libraryItem.updatedAt > mostRecent ? book.libraryItem.updatedAt : mostRecent
+      const updatedAt = book.libraryItem.updatedAt > book.updatedAt ? book.libraryItem.updatedAt : book.updatedAt
+      return updatedAt > mostRecent ? updatedAt : mostRecent
     }, seriesExpanded.updatedAt)
 
     const firstBookWithCover = booksWithTracks.find((book) => book.coverPath)
@@ -516,17 +521,24 @@ class Feed extends Model {
     try {
       const updatedFeed = await this.update(feedObj, { transaction })
 
-      // Remove existing feed episodes
+      const existingFeedEpisodeIds = this.feedEpisodes.map((ep) => ep.id)
-      await feedEpisodeModel.destroy({
-        where: {
-          feedId: this.id
-        },
-        transaction
-      })
 
       // Create new feed episodes
       updatedFeed.feedEpisodes = await feedEpisodeCreateFunc(feedEpisodeCreateFuncEntity, updatedFeed, this.slug, transaction)
 
+      const newFeedEpisodeIds = updatedFeed.feedEpisodes.map((ep) => ep.id)
+      const feedEpisodeIdsToRemove = existingFeedEpisodeIds.filter((epid) => !newFeedEpisodeIds.includes(epid))
+
+      if (feedEpisodeIdsToRemove.length) {
+        Logger.info(`[Feed] Removing ${feedEpisodeIdsToRemove.length} episodes from feed ${this.id}`)
+        await feedEpisodeModel.destroy({
+          where: {
+            id: feedEpisodeIdsToRemove
+          },
+          transaction
+        })
+      }
+
       await transaction.commit()
 
       return updatedFeed

server/models/FeedEpisode.js

@@ -53,9 +53,10 @@ class FeedEpisode extends Model {
    * @param {import('./Feed')} feed
    * @param {string} slug
    * @param {import('./PodcastEpisode')} episode
+   * @param {string} [existingEpisodeId]
    */
-  static getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode) {
+  static getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode, existingEpisodeId = null) {
-    const episodeId = uuidv4()
+    const episodeId = existingEpisodeId || uuidv4()
     return {
       id: episodeId,
       title: episode.title,
@@ -94,11 +95,18 @@ class FeedEpisode extends Model {
       libraryItemExpanded.media.podcastEpisodes.sort((a, b) => new Date(a.pubDate) - new Date(b.pubDate))
     }
 
+    let numExisting = 0
     for (const episode of libraryItemExpanded.media.podcastEpisodes) {
-      feedEpisodeObjs.push(this.getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode))
+      // Check for existing episode by filepath
+      const existingEpisode = feed.feedEpisodes?.find((feedEpisode) => {
+        return feedEpisode.filePath === episode.audioFile.metadata.path
+      })
+      numExisting = existingEpisode ? numExisting + 1 : numExisting
+
+      feedEpisodeObjs.push(this.getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode, existingEpisode?.id))
     }
-    Logger.info(`[FeedEpisode] Creating ${feedEpisodeObjs.length} episodes for feed ${feed.id}`)
+    Logger.info(`[FeedEpisode] Upserting ${feedEpisodeObjs.length} episodes for feed ${feed.id} (${numExisting} existing)`)
-    return this.bulkCreate(feedEpisodeObjs, { transaction })
+    return this.bulkCreate(feedEpisodeObjs, { transaction, updateOnDuplicate: ['title', 'author', 'description', 'siteURL', 'enclosureURL', 'enclosureType', 'enclosureSize', 'pubDate', 'season', 'episode', 'episodeType', 'duration', 'filePath', 'explicit'] })
   }
 
   /**
@@ -127,11 +135,12 @@ class FeedEpisode extends Model {
    * @param {string} slug
    * @param {import('./Book').AudioFileObject} audioTrack
    * @param {boolean} useChapterTitles
+   * @param {string} [existingEpisodeId]
    */
-  static getFeedEpisodeObjFromAudiobookTrack(book, pubDateStart, feed, slug, audioTrack, useChapterTitles) {
+  static getFeedEpisodeObjFromAudiobookTrack(book, pubDateStart, feed, slug, audioTrack, useChapterTitles, existingEpisodeId = null) {
     // Example: <pubDate>Fri, 04 Feb 2015 00:00:00 GMT</pubDate>
     let timeOffset = isNaN(audioTrack.index) ? 0 : Number(audioTrack.index) * 1000 // Offset pubdate to ensure correct order
-    let episodeId = uuidv4()
+    let episodeId = existingEpisodeId || uuidv4()
 
     // e.g. Track 1 will have a pub date before Track 2
     const audiobookPubDate = date.format(new Date(pubDateStart.valueOf() + timeOffset), 'ddd, DD MMM YYYY HH:mm:ss [GMT]')
@@ -179,11 +188,18 @@ class FeedEpisode extends Model {
     const useChapterTitles = this.checkUseChapterTitlesForEpisodes(libraryItemExpanded.media)
 
     const feedEpisodeObjs = []
+    let numExisting = 0
     for (const track of libraryItemExpanded.media.trackList) {
-      feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(libraryItemExpanded.media, libraryItemExpanded.createdAt, feed, slug, track, useChapterTitles))
+      // Check for existing episode by filepath
+      const existingEpisode = feed.feedEpisodes?.find((episode) => {
+        return episode.filePath === track.metadata.path
+      })
+      numExisting = existingEpisode ? numExisting + 1 : numExisting
+
+      feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(libraryItemExpanded.media, libraryItemExpanded.createdAt, feed, slug, track, useChapterTitles, existingEpisode?.id))
     }
-    Logger.info(`[FeedEpisode] Creating ${feedEpisodeObjs.length} episodes for feed ${feed.id}`)
+    Logger.info(`[FeedEpisode] Upserting ${feedEpisodeObjs.length} episodes for feed ${feed.id} (${numExisting} existing)`)
-    return this.bulkCreate(feedEpisodeObjs, { transaction })
+    return this.bulkCreate(feedEpisodeObjs, { transaction, updateOnDuplicate: ['title', 'author', 'description', 'siteURL', 'enclosureURL', 'enclosureType', 'enclosureSize', 'pubDate', 'season', 'episode', 'episodeType', 'duration', 'filePath', 'explicit'] })
   }
 
   /**
@@ -200,14 +216,21 @@ class FeedEpisode extends Model {
     }).libraryItem.createdAt
 
     const feedEpisodeObjs = []
+    let numExisting = 0
     for (const book of books) {
       const useChapterTitles = this.checkUseChapterTitlesForEpisodes(book)
       for (const track of book.trackList) {
-        feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(book, earliestLibraryItemCreatedAt, feed, slug, track, useChapterTitles))
+        // Check for existing episode by filepath
+        const existingEpisode = feed.feedEpisodes?.find((episode) => {
+          return episode.filePath === track.metadata.path
+        })
+        numExisting = existingEpisode ? numExisting + 1 : numExisting
+
+        feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(book, earliestLibraryItemCreatedAt, feed, slug, track, useChapterTitles, existingEpisode?.id))
       }
     }
-    Logger.info(`[FeedEpisode] Creating ${feedEpisodeObjs.length} episodes for feed ${feed.id}`)
+    Logger.info(`[FeedEpisode] Upserting ${feedEpisodeObjs.length} episodes for feed ${feed.id} (${numExisting} existing)`)
-    return this.bulkCreate(feedEpisodeObjs, { transaction })
+    return this.bulkCreate(feedEpisodeObjs, { transaction, updateOnDuplicate: ['title', 'author', 'description', 'siteURL', 'enclosureURL', 'enclosureType', 'enclosureSize', 'pubDate', 'season', 'episode', 'episodeType', 'duration', 'filePath', 'explicit'] })
   }
 
   /**