From 0d5792405f54efa75ad098acf17de0113aad178e Mon Sep 17 00:00:00 2001 From: advplyr Date: Mon, 16 Oct 2023 17:47:44 -0500 Subject: [PATCH] Fix:Podcast episodes store RSS feed guid so they can be matched if the RSS feed changes the episode URL #2207 --- .../components/modals/podcast/EpisodeFeed.vue | 38 +++++++++---------- server/controllers/PodcastController.js | 7 ++-- server/managers/PodcastManager.js | 10 ++--- server/models/PodcastEpisode.js | 4 ++ server/objects/entities/PodcastEpisode.js | 5 +++ server/utils/podcastUtils.js | 34 +++++++++++------ 6 files changed, 59 insertions(+), 39 deletions(-) diff --git a/client/components/modals/podcast/EpisodeFeed.vue b/client/components/modals/podcast/EpisodeFeed.vue index 0f75644b..1378dbe5 100644 --- a/client/components/modals/podcast/EpisodeFeed.vue +++ b/client/components/modals/podcast/EpisodeFeed.vue @@ -16,11 +16,11 @@ v-for="(episode, index) in episodesList" :key="index" class="relative" - :class="itemEpisodeMap[episode.cleanUrl] ? 'bg-primary bg-opacity-40' : selectedEpisodes[episode.cleanUrl] ? 'cursor-pointer bg-success bg-opacity-10' : index % 2 == 0 ? 'cursor-pointer bg-primary bg-opacity-25 hover:bg-opacity-40' : 'cursor-pointer bg-primary bg-opacity-5 hover:bg-opacity-25'" + :class="getIsEpisodeDownloaded(episode) ? 'bg-primary bg-opacity-40' : selectedEpisodes[episode.cleanUrl] ? 'cursor-pointer bg-success bg-opacity-10' : index % 2 == 0 ? 'cursor-pointer bg-primary bg-opacity-25 hover:bg-opacity-40' : 'cursor-pointer bg-primary bg-opacity-5 hover:bg-opacity-25'" @click="toggleSelectEpisode(episode)" >
- download_done + download_done
@@ -93,7 +93,7 @@ export default { return this.libraryItem.media.metadata.title || 'Unknown' }, allDownloaded() { - return !this.episodesCleaned.some((episode) => !this.itemEpisodeMap[episode.cleanUrl]) + return !this.episodesCleaned.some((episode) => this.getIsEpisodeDownloaded(episode)) }, episodesSelected() { return Object.keys(this.selectedEpisodes).filter((key) => !!this.selectedEpisodes[key]) @@ -104,18 +104,7 @@ export default { return this.$getString('LabelDownloadNEpisodes', [this.episodesSelected.length]) }, itemEpisodes() { - if (!this.libraryItem) return [] - return this.libraryItem.media.episodes || [] - }, - itemEpisodeMap() { - const map = {} - this.itemEpisodes.forEach((item) => { - if (item.enclosure) { - const cleanUrl = this.getCleanEpisodeUrl(item.enclosure.url) - map[cleanUrl] = true - } - }) - return map + return this.libraryItem?.media.episodes || [] }, episodesList() { return this.episodesCleaned.filter((episode) => { @@ -127,12 +116,23 @@ export default { if (this.episodesList.length === this.episodesCleaned.length) { return this.$strings.LabelSelectAllEpisodes } - const episodesNotDownloaded = this.episodesList.filter((ep) => !this.itemEpisodeMap[ep.cleanUrl]).length + const episodesNotDownloaded = this.episodesList.filter((ep) => !this.getIsEpisodeDownloaded(ep)).length return this.$getString('LabelSelectEpisodesShowing', [episodesNotDownloaded]) } }, methods: { + getIsEpisodeDownloaded(episode) { + return this.itemEpisodes.some((downloadedEpisode) => { + if (episode.guid && downloadedEpisode.guid === episode.guid) return true + if (!downloadedEpisode.enclosure?.url) return false + return this.getCleanEpisodeUrl(downloadedEpisode.enclosure.url) === episode.cleanUrl + }) + }, /** + * UPDATE: As of v2.4.5 guid is used for matching existing downloaded episodes if it is found on the RSS feed. + * Fallback to checking the clean url + * @see https://github.com/advplyr/audiobookshelf/issues/2207 + * * RSS feed episode url is used for matching with existing downloaded episodes. * Some RSS feeds include timestamps in the episode url (e.g. patreon) that can change on requests. * These need to be removed in order to detect the same episode each time the feed is pulled. @@ -169,13 +169,13 @@ export default { }, toggleSelectAll(val) { for (const episode of this.episodesList) { - if (this.itemEpisodeMap[episode.cleanUrl]) this.selectedEpisodes[episode.cleanUrl] = false + if (this.getIsEpisodeDownloaded(episode)) this.selectedEpisodes[episode.cleanUrl] = false else this.$set(this.selectedEpisodes, episode.cleanUrl, val) } }, checkSetIsSelectedAll() { for (const episode of this.episodesList) { - if (!this.itemEpisodeMap[episode.cleanUrl] && !this.selectedEpisodes[episode.cleanUrl]) { + if (!this.getIsEpisodeDownloaded(episode) && !this.selectedEpisodes[episode.cleanUrl]) { this.selectAll = false return } @@ -183,7 +183,7 @@ export default { this.selectAll = true }, toggleSelectEpisode(episode) { - if (this.itemEpisodeMap[episode.cleanUrl]) return + if (this.getIsEpisodeDownloaded(episode)) return this.$set(this.selectedEpisodes, episode.cleanUrl, !this.selectedEpisodes[episode.cleanUrl]) this.checkSetIsSelectedAll() }, diff --git a/server/controllers/PodcastController.js b/server/controllers/PodcastController.js index c4112db6..22c3cafa 100644 --- a/server/controllers/PodcastController.js +++ b/server/controllers/PodcastController.js @@ -184,10 +184,9 @@ class PodcastController { Logger.error(`[PodcastController] Non-admin user attempted to download episodes`, req.user) return res.sendStatus(403) } - var libraryItem = req.libraryItem - - var episodes = req.body - if (!episodes || !episodes.length) { + const libraryItem = req.libraryItem + const episodes = req.body + if (!episodes?.length) { return res.sendStatus(400) } diff --git a/server/managers/PodcastManager.js b/server/managers/PodcastManager.js index 5dec2152..b88a38af 100644 --- a/server/managers/PodcastManager.js +++ b/server/managers/PodcastManager.js @@ -201,7 +201,7 @@ class PodcastManager { }) // TODO: Should we check for open playback sessions for this episode? // TODO: remove all user progress for this episode - if (oldestEpisode && oldestEpisode.audioFile) { + if (oldestEpisode?.audioFile) { Logger.info(`[PodcastManager] Deleting oldest episode "${oldestEpisode.title}"`) const successfullyDeleted = await removeFile(oldestEpisode.audioFile.metadata.path) if (successfullyDeleted) { @@ -246,7 +246,7 @@ class PodcastManager { Logger.debug(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" checking for episodes after ${new Date(dateToCheckForEpisodesAfter)}`) var newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, dateToCheckForEpisodesAfter, libraryItem.media.maxNewEpisodesToDownload) - Logger.debug(`[PodcastManager] runEpisodeCheck: ${newEpisodes ? newEpisodes.length : 'N/A'} episodes found`) + Logger.debug(`[PodcastManager] runEpisodeCheck: ${newEpisodes?.length || 'N/A'} episodes found`) if (!newEpisodes) { // Failed // Allow up to MaxFailedEpisodeChecks failed attempts before disabling auto download @@ -280,14 +280,14 @@ class PodcastManager { Logger.error(`[PodcastManager] checkPodcastForNewEpisodes no feed url for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`) return false } - var feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl) - if (!feed || !feed.episodes) { + const feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl) + if (!feed?.episodes) { Logger.error(`[PodcastManager] checkPodcastForNewEpisodes invalid feed payload for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`, feed) return false } // Filter new and not already has - var newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url)) + let newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url)) if (maxNewEpisodes > 0) { newEpisodes = newEpisodes.slice(0, maxNewEpisodes) diff --git a/server/models/PodcastEpisode.js b/server/models/PodcastEpisode.js index 6416627a..55b2f9d4 100644 --- a/server/models/PodcastEpisode.js +++ b/server/models/PodcastEpisode.js @@ -79,6 +79,7 @@ class PodcastEpisode extends Model { subtitle: this.subtitle, description: this.description, enclosure, + guid: this.extraData?.guid || null, pubDate: this.pubDate, chapters: this.chapters, audioFile: this.audioFile, @@ -98,6 +99,9 @@ class PodcastEpisode extends Model { if (oldEpisode.oldEpisodeId) { extraData.oldEpisodeId = oldEpisode.oldEpisodeId } + if (oldEpisode.guid) { + extraData.guid = oldEpisode.guid + } return { id: oldEpisode.id, index: oldEpisode.index, diff --git a/server/objects/entities/PodcastEpisode.js b/server/objects/entities/PodcastEpisode.js index 2b91aeb6..0a8f3349 100644 --- a/server/objects/entities/PodcastEpisode.js +++ b/server/objects/entities/PodcastEpisode.js @@ -20,6 +20,7 @@ class PodcastEpisode { this.subtitle = null this.description = null this.enclosure = null + this.guid = null this.pubDate = null this.chapters = [] @@ -46,6 +47,7 @@ class PodcastEpisode { this.subtitle = episode.subtitle this.description = episode.description this.enclosure = episode.enclosure ? { ...episode.enclosure } : null + this.guid = episode.guid || null this.pubDate = episode.pubDate this.chapters = episode.chapters?.map(ch => ({ ...ch })) || [] this.audioFile = new AudioFile(episode.audioFile) @@ -70,6 +72,7 @@ class PodcastEpisode { subtitle: this.subtitle, description: this.description, enclosure: this.enclosure ? { ...this.enclosure } : null, + guid: this.guid, pubDate: this.pubDate, chapters: this.chapters.map(ch => ({ ...ch })), audioFile: this.audioFile.toJSON(), @@ -93,6 +96,7 @@ class PodcastEpisode { subtitle: this.subtitle, description: this.description, enclosure: this.enclosure ? { ...this.enclosure } : null, + guid: this.guid, pubDate: this.pubDate, chapters: this.chapters.map(ch => ({ ...ch })), audioFile: this.audioFile.toJSON(), @@ -133,6 +137,7 @@ class PodcastEpisode { this.pubDate = data.pubDate || '' this.description = data.description || '' this.enclosure = data.enclosure ? { ...data.enclosure } : null + this.guid = data.guid || null this.season = data.season || '' this.episode = data.episode || '' this.episodeType = data.episodeType || 'full' diff --git a/server/utils/podcastUtils.js b/server/utils/podcastUtils.js index 2fd684ea..0e68a0a4 100644 --- a/server/utils/podcastUtils.js +++ b/server/utils/podcastUtils.js @@ -4,7 +4,7 @@ const { xmlToJSON, levenshteinDistance } = require('./index') const htmlSanitizer = require('../utils/htmlSanitizer') function extractFirstArrayItem(json, key) { - if (!json[key] || !json[key].length) return null + if (!json[key]?.length) return null return json[key][0] } @@ -110,13 +110,24 @@ function extractEpisodeData(item) { const pubDate = extractFirstArrayItem(item, 'pubDate') if (typeof pubDate === 'string') { episode.pubDate = pubDate - } else if (pubDate && typeof pubDate._ === 'string') { + } else if (typeof pubDate?._ === 'string') { episode.pubDate = pubDate._ } else { Logger.error(`[podcastUtils] Invalid pubDate ${item['pubDate']} for ${episode.enclosure.url}`) } } + if (item['guid']) { + const guidItem = extractFirstArrayItem(item, 'guid') + if (typeof guidItem === 'string') { + episode.guid = guidItem + } else if (typeof guidItem?._ === 'string') { + episode.guid = guidItem._ + } else { + Logger.error(`[podcastUtils] Invalid guid ${item['guid']} for ${episode.enclosure.url}`) + } + } + const arrayFields = ['title', 'itunes:episodeType', 'itunes:season', 'itunes:episode', 'itunes:author', 'itunes:duration', 'itunes:explicit', 'itunes:subtitle'] arrayFields.forEach((key) => { const cleanKey = key.split(':').pop() @@ -142,6 +153,7 @@ function cleanEpisodeData(data) { explicit: data.explicit || '', publishedAt, enclosure: data.enclosure, + guid: data.guid || null, chaptersUrl: data.chaptersUrl || null, chaptersType: data.chaptersType || null } @@ -159,16 +171,16 @@ function extractPodcastEpisodes(items) { } function cleanPodcastJson(rssJson, excludeEpisodeMetadata) { - if (!rssJson.channel || !rssJson.channel.length) { + if (!rssJson.channel?.length) { Logger.error(`[podcastUtil] Invalid podcast no channel object`) return null } - var channel = rssJson.channel[0] - if (!channel.item || !channel.item.length) { + const channel = rssJson.channel[0] + if (!channel.item?.length) { Logger.error(`[podcastUtil] Invalid podcast no episodes`) return null } - var podcast = { + const podcast = { metadata: extractPodcastMetadata(channel) } if (!excludeEpisodeMetadata) { @@ -181,8 +193,8 @@ function cleanPodcastJson(rssJson, excludeEpisodeMetadata) { module.exports.parsePodcastRssFeedXml = async (xml, excludeEpisodeMetadata = false, includeRaw = false) => { if (!xml) return null - var json = await xmlToJSON(xml) - if (!json || !json.rss) { + const json = await xmlToJSON(xml) + if (!json?.rss) { Logger.error('[podcastUtils] Invalid XML or RSS feed') return null } @@ -215,12 +227,12 @@ module.exports.getPodcastFeed = (feedUrl, excludeEpisodeMetadata = false) => { data.data = data.data.toString() } - if (!data || !data.data) { + if (!data?.data) { Logger.error(`[podcastUtils] getPodcastFeed: Invalid podcast feed request response (${feedUrl})`) return false } Logger.debug(`[podcastUtils] getPodcastFeed for "${feedUrl}" success - parsing xml`) - var payload = await this.parsePodcastRssFeedXml(data.data, excludeEpisodeMetadata) + const payload = await this.parsePodcastRssFeedXml(data.data, excludeEpisodeMetadata) if (!payload) { return false } @@ -246,7 +258,7 @@ module.exports.findMatchingEpisodes = async (feedUrl, searchTitle) => { module.exports.findMatchingEpisodesInFeed = (feed, searchTitle) => { searchTitle = searchTitle.toLowerCase().trim() - if (!feed || !feed.episodes) { + if (!feed?.episodes) { return null }