From 2b3c3cd088db9fe4ac98f19ad0e5b85c048382d2 Mon Sep 17 00:00:00 2001 From: mfcar Date: Sat, 23 Sep 2023 20:38:03 +0100 Subject: [PATCH 01/19] Add infra to handle with feed healthy --- client/pages/item/_id/index.vue | 8 +- server/controllers/PodcastController.js | 13 + server/managers/PodcastManager.js | 731 ++++++++++++------------ server/models/Podcast.js | 18 +- server/routers/ApiRouter.js | 1 + server/utils/migrations/dbMigration.js | 6 +- 6 files changed, 412 insertions(+), 365 deletions(-) diff --git a/client/pages/item/_id/index.vue b/client/pages/item/_id/index.vue index 176725b91..cd333cdc8 100644 --- a/client/pages/item/_id/index.vue +++ b/client/pages/item/_id/index.vue @@ -485,16 +485,16 @@ export default { return this.$toast.error('Podcast does not have an RSS Feed') } this.fetchingRSSFeed = true - var payload = await this.$axios.$post(`/api/podcasts/feed`, { rssFeed: this.mediaMetadata.feedUrl }).catch((error) => { + var payload = await this.$axios.get(`/api/podcasts/${this.libraryItemId}/feed`).catch((error) => { console.error('Failed to get feed', error) this.$toast.error('Failed to get podcast feed') return null }) this.fetchingRSSFeed = false - if (!payload) return + if (!payload || !payload.data) return - console.log('Podcast feed', payload) - const podcastfeed = payload.podcast + console.log('Podcast feed', payload.data) + const podcastfeed = payload.data.podcast if (!podcastfeed.episodes || !podcastfeed.episodes.length) { this.$toast.info('No episodes found in RSS feed') return diff --git a/server/controllers/PodcastController.js b/server/controllers/PodcastController.js index 0e1ebcd3a..19f16476a 100644 --- a/server/controllers/PodcastController.js +++ b/server/controllers/PodcastController.js @@ -115,6 +115,19 @@ class PodcastController { res.json({ podcast }) } + async checkPodcastFeed(req, res) { + const libraryItem = req.libraryItem + const podcast = await getPodcastFeed(libraryItem.media.metadata.feedUrl) + + if (!podcast) { + this.podcastManager.setFeedHealthStatus(libraryItem, false) + return res.status(404).send('Podcast RSS feed request failed or invalid response data') + } + + this.podcastManager.setFeedHealthStatus(libraryItem, true) + res.json({ podcast }) + } + async getFeedsFromOPMLText(req, res) { if (!req.body.opmlText) { return res.sendStatus(400) diff --git a/server/managers/PodcastManager.js b/server/managers/PodcastManager.js index 5dec21523..19254e170 100644 --- a/server/managers/PodcastManager.js +++ b/server/managers/PodcastManager.js @@ -4,9 +4,9 @@ const Database = require('../Database') const fs = require('../libs/fsExtra') -const { getPodcastFeed } = require('../utils/podcastUtils') -const { removeFile, downloadFile } = require('../utils/fileUtils') -const { levenshteinDistance } = require('../utils/index') +const {getPodcastFeed} = require('../utils/podcastUtils') +const {removeFile, downloadFile} = require('../utils/fileUtils') +const {levenshteinDistance} = require('../utils/index') const opmlParser = require('../utils/parsers/parseOPML') const opmlGenerator = require('../utils/generators/opmlGenerator') const prober = require('../utils/prober') @@ -19,374 +19,395 @@ const AudioFile = require('../objects/files/AudioFile') const Task = require("../objects/Task") class PodcastManager { - constructor(watcher, notificationManager, taskManager) { - this.watcher = watcher - this.notificationManager = notificationManager - this.taskManager = taskManager + constructor(watcher, notificationManager, taskManager) { + this.watcher = watcher + this.notificationManager = notificationManager + this.taskManager = taskManager - this.downloadQueue = [] - this.currentDownload = null + this.downloadQueue = [] + this.currentDownload = null - this.failedCheckMap = {} - this.MaxFailedEpisodeChecks = 24 - } - - getEpisodeDownloadsInQueue(libraryItemId) { - return this.downloadQueue.filter(d => d.libraryItemId === libraryItemId) - } - - clearDownloadQueue(libraryItemId = null) { - if (!this.downloadQueue.length) return - - if (!libraryItemId) { - Logger.info(`[PodcastManager] Clearing all downloads in queue (${this.downloadQueue.length})`) - this.downloadQueue = [] - } else { - var itemDownloads = this.getEpisodeDownloadsInQueue(libraryItemId) - Logger.info(`[PodcastManager] Clearing downloads in queue for item "${libraryItemId}" (${itemDownloads.length})`) - this.downloadQueue = this.downloadQueue.filter(d => d.libraryItemId !== libraryItemId) - } - } - - async downloadPodcastEpisodes(libraryItem, episodesToDownload, isAutoDownload) { - let index = Math.max(...libraryItem.media.episodes.filter(ep => ep.index == null || isNaN(ep.index)).map(ep => Number(ep.index))) + 1 - for (const ep of episodesToDownload) { - const newPe = new PodcastEpisode() - newPe.setData(ep, index++) - newPe.libraryItemId = libraryItem.id - newPe.podcastId = libraryItem.media.id - const newPeDl = new PodcastEpisodeDownload() - newPeDl.setData(newPe, libraryItem, isAutoDownload, libraryItem.libraryId) - this.startPodcastEpisodeDownload(newPeDl) - } - } - - async startPodcastEpisodeDownload(podcastEpisodeDownload) { - SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) - if (this.currentDownload) { - this.downloadQueue.push(podcastEpisodeDownload) - SocketAuthority.emitter('episode_download_queued', podcastEpisodeDownload.toJSONForClient()) - return + this.failedCheckMap = {} + this.MaxFailedEpisodeChecks = 24 } - const task = new Task() - const taskDescription = `Downloading episode "${podcastEpisodeDownload.podcastEpisode.title}".` - const taskData = { - libraryId: podcastEpisodeDownload.libraryId, - libraryItemId: podcastEpisodeDownload.libraryItemId, - } - task.setData('download-podcast-episode', 'Downloading Episode', taskDescription, false, taskData) - this.taskManager.addTask(task) - - SocketAuthority.emitter('episode_download_started', podcastEpisodeDownload.toJSONForClient()) - this.currentDownload = podcastEpisodeDownload - - // If this file already exists then append the episode id to the filename - // e.g. "/tagesschau 20 Uhr.mp3" becomes "/tagesschau 20 Uhr (ep_asdfasdf).mp3" - // this handles podcasts where every title is the same (ref https://github.com/advplyr/audiobookshelf/issues/1802) - if (await fs.pathExists(this.currentDownload.targetPath)) { - this.currentDownload.appendEpisodeId = true + getEpisodeDownloadsInQueue(libraryItemId) { + return this.downloadQueue.filter(d => d.libraryItemId === libraryItemId) } - // Ignores all added files to this dir - this.watcher.addIgnoreDir(this.currentDownload.libraryItem.path) + clearDownloadQueue(libraryItemId = null) { + if (!this.downloadQueue.length) return - // Make sure podcast library item folder exists - if (!(await fs.pathExists(this.currentDownload.libraryItem.path))) { - Logger.warn(`[PodcastManager] Podcast episode download: Podcast folder no longer exists at "${this.currentDownload.libraryItem.path}" - Creating it`) - await fs.mkdir(this.currentDownload.libraryItem.path) - } - - let success = false - if (this.currentDownload.urlFileExtension === 'mp3') { - // Download episode and tag it - success = await ffmpegHelpers.downloadPodcastEpisode(this.currentDownload).catch((error) => { - Logger.error(`[PodcastManager] Podcast Episode download failed`, error) - return false - }) - } else { - // Download episode only - success = await downloadFile(this.currentDownload.url, this.currentDownload.targetPath).then(() => true).catch((error) => { - Logger.error(`[PodcastManager] Podcast Episode download failed`, error) - return false - }) - } - - if (success) { - success = await this.scanAddPodcastEpisodeAudioFile() - if (!success) { - await fs.remove(this.currentDownload.targetPath) - this.currentDownload.setFinished(false) - task.setFailed('Failed to download episode') - } else { - Logger.info(`[PodcastManager] Successfully downloaded podcast episode "${this.currentDownload.podcastEpisode.title}"`) - this.currentDownload.setFinished(true) - task.setFinished() - } - } else { - task.setFailed('Failed to download episode') - this.currentDownload.setFinished(false) - } - - this.taskManager.taskFinished(task) - - SocketAuthority.emitter('episode_download_finished', this.currentDownload.toJSONForClient()) - SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) - - this.watcher.removeIgnoreDir(this.currentDownload.libraryItem.path) - this.currentDownload = null - if (this.downloadQueue.length) { - this.startPodcastEpisodeDownload(this.downloadQueue.shift()) - } - } - - async scanAddPodcastEpisodeAudioFile() { - const libraryFile = await this.getLibraryFile(this.currentDownload.targetPath, this.currentDownload.targetRelPath) - - const audioFile = await this.probeAudioFile(libraryFile) - if (!audioFile) { - return false - } - - const libraryItem = await Database.libraryItemModel.getOldById(this.currentDownload.libraryItem.id) - if (!libraryItem) { - Logger.error(`[PodcastManager] Podcast Episode finished but library item was not found ${this.currentDownload.libraryItem.id}`) - return false - } - - const podcastEpisode = this.currentDownload.podcastEpisode - podcastEpisode.audioFile = audioFile - - if (audioFile.chapters?.length) { - podcastEpisode.chapters = audioFile.chapters.map(ch => ({ ...ch })) - } - - libraryItem.media.addPodcastEpisode(podcastEpisode) - if (libraryItem.isInvalid) { - // First episode added to an empty podcast - libraryItem.isInvalid = false - } - libraryItem.libraryFiles.push(libraryFile) - - if (this.currentDownload.isAutoDownload) { - // Check setting maxEpisodesToKeep and remove episode if necessary - if (libraryItem.media.maxEpisodesToKeep && libraryItem.media.episodesWithPubDate.length > libraryItem.media.maxEpisodesToKeep) { - Logger.info(`[PodcastManager] # of episodes (${libraryItem.media.episodesWithPubDate.length}) exceeds max episodes to keep (${libraryItem.media.maxEpisodesToKeep})`) - await this.removeOldestEpisode(libraryItem, podcastEpisode.id) - } - } - - libraryItem.updatedAt = Date.now() - await Database.updateLibraryItem(libraryItem) - SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) - const podcastEpisodeExpanded = podcastEpisode.toJSONExpanded() - podcastEpisodeExpanded.libraryItem = libraryItem.toJSONExpanded() - SocketAuthority.emitter('episode_added', podcastEpisodeExpanded) - - if (this.currentDownload.isAutoDownload) { // Notifications only for auto downloaded episodes - this.notificationManager.onPodcastEpisodeDownloaded(libraryItem, podcastEpisode) - } - - return true - } - - async removeOldestEpisode(libraryItem, episodeIdJustDownloaded) { - var smallestPublishedAt = 0 - var oldestEpisode = null - libraryItem.media.episodesWithPubDate.filter(ep => ep.id !== episodeIdJustDownloaded).forEach((ep) => { - if (!smallestPublishedAt || ep.publishedAt < smallestPublishedAt) { - smallestPublishedAt = ep.publishedAt - oldestEpisode = ep - } - }) - // TODO: Should we check for open playback sessions for this episode? - // TODO: remove all user progress for this episode - if (oldestEpisode && oldestEpisode.audioFile) { - Logger.info(`[PodcastManager] Deleting oldest episode "${oldestEpisode.title}"`) - const successfullyDeleted = await removeFile(oldestEpisode.audioFile.metadata.path) - if (successfullyDeleted) { - libraryItem.media.removeEpisode(oldestEpisode.id) - libraryItem.removeLibraryFile(oldestEpisode.audioFile.ino) - return true - } else { - Logger.warn(`[PodcastManager] Failed to remove oldest episode "${oldestEpisode.title}"`) - } - } - return false - } - - async getLibraryFile(path, relPath) { - var newLibFile = new LibraryFile() - await newLibFile.setDataFromPath(path, relPath) - return newLibFile - } - - async probeAudioFile(libraryFile) { - const path = libraryFile.metadata.path - const mediaProbeData = await prober.probe(path) - if (mediaProbeData.error) { - Logger.error(`[PodcastManager] Podcast Episode downloaded but failed to probe "${path}"`, mediaProbeData.error) - return false - } - const newAudioFile = new AudioFile() - newAudioFile.setDataFromProbe(libraryFile, mediaProbeData) - newAudioFile.index = 1 - return newAudioFile - } - - // Returns false if auto download episodes was disabled (disabled if reaches max failed checks) - async runEpisodeCheck(libraryItem) { - const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) - const latestEpisodePublishedAt = libraryItem.media.latestEpisodePublished - Logger.info(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" | Last check: ${lastEpisodeCheckDate} | ${latestEpisodePublishedAt ? `Latest episode pubDate: ${new Date(latestEpisodePublishedAt)}` : 'No latest episode'}`) - - // Use latest episode pubDate if exists OR fallback to using lastEpisodeCheckDate - // lastEpisodeCheckDate will be the current time when adding a new podcast - const dateToCheckForEpisodesAfter = latestEpisodePublishedAt || lastEpisodeCheckDate - Logger.debug(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" checking for episodes after ${new Date(dateToCheckForEpisodesAfter)}`) - - var newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, dateToCheckForEpisodesAfter, libraryItem.media.maxNewEpisodesToDownload) - Logger.debug(`[PodcastManager] runEpisodeCheck: ${newEpisodes ? newEpisodes.length : 'N/A'} episodes found`) - - if (!newEpisodes) { // Failed - // Allow up to MaxFailedEpisodeChecks failed attempts before disabling auto download - if (!this.failedCheckMap[libraryItem.id]) this.failedCheckMap[libraryItem.id] = 0 - this.failedCheckMap[libraryItem.id]++ - if (this.failedCheckMap[libraryItem.id] >= this.MaxFailedEpisodeChecks) { - Logger.error(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}" - disabling auto download`) - libraryItem.media.autoDownloadEpisodes = false - delete this.failedCheckMap[libraryItem.id] - } else { - Logger.warn(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}"`) - } - } else if (newEpisodes.length) { - delete this.failedCheckMap[libraryItem.id] - Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) - this.downloadPodcastEpisodes(libraryItem, newEpisodes, true) - } else { - delete this.failedCheckMap[libraryItem.id] - Logger.debug(`[PodcastManager] No new episodes for "${libraryItem.media.metadata.title}"`) - } - - libraryItem.media.lastEpisodeCheck = Date.now() - libraryItem.updatedAt = Date.now() - await Database.updateLibraryItem(libraryItem) - SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) - return libraryItem.media.autoDownloadEpisodes - } - - async checkPodcastForNewEpisodes(podcastLibraryItem, dateToCheckForEpisodesAfter, maxNewEpisodes = 3) { - if (!podcastLibraryItem.media.metadata.feedUrl) { - Logger.error(`[PodcastManager] checkPodcastForNewEpisodes no feed url for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`) - return false - } - var feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl) - if (!feed || !feed.episodes) { - Logger.error(`[PodcastManager] checkPodcastForNewEpisodes invalid feed payload for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`, feed) - return false - } - - // Filter new and not already has - var newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url)) - - if (maxNewEpisodes > 0) { - newEpisodes = newEpisodes.slice(0, maxNewEpisodes) - } - - return newEpisodes - } - - async checkAndDownloadNewEpisodes(libraryItem, maxEpisodesToDownload) { - const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) - Logger.info(`[PodcastManager] checkAndDownloadNewEpisodes for "${libraryItem.media.metadata.title}" - Last episode check: ${lastEpisodeCheckDate}`) - var newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, libraryItem.media.lastEpisodeCheck, maxEpisodesToDownload) - if (newEpisodes.length) { - Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) - this.downloadPodcastEpisodes(libraryItem, newEpisodes, false) - } else { - Logger.info(`[PodcastManager] No new episodes found for podcast "${libraryItem.media.metadata.title}"`) - } - - libraryItem.media.lastEpisodeCheck = Date.now() - libraryItem.updatedAt = Date.now() - await Database.updateLibraryItem(libraryItem) - SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) - - return newEpisodes - } - - async findEpisode(rssFeedUrl, searchTitle) { - const feed = await getPodcastFeed(rssFeedUrl).catch(() => { - return null - }) - if (!feed || !feed.episodes) { - return null - } - - const matches = [] - feed.episodes.forEach(ep => { - if (!ep.title) return - - const epTitle = ep.title.toLowerCase().trim() - if (epTitle === searchTitle) { - matches.push({ - episode: ep, - levenshtein: 0 - }) - } else { - const levenshtein = levenshteinDistance(searchTitle, epTitle, true) - if (levenshtein <= 6 && epTitle.length > levenshtein) { - matches.push({ - episode: ep, - levenshtein - }) + if (!libraryItemId) { + Logger.info(`[PodcastManager] Clearing all downloads in queue (${this.downloadQueue.length})`) + this.downloadQueue = [] + } else { + var itemDownloads = this.getEpisodeDownloadsInQueue(libraryItemId) + Logger.info(`[PodcastManager] Clearing downloads in queue for item "${libraryItemId}" (${itemDownloads.length})`) + this.downloadQueue = this.downloadQueue.filter(d => d.libraryItemId !== libraryItemId) } - } - }) - return matches.sort((a, b) => a.levenshtein - b.levenshtein) - } - - async getOPMLFeeds(opmlText) { - var extractedFeeds = opmlParser.parse(opmlText) - if (!extractedFeeds || !extractedFeeds.length) { - Logger.error('[PodcastManager] getOPMLFeeds: No RSS feeds found in OPML') - return { - error: 'No RSS feeds found in OPML' - } } - var rssFeedData = [] - - for (let feed of extractedFeeds) { - var feedData = await getPodcastFeed(feed.feedUrl, true) - if (feedData) { - feedData.metadata.feedUrl = feed.feedUrl - rssFeedData.push(feedData) - } + async downloadPodcastEpisodes(libraryItem, episodesToDownload, isAutoDownload) { + let index = Math.max(...libraryItem.media.episodes.filter(ep => ep.index == null || isNaN(ep.index)).map(ep => Number(ep.index))) + 1 + for (const ep of episodesToDownload) { + const newPe = new PodcastEpisode() + newPe.setData(ep, index++) + newPe.libraryItemId = libraryItem.id + newPe.podcastId = libraryItem.media.id + const newPeDl = new PodcastEpisodeDownload() + newPeDl.setData(newPe, libraryItem, isAutoDownload, libraryItem.libraryId) + this.startPodcastEpisodeDownload(newPeDl) + } } - return { - feeds: rssFeedData + async startPodcastEpisodeDownload(podcastEpisodeDownload) { + SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) + if (this.currentDownload) { + this.downloadQueue.push(podcastEpisodeDownload) + SocketAuthority.emitter('episode_download_queued', podcastEpisodeDownload.toJSONForClient()) + return + } + + const task = new Task() + const taskDescription = `Downloading episode "${podcastEpisodeDownload.podcastEpisode.title}".` + const taskData = { + libraryId: podcastEpisodeDownload.libraryId, + libraryItemId: podcastEpisodeDownload.libraryItemId, + } + task.setData('download-podcast-episode', 'Downloading Episode', taskDescription, false, taskData) + this.taskManager.addTask(task) + + SocketAuthority.emitter('episode_download_started', podcastEpisodeDownload.toJSONForClient()) + this.currentDownload = podcastEpisodeDownload + + // If this file already exists then append the episode id to the filename + // e.g. "/tagesschau 20 Uhr.mp3" becomes "/tagesschau 20 Uhr (ep_asdfasdf).mp3" + // this handles podcasts where every title is the same (ref https://github.com/advplyr/audiobookshelf/issues/1802) + if (await fs.pathExists(this.currentDownload.targetPath)) { + this.currentDownload.appendEpisodeId = true + } + + // Ignores all added files to this dir + this.watcher.addIgnoreDir(this.currentDownload.libraryItem.path) + + // Make sure podcast library item folder exists + if (!(await fs.pathExists(this.currentDownload.libraryItem.path))) { + Logger.warn(`[PodcastManager] Podcast episode download: Podcast folder no longer exists at "${this.currentDownload.libraryItem.path}" - Creating it`) + await fs.mkdir(this.currentDownload.libraryItem.path) + } + + let success = false + if (this.currentDownload.urlFileExtension === 'mp3') { + // Download episode and tag it + success = await ffmpegHelpers.downloadPodcastEpisode(this.currentDownload).catch((error) => { + Logger.error(`[PodcastManager] Podcast Episode download failed`, error) + return false + }) + } else { + // Download episode only + success = await downloadFile(this.currentDownload.url, this.currentDownload.targetPath).then(() => true).catch((error) => { + Logger.error(`[PodcastManager] Podcast Episode download failed`, error) + return false + }) + } + + if (success) { + success = await this.scanAddPodcastEpisodeAudioFile() + if (!success) { + await fs.remove(this.currentDownload.targetPath) + this.currentDownload.setFinished(false) + task.setFailed('Failed to download episode') + } else { + Logger.info(`[PodcastManager] Successfully downloaded podcast episode "${this.currentDownload.podcastEpisode.title}"`) + this.currentDownload.setFinished(true) + task.setFinished() + } + } else { + task.setFailed('Failed to download episode') + this.currentDownload.setFinished(false) + } + + this.taskManager.taskFinished(task) + + SocketAuthority.emitter('episode_download_finished', this.currentDownload.toJSONForClient()) + SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) + + this.watcher.removeIgnoreDir(this.currentDownload.libraryItem.path) + this.currentDownload = null + if (this.downloadQueue.length) { + this.startPodcastEpisodeDownload(this.downloadQueue.shift()) + } } - } - /** - * OPML file string for podcasts in a library - * @param {import('../models/Podcast')[]} podcasts - * @returns {string} XML string - */ - generateOPMLFileText(podcasts) { - return opmlGenerator.generate(podcasts) - } + async scanAddPodcastEpisodeAudioFile() { + const libraryFile = await this.getLibraryFile(this.currentDownload.targetPath, this.currentDownload.targetRelPath) - getDownloadQueueDetails(libraryId = null) { - let _currentDownload = this.currentDownload - if (libraryId && _currentDownload?.libraryId !== libraryId) _currentDownload = null + const audioFile = await this.probeAudioFile(libraryFile) + if (!audioFile) { + return false + } - return { - currentDownload: _currentDownload?.toJSONForClient(), - queue: this.downloadQueue.filter(item => !libraryId || item.libraryId === libraryId).map(item => item.toJSONForClient()) + const libraryItem = await Database.libraryItemModel.getOldById(this.currentDownload.libraryItem.id) + if (!libraryItem) { + Logger.error(`[PodcastManager] Podcast Episode finished but library item was not found ${this.currentDownload.libraryItem.id}`) + return false + } + + const podcastEpisode = this.currentDownload.podcastEpisode + podcastEpisode.audioFile = audioFile + + if (audioFile.chapters?.length) { + podcastEpisode.chapters = audioFile.chapters.map(ch => ({...ch})) + } + + libraryItem.media.addPodcastEpisode(podcastEpisode) + if (libraryItem.isInvalid) { + // First episode added to an empty podcast + libraryItem.isInvalid = false + } + libraryItem.libraryFiles.push(libraryFile) + + if (this.currentDownload.isAutoDownload) { + // Check setting maxEpisodesToKeep and remove episode if necessary + if (libraryItem.media.maxEpisodesToKeep && libraryItem.media.episodesWithPubDate.length > libraryItem.media.maxEpisodesToKeep) { + Logger.info(`[PodcastManager] # of episodes (${libraryItem.media.episodesWithPubDate.length}) exceeds max episodes to keep (${libraryItem.media.maxEpisodesToKeep})`) + await this.removeOldestEpisode(libraryItem, podcastEpisode.id) + } + } + + libraryItem.updatedAt = Date.now() + await Database.updateLibraryItem(libraryItem) + SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) + const podcastEpisodeExpanded = podcastEpisode.toJSONExpanded() + podcastEpisodeExpanded.libraryItem = libraryItem.toJSONExpanded() + SocketAuthority.emitter('episode_added', podcastEpisodeExpanded) + + if (this.currentDownload.isAutoDownload) { // Notifications only for auto downloaded episodes + this.notificationManager.onPodcastEpisodeDownloaded(libraryItem, podcastEpisode) + } + + return true + } + + async removeOldestEpisode(libraryItem, episodeIdJustDownloaded) { + var smallestPublishedAt = 0 + var oldestEpisode = null + libraryItem.media.episodesWithPubDate.filter(ep => ep.id !== episodeIdJustDownloaded).forEach((ep) => { + if (!smallestPublishedAt || ep.publishedAt < smallestPublishedAt) { + smallestPublishedAt = ep.publishedAt + oldestEpisode = ep + } + }) + // TODO: Should we check for open playback sessions for this episode? + // TODO: remove all user progress for this episode + if (oldestEpisode && oldestEpisode.audioFile) { + Logger.info(`[PodcastManager] Deleting oldest episode "${oldestEpisode.title}"`) + const successfullyDeleted = await removeFile(oldestEpisode.audioFile.metadata.path) + if (successfullyDeleted) { + libraryItem.media.removeEpisode(oldestEpisode.id) + libraryItem.removeLibraryFile(oldestEpisode.audioFile.ino) + return true + } else { + Logger.warn(`[PodcastManager] Failed to remove oldest episode "${oldestEpisode.title}"`) + } + } + return false + } + + async getLibraryFile(path, relPath) { + var newLibFile = new LibraryFile() + await newLibFile.setDataFromPath(path, relPath) + return newLibFile + } + + async probeAudioFile(libraryFile) { + const path = libraryFile.metadata.path + const mediaProbeData = await prober.probe(path) + if (mediaProbeData.error) { + Logger.error(`[PodcastManager] Podcast Episode downloaded but failed to probe "${path}"`, mediaProbeData.error) + return false + } + const newAudioFile = new AudioFile() + newAudioFile.setDataFromProbe(libraryFile, mediaProbeData) + newAudioFile.index = 1 + return newAudioFile + } + + // Returns false if auto download episodes was disabled (disabled if reaches max failed checks) + async runEpisodeCheck(libraryItem) { + const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) + const latestEpisodePublishedAt = libraryItem.media.latestEpisodePublished + Logger.info(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" | Last check: ${lastEpisodeCheckDate} | ${latestEpisodePublishedAt ? `Latest episode pubDate: ${new Date(latestEpisodePublishedAt)}` : 'No latest episode'}`) + + // Use latest episode pubDate if exists OR fallback to using lastEpisodeCheckDate + // lastEpisodeCheckDate will be the current time when adding a new podcast + const dateToCheckForEpisodesAfter = latestEpisodePublishedAt || lastEpisodeCheckDate + Logger.debug(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" checking for episodes after ${new Date(dateToCheckForEpisodesAfter)}`) + + let newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, dateToCheckForEpisodesAfter, libraryItem.media.maxNewEpisodesToDownload) + Logger.debug(`[PodcastManager] runEpisodeCheck: ${newEpisodes ? newEpisodes.length : 'N/A'} episodes found`) + + if (!newEpisodes) { // Failed + // Allow up to MaxFailedEpisodeChecks failed attempts before disabling auto download + if (!this.failedCheckMap[libraryItem.id]) this.failedCheckMap[libraryItem.id] = 0 + this.failedCheckMap[libraryItem.id]++ + if (this.failedCheckMap[libraryItem.id] >= this.MaxFailedEpisodeChecks) { + Logger.error(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}" - disabling auto download`) + libraryItem.media.autoDownloadEpisodes = false + delete this.failedCheckMap[libraryItem.id] + } else { + Logger.warn(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}"`) + } + libraryItem.media.metadata.feedHealthy = false + } else if (newEpisodes.length) { + delete this.failedCheckMap[libraryItem.id] + Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) + this.downloadPodcastEpisodes(libraryItem, newEpisodes, true) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } else { + delete this.failedCheckMap[libraryItem.id] + Logger.debug(`[PodcastManager] No new episodes for "${libraryItem.media.metadata.title}"`) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } + + libraryItem.media.lastEpisodeCheck = Date.now() + libraryItem.updatedAt = Date.now() + await Database.updateLibraryItem(libraryItem) + SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) + return libraryItem.media.autoDownloadEpisodes + } + + async checkPodcastForNewEpisodes(podcastLibraryItem, dateToCheckForEpisodesAfter, maxNewEpisodes = 3) { + if (!podcastLibraryItem.media.metadata.feedUrl) { + Logger.error(`[PodcastManager] checkPodcastForNewEpisodes no feed url for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`) + return false + } + var feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl) + if (!feed || !feed.episodes) { + Logger.error(`[PodcastManager] checkPodcastForNewEpisodes invalid feed payload for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id}, URL: ${podcastLibraryItem.media.metadata.feedUrl})`, feed) + return false + } + + // Filter new and not already has + var newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url)) + + if (maxNewEpisodes > 0) { + newEpisodes = newEpisodes.slice(0, maxNewEpisodes) + } + + return newEpisodes + } + + async checkAndDownloadNewEpisodes(libraryItem, maxEpisodesToDownload) { + const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) + Logger.info(`[PodcastManager] checkAndDownloadNewEpisodes for "${libraryItem.media.metadata.title}" - Last episode check: ${lastEpisodeCheckDate}`) + let newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, libraryItem.media.lastEpisodeCheck, maxEpisodesToDownload) + if (!newEpisodes) { + libraryItem.media.metadata.feedHealthy = false + } else if (newEpisodes.length) { + Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) + this.downloadPodcastEpisodes(libraryItem, newEpisodes, false) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } else { + Logger.info(`[PodcastManager] No new episodes found for podcast "${libraryItem.media.metadata.title}"`) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } + + libraryItem.media.lastEpisodeCheck = Date.now() + libraryItem.updatedAt = Date.now() + await Database.updateLibraryItem(libraryItem) + SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) + + return newEpisodes + } + + setFeedHealthStatus(libraryItem, isHealthy) { + libraryItem.media.metadata.feedHealthy = isHealthy + if (isHealthy) { + libraryItem.media.lastSuccessfulFetchAt = Date.now() + } + libraryItem.updatedAt = Date.now() + Database.updateLibraryItem(libraryItem) + } + + async findEpisode(rssFeedUrl, searchTitle) { + const feed = await getPodcastFeed(rssFeedUrl).catch(() => { + return null + }) + if (!feed || !feed.episodes) { + return null + } + + const matches = [] + feed.episodes.forEach(ep => { + if (!ep.title) return + + const epTitle = ep.title.toLowerCase().trim() + if (epTitle === searchTitle) { + matches.push({ + episode: ep, + levenshtein: 0 + }) + } else { + const levenshtein = levenshteinDistance(searchTitle, epTitle, true) + if (levenshtein <= 6 && epTitle.length > levenshtein) { + matches.push({ + episode: ep, + levenshtein + }) + } + } + }) + return matches.sort((a, b) => a.levenshtein - b.levenshtein) + } + + async getOPMLFeeds(opmlText) { + var extractedFeeds = opmlParser.parse(opmlText) + if (!extractedFeeds || !extractedFeeds.length) { + Logger.error('[PodcastManager] getOPMLFeeds: No RSS feeds found in OPML') + return { + error: 'No RSS feeds found in OPML' + } + } + + var rssFeedData = [] + + for (let feed of extractedFeeds) { + var feedData = await getPodcastFeed(feed.feedUrl, true) + if (feedData) { + feedData.metadata.feedUrl = feed.feedUrl + rssFeedData.push(feedData) + } + } + + return { + feeds: rssFeedData + } + } + + /** + * OPML file string for podcasts in a library + * @param {import('../models/Podcast')[]} podcasts + * @returns {string} XML string + */ + generateOPMLFileText(podcasts) { + return opmlGenerator.generate(podcasts) + } + + getDownloadQueueDetails(libraryId = null) { + let _currentDownload = this.currentDownload + if (libraryId && _currentDownload?.libraryId !== libraryId) _currentDownload = null + + return { + currentDownload: _currentDownload?.toJSONForClient(), + queue: this.downloadQueue.filter(item => !libraryId || item.libraryId === libraryId).map(item => item.toJSONForClient()) + } } - } } + module.exports = PodcastManager diff --git a/server/models/Podcast.js b/server/models/Podcast.js index 60311bfde..c2ab917d3 100644 --- a/server/models/Podcast.js +++ b/server/models/Podcast.js @@ -50,6 +50,10 @@ class Podcast extends Model { this.createdAt /** @type {Date} */ this.updatedAt + /** @type {Date} */ + this.lastSuccessfulFetchAt + /** @type {boolean} */ + this.feedHealthy } static getOldPodcast(libraryItemExpanded) { @@ -71,7 +75,9 @@ class Podcast extends Model { itunesArtistId: podcastExpanded.itunesArtistId, explicit: podcastExpanded.explicit, language: podcastExpanded.language, - type: podcastExpanded.podcastType + type: podcastExpanded.podcastType, + lastSuccessfulFetchAt: podcastExpanded.lastSuccessfulFetchAt?.valueOf() || null, + feedHealthy: !!podcastExpanded.feedHealthy }, coverPath: podcastExpanded.coverPath, tags: podcastExpanded.tags, @@ -108,7 +114,9 @@ class Podcast extends Model { maxNewEpisodesToDownload: oldPodcast.maxNewEpisodesToDownload, coverPath: oldPodcast.coverPath, tags: oldPodcast.tags, - genres: oldPodcastMetadata.genres + genres: oldPodcastMetadata.genres, + lastSuccessfulFetchAt: oldPodcastMetadata.lastSuccessfulFetchAt, + feedHealthy: !!oldPodcastMetadata.feedHealthy } } @@ -144,7 +152,9 @@ class Podcast extends Model { maxNewEpisodesToDownload: DataTypes.INTEGER, coverPath: DataTypes.STRING, tags: DataTypes.JSON, - genres: DataTypes.JSON + genres: DataTypes.JSON, + lastSuccessfulFetchAt: DataTypes.DATE, + feedHealthy: DataTypes.BOOLEAN }, { sequelize, modelName: 'podcast' @@ -152,4 +162,4 @@ class Podcast extends Model { } } -module.exports = Podcast \ No newline at end of file +module.exports = Podcast diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index 74d8aa566..67566cbfb 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -226,6 +226,7 @@ class ApiRouter { // this.router.post('/podcasts', PodcastController.create.bind(this)) this.router.post('/podcasts/feed', PodcastController.getPodcastFeed.bind(this)) + this.router.get('/podcasts/:id/feed', PodcastController.middleware.bind(this), PodcastController.checkPodcastFeed.bind(this)) this.router.post('/podcasts/opml', PodcastController.getFeedsFromOPMLText.bind(this)) this.router.get('/podcasts/:id/checknew', PodcastController.middleware.bind(this), PodcastController.checkNewEpisodes.bind(this)) this.router.get('/podcasts/:id/downloads', PodcastController.middleware.bind(this), PodcastController.getEpisodeDownloads.bind(this)) diff --git a/server/utils/migrations/dbMigration.js b/server/utils/migrations/dbMigration.js index 3d38cca6a..dd1360889 100644 --- a/server/utils/migrations/dbMigration.js +++ b/server/utils/migrations/dbMigration.js @@ -198,7 +198,9 @@ function migratePodcast(oldLibraryItem, LibraryItem) { updatedAt: LibraryItem.updatedAt, coverPath: oldPodcast.coverPath, tags: oldPodcast.tags, - genres: oldPodcastMetadata.genres + genres: oldPodcastMetadata.genres, + lastSuccessfulFetchAt: oldPodcastMetadata.lastSuccessfulFetchAt || null, + feedHealthy: !!oldPodcastMetadata.feedHealthy || null } _newRecords.podcast = Podcast oldDbIdMap.podcasts[oldLibraryItem.id] = Podcast.id @@ -1708,4 +1710,4 @@ module.exports.migrationPatch2 = async (ctx) => { Logger.error(`[dbMigration] Migration from 2.3.3+ column creation failed`, error) throw new Error('Migration 2.3.3+ failed ' + error) } -} \ No newline at end of file +} From 9872937a9904fe033d843d7cf07dc6c25203a24f Mon Sep 17 00:00:00 2001 From: mfcar Date: Sun, 24 Sep 2023 21:20:01 +0100 Subject: [PATCH 02/19] Creating a controller to return only the podcasts with feedURL --- server/controllers/PodcastController.js | 7 +++++++ server/managers/PodcastManager.js | 6 ++++++ server/models/Podcast.js | 5 +++++ server/routers/ApiRouter.js | 1 + 4 files changed, 19 insertions(+) diff --git a/server/controllers/PodcastController.js b/server/controllers/PodcastController.js index 19f16476a..afd2c32f1 100644 --- a/server/controllers/PodcastController.js +++ b/server/controllers/PodcastController.js @@ -115,6 +115,13 @@ class PodcastController { res.json({ podcast }) } + async getPodcastsWithInboundFeed(req, res) { + const podcasts = await Database.podcastModel.getAllInboundFeeds() + res.json({ + podcasts + }) + } + async checkPodcastFeed(req, res) { const libraryItem = req.libraryItem const podcast = await getPodcastFeed(libraryItem.media.metadata.feedUrl) diff --git a/server/managers/PodcastManager.js b/server/managers/PodcastManager.js index 19254e170..4ac810443 100644 --- a/server/managers/PodcastManager.js +++ b/server/managers/PodcastManager.js @@ -408,6 +408,12 @@ class PodcastManager { queue: this.downloadQueue.filter(item => !libraryId || item.libraryId === libraryId).map(item => item.toJSONForClient()) } } + + async getPodcastsWithInboundFeed() { + const podcasts = await Database.models.podcast.getAllInboundFeeds() + Logger.info(`[PodcastManager] Fetched all podcasts with feed`) + return podcasts + } } module.exports = PodcastManager diff --git a/server/models/Podcast.js b/server/models/Podcast.js index c2ab917d3..00f0f7b99 100644 --- a/server/models/Podcast.js +++ b/server/models/Podcast.js @@ -120,6 +120,11 @@ class Podcast extends Model { } } + static async getAllInboundFeeds(){ + const podcasts = await this.findAll() + return podcasts.map(p => this.getFromOld(p)) + } + /** * Initialize model * @param {import('../Database').sequelize} sequelize diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index 67566cbfb..ad5ef44cc 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -226,6 +226,7 @@ class ApiRouter { // this.router.post('/podcasts', PodcastController.create.bind(this)) this.router.post('/podcasts/feed', PodcastController.getPodcastFeed.bind(this)) + this.router.get('/podcasts/inboundFeeds', PodcastController.getPodcastsWithInboundFeed.bind(this)) this.router.get('/podcasts/:id/feed', PodcastController.middleware.bind(this), PodcastController.checkPodcastFeed.bind(this)) this.router.post('/podcasts/opml', PodcastController.getFeedsFromOPMLText.bind(this)) this.router.get('/podcasts/:id/checknew', PodcastController.middleware.bind(this), PodcastController.checkNewEpisodes.bind(this)) From 7f5fde1de25b222aa7cdbeea967ca598c739bb9c Mon Sep 17 00:00:00 2001 From: mfcar Date: Sun, 1 Oct 2023 01:21:07 +0100 Subject: [PATCH 03/19] Fixing the rss incoming server side --- server/controllers/PodcastController.js | 4 ++-- server/managers/PodcastManager.js | 6 +++--- server/models/Podcast.js | 4 ++-- server/routers/ApiRouter.js | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/server/controllers/PodcastController.js b/server/controllers/PodcastController.js index 0c12620bc..6f1138fb5 100644 --- a/server/controllers/PodcastController.js +++ b/server/controllers/PodcastController.js @@ -115,8 +115,8 @@ class PodcastController { res.json({ podcast }) } - async getPodcastsWithInboundFeed(req, res) { - const podcasts = await Database.podcastModel.getAllInboundFeeds() + async getPodcastsWithIncomingFeeds(req, res) { + const podcasts = await Database.podcastModel.getAllIncomingFeeds() res.json({ podcasts }) diff --git a/server/managers/PodcastManager.js b/server/managers/PodcastManager.js index 4ac810443..45affb24f 100644 --- a/server/managers/PodcastManager.js +++ b/server/managers/PodcastManager.js @@ -409,9 +409,9 @@ class PodcastManager { } } - async getPodcastsWithInboundFeed() { - const podcasts = await Database.models.podcast.getAllInboundFeeds() - Logger.info(`[PodcastManager] Fetched all podcasts with feed`) + async getPodcastsWithIncomingFeeds() { + const podcasts = await Database.models.podcast.getAllIncomingFeeds() + Logger.info(`[PodcastManager] Fetched ${podcasts.length} podcasts with incoming feeds`) return podcasts } } diff --git a/server/models/Podcast.js b/server/models/Podcast.js index 00f0f7b99..a1c58d4bb 100644 --- a/server/models/Podcast.js +++ b/server/models/Podcast.js @@ -120,9 +120,9 @@ class Podcast extends Model { } } - static async getAllInboundFeeds(){ + static async getAllIncomingFeeds(){ const podcasts = await this.findAll() - return podcasts.map(p => this.getFromOld(p)) + return podcasts.map(p => this.getFromOld({metadata: p.dataValues})) } /** diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index 421eec00e..f58c0d3ae 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -227,7 +227,7 @@ class ApiRouter { // this.router.post('/podcasts', PodcastController.create.bind(this)) this.router.post('/podcasts/feed', PodcastController.getPodcastFeed.bind(this)) - this.router.get('/podcasts/inboundFeeds', PodcastController.getPodcastsWithInboundFeed.bind(this)) + this.router.get('/podcasts/incomingFeeds', PodcastController.getPodcastsWithIncomingFeeds.bind(this)) this.router.get('/podcasts/:id/feed', PodcastController.middleware.bind(this), PodcastController.checkPodcastFeed.bind(this)) this.router.post('/podcasts/opml', PodcastController.getFeedsFromOPMLText.bind(this)) this.router.get('/podcasts/:id/checknew', PodcastController.middleware.bind(this), PodcastController.checkNewEpisodes.bind(this)) From 2ada293e8a02473fa6be1d69ce8e0a6b07f4292b Mon Sep 17 00:00:00 2001 From: mfcar Date: Sun, 1 Oct 2023 22:04:07 +0100 Subject: [PATCH 04/19] Feed Healthy indicator --- .../widgets/FeedHealthyIndicator.vue | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 client/components/widgets/FeedHealthyIndicator.vue diff --git a/client/components/widgets/FeedHealthyIndicator.vue b/client/components/widgets/FeedHealthyIndicator.vue new file mode 100644 index 000000000..f7e34211e --- /dev/null +++ b/client/components/widgets/FeedHealthyIndicator.vue @@ -0,0 +1,28 @@ + + + From aefcc4490f65bdc629dd68b13fc8ab26af11d42d Mon Sep 17 00:00:00 2001 From: mfcar Date: Tue, 3 Oct 2023 12:32:39 +0100 Subject: [PATCH 05/19] RSS Manager code --- .../components/widgets/PodcastDetailsEdit.vue | 16 +- client/pages/config/rss-feeds.vue | 358 ++++++++++++++---- client/strings/en-us.json | 13 +- server/controllers/PodcastController.js | 22 +- server/models/Podcast.js | 317 ++++++++-------- server/objects/metadata/PodcastMetadata.js | 14 +- server/routers/ApiRouter.js | 1 + 7 files changed, 508 insertions(+), 233 deletions(-) diff --git a/client/components/widgets/PodcastDetailsEdit.vue b/client/components/widgets/PodcastDetailsEdit.vue index 4c2fd739a..c93d7f84c 100644 --- a/client/components/widgets/PodcastDetailsEdit.vue +++ b/client/components/widgets/PodcastDetailsEdit.vue @@ -10,7 +10,15 @@ - +
+
+ +
+ +
+
+
+

{{ $strings.LabelFeedLastSuccessfulCheck }}: {{$dateDistanceFromNow(details.lastSuccessfulFetchAt)}}

@@ -71,7 +79,9 @@ export default { itunesArtistId: null, explicit: false, language: null, - type: null + type: null, + feedHealthy: false, + lastSuccessfulFetchAt: null }, newTags: [] } @@ -229,6 +239,8 @@ export default { this.details.language = this.mediaMetadata.language || '' this.details.explicit = !!this.mediaMetadata.explicit this.details.type = this.mediaMetadata.type || 'episodic' + this.details.feedHealthy = !!this.mediaMetadata.feedHealthy + this.details.lastSuccessfulFetchAt = this.mediaMetadata.lastSuccessfulFetchAt || null this.newTags = [...(this.media.tags || [])] }, diff --git a/client/pages/config/rss-feeds.vue b/client/pages/config/rss-feeds.vue index 28dba6704..d62cf6d08 100644 --- a/client/pages/config/rss-feeds.vue +++ b/client/pages/config/rss-feeds.vue @@ -1,61 +1,180 @@ @@ -63,9 +182,18 @@ export default { data() { return { + showIncomingFeedsView: false, showFeedModal: false, selectedFeed: null, - feeds: [] + feeds: [], + incomingFeeds: [], + feedsSearch: null, + feedsSearchTimeout: null, + feedsSearchText: null, + incomingFeedsSearch: null, + incomingFeedsSearchTimeout: null, + incomingFeedsSearchText: null, + incomingFeedShowOnlyUnhealthy: false, } }, computed: { @@ -74,13 +202,63 @@ export default { }, timeFormat() { return this.$store.state.serverSettings.timeFormat - } + }, + noCoverUrl() { + return `${this.$config.routerBasePath}/Logo.png` + }, + bookCoverAspectRatio() { + return this.$store.getters['libraries/getBookCoverAspectRatio'] + }, + feedsList() { + return this.feeds.filter((feed) => { + if (!this.feedsSearchText) return true + return feed?.meta?.title?.toLowerCase().includes(this.feedsSearchText) || feed?.slug?.toLowerCase().includes(this.feedsSearchText) + }) + }, + incomingFeedsSorted() { + return this.incomingFeedShowOnlyUnhealthy + ? this.incomingFeeds.filter(incomingFeed => !incomingFeed.metadata.feedHealthy) + : this.incomingFeeds; + }, + incomingFeedsList() { + return this.incomingFeedsSorted.filter((incomingFeed) => { + if (!this.incomingFeedsSearchText) return true + if (this.incomingFeedShowOnlyUnhealthy && incomingFeed?.metadata?.feedHealthy) return false + return incomingFeed?.metadata?.title?.toLowerCase().includes(this.incomingFeedsSearchText) || + incomingFeed?.metadata?.feedUrl?.toLowerCase().includes(this.incomingFeedsSearchText) + }) + }, }, methods: { + feedsSubmit() {}, + feedsInputUpdate() { + clearTimeout(this.feedsSearchTimeout) + this.feedsSearchTimeout = setTimeout(() => { + if (!this.feedsSearch || !this.feedsSearch.trim()) { + this.feedsSearchText = '' + return + } + this.feedsSearchText = this.feedsSearch.toLowerCase().trim() + }, 500) + }, + incomingFeedsSubmit() {}, + incomingFeedsInputUpdate() { + clearTimeout(this.incomingFeedsSearchTimeout) + this.incomingFeedsSearchTimeout = setTimeout(() => { + if (!this.incomingFeedsSearch || !this.incomingFeedsSearch.trim()) { + this.incomingFeedsSearchText = '' + return + } + this.incomingFeedsSearchText = this.incomingFeedsSearch.toLowerCase().trim() + }, 500) + }, showFeed(feed) { this.selectedFeed = feed this.showFeedModal = true }, + copyToClipboard(str) { + this.$copyToClipboard(str, this) + }, deleteFeedClick(feed) { const payload = { message: this.$strings.MessageConfirmCloseFeed, @@ -96,19 +274,19 @@ export default { deleteFeed(feed) { this.processing = true this.$axios - .$post(`/api/feeds/${feed.id}/close`) - .then(() => { - this.$toast.success(this.$strings.ToastRSSFeedCloseSuccess) - this.show = false - this.loadFeeds() - }) - .catch((error) => { - console.error('Failed to close RSS feed', error) - this.$toast.error(this.$strings.ToastRSSFeedCloseFailed) - }) - .finally(() => { - this.processing = false - }) + .$post(`/api/feeds/${feed.id}/close`) + .then(() => { + this.$toast.success(this.$strings.ToastRSSFeedCloseSuccess) + this.show = false + this.loadFeeds() + }) + .catch((error) => { + console.error('Failed to close RSS feed', error) + this.$toast.error(this.$strings.ToastRSSFeedCloseFailed) + }) + .finally(() => { + this.processing = false + }) }, getEntityType(entityType) { if (entityType === 'libraryItem') return this.$strings.LabelItem @@ -117,9 +295,40 @@ export default { return this.$strings.LabelUnknown }, coverUrl(feed) { - if (!feed.coverPath) return `${this.$config.routerBasePath}/Logo.png` + if (!feed.coverPath) return this.noCoverUrl return `${feed.feedUrl}/cover` }, + nextRun(cronExpression) { + if (!cronExpression) return '' + const parsed = this.$getNextScheduledDate(cronExpression) + return this.$formatJsDatetime(parsed, this.$store.state.serverSettings.dateFormat, this.$store.state.serverSettings.timeFormat) || '' + }, + async forceRecheckFeed(podcast) { + podcast.isLoading = true + let podcastResult; + + try { + podcastResult = await this.$axios.$get(`/api/podcasts/${podcast.id}/check-feed-url`) + + if (!podcastResult?.feedHealthy) { + this.$toast.error('Podcast feed url is not healthy') + } else { + this.$toast.success('Podcast feed url is healthy') + } + + podcast.lastEpisodeCheck = Date.parse(podcastResult.lastEpisodeCheck) + if (podcastResult.lastSuccessfulFetchAt) { + podcast.metadata.lastSuccessfulFetchAt = Date.parse(podcastResult.lastSuccessfulFetchAt) + } + podcast.metadata.feedHealthy = podcastResult.feedHealthy + } catch (error) { + console.error('Podcast feed url is not healthy', error) + this.$toast.error('Podcast feed url is not healthy') + podcastResult = null + } finally { + podcast.isLoading = false + } + }, async loadFeeds() { const data = await this.$axios.$get(`/api/feeds`).catch((err) => { console.error('Failed to load RSS feeds', err) @@ -131,8 +340,21 @@ export default { } this.feeds = data.feeds }, + async loadIncomingFeeds() { + const data = await this.$axios.$get(`/api/podcasts/incomingFeeds`).catch((err) => { + console.error('Failed to load incoming RSS feeds', err) + return null + }) + if (!data) { + this.$toast.error('Failed to load incoming RSS feeds') + return + } + + this.incomingFeeds = data.podcasts.map(podcast => ({...podcast, isLoading: false})); + }, init() { this.loadFeeds() + this.loadIncomingFeeds() } }, mounted() { diff --git a/client/strings/en-us.json b/client/strings/en-us.json index ae018d4bb..92bd6337e 100644 --- a/client/strings/en-us.json +++ b/client/strings/en-us.json @@ -17,6 +17,7 @@ "ButtonCloseFeed": "Close Feed", "ButtonCollections": "Collections", "ButtonConfigureScanner": "Configure Scanner", + "ButtonCopyFeedURL": "Copy Feed URL", "ButtonCreate": "Create", "ButtonCreateBackup": "Create Backup", "ButtonDelete": "Delete", @@ -24,6 +25,7 @@ "ButtonEdit": "Edit", "ButtonEditChapters": "Edit Chapters", "ButtonEditPodcast": "Edit Podcast", + "ButtonForceReCheckFeed": "Force Re-Check Feed", "ButtonForceReScan": "Force Re-Scan", "ButtonFullPath": "Full Path", "ButtonHide": "Hide", @@ -246,7 +248,14 @@ "LabelEpisodeType": "Episode Type", "LabelExample": "Example", "LabelExplicit": "Explicit", + "LabelFeedHealthy": "Feed Healthy", "LabelFeedURL": "Feed URL", + "LabelFeedLastChecked": "Last Checked", + "LabelFeedLastSuccessfulCheck": "Last Successful Check", + "LabelFeedShowOnlyUnhealthy": "Show only unhealthy", + "LabelFeedNextAutomaticCheck": "Next Automatic Check", + "LabelFeedNotWorking": "Feed is returning errors, check the logs for more information", + "LabelFeedWorking": "Feed working as expected", "LabelFile": "File", "LabelFileBirthtime": "File Birthtime", "LabelFileModified": "File Modified", @@ -572,6 +581,7 @@ "MessageMatchBooksDescription": "will attempt to match books in the library with a book from the selected search provider and fill in empty details and cover art. Does not overwrite details.", "MessageNoAudioTracks": "No audio tracks", "MessageNoAuthors": "No Authors", + "MessageNoAvailable": "N/A", "MessageNoBackups": "No Backups", "MessageNoBookmarks": "No Bookmarks", "MessageNoChapters": "No Chapters", @@ -642,6 +652,7 @@ "PlaceholderNewPlaylist": "New playlist name", "PlaceholderSearch": "Search..", "PlaceholderSearchEpisode": "Search episode..", + "PlaceholderSearchTitle": "Search title..", "ToastAccountUpdateFailed": "Failed to update account", "ToastAccountUpdateSuccess": "Account updated", "ToastAuthorImageRemoveFailed": "Failed to remove image", @@ -713,4 +724,4 @@ "ToastSocketFailedToConnect": "Socket failed to connect", "ToastUserDeleteFailed": "Failed to delete user", "ToastUserDeleteSuccess": "User deleted" -} \ No newline at end of file +} diff --git a/server/controllers/PodcastController.js b/server/controllers/PodcastController.js index 6f1138fb5..0b16ce719 100644 --- a/server/controllers/PodcastController.js +++ b/server/controllers/PodcastController.js @@ -127,14 +127,32 @@ class PodcastController { const podcast = await getPodcastFeed(libraryItem.media.metadata.feedUrl) if (!podcast) { - this.podcastManager.setFeedHealthStatus(libraryItem, false) + this.podcastManager.setFeedHealthStatus(libraryItem.media.id, false) return res.status(404).send('Podcast RSS feed request failed or invalid response data') } - this.podcastManager.setFeedHealthStatus(libraryItem, true) + this.podcastManager.setFeedHealthStatus(libraryItem.media.id, true) res.json({ podcast }) } + async checkPodcastFeedUrl(req, res) { + const podcastId = req.params.id; + + try { + const podcast = await Database.podcastModel.findByPk(req.params.id) + + const podcastResult = await getPodcastFeed(podcast.feedURL); + const podcastNewStatus = await this.podcastManager.setFeedHealthStatus(podcastId, !!podcastResult); + + Logger.info(podcastNewStatus); + + return res.json(podcastNewStatus); + } catch (error) { + Logger.error(`[PodcastController] checkPodcastFeed: Error checking podcast feed for podcast ${podcastId}`, error) + res.status(500).json({ error: 'An error occurred while checking the podcast feed.' }); + } + } + async getFeedsFromOPMLText(req, res) { if (!req.body.opmlText) { return res.sendStatus(400) diff --git a/server/models/Podcast.js b/server/models/Podcast.js index a1c58d4bb..d5dd5c65a 100644 --- a/server/models/Podcast.js +++ b/server/models/Podcast.js @@ -1,170 +1,171 @@ -const { DataTypes, Model } = require('sequelize') +const {DataTypes, Model} = require('sequelize') class Podcast extends Model { - constructor(values, options) { - super(values, options) + constructor(values, options) { + super(values, options) - /** @type {string} */ - this.id - /** @type {string} */ - this.title - /** @type {string} */ - this.titleIgnorePrefix - /** @type {string} */ - this.author - /** @type {string} */ - this.releaseDate - /** @type {string} */ - this.feedURL - /** @type {string} */ - this.imageURL - /** @type {string} */ - this.description - /** @type {string} */ - this.itunesPageURL - /** @type {string} */ - this.itunesId - /** @type {string} */ - this.itunesArtistId - /** @type {string} */ - this.language - /** @type {string} */ - this.podcastType - /** @type {boolean} */ - this.explicit - /** @type {boolean} */ - this.autoDownloadEpisodes - /** @type {string} */ - this.autoDownloadSchedule - /** @type {Date} */ - this.lastEpisodeCheck - /** @type {number} */ - this.maxEpisodesToKeep - /** @type {string} */ - this.coverPath - /** @type {string[]} */ - this.tags - /** @type {string[]} */ - this.genres - /** @type {Date} */ - this.createdAt - /** @type {Date} */ - this.updatedAt - /** @type {Date} */ - this.lastSuccessfulFetchAt - /** @type {boolean} */ - this.feedHealthy - } - - static getOldPodcast(libraryItemExpanded) { - const podcastExpanded = libraryItemExpanded.media - const podcastEpisodes = podcastExpanded.podcastEpisodes?.map(ep => ep.getOldPodcastEpisode(libraryItemExpanded.id).toJSON()).sort((a, b) => a.index - b.index) - return { - id: podcastExpanded.id, - libraryItemId: libraryItemExpanded.id, - metadata: { - title: podcastExpanded.title, - author: podcastExpanded.author, - description: podcastExpanded.description, - releaseDate: podcastExpanded.releaseDate, - genres: podcastExpanded.genres, - feedUrl: podcastExpanded.feedURL, - imageUrl: podcastExpanded.imageURL, - itunesPageUrl: podcastExpanded.itunesPageURL, - itunesId: podcastExpanded.itunesId, - itunesArtistId: podcastExpanded.itunesArtistId, - explicit: podcastExpanded.explicit, - language: podcastExpanded.language, - type: podcastExpanded.podcastType, - lastSuccessfulFetchAt: podcastExpanded.lastSuccessfulFetchAt?.valueOf() || null, - feedHealthy: !!podcastExpanded.feedHealthy - }, - coverPath: podcastExpanded.coverPath, - tags: podcastExpanded.tags, - episodes: podcastEpisodes || [], - autoDownloadEpisodes: podcastExpanded.autoDownloadEpisodes, - autoDownloadSchedule: podcastExpanded.autoDownloadSchedule, - lastEpisodeCheck: podcastExpanded.lastEpisodeCheck?.valueOf() || null, - maxEpisodesToKeep: podcastExpanded.maxEpisodesToKeep, - maxNewEpisodesToDownload: podcastExpanded.maxNewEpisodesToDownload + /** @type {string} */ + this.id + /** @type {string} */ + this.title + /** @type {string} */ + this.titleIgnorePrefix + /** @type {string} */ + this.author + /** @type {string} */ + this.releaseDate + /** @type {string} */ + this.feedURL + /** @type {string} */ + this.imageURL + /** @type {string} */ + this.description + /** @type {string} */ + this.itunesPageURL + /** @type {string} */ + this.itunesId + /** @type {string} */ + this.itunesArtistId + /** @type {string} */ + this.language + /** @type {string} */ + this.podcastType + /** @type {boolean} */ + this.explicit + /** @type {boolean} */ + this.autoDownloadEpisodes + /** @type {string} */ + this.autoDownloadSchedule + /** @type {Date} */ + this.lastEpisodeCheck + /** @type {number} */ + this.maxEpisodesToKeep + /** @type {string} */ + this.coverPath + /** @type {string[]} */ + this.tags + /** @type {string[]} */ + this.genres + /** @type {Date} */ + this.createdAt + /** @type {Date} */ + this.updatedAt + /** @type {Date} */ + this.lastSuccessfulFetchAt + /** @type {boolean} */ + this.feedHealthy } - } - static getFromOld(oldPodcast) { - const oldPodcastMetadata = oldPodcast.metadata - return { - id: oldPodcast.id, - title: oldPodcastMetadata.title, - titleIgnorePrefix: oldPodcastMetadata.titleIgnorePrefix, - author: oldPodcastMetadata.author, - releaseDate: oldPodcastMetadata.releaseDate, - feedURL: oldPodcastMetadata.feedUrl, - imageURL: oldPodcastMetadata.imageUrl, - description: oldPodcastMetadata.description, - itunesPageURL: oldPodcastMetadata.itunesPageUrl, - itunesId: oldPodcastMetadata.itunesId, - itunesArtistId: oldPodcastMetadata.itunesArtistId, - language: oldPodcastMetadata.language, - podcastType: oldPodcastMetadata.type, - explicit: !!oldPodcastMetadata.explicit, - autoDownloadEpisodes: !!oldPodcast.autoDownloadEpisodes, - autoDownloadSchedule: oldPodcast.autoDownloadSchedule, - lastEpisodeCheck: oldPodcast.lastEpisodeCheck, - maxEpisodesToKeep: oldPodcast.maxEpisodesToKeep, - maxNewEpisodesToDownload: oldPodcast.maxNewEpisodesToDownload, - coverPath: oldPodcast.coverPath, - tags: oldPodcast.tags, - genres: oldPodcastMetadata.genres, - lastSuccessfulFetchAt: oldPodcastMetadata.lastSuccessfulFetchAt, - feedHealthy: !!oldPodcastMetadata.feedHealthy + static getOldPodcast(libraryItemExpanded) { + const podcastExpanded = libraryItemExpanded.media + const podcastEpisodes = podcastExpanded.podcastEpisodes?.map(ep => ep.getOldPodcastEpisode(libraryItemExpanded.id).toJSON()).sort((a, b) => a.index - b.index) + return { + id: podcastExpanded.id, + libraryItemId: libraryItemExpanded.id, + metadata: { + title: podcastExpanded.title, + author: podcastExpanded.author, + description: podcastExpanded.description, + releaseDate: podcastExpanded.releaseDate, + genres: podcastExpanded.genres, + feedUrl: podcastExpanded.feedURL, + imageUrl: podcastExpanded.imageURL, + itunesPageUrl: podcastExpanded.itunesPageURL, + itunesId: podcastExpanded.itunesId, + itunesArtistId: podcastExpanded.itunesArtistId, + explicit: podcastExpanded.explicit, + language: podcastExpanded.language, + type: podcastExpanded.podcastType, + lastSuccessfulFetchAt: podcastExpanded.lastSuccessfulFetchAt?.valueOf() || null, + feedHealthy: !!podcastExpanded.feedHealthy + }, + coverPath: podcastExpanded.coverPath, + tags: podcastExpanded.tags, + episodes: podcastEpisodes || [], + autoDownloadEpisodes: podcastExpanded.autoDownloadEpisodes, + autoDownloadSchedule: podcastExpanded.autoDownloadSchedule, + lastEpisodeCheck: podcastExpanded.lastEpisodeCheck?.valueOf() || null, + maxEpisodesToKeep: podcastExpanded.maxEpisodesToKeep, + maxNewEpisodesToDownload: podcastExpanded.maxNewEpisodesToDownload + } } - } - static async getAllIncomingFeeds(){ - const podcasts = await this.findAll() - return podcasts.map(p => this.getFromOld({metadata: p.dataValues})) - } + static getFromOld(oldPodcast) { + const oldPodcastMetadata = oldPodcast.metadata + return { + id: oldPodcast.id, + title: oldPodcastMetadata.title, + titleIgnorePrefix: oldPodcastMetadata.titleIgnorePrefix, + author: oldPodcastMetadata.author, + releaseDate: oldPodcastMetadata.releaseDate, + feedURL: oldPodcastMetadata.feedUrl, + imageURL: oldPodcastMetadata.imageUrl, + description: oldPodcastMetadata.description, + itunesPageURL: oldPodcastMetadata.itunesPageUrl, + itunesId: oldPodcastMetadata.itunesId, + itunesArtistId: oldPodcastMetadata.itunesArtistId, + language: oldPodcastMetadata.language, + podcastType: oldPodcastMetadata.type, + explicit: !!oldPodcastMetadata.explicit, + autoDownloadEpisodes: !!oldPodcast.autoDownloadEpisodes, + autoDownloadSchedule: oldPodcast.autoDownloadSchedule, + lastEpisodeCheck: oldPodcast.lastEpisodeCheck, + maxEpisodesToKeep: oldPodcast.maxEpisodesToKeep, + maxNewEpisodesToDownload: oldPodcast.maxNewEpisodesToDownload, + coverPath: oldPodcast.coverPath, + tags: oldPodcast.tags, + genres: oldPodcastMetadata.genres, + lastSuccessfulFetchAt: oldPodcastMetadata.lastSuccessfulFetchAt, + feedHealthy: !!oldPodcastMetadata.feedHealthy + } + } - /** - * Initialize model - * @param {import('../Database').sequelize} sequelize - */ - static init(sequelize) { - super.init({ - id: { - type: DataTypes.UUID, - defaultValue: DataTypes.UUIDV4, - primaryKey: true - }, - title: DataTypes.STRING, - titleIgnorePrefix: DataTypes.STRING, - author: DataTypes.STRING, - releaseDate: DataTypes.STRING, - feedURL: DataTypes.STRING, - imageURL: DataTypes.STRING, - description: DataTypes.TEXT, - itunesPageURL: DataTypes.STRING, - itunesId: DataTypes.STRING, - itunesArtistId: DataTypes.STRING, - language: DataTypes.STRING, - podcastType: DataTypes.STRING, - explicit: DataTypes.BOOLEAN, + static async getAllIncomingFeeds() { + const podcasts = await this.findAll() + const podcastsFiltered = podcasts.filter(p => p.dataValues.feedURL !== null); + return podcastsFiltered.map(p => this.getOldPodcast({media: p.dataValues})) + } - autoDownloadEpisodes: DataTypes.BOOLEAN, - autoDownloadSchedule: DataTypes.STRING, - lastEpisodeCheck: DataTypes.DATE, - maxEpisodesToKeep: DataTypes.INTEGER, - maxNewEpisodesToDownload: DataTypes.INTEGER, - coverPath: DataTypes.STRING, - tags: DataTypes.JSON, - genres: DataTypes.JSON, - lastSuccessfulFetchAt: DataTypes.DATE, - feedHealthy: DataTypes.BOOLEAN - }, { - sequelize, - modelName: 'podcast' - }) - } + /** + * Initialize model + * @param {import('../Database').sequelize} sequelize + */ + static init(sequelize) { + super.init({ + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true + }, + title: DataTypes.STRING, + titleIgnorePrefix: DataTypes.STRING, + author: DataTypes.STRING, + releaseDate: DataTypes.STRING, + feedURL: DataTypes.STRING, + imageURL: DataTypes.STRING, + description: DataTypes.TEXT, + itunesPageURL: DataTypes.STRING, + itunesId: DataTypes.STRING, + itunesArtistId: DataTypes.STRING, + language: DataTypes.STRING, + podcastType: DataTypes.STRING, + explicit: DataTypes.BOOLEAN, + + autoDownloadEpisodes: DataTypes.BOOLEAN, + autoDownloadSchedule: DataTypes.STRING, + lastEpisodeCheck: DataTypes.DATE, + maxEpisodesToKeep: DataTypes.INTEGER, + maxNewEpisodesToDownload: DataTypes.INTEGER, + coverPath: DataTypes.STRING, + tags: DataTypes.JSON, + genres: DataTypes.JSON, + lastSuccessfulFetchAt: DataTypes.DATE, + feedHealthy: DataTypes.BOOLEAN + }, { + sequelize, + modelName: 'podcast' + }) + } } module.exports = Podcast diff --git a/server/objects/metadata/PodcastMetadata.js b/server/objects/metadata/PodcastMetadata.js index 2c371c6cb..3655d78c7 100644 --- a/server/objects/metadata/PodcastMetadata.js +++ b/server/objects/metadata/PodcastMetadata.js @@ -16,6 +16,8 @@ class PodcastMetadata { this.explicit = false this.language = null this.type = null + this.lastSuccessfulFetchAt = null + this.feedHealthy = null if (metadata) { this.construct(metadata) @@ -36,6 +38,8 @@ class PodcastMetadata { this.explicit = metadata.explicit this.language = metadata.language || null this.type = metadata.type || 'episodic' + this.lastSuccessfulFetchAt = metadata.lastSuccessfulFetchAt || null + this.feedHealthy = metadata.feedHealthy || null } toJSON() { @@ -52,7 +56,9 @@ class PodcastMetadata { itunesArtistId: this.itunesArtistId, explicit: this.explicit, language: this.language, - type: this.type + type: this.type, + lastSuccessfulFetchAt: this.lastSuccessfulFetchAt, + feedHealthy: this.feedHealthy } } @@ -71,7 +77,9 @@ class PodcastMetadata { itunesArtistId: this.itunesArtistId, explicit: this.explicit, language: this.language, - type: this.type + type: this.type, + lastSuccessfulFetchAt: this.lastSuccessfulFetchAt, + feedHealthy: this.feedHealthy } } @@ -120,6 +128,8 @@ class PodcastMetadata { if (mediaMetadata.genres && mediaMetadata.genres.length) { this.genres = [...mediaMetadata.genres] } + this.lastSuccessfulFetchAt = mediaMetadata.lastSuccessfulFetchAt || null + this.feedHealthy = mediaMetadata.feedHealthy || null } update(payload) { diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index f58c0d3ae..9ee2c75be 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -229,6 +229,7 @@ class ApiRouter { this.router.post('/podcasts/feed', PodcastController.getPodcastFeed.bind(this)) this.router.get('/podcasts/incomingFeeds', PodcastController.getPodcastsWithIncomingFeeds.bind(this)) this.router.get('/podcasts/:id/feed', PodcastController.middleware.bind(this), PodcastController.checkPodcastFeed.bind(this)) + this.router.get('/podcasts/:id/check-feed-url', PodcastController.checkPodcastFeedUrl.bind(this)) this.router.post('/podcasts/opml', PodcastController.getFeedsFromOPMLText.bind(this)) this.router.get('/podcasts/:id/checknew', PodcastController.middleware.bind(this), PodcastController.checkNewEpisodes.bind(this)) this.router.get('/podcasts/:id/downloads', PodcastController.middleware.bind(this), PodcastController.getEpisodeDownloads.bind(this)) From 309f5ef5e30d6ee8eefa66f502519130fb02ac4d Mon Sep 17 00:00:00 2001 From: mfcar Date: Tue, 3 Oct 2023 12:33:46 +0100 Subject: [PATCH 06/19] Update the podcastmanager --- server/managers/PodcastManager.js | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/server/managers/PodcastManager.js b/server/managers/PodcastManager.js index 45affb24f..ac63af0ab 100644 --- a/server/managers/PodcastManager.js +++ b/server/managers/PodcastManager.js @@ -285,14 +285,14 @@ class PodcastManager { Logger.error(`[PodcastManager] checkPodcastForNewEpisodes no feed url for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`) return false } - var feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl) + let feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl) if (!feed || !feed.episodes) { Logger.error(`[PodcastManager] checkPodcastForNewEpisodes invalid feed payload for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id}, URL: ${podcastLibraryItem.media.metadata.feedUrl})`, feed) return false } // Filter new and not already has - var newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url)) + let newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url)) if (maxNewEpisodes > 0) { newEpisodes = newEpisodes.slice(0, maxNewEpisodes) @@ -326,13 +326,22 @@ class PodcastManager { return newEpisodes } - setFeedHealthStatus(libraryItem, isHealthy) { - libraryItem.media.metadata.feedHealthy = isHealthy + async setFeedHealthStatus(podcastId, isHealthy) { + const podcast = await Database.podcastModel.findByPk(podcastId) + + if (!podcast) return + + podcast.feedHealthy = isHealthy if (isHealthy) { - libraryItem.media.lastSuccessfulFetchAt = Date.now() + podcast.lastSuccessfulFetchAt = Date.now() } - libraryItem.updatedAt = Date.now() - Database.updateLibraryItem(libraryItem) + podcast.lastEpisodeCheck = Date.now() + podcast.updatedAt = Date.now() + await Database.podcastModel.update(podcast, { + where: { id: podcastId } + }) + + return {lastEpisodeCheck: podcast.lastEpisodeCheck, lastSuccessfulFetchAt: podcast.lastSuccessfulFetchAt, feedHealthy: podcast.feedHealthy} } async findEpisode(rssFeedUrl, searchTitle) { @@ -408,12 +417,6 @@ class PodcastManager { queue: this.downloadQueue.filter(item => !libraryId || item.libraryId === libraryId).map(item => item.toJSONForClient()) } } - - async getPodcastsWithIncomingFeeds() { - const podcasts = await Database.models.podcast.getAllIncomingFeeds() - Logger.info(`[PodcastManager] Fetched ${podcasts.length} podcasts with incoming feeds`) - return podcasts - } } module.exports = PodcastManager From 56ff12eaaf4125fb794cf8ee6bcd3b41c3e51b1a Mon Sep 17 00:00:00 2001 From: Marcos Carvalho Date: Tue, 3 Oct 2023 22:42:34 +0100 Subject: [PATCH 07/19] Fix Podcast.js indentation --- server/models/Podcast.js | 318 +++++++++++++++++++-------------------- 1 file changed, 159 insertions(+), 159 deletions(-) diff --git a/server/models/Podcast.js b/server/models/Podcast.js index d5dd5c65a..57fb59ee7 100644 --- a/server/models/Podcast.js +++ b/server/models/Podcast.js @@ -1,171 +1,171 @@ -const {DataTypes, Model} = require('sequelize') +const { DataTypes, Model } = require('sequelize') class Podcast extends Model { - constructor(values, options) { - super(values, options) + constructor(values, options) { + super(values, options) - /** @type {string} */ - this.id - /** @type {string} */ - this.title - /** @type {string} */ - this.titleIgnorePrefix - /** @type {string} */ - this.author - /** @type {string} */ - this.releaseDate - /** @type {string} */ - this.feedURL - /** @type {string} */ - this.imageURL - /** @type {string} */ - this.description - /** @type {string} */ - this.itunesPageURL - /** @type {string} */ - this.itunesId - /** @type {string} */ - this.itunesArtistId - /** @type {string} */ - this.language - /** @type {string} */ - this.podcastType - /** @type {boolean} */ - this.explicit - /** @type {boolean} */ - this.autoDownloadEpisodes - /** @type {string} */ - this.autoDownloadSchedule - /** @type {Date} */ - this.lastEpisodeCheck - /** @type {number} */ - this.maxEpisodesToKeep - /** @type {string} */ - this.coverPath - /** @type {string[]} */ - this.tags - /** @type {string[]} */ - this.genres - /** @type {Date} */ - this.createdAt - /** @type {Date} */ - this.updatedAt - /** @type {Date} */ - this.lastSuccessfulFetchAt - /** @type {boolean} */ - this.feedHealthy + /** @type {string} */ + this.id + /** @type {string} */ + this.title + /** @type {string} */ + this.titleIgnorePrefix + /** @type {string} */ + this.author + /** @type {string} */ + this.releaseDate + /** @type {string} */ + this.feedURL + /** @type {string} */ + this.imageURL + /** @type {string} */ + this.description + /** @type {string} */ + this.itunesPageURL + /** @type {string} */ + this.itunesId + /** @type {string} */ + this.itunesArtistId + /** @type {string} */ + this.language + /** @type {string} */ + this.podcastType + /** @type {boolean} */ + this.explicit + /** @type {boolean} */ + this.autoDownloadEpisodes + /** @type {string} */ + this.autoDownloadSchedule + /** @type {Date} */ + this.lastEpisodeCheck + /** @type {number} */ + this.maxEpisodesToKeep + /** @type {string} */ + this.coverPath + /** @type {string[]} */ + this.tags + /** @type {string[]} */ + this.genres + /** @type {Date} */ + this.createdAt + /** @type {Date} */ + this.updatedAt + /** @type {Date} */ + this.lastSuccessfulFetchAt + /** @type {boolean} */ + this.feedHealthy + } + + static getOldPodcast(libraryItemExpanded) { + const podcastExpanded = libraryItemExpanded.media + const podcastEpisodes = podcastExpanded.podcastEpisodes?.map(ep => ep.getOldPodcastEpisode(libraryItemExpanded.id).toJSON()).sort((a, b) => a.index - b.index) + return { + id: podcastExpanded.id, + libraryItemId: libraryItemExpanded.id, + metadata: { + title: podcastExpanded.title, + author: podcastExpanded.author, + description: podcastExpanded.description, + releaseDate: podcastExpanded.releaseDate, + genres: podcastExpanded.genres, + feedUrl: podcastExpanded.feedURL, + imageUrl: podcastExpanded.imageURL, + itunesPageUrl: podcastExpanded.itunesPageURL, + itunesId: podcastExpanded.itunesId, + itunesArtistId: podcastExpanded.itunesArtistId, + explicit: podcastExpanded.explicit, + language: podcastExpanded.language, + type: podcastExpanded.podcastType, + lastSuccessfulFetchAt: podcastExpanded.lastSuccessfulFetchAt?.valueOf() || null, + feedHealthy: !!podcastExpanded.feedHealthy + }, + coverPath: podcastExpanded.coverPath, + tags: podcastExpanded.tags, + episodes: podcastEpisodes || [], + autoDownloadEpisodes: podcastExpanded.autoDownloadEpisodes, + autoDownloadSchedule: podcastExpanded.autoDownloadSchedule, + lastEpisodeCheck: podcastExpanded.lastEpisodeCheck?.valueOf() || null, + maxEpisodesToKeep: podcastExpanded.maxEpisodesToKeep, + maxNewEpisodesToDownload: podcastExpanded.maxNewEpisodesToDownload } + } - static getOldPodcast(libraryItemExpanded) { - const podcastExpanded = libraryItemExpanded.media - const podcastEpisodes = podcastExpanded.podcastEpisodes?.map(ep => ep.getOldPodcastEpisode(libraryItemExpanded.id).toJSON()).sort((a, b) => a.index - b.index) - return { - id: podcastExpanded.id, - libraryItemId: libraryItemExpanded.id, - metadata: { - title: podcastExpanded.title, - author: podcastExpanded.author, - description: podcastExpanded.description, - releaseDate: podcastExpanded.releaseDate, - genres: podcastExpanded.genres, - feedUrl: podcastExpanded.feedURL, - imageUrl: podcastExpanded.imageURL, - itunesPageUrl: podcastExpanded.itunesPageURL, - itunesId: podcastExpanded.itunesId, - itunesArtistId: podcastExpanded.itunesArtistId, - explicit: podcastExpanded.explicit, - language: podcastExpanded.language, - type: podcastExpanded.podcastType, - lastSuccessfulFetchAt: podcastExpanded.lastSuccessfulFetchAt?.valueOf() || null, - feedHealthy: !!podcastExpanded.feedHealthy - }, - coverPath: podcastExpanded.coverPath, - tags: podcastExpanded.tags, - episodes: podcastEpisodes || [], - autoDownloadEpisodes: podcastExpanded.autoDownloadEpisodes, - autoDownloadSchedule: podcastExpanded.autoDownloadSchedule, - lastEpisodeCheck: podcastExpanded.lastEpisodeCheck?.valueOf() || null, - maxEpisodesToKeep: podcastExpanded.maxEpisodesToKeep, - maxNewEpisodesToDownload: podcastExpanded.maxNewEpisodesToDownload - } + static getFromOld(oldPodcast) { + const oldPodcastMetadata = oldPodcast.metadata + return { + id: oldPodcast.id, + title: oldPodcastMetadata.title, + titleIgnorePrefix: oldPodcastMetadata.titleIgnorePrefix, + author: oldPodcastMetadata.author, + releaseDate: oldPodcastMetadata.releaseDate, + feedURL: oldPodcastMetadata.feedUrl, + imageURL: oldPodcastMetadata.imageUrl, + description: oldPodcastMetadata.description, + itunesPageURL: oldPodcastMetadata.itunesPageUrl, + itunesId: oldPodcastMetadata.itunesId, + itunesArtistId: oldPodcastMetadata.itunesArtistId, + language: oldPodcastMetadata.language, + podcastType: oldPodcastMetadata.type, + explicit: !!oldPodcastMetadata.explicit, + autoDownloadEpisodes: !!oldPodcast.autoDownloadEpisodes, + autoDownloadSchedule: oldPodcast.autoDownloadSchedule, + lastEpisodeCheck: oldPodcast.lastEpisodeCheck, + maxEpisodesToKeep: oldPodcast.maxEpisodesToKeep, + maxNewEpisodesToDownload: oldPodcast.maxNewEpisodesToDownload, + coverPath: oldPodcast.coverPath, + tags: oldPodcast.tags, + genres: oldPodcastMetadata.genres, + lastSuccessfulFetchAt: oldPodcastMetadata.lastSuccessfulFetchAt, + feedHealthy: !!oldPodcastMetadata.feedHealthy } + } - static getFromOld(oldPodcast) { - const oldPodcastMetadata = oldPodcast.metadata - return { - id: oldPodcast.id, - title: oldPodcastMetadata.title, - titleIgnorePrefix: oldPodcastMetadata.titleIgnorePrefix, - author: oldPodcastMetadata.author, - releaseDate: oldPodcastMetadata.releaseDate, - feedURL: oldPodcastMetadata.feedUrl, - imageURL: oldPodcastMetadata.imageUrl, - description: oldPodcastMetadata.description, - itunesPageURL: oldPodcastMetadata.itunesPageUrl, - itunesId: oldPodcastMetadata.itunesId, - itunesArtistId: oldPodcastMetadata.itunesArtistId, - language: oldPodcastMetadata.language, - podcastType: oldPodcastMetadata.type, - explicit: !!oldPodcastMetadata.explicit, - autoDownloadEpisodes: !!oldPodcast.autoDownloadEpisodes, - autoDownloadSchedule: oldPodcast.autoDownloadSchedule, - lastEpisodeCheck: oldPodcast.lastEpisodeCheck, - maxEpisodesToKeep: oldPodcast.maxEpisodesToKeep, - maxNewEpisodesToDownload: oldPodcast.maxNewEpisodesToDownload, - coverPath: oldPodcast.coverPath, - tags: oldPodcast.tags, - genres: oldPodcastMetadata.genres, - lastSuccessfulFetchAt: oldPodcastMetadata.lastSuccessfulFetchAt, - feedHealthy: !!oldPodcastMetadata.feedHealthy - } - } + static async getAllIncomingFeeds() { + const podcasts = await this.findAll() + const podcastsFiltered = podcasts.filter(p => p.dataValues.feedURL !== null); + return podcastsFiltered.map(p => this.getOldPodcast({media: p.dataValues})) + } - static async getAllIncomingFeeds() { - const podcasts = await this.findAll() - const podcastsFiltered = podcasts.filter(p => p.dataValues.feedURL !== null); - return podcastsFiltered.map(p => this.getOldPodcast({media: p.dataValues})) - } + /** + * Initialize model + * @param {import('../Database').sequelize} sequelize + */ + static init(sequelize) { + super.init({ + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true + }, + title: DataTypes.STRING, + titleIgnorePrefix: DataTypes.STRING, + author: DataTypes.STRING, + releaseDate: DataTypes.STRING, + feedURL: DataTypes.STRING, + imageURL: DataTypes.STRING, + description: DataTypes.TEXT, + itunesPageURL: DataTypes.STRING, + itunesId: DataTypes.STRING, + itunesArtistId: DataTypes.STRING, + language: DataTypes.STRING, + podcastType: DataTypes.STRING, + explicit: DataTypes.BOOLEAN, - /** - * Initialize model - * @param {import('../Database').sequelize} sequelize - */ - static init(sequelize) { - super.init({ - id: { - type: DataTypes.UUID, - defaultValue: DataTypes.UUIDV4, - primaryKey: true - }, - title: DataTypes.STRING, - titleIgnorePrefix: DataTypes.STRING, - author: DataTypes.STRING, - releaseDate: DataTypes.STRING, - feedURL: DataTypes.STRING, - imageURL: DataTypes.STRING, - description: DataTypes.TEXT, - itunesPageURL: DataTypes.STRING, - itunesId: DataTypes.STRING, - itunesArtistId: DataTypes.STRING, - language: DataTypes.STRING, - podcastType: DataTypes.STRING, - explicit: DataTypes.BOOLEAN, - - autoDownloadEpisodes: DataTypes.BOOLEAN, - autoDownloadSchedule: DataTypes.STRING, - lastEpisodeCheck: DataTypes.DATE, - maxEpisodesToKeep: DataTypes.INTEGER, - maxNewEpisodesToDownload: DataTypes.INTEGER, - coverPath: DataTypes.STRING, - tags: DataTypes.JSON, - genres: DataTypes.JSON, - lastSuccessfulFetchAt: DataTypes.DATE, - feedHealthy: DataTypes.BOOLEAN - }, { - sequelize, - modelName: 'podcast' - }) - } + autoDownloadEpisodes: DataTypes.BOOLEAN, + autoDownloadSchedule: DataTypes.STRING, + lastEpisodeCheck: DataTypes.DATE, + maxEpisodesToKeep: DataTypes.INTEGER, + maxNewEpisodesToDownload: DataTypes.INTEGER, + coverPath: DataTypes.STRING, + tags: DataTypes.JSON, + genres: DataTypes.JSON, + lastSuccessfulFetchAt: DataTypes.DATE, + feedHealthy: DataTypes.BOOLEAN + }, { + sequelize, + modelName: 'podcast' + }) + } } module.exports = Podcast From ecdafcdcc531f2773616ca8b7a41b7cd312561b1 Mon Sep 17 00:00:00 2001 From: Marcos Carvalho Date: Tue, 3 Oct 2023 23:00:03 +0100 Subject: [PATCH 08/19] Fix PodcastManager.js indentation --- server/managers/PodcastManager.js | 729 +++++++++++++++--------------- 1 file changed, 363 insertions(+), 366 deletions(-) diff --git a/server/managers/PodcastManager.js b/server/managers/PodcastManager.js index ac63af0ab..3699a735a 100644 --- a/server/managers/PodcastManager.js +++ b/server/managers/PodcastManager.js @@ -4,9 +4,9 @@ const Database = require('../Database') const fs = require('../libs/fsExtra') -const {getPodcastFeed} = require('../utils/podcastUtils') -const {removeFile, downloadFile} = require('../utils/fileUtils') -const {levenshteinDistance} = require('../utils/index') +const { getPodcastFeed } = require('../utils/podcastUtils') +const { removeFile, downloadFile } = require('../utils/fileUtils') +const { levenshteinDistance } = require('../utils/index') const opmlParser = require('../utils/parsers/parseOPML') const opmlGenerator = require('../utils/generators/opmlGenerator') const prober = require('../utils/prober') @@ -19,404 +19,401 @@ const AudioFile = require('../objects/files/AudioFile') const Task = require("../objects/Task") class PodcastManager { - constructor(watcher, notificationManager, taskManager) { - this.watcher = watcher - this.notificationManager = notificationManager - this.taskManager = taskManager + constructor(watcher, notificationManager, taskManager) { + this.watcher = watcher + this.notificationManager = notificationManager + this.taskManager = taskManager - this.downloadQueue = [] - this.currentDownload = null + this.downloadQueue = [] + this.currentDownload = null - this.failedCheckMap = {} - this.MaxFailedEpisodeChecks = 24 + this.failedCheckMap = {} + this.MaxFailedEpisodeChecks = 24 + } + + getEpisodeDownloadsInQueue(libraryItemId) { + return this.downloadQueue.filter(d => d.libraryItemId === libraryItemId) + } + + clearDownloadQueue(libraryItemId = null) { + if (!this.downloadQueue.length) return + + if (!libraryItemId) { + Logger.info(`[PodcastManager] Clearing all downloads in queue (${this.downloadQueue.length})`) + this.downloadQueue = [] + } else { + var itemDownloads = this.getEpisodeDownloadsInQueue(libraryItemId) + Logger.info(`[PodcastManager] Clearing downloads in queue for item "${libraryItemId}" (${itemDownloads.length})`) + this.downloadQueue = this.downloadQueue.filter(d => d.libraryItemId !== libraryItemId) + } + } + + async downloadPodcastEpisodes(libraryItem, episodesToDownload, isAutoDownload) { + let index = Math.max(...libraryItem.media.episodes.filter(ep => ep.index == null || isNaN(ep.index)).map(ep => Number(ep.index))) + 1 + for (const ep of episodesToDownload) { + const newPe = new PodcastEpisode() + newPe.setData(ep, index++) + newPe.libraryItemId = libraryItem.id + newPe.podcastId = libraryItem.media.id + const newPeDl = new PodcastEpisodeDownload() + newPeDl.setData(newPe, libraryItem, isAutoDownload, libraryItem.libraryId) + this.startPodcastEpisodeDownload(newPeDl) + } + } + + async startPodcastEpisodeDownload(podcastEpisodeDownload) { + SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) + if (this.currentDownload) { + this.downloadQueue.push(podcastEpisodeDownload) + SocketAuthority.emitter('episode_download_queued', podcastEpisodeDownload.toJSONForClient()) + return } - getEpisodeDownloadsInQueue(libraryItemId) { - return this.downloadQueue.filter(d => d.libraryItemId === libraryItemId) + const task = new Task() + const taskDescription = `Downloading episode "${podcastEpisodeDownload.podcastEpisode.title}".` + const taskData = { + libraryId: podcastEpisodeDownload.libraryId, + libraryItemId: podcastEpisodeDownload.libraryItemId, + } + task.setData('download-podcast-episode', 'Downloading Episode', taskDescription, false, taskData) + this.taskManager.addTask(task) + + SocketAuthority.emitter('episode_download_started', podcastEpisodeDownload.toJSONForClient()) + this.currentDownload = podcastEpisodeDownload + + // If this file already exists then append the episode id to the filename + // e.g. "/tagesschau 20 Uhr.mp3" becomes "/tagesschau 20 Uhr (ep_asdfasdf).mp3" + // this handles podcasts where every title is the same (ref https://github.com/advplyr/audiobookshelf/issues/1802) + if (await fs.pathExists(this.currentDownload.targetPath)) { + this.currentDownload.appendEpisodeId = true } - clearDownloadQueue(libraryItemId = null) { - if (!this.downloadQueue.length) return + // Ignores all added files to this dir + this.watcher.addIgnoreDir(this.currentDownload.libraryItem.path) - if (!libraryItemId) { - Logger.info(`[PodcastManager] Clearing all downloads in queue (${this.downloadQueue.length})`) - this.downloadQueue = [] - } else { - var itemDownloads = this.getEpisodeDownloadsInQueue(libraryItemId) - Logger.info(`[PodcastManager] Clearing downloads in queue for item "${libraryItemId}" (${itemDownloads.length})`) - this.downloadQueue = this.downloadQueue.filter(d => d.libraryItemId !== libraryItemId) - } + // Make sure podcast library item folder exists + if (!(await fs.pathExists(this.currentDownload.libraryItem.path))) { + Logger.warn(`[PodcastManager] Podcast episode download: Podcast folder no longer exists at "${this.currentDownload.libraryItem.path}" - Creating it`) + await fs.mkdir(this.currentDownload.libraryItem.path) } - async downloadPodcastEpisodes(libraryItem, episodesToDownload, isAutoDownload) { - let index = Math.max(...libraryItem.media.episodes.filter(ep => ep.index == null || isNaN(ep.index)).map(ep => Number(ep.index))) + 1 - for (const ep of episodesToDownload) { - const newPe = new PodcastEpisode() - newPe.setData(ep, index++) - newPe.libraryItemId = libraryItem.id - newPe.podcastId = libraryItem.media.id - const newPeDl = new PodcastEpisodeDownload() - newPeDl.setData(newPe, libraryItem, isAutoDownload, libraryItem.libraryId) - this.startPodcastEpisodeDownload(newPeDl) - } - } - - async startPodcastEpisodeDownload(podcastEpisodeDownload) { - SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) - if (this.currentDownload) { - this.downloadQueue.push(podcastEpisodeDownload) - SocketAuthority.emitter('episode_download_queued', podcastEpisodeDownload.toJSONForClient()) - return - } - - const task = new Task() - const taskDescription = `Downloading episode "${podcastEpisodeDownload.podcastEpisode.title}".` - const taskData = { - libraryId: podcastEpisodeDownload.libraryId, - libraryItemId: podcastEpisodeDownload.libraryItemId, - } - task.setData('download-podcast-episode', 'Downloading Episode', taskDescription, false, taskData) - this.taskManager.addTask(task) - - SocketAuthority.emitter('episode_download_started', podcastEpisodeDownload.toJSONForClient()) - this.currentDownload = podcastEpisodeDownload - - // If this file already exists then append the episode id to the filename - // e.g. "/tagesschau 20 Uhr.mp3" becomes "/tagesschau 20 Uhr (ep_asdfasdf).mp3" - // this handles podcasts where every title is the same (ref https://github.com/advplyr/audiobookshelf/issues/1802) - if (await fs.pathExists(this.currentDownload.targetPath)) { - this.currentDownload.appendEpisodeId = true - } - - // Ignores all added files to this dir - this.watcher.addIgnoreDir(this.currentDownload.libraryItem.path) - - // Make sure podcast library item folder exists - if (!(await fs.pathExists(this.currentDownload.libraryItem.path))) { - Logger.warn(`[PodcastManager] Podcast episode download: Podcast folder no longer exists at "${this.currentDownload.libraryItem.path}" - Creating it`) - await fs.mkdir(this.currentDownload.libraryItem.path) - } - - let success = false - if (this.currentDownload.urlFileExtension === 'mp3') { - // Download episode and tag it - success = await ffmpegHelpers.downloadPodcastEpisode(this.currentDownload).catch((error) => { - Logger.error(`[PodcastManager] Podcast Episode download failed`, error) - return false - }) - } else { - // Download episode only - success = await downloadFile(this.currentDownload.url, this.currentDownload.targetPath).then(() => true).catch((error) => { - Logger.error(`[PodcastManager] Podcast Episode download failed`, error) - return false - }) - } - - if (success) { - success = await this.scanAddPodcastEpisodeAudioFile() - if (!success) { - await fs.remove(this.currentDownload.targetPath) - this.currentDownload.setFinished(false) - task.setFailed('Failed to download episode') - } else { - Logger.info(`[PodcastManager] Successfully downloaded podcast episode "${this.currentDownload.podcastEpisode.title}"`) - this.currentDownload.setFinished(true) - task.setFinished() - } - } else { - task.setFailed('Failed to download episode') - this.currentDownload.setFinished(false) - } - - this.taskManager.taskFinished(task) - - SocketAuthority.emitter('episode_download_finished', this.currentDownload.toJSONForClient()) - SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) - - this.watcher.removeIgnoreDir(this.currentDownload.libraryItem.path) - this.currentDownload = null - if (this.downloadQueue.length) { - this.startPodcastEpisodeDownload(this.downloadQueue.shift()) - } - } - - async scanAddPodcastEpisodeAudioFile() { - const libraryFile = await this.getLibraryFile(this.currentDownload.targetPath, this.currentDownload.targetRelPath) - - const audioFile = await this.probeAudioFile(libraryFile) - if (!audioFile) { - return false - } - - const libraryItem = await Database.libraryItemModel.getOldById(this.currentDownload.libraryItem.id) - if (!libraryItem) { - Logger.error(`[PodcastManager] Podcast Episode finished but library item was not found ${this.currentDownload.libraryItem.id}`) - return false - } - - const podcastEpisode = this.currentDownload.podcastEpisode - podcastEpisode.audioFile = audioFile - - if (audioFile.chapters?.length) { - podcastEpisode.chapters = audioFile.chapters.map(ch => ({...ch})) - } - - libraryItem.media.addPodcastEpisode(podcastEpisode) - if (libraryItem.isInvalid) { - // First episode added to an empty podcast - libraryItem.isInvalid = false - } - libraryItem.libraryFiles.push(libraryFile) - - if (this.currentDownload.isAutoDownload) { - // Check setting maxEpisodesToKeep and remove episode if necessary - if (libraryItem.media.maxEpisodesToKeep && libraryItem.media.episodesWithPubDate.length > libraryItem.media.maxEpisodesToKeep) { - Logger.info(`[PodcastManager] # of episodes (${libraryItem.media.episodesWithPubDate.length}) exceeds max episodes to keep (${libraryItem.media.maxEpisodesToKeep})`) - await this.removeOldestEpisode(libraryItem, podcastEpisode.id) - } - } - - libraryItem.updatedAt = Date.now() - await Database.updateLibraryItem(libraryItem) - SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) - const podcastEpisodeExpanded = podcastEpisode.toJSONExpanded() - podcastEpisodeExpanded.libraryItem = libraryItem.toJSONExpanded() - SocketAuthority.emitter('episode_added', podcastEpisodeExpanded) - - if (this.currentDownload.isAutoDownload) { // Notifications only for auto downloaded episodes - this.notificationManager.onPodcastEpisodeDownloaded(libraryItem, podcastEpisode) - } - - return true - } - - async removeOldestEpisode(libraryItem, episodeIdJustDownloaded) { - var smallestPublishedAt = 0 - var oldestEpisode = null - libraryItem.media.episodesWithPubDate.filter(ep => ep.id !== episodeIdJustDownloaded).forEach((ep) => { - if (!smallestPublishedAt || ep.publishedAt < smallestPublishedAt) { - smallestPublishedAt = ep.publishedAt - oldestEpisode = ep - } - }) - // TODO: Should we check for open playback sessions for this episode? - // TODO: remove all user progress for this episode - if (oldestEpisode && oldestEpisode.audioFile) { - Logger.info(`[PodcastManager] Deleting oldest episode "${oldestEpisode.title}"`) - const successfullyDeleted = await removeFile(oldestEpisode.audioFile.metadata.path) - if (successfullyDeleted) { - libraryItem.media.removeEpisode(oldestEpisode.id) - libraryItem.removeLibraryFile(oldestEpisode.audioFile.ino) - return true - } else { - Logger.warn(`[PodcastManager] Failed to remove oldest episode "${oldestEpisode.title}"`) - } - } + let success = false + if (this.currentDownload.urlFileExtension === 'mp3') { + // Download episode and tag it + success = await ffmpegHelpers.downloadPodcastEpisode(this.currentDownload).catch((error) => { + Logger.error(`[PodcastManager] Podcast Episode download failed`, error) return false + }) + } else { + // Download episode only + success = await downloadFile(this.currentDownload.url, this.currentDownload.targetPath).then(() => true).catch((error) => { + Logger.error(`[PodcastManager] Podcast Episode download failed`, error) + return false + }) } - async getLibraryFile(path, relPath) { - var newLibFile = new LibraryFile() - await newLibFile.setDataFromPath(path, relPath) - return newLibFile + if (success) { + success = await this.scanAddPodcastEpisodeAudioFile() + if (!success) { + await fs.remove(this.currentDownload.targetPath) + this.currentDownload.setFinished(false) + task.setFailed('Failed to download episode') + } else { + Logger.info(`[PodcastManager] Successfully downloaded podcast episode "${this.currentDownload.podcastEpisode.title}"`) + this.currentDownload.setFinished(true) + task.setFinished() + } + } else { + task.setFailed('Failed to download episode') + this.currentDownload.setFinished(false) } - async probeAudioFile(libraryFile) { - const path = libraryFile.metadata.path - const mediaProbeData = await prober.probe(path) - if (mediaProbeData.error) { - Logger.error(`[PodcastManager] Podcast Episode downloaded but failed to probe "${path}"`, mediaProbeData.error) - return false - } - const newAudioFile = new AudioFile() - newAudioFile.setDataFromProbe(libraryFile, mediaProbeData) - newAudioFile.index = 1 - return newAudioFile + this.taskManager.taskFinished(task) + + SocketAuthority.emitter('episode_download_finished', this.currentDownload.toJSONForClient()) + SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) + + this.watcher.removeIgnoreDir(this.currentDownload.libraryItem.path) + this.currentDownload = null + if (this.downloadQueue.length) { + this.startPodcastEpisodeDownload(this.downloadQueue.shift()) + } + } + + async scanAddPodcastEpisodeAudioFile() { + const libraryFile = await this.getLibraryFile(this.currentDownload.targetPath, this.currentDownload.targetRelPath) + + const audioFile = await this.probeAudioFile(libraryFile) + if (!audioFile) { + return false } - // Returns false if auto download episodes was disabled (disabled if reaches max failed checks) - async runEpisodeCheck(libraryItem) { - const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) - const latestEpisodePublishedAt = libraryItem.media.latestEpisodePublished - Logger.info(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" | Last check: ${lastEpisodeCheckDate} | ${latestEpisodePublishedAt ? `Latest episode pubDate: ${new Date(latestEpisodePublishedAt)}` : 'No latest episode'}`) - - // Use latest episode pubDate if exists OR fallback to using lastEpisodeCheckDate - // lastEpisodeCheckDate will be the current time when adding a new podcast - const dateToCheckForEpisodesAfter = latestEpisodePublishedAt || lastEpisodeCheckDate - Logger.debug(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" checking for episodes after ${new Date(dateToCheckForEpisodesAfter)}`) - - let newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, dateToCheckForEpisodesAfter, libraryItem.media.maxNewEpisodesToDownload) - Logger.debug(`[PodcastManager] runEpisodeCheck: ${newEpisodes ? newEpisodes.length : 'N/A'} episodes found`) - - if (!newEpisodes) { // Failed - // Allow up to MaxFailedEpisodeChecks failed attempts before disabling auto download - if (!this.failedCheckMap[libraryItem.id]) this.failedCheckMap[libraryItem.id] = 0 - this.failedCheckMap[libraryItem.id]++ - if (this.failedCheckMap[libraryItem.id] >= this.MaxFailedEpisodeChecks) { - Logger.error(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}" - disabling auto download`) - libraryItem.media.autoDownloadEpisodes = false - delete this.failedCheckMap[libraryItem.id] - } else { - Logger.warn(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}"`) - } - libraryItem.media.metadata.feedHealthy = false - } else if (newEpisodes.length) { - delete this.failedCheckMap[libraryItem.id] - Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) - this.downloadPodcastEpisodes(libraryItem, newEpisodes, true) - libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() - libraryItem.media.metadata.feedHealthy = true - } else { - delete this.failedCheckMap[libraryItem.id] - Logger.debug(`[PodcastManager] No new episodes for "${libraryItem.media.metadata.title}"`) - libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() - libraryItem.media.metadata.feedHealthy = true - } - - libraryItem.media.lastEpisodeCheck = Date.now() - libraryItem.updatedAt = Date.now() - await Database.updateLibraryItem(libraryItem) - SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) - return libraryItem.media.autoDownloadEpisodes + const libraryItem = await Database.libraryItemModel.getOldById(this.currentDownload.libraryItem.id) + if (!libraryItem) { + Logger.error(`[PodcastManager] Podcast Episode finished but library item was not found ${this.currentDownload.libraryItem.id}`) + return false } - async checkPodcastForNewEpisodes(podcastLibraryItem, dateToCheckForEpisodesAfter, maxNewEpisodes = 3) { - if (!podcastLibraryItem.media.metadata.feedUrl) { - Logger.error(`[PodcastManager] checkPodcastForNewEpisodes no feed url for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`) - return false - } - let feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl) - if (!feed || !feed.episodes) { - Logger.error(`[PodcastManager] checkPodcastForNewEpisodes invalid feed payload for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id}, URL: ${podcastLibraryItem.media.metadata.feedUrl})`, feed) - return false - } + const podcastEpisode = this.currentDownload.podcastEpisode + podcastEpisode.audioFile = audioFile - // Filter new and not already has - let newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url)) - - if (maxNewEpisodes > 0) { - newEpisodes = newEpisodes.slice(0, maxNewEpisodes) - } - - return newEpisodes + if (audioFile.chapters?.length) { + podcastEpisode.chapters = audioFile.chapters.map(ch => ({ ...ch })) } - async checkAndDownloadNewEpisodes(libraryItem, maxEpisodesToDownload) { - const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) - Logger.info(`[PodcastManager] checkAndDownloadNewEpisodes for "${libraryItem.media.metadata.title}" - Last episode check: ${lastEpisodeCheckDate}`) - let newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, libraryItem.media.lastEpisodeCheck, maxEpisodesToDownload) - if (!newEpisodes) { - libraryItem.media.metadata.feedHealthy = false - } else if (newEpisodes.length) { - Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) - this.downloadPodcastEpisodes(libraryItem, newEpisodes, false) - libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() - libraryItem.media.metadata.feedHealthy = true - } else { - Logger.info(`[PodcastManager] No new episodes found for podcast "${libraryItem.media.metadata.title}"`) - libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() - libraryItem.media.metadata.feedHealthy = true - } + libraryItem.media.addPodcastEpisode(podcastEpisode) + if (libraryItem.isInvalid) { + // First episode added to an empty podcast + libraryItem.isInvalid = false + } + libraryItem.libraryFiles.push(libraryFile) - libraryItem.media.lastEpisodeCheck = Date.now() - libraryItem.updatedAt = Date.now() - await Database.updateLibraryItem(libraryItem) - SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) - - return newEpisodes + if (this.currentDownload.isAutoDownload) { + // Check setting maxEpisodesToKeep and remove episode if necessary + if (libraryItem.media.maxEpisodesToKeep && libraryItem.media.episodesWithPubDate.length > libraryItem.media.maxEpisodesToKeep) { + Logger.info(`[PodcastManager] # of episodes (${libraryItem.media.episodesWithPubDate.length}) exceeds max episodes to keep (${libraryItem.media.maxEpisodesToKeep})`) + await this.removeOldestEpisode(libraryItem, podcastEpisode.id) + } } - async setFeedHealthStatus(podcastId, isHealthy) { - const podcast = await Database.podcastModel.findByPk(podcastId) + libraryItem.updatedAt = Date.now() + await Database.updateLibraryItem(libraryItem) + SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) + const podcastEpisodeExpanded = podcastEpisode.toJSONExpanded() + podcastEpisodeExpanded.libraryItem = libraryItem.toJSONExpanded() + SocketAuthority.emitter('episode_added', podcastEpisodeExpanded) - if (!podcast) return + if (this.currentDownload.isAutoDownload) { // Notifications only for auto downloaded episodes + this.notificationManager.onPodcastEpisodeDownloaded(libraryItem, podcastEpisode) + } - podcast.feedHealthy = isHealthy - if (isHealthy) { - podcast.lastSuccessfulFetchAt = Date.now() - } - podcast.lastEpisodeCheck = Date.now() - podcast.updatedAt = Date.now() - await Database.podcastModel.update(podcast, { - where: { id: podcastId } + return true + } + + async removeOldestEpisode(libraryItem, episodeIdJustDownloaded) { + var smallestPublishedAt = 0 + var oldestEpisode = null + libraryItem.media.episodesWithPubDate.filter(ep => ep.id !== episodeIdJustDownloaded).forEach((ep) => { + if (!smallestPublishedAt || ep.publishedAt < smallestPublishedAt) { + smallestPublishedAt = ep.publishedAt + oldestEpisode = ep + } + }) + // TODO: Should we check for open playback sessions for this episode? + // TODO: remove all user progress for this episode + if (oldestEpisode && oldestEpisode.audioFile) { + Logger.info(`[PodcastManager] Deleting oldest episode "${oldestEpisode.title}"`) + const successfullyDeleted = await removeFile(oldestEpisode.audioFile.metadata.path) + if (successfullyDeleted) { + libraryItem.media.removeEpisode(oldestEpisode.id) + libraryItem.removeLibraryFile(oldestEpisode.audioFile.ino) + return true + } else { + Logger.warn(`[PodcastManager] Failed to remove oldest episode "${oldestEpisode.title}"`) + } + } + return false + } + + async getLibraryFile(path, relPath) { + var newLibFile = new LibraryFile() + await newLibFile.setDataFromPath(path, relPath) + return newLibFile + } + + async probeAudioFile(libraryFile) { + const path = libraryFile.metadata.path + const mediaProbeData = await prober.probe(path) + if (mediaProbeData.error) { + Logger.error(`[PodcastManager] Podcast Episode downloaded but failed to probe "${path}"`, mediaProbeData.error) + return false + } + const newAudioFile = new AudioFile() + newAudioFile.setDataFromProbe(libraryFile, mediaProbeData) + newAudioFile.index = 1 + return newAudioFile + } + + // Returns false if auto download episodes was disabled (disabled if reaches max failed checks) + async runEpisodeCheck(libraryItem) { + const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) + const latestEpisodePublishedAt = libraryItem.media.latestEpisodePublished + Logger.info(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" | Last check: ${lastEpisodeCheckDate} | ${latestEpisodePublishedAt ? `Latest episode pubDate: ${new Date(latestEpisodePublishedAt)}` : 'No latest episode'}`) + + // Use latest episode pubDate if exists OR fallback to using lastEpisodeCheckDate + // lastEpisodeCheckDate will be the current time when adding a new podcast + const dateToCheckForEpisodesAfter = latestEpisodePublishedAt || lastEpisodeCheckDate + Logger.debug(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" checking for episodes after ${new Date(dateToCheckForEpisodesAfter)}`) + + let newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, dateToCheckForEpisodesAfter, libraryItem.media.maxNewEpisodesToDownload) + Logger.debug(`[PodcastManager] runEpisodeCheck: ${newEpisodes ? newEpisodes.length : 'N/A'} episodes found`) + + if (!newEpisodes) { // Failed + // Allow up to MaxFailedEpisodeChecks failed attempts before disabling auto download + if (!this.failedCheckMap[libraryItem.id]) this.failedCheckMap[libraryItem.id] = 0 + this.failedCheckMap[libraryItem.id]++ + if (this.failedCheckMap[libraryItem.id] >= this.MaxFailedEpisodeChecks) { + Logger.error(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}" - disabling auto download`) + libraryItem.media.autoDownloadEpisodes = false + delete this.failedCheckMap[libraryItem.id] + } else { + Logger.warn(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}"`) + } + libraryItem.media.metadata.feedHealthy = false + } else if (newEpisodes.length) { + delete this.failedCheckMap[libraryItem.id] + Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) + this.downloadPodcastEpisodes(libraryItem, newEpisodes, true) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } else { + delete this.failedCheckMap[libraryItem.id] + Logger.debug(`[PodcastManager] No new episodes for "${libraryItem.media.metadata.title}"`) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } + + libraryItem.media.lastEpisodeCheck = Date.now() + libraryItem.updatedAt = Date.now() + await Database.updateLibraryItem(libraryItem) + SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) + return libraryItem.media.autoDownloadEpisodes + } + + async checkPodcastForNewEpisodes(podcastLibraryItem, dateToCheckForEpisodesAfter, maxNewEpisodes = 3) { + if (!podcastLibraryItem.media.metadata.feedUrl) { + Logger.error(`[PodcastManager] checkPodcastForNewEpisodes no feed url for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`) + return false + } + let feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl) + if (!feed || !feed.episodes) { + Logger.error(`[PodcastManager] checkPodcastForNewEpisodes invalid feed payload for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id}, URL: ${podcastLibraryItem.media.metadata.feedUrl})`, feed) + return false + } + + // Filter new and not already has + let newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url)) + + if (maxNewEpisodes > 0) { + newEpisodes = newEpisodes.slice(0, maxNewEpisodes) + } + + return newEpisodes + } + + async checkAndDownloadNewEpisodes(libraryItem, maxEpisodesToDownload) { + const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) + Logger.info(`[PodcastManager] checkAndDownloadNewEpisodes for "${libraryItem.media.metadata.title}" - Last episode check: ${lastEpisodeCheckDate}`) + let newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, libraryItem.media.lastEpisodeCheck, maxEpisodesToDownload) + if (!newEpisodes) { + libraryItem.media.metadata.feedHealthy = false + } else if (newEpisodes.length) { + Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) + this.downloadPodcastEpisodes(libraryItem, newEpisodes, false) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } else { + Logger.info(`[PodcastManager] No new episodes found for podcast "${libraryItem.media.metadata.title}"`) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } + + libraryItem.media.lastEpisodeCheck = Date.now() + libraryItem.updatedAt = Date.now() + await Database.updateLibraryItem(libraryItem) + SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) + + return newEpisodes + } + + async setFeedHealthStatus(podcastId, isHealthy) { + const podcast = await Database.podcastModel.findByPk(podcastId) + + if (!podcast) return + + podcast.feedHealthy = isHealthy + if (isHealthy) { + podcast.lastSuccessfulFetchAt = Date.now() + } + podcast.lastEpisodeCheck = Date.now() + podcast.updatedAt = Date.now() + await Database.podcastModel.update(podcast, {where: { id: podcastId }}) + + return {lastEpisodeCheck: podcast.lastEpisodeCheck, lastSuccessfulFetchAt: podcast.lastSuccessfulFetchAt, feedHealthy: podcast.feedHealthy} + } + + async findEpisode(rssFeedUrl, searchTitle) { + const feed = await getPodcastFeed(rssFeedUrl).catch(() => { + return null + }) + if (!feed || !feed.episodes) { + return null + } + + const matches = [] + feed.episodes.forEach(ep => { + if (!ep.title) return + + const epTitle = ep.title.toLowerCase().trim() + if (epTitle === searchTitle) { + matches.push({ + episode: ep, + levenshtein: 0 }) + } else { + const levenshtein = levenshteinDistance(searchTitle, epTitle, true) + if (levenshtein <= 6 && epTitle.length > levenshtein) { + matches.push({ + episode: ep, + levenshtein + }) + } + } + }) + return matches.sort((a, b) => a.levenshtein - b.levenshtein) + } - return {lastEpisodeCheck: podcast.lastEpisodeCheck, lastSuccessfulFetchAt: podcast.lastSuccessfulFetchAt, feedHealthy: podcast.feedHealthy} + async getOPMLFeeds(opmlText) { + var extractedFeeds = opmlParser.parse(opmlText) + if (!extractedFeeds || !extractedFeeds.length) { + Logger.error('[PodcastManager] getOPMLFeeds: No RSS feeds found in OPML') + return { + error: 'No RSS feeds found in OPML' + } } - async findEpisode(rssFeedUrl, searchTitle) { - const feed = await getPodcastFeed(rssFeedUrl).catch(() => { - return null - }) - if (!feed || !feed.episodes) { - return null - } + var rssFeedData = [] - const matches = [] - feed.episodes.forEach(ep => { - if (!ep.title) return - - const epTitle = ep.title.toLowerCase().trim() - if (epTitle === searchTitle) { - matches.push({ - episode: ep, - levenshtein: 0 - }) - } else { - const levenshtein = levenshteinDistance(searchTitle, epTitle, true) - if (levenshtein <= 6 && epTitle.length > levenshtein) { - matches.push({ - episode: ep, - levenshtein - }) - } - } - }) - return matches.sort((a, b) => a.levenshtein - b.levenshtein) + for (let feed of extractedFeeds) { + var feedData = await getPodcastFeed(feed.feedUrl, true) + if (feedData) { + feedData.metadata.feedUrl = feed.feedUrl + rssFeedData.push(feedData) + } } - async getOPMLFeeds(opmlText) { - var extractedFeeds = opmlParser.parse(opmlText) - if (!extractedFeeds || !extractedFeeds.length) { - Logger.error('[PodcastManager] getOPMLFeeds: No RSS feeds found in OPML') - return { - error: 'No RSS feeds found in OPML' - } - } - - var rssFeedData = [] - - for (let feed of extractedFeeds) { - var feedData = await getPodcastFeed(feed.feedUrl, true) - if (feedData) { - feedData.metadata.feedUrl = feed.feedUrl - rssFeedData.push(feedData) - } - } - - return { - feeds: rssFeedData - } + return { + feeds: rssFeedData } + } - /** - * OPML file string for podcasts in a library - * @param {import('../models/Podcast')[]} podcasts - * @returns {string} XML string - */ - generateOPMLFileText(podcasts) { - return opmlGenerator.generate(podcasts) - } - - getDownloadQueueDetails(libraryId = null) { - let _currentDownload = this.currentDownload - if (libraryId && _currentDownload?.libraryId !== libraryId) _currentDownload = null - - return { - currentDownload: _currentDownload?.toJSONForClient(), - queue: this.downloadQueue.filter(item => !libraryId || item.libraryId === libraryId).map(item => item.toJSONForClient()) - } + /** + * OPML file string for podcasts in a library + * @param {import('../models/Podcast')[]} podcasts + * @returns {string} XML string + */ + generateOPMLFileText(podcasts) { + return opmlGenerator.generate(podcasts) + } + + getDownloadQueueDetails(libraryId = null) { + let _currentDownload = this.currentDownload + if (libraryId && _currentDownload?.libraryId !== libraryId) _currentDownload = null + + return { + currentDownload: _currentDownload?.toJSONForClient(), + queue: this.downloadQueue.filter(item => !libraryId || item.libraryId === libraryId).map(item => item.toJSONForClient()) } + } } - module.exports = PodcastManager From 2d98c27e62113fad9217c9d10447406274742839 Mon Sep 17 00:00:00 2001 From: mfcar Date: Wed, 21 Feb 2024 20:55:53 +0000 Subject: [PATCH 09/19] Working on the rss --- client/pages/config/rss-feeds.vue | 18 ++++++++++++-- .../queries/libraryItemsPodcastFilters.js | 24 ++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/client/pages/config/rss-feeds.vue b/client/pages/config/rss-feeds.vue index 79be2d87a..80a8763d2 100644 --- a/client/pages/config/rss-feeds.vue +++ b/client/pages/config/rss-feeds.vue @@ -15,13 +15,13 @@ class="w-1/2 h-8 rounded-tl-md relative border border-black-200 flex items-center justify-center cursor-pointer" :class="!showIncomingFeedsView ? 'text-white bg-bg hover:bg-opacity-60 border-b-bg' : 'text-gray-400 hover:text-gray-300 bg-primary bg-opacity-70 hover:bg-opacity-60'" @click="showIncomingFeedsView = false"> -

Opened Feeds

+

{{$strings.HeaderOpenedRSSFeeds}}

-

Subscribed Feeds

+

{{$strings.HeaderExternalFeedURLHealthChecker}}

@@ -45,6 +45,7 @@ {{ $strings.LabelFeedLastChecked }} {{ $strings.LabelFeedLastSuccessfulCheck }} {{ $strings.LabelFeedHealthy }} + {{ $strings.LabelFeedAutoDownloadEnabled}} {{ $strings.LabelFeedNextAutomaticCheck }} @@ -84,6 +85,11 @@ + + check + close + + @@ -113,6 +119,10 @@ + +
+ 
{{incomingFeedsList}}
+