Fix:Large OPML import timeouts #3118

- Added OPML Api endpoints for /parse and /create, removed old - Show task for OPML import and create failed tasks for failed feeds
2024-12-20 19:06:06 +01:00 · 2024-07-16 17:05:52 -05:00 · 2024-07-16 17:05:52 -05:00 · 37ad1cced2
commit 37ad1cced2
parent b1bc472205
9 changed files with 258 additions and 93 deletions
--- a/client/components/modals/podcast/OpmlFeedsModal.vue
+++ b/client/components/modals/podcast/OpmlFeedsModal.vue
@ -16,11 +16,18 @@
          </div>
        </div>

-        <p class="text-lg font-semibold mb-2">{{ $strings.HeaderPodcastsToAdd }}</p>
+        <p class="text-lg font-semibold mb-1">{{ $strings.HeaderPodcastsToAdd }}</p>
+        <p class="text-sm text-gray-300 mb-4">{{ $strings.MessageOpmlPreviewNote }}</p>

        <div class="w-full overflow-y-auto" style="max-height: 50vh">
-          <template v-for="(feed, index) in feedMetadata">
-            <cards-podcast-feed-summary-card :key="index" :feed="feed" :library-folder-path="selectedFolderPath" class="my-1" />
+          <template v-for="(feed, index) in feeds">
+            <div :key="index" class="py-1 flex items-center">
+              <p class="text-lg font-semibold">{{ index + 1 }}.</p>
+              <div class="pl-2">
+                <p v-if="feed.title" class="text-sm font-semibold">{{ feed.title }}</p>
+                <p class="text-xs text-gray-400">{{ feed.feedUrl }}</p>
+              </div>
+            </div>
          </template>
        </div>
      </div>
@ -45,9 +52,7 @@ export default {
    return {
      processing: false,
      selectedFolderId: null,
-      fullPath: null,
-      autoDownloadEpisodes: false,
-      feedMetadata: []
+      autoDownloadEpisodes: false
    }
  },
  watch: {
@ -96,73 +101,36 @@ export default {
    }
  },
  methods: {
-    toFeedMetadata(feed) {
-      const metadata = feed.metadata
-      return {
-        title: metadata.title,
-        author: metadata.author,
-        description: metadata.description,
-        releaseDate: '',
-        genres: [...metadata.categories],
-        feedUrl: metadata.feedUrl,
-        imageUrl: metadata.image,
-        itunesPageUrl: '',
-        itunesId: '',
-        itunesArtistId: '',
-        language: '',
-        numEpisodes: feed.numEpisodes
-      }
-    },
    init() {
-      this.feedMetadata = this.feeds.map(this.toFeedMetadata)
-
      if (this.folderItems[0]) {
        this.selectedFolderId = this.folderItems[0].value
      }
    },
    async submit() {
      this.processing = true
-      const newFeedPayloads = this.feedMetadata.map((metadata) => {
-        return {
-          path: `${this.selectedFolderPath}/${this.$sanitizeFilename(metadata.title)}`,
-          folderId: this.selectedFolderId,
-          libraryId: this.currentLibrary.id,
-          media: {
-            metadata: {
-              ...metadata
-            },
-            autoDownloadEpisodes: this.autoDownloadEpisodes
-          }
-        }
-      })
-      console.log('New feed payloads', newFeedPayloads)

-      for (const podcastPayload of newFeedPayloads) {
-        await this.$axios
-          .$post('/api/podcasts', podcastPayload)
-          .then(() => {
-            this.$toast.success(`${podcastPayload.media.metadata.title}: ${this.$strings.ToastPodcastCreateSuccess}`)
-          })
-          .catch((error) => {
-            var errorMsg = error.response && error.response.data ? error.response.data : this.$strings.ToastPodcastCreateFailed
-            console.error('Failed to create podcast', podcastPayload, error)
-            this.$toast.error(`${podcastPayload.media.metadata.title}: ${errorMsg}`)
-          })
+      const payload = {
+        feeds: this.feeds.map((f) => f.feedUrl),
+        folderId: this.selectedFolderId,
+        libraryId: this.currentLibrary.id,
+        autoDownloadEpisodes: this.autoDownloadEpisodes
      }
-      this.processing = false
-      this.show = false
+      this.$axios
+        .$post('/api/podcasts/opml/create', payload)
+        .then(() => {
+          this.show = false
+        })
+        .catch((error) => {
+          const errorMsg = error.response?.data || this.$strings.ToastPodcastCreateFailed
+          console.error('Failed to create podcast', payload, error)
+          this.$toast.error(errorMsg)
+        })
+        .finally(() => {
+          this.processing = false
+        })
    }
  },
  mounted() {}
 }
 </script>

-<style scoped>
-#podcast-wrapper {
-  min-height: 400px;
-  max-height: 80vh;
-}
-#episodes-scroll {
-  max-height: calc(80vh - 200px);
-}
-</style>
--- a/client/pages/library/_library/podcast/search.vue
+++ b/client/pages/library/_library/podcast/search.vue
@ -113,18 +113,23 @@ export default {
        return
      }

-      await this.$axios
-        .$post(`/api/podcasts/opml`, { opmlText: txt })
+      this.$axios
+        .$post(`/api/podcasts/opml/parse`, { opmlText: txt })
        .then((data) => {
-          console.log(data)
-          this.opmlFeeds = data.feeds || []
-          this.showOPMLFeedsModal = true
+          if (!data.feeds?.length) {
+            this.$toast.error('No feeds found in OPML file')
+          } else {
+            this.opmlFeeds = data.feeds || []
+            this.showOPMLFeedsModal = true
+          }
        })
        .catch((error) => {
          console.error('Failed', error)
          this.$toast.error('Failed to parse OPML file')
        })
-      this.processing = false
+        .finally(() => {
+          this.processing = false
+        })
    },
    submit() {
      if (!this.searchInput) return
--- a/client/strings/en-us.json
+++ b/client/strings/en-us.json
@ -722,6 +722,7 @@
  "MessageNoUpdatesWereNecessary": "No updates were necessary",
  "MessageNoUserPlaylists": "You have no playlists",
  "MessageNotYetImplemented": "Not yet implemented",
+  "MessageOpmlPreviewNote": "Note: This is a preview of the parsed OPML file. The actual podcast title will be taken from the RSS feed.",
  "MessageOr": "or",
  "MessagePauseChapter": "Pause chapter playback",
  "MessagePlayChapter": "Listen to beginning of chapter",
--- a/server/controllers/PodcastController.js
+++ b/server/controllers/PodcastController.js
@ -14,6 +14,15 @@ const CoverManager = require('../managers/CoverManager')
 const LibraryItem = require('../objects/LibraryItem')

 class PodcastController {
+  /**
+   * POST /api/podcasts
+   * Create podcast
+   *
+   * @this import('../routers/ApiRouter')
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
  async create(req, res) {
    if (!req.user.isAdminOrUp) {
      Logger.error(`[PodcastController] Non-admin user "${req.user.username}" attempted to create podcast`)
@ -133,6 +142,14 @@ class PodcastController {
    res.json({ podcast })
  }

+  /**
+   * POST: /api/podcasts/opml
+   *
+   * @this import('../routers/ApiRouter')
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
  async getFeedsFromOPMLText(req, res) {
    if (!req.user.isAdminOrUp) {
      Logger.error(`[PodcastController] Non-admin user "${req.user.username}" attempted to get feeds from opml`)
@ -143,8 +160,44 @@ class PodcastController {
      return res.sendStatus(400)
    }

-    const rssFeedsData = await this.podcastManager.getOPMLFeeds(req.body.opmlText)
-    res.json(rssFeedsData)
+    res.json({
+      feeds: this.podcastManager.getParsedOPMLFileFeeds(req.body.opmlText)
+    })
+  }
+
+  /**
+   * POST: /api/podcasts/opml/create
+   *
+   * @this import('../routers/ApiRouter')
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
+  async bulkCreatePodcastsFromOpmlFeedUrls(req, res) {
+    if (!req.user.isAdminOrUp) {
+      Logger.error(`[PodcastController] Non-admin user "${req.user.username}" attempted to bulk create podcasts`)
+      return res.sendStatus(403)
+    }
+
+    const rssFeeds = req.body.feeds
+    if (!Array.isArray(rssFeeds) || !rssFeeds.length || rssFeeds.some((feed) => !validateUrl(feed))) {
+      return res.status(400).send('Invalid request body. "feeds" must be an array of RSS feed URLs')
+    }
+
+    const libraryId = req.body.libraryId
+    const folderId = req.body.folderId
+    if (!libraryId || !folderId) {
+      return res.status(400).send('Invalid request body. "libraryId" and "folderId" are required')
+    }
+
+    const folder = await Database.libraryFolderModel.findByPk(folderId)
+    if (!folder || folder.libraryId !== libraryId) {
+      return res.status(404).send('Folder not found')
+    }
+    const autoDownloadEpisodes = !!req.body.autoDownloadEpisodes
+    this.podcastManager.createPodcastsFromFeedUrls(rssFeeds, folder, autoDownloadEpisodes, this.cronManager)
+
+    res.sendStatus(200)
  }

  async checkNewEpisodes(req, res) {
--- a/server/managers/PodcastManager.js
+++ b/server/managers/PodcastManager.js
@ -5,7 +5,7 @@ const Database = require('../Database')
 const fs = require('../libs/fsExtra')

 const { getPodcastFeed } = require('../utils/podcastUtils')
-const { removeFile, downloadFile } = require('../utils/fileUtils')
+const { removeFile, downloadFile, sanitizeFilename, filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils')
 const { levenshteinDistance } = require('../utils/index')
 const opmlParser = require('../utils/parsers/parseOPML')
 const opmlGenerator = require('../utils/generators/opmlGenerator')
@ -13,11 +13,13 @@ const prober = require('../utils/prober')
 const ffmpegHelpers = require('../utils/ffmpegHelpers')

 const TaskManager = require('./TaskManager')
+const CoverManager = require('../managers/CoverManager')

 const LibraryFile = require('../objects/files/LibraryFile')
 const PodcastEpisodeDownload = require('../objects/PodcastEpisodeDownload')
 const PodcastEpisode = require('../objects/entities/PodcastEpisode')
 const AudioFile = require('../objects/files/AudioFile')
+const LibraryItem = require('../objects/LibraryItem')

 class PodcastManager {
  constructor(watcher, notificationManager) {
@ -350,19 +352,23 @@ class PodcastManager {
    return matches.sort((a, b) => a.levenshtein - b.levenshtein)
  }

+  getParsedOPMLFileFeeds(opmlText) {
+    return opmlParser.parse(opmlText)
+  }
+
  async getOPMLFeeds(opmlText) {
-    var extractedFeeds = opmlParser.parse(opmlText)
-    if (!extractedFeeds || !extractedFeeds.length) {
+    const extractedFeeds = opmlParser.parse(opmlText)
+    if (!extractedFeeds?.length) {
      Logger.error('[PodcastManager] getOPMLFeeds: No RSS feeds found in OPML')
      return {
        error: 'No RSS feeds found in OPML'
      }
    }

-    var rssFeedData = []
+    const rssFeedData = []

    for (let feed of extractedFeeds) {
-      var feedData = await getPodcastFeed(feed.feedUrl, true)
+      const feedData = await getPodcastFeed(feed.feedUrl, true)
      if (feedData) {
        feedData.metadata.feedUrl = feed.feedUrl
        rssFeedData.push(feedData)
@ -392,5 +398,115 @@ class PodcastManager {
      queue: this.downloadQueue.filter((item) => !libraryId || item.libraryId === libraryId).map((item) => item.toJSONForClient())
    }
  }
+
+  /**
+   *
+   * @param {string[]} rssFeedUrls
+   * @param {import('../models/LibraryFolder')} folder
+   * @param {boolean} autoDownloadEpisodes
+   * @param {import('../managers/CronManager')} cronManager
+   */
+  async createPodcastsFromFeedUrls(rssFeedUrls, folder, autoDownloadEpisodes, cronManager) {
+    const task = TaskManager.createAndAddTask('opml-import', 'OPML import', `Creating podcasts from ${rssFeedUrls.length} RSS feeds`, true, null)
+    let numPodcastsAdded = 0
+    Logger.info(`[PodcastManager] createPodcastsFromFeedUrls: Importing ${rssFeedUrls.length} RSS feeds to folder "${folder.path}"`)
+    for (const feedUrl of rssFeedUrls) {
+      const feed = await getPodcastFeed(feedUrl).catch(() => null)
+      if (!feed?.episodes) {
+        TaskManager.createAndEmitFailedTask('opml-import-feed', 'OPML import feed', `Importing RSS feed "${feedUrl}"`, 'Failed to get podcast feed')
+        Logger.error(`[PodcastManager] createPodcastsFromFeedUrls: Failed to get podcast feed for "${feedUrl}"`)
+        continue
+      }
+
+      const podcastFilename = sanitizeFilename(feed.metadata.title)
+      const podcastPath = filePathToPOSIX(`${folder.path}/${podcastFilename}`)
+      // Check if a library item with this podcast folder exists already
+      const existingLibraryItem =
+        (await Database.libraryItemModel.count({
+          where: {
+            path: podcastPath
+          }
+        })) > 0
+      if (existingLibraryItem) {
+        Logger.error(`[PodcastManager] createPodcastsFromFeedUrls: Podcast already exists at path "${podcastPath}"`)
+        TaskManager.createAndEmitFailedTask('opml-import-feed', 'OPML import feed', `Creating podcast "${feed.metadata.title}"`, 'Podcast already exists at path')
+        continue
+      }
+
+      const successCreatingPath = await fs
+        .ensureDir(podcastPath)
+        .then(() => true)
+        .catch((error) => {
+          Logger.error(`[PodcastManager] Failed to ensure podcast dir "${podcastPath}"`, error)
+          return false
+        })
+      if (!successCreatingPath) {
+        Logger.error(`[PodcastManager] createPodcastsFromFeedUrls: Failed to create podcast folder at "${podcastPath}"`)
+        TaskManager.createAndEmitFailedTask('opml-import-feed', 'OPML import feed', `Creating podcast "${feed.metadata.title}"`, 'Failed to create podcast folder')
+        continue
+      }
+
+      const newPodcastMetadata = {
+        title: feed.metadata.title,
+        author: feed.metadata.author,
+        description: feed.metadata.description,
+        releaseDate: '',
+        genres: [...feed.metadata.categories],
+        feedUrl: feed.metadata.feedUrl,
+        imageUrl: feed.metadata.image,
+        itunesPageUrl: '',
+        itunesId: '',
+        itunesArtistId: '',
+        language: '',
+        numEpisodes: feed.numEpisodes
+      }
+
+      const libraryItemFolderStats = await getFileTimestampsWithIno(podcastPath)
+      const libraryItemPayload = {
+        path: podcastPath,
+        relPath: podcastFilename,
+        folderId: folder.id,
+        libraryId: folder.libraryId,
+        ino: libraryItemFolderStats.ino,
+        mtimeMs: libraryItemFolderStats.mtimeMs || 0,
+        ctimeMs: libraryItemFolderStats.ctimeMs || 0,
+        birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
+        media: {
+          metadata: newPodcastMetadata,
+          autoDownloadEpisodes
+        }
+      }
+
+      const libraryItem = new LibraryItem()
+      libraryItem.setData('podcast', libraryItemPayload)
+
+      // Download and save cover image
+      if (newPodcastMetadata.imageUrl) {
+        // TODO: Scan cover image to library files
+        // Podcast cover will always go into library item folder
+        const coverResponse = await CoverManager.downloadCoverFromUrl(libraryItem, newPodcastMetadata.imageUrl, true)
+        if (coverResponse) {
+          if (coverResponse.error) {
+            Logger.error(`[PodcastManager] createPodcastsFromFeedUrls: Download cover error from "${newPodcastMetadata.imageUrl}": ${coverResponse.error}`)
+          } else if (coverResponse.cover) {
+            libraryItem.media.coverPath = coverResponse.cover
+          }
+        }
+      }
+
+      await Database.createLibraryItem(libraryItem)
+      SocketAuthority.emitter('item_added', libraryItem.toJSONExpanded())
+
+      // Turn on podcast auto download cron if not already on
+      if (libraryItem.media.autoDownloadEpisodes) {
+        cronManager.checkUpdatePodcastCron(libraryItem)
+      }
+
+      numPodcastsAdded++
+    }
+    task.setFinished(`Added ${numPodcastsAdded} podcasts`)
+    TaskManager.taskFinished(task)
+    Logger.info(`[PodcastManager] createPodcastsFromFeedUrls: Finished OPML import. Created ${numPodcastsAdded} podcasts out of ${rssFeedUrls.length} RSS feed URLs`)
+  }
 }
 module.exports = PodcastManager
--- a/server/managers/TaskManager.js
+++ b/server/managers/TaskManager.js
@ -23,8 +23,8 @@ class TaskManager {
   * @param {Task} task
   */
  taskFinished(task) {
-    if (this.tasks.some(t => t.id === task.id)) {
-      this.tasks = this.tasks.filter(t => t.id !== task.id)
+    if (this.tasks.some((t) => t.id === task.id)) {
+      this.tasks = this.tasks.filter((t) => t.id !== task.id)
      SocketAuthority.emitter('task_finished', task.toJSON())
    }
  }
@ -44,5 +44,21 @@ class TaskManager {
    this.addTask(task)
    return task
  }
+
+  /**
+   * Create new failed task and add
+   *
+   * @param {string} action
+   * @param {string} title
+   * @param {string} description
+   * @param {string} errorMessage
+   */
+  createAndEmitFailedTask(action, title, description, errorMessage) {
+    const task = new Task()
+    task.setData(action, title, description, false)
+    task.setFailed(errorMessage)
+    SocketAuthority.emitter('task_started', task.toJSON())
+    return task
+  }
 }
 module.exports = new TaskManager()
--- a/server/models/Library.js
+++ b/server/models/Library.js
@ -60,7 +60,7 @@ class Library extends Model {
  /**
   * Convert expanded Library to oldLibrary
   * @param {Library} libraryExpanded
-   * @returns {Promise<oldLibrary>}
+   * @returns {oldLibrary}
   */
  static getOldLibrary(libraryExpanded) {
    const folders = libraryExpanded.libraryFolders.map((folder) => {
--- a/server/routers/ApiRouter.js
+++ b/server/routers/ApiRouter.js
@ -45,6 +45,7 @@ class ApiRouter {
    this.backupManager = Server.backupManager
    /** @type {import('../Watcher')} */
    this.watcher = Server.watcher
+    /** @type {import('../managers/PodcastManager')} */
    this.podcastManager = Server.podcastManager
    this.audioMetadataManager = Server.audioMetadataManager
    this.rssFeedManager = Server.rssFeedManager
@ -239,7 +240,8 @@ class ApiRouter {
    //
    this.router.post('/podcasts', PodcastController.create.bind(this))
    this.router.post('/podcasts/feed', PodcastController.getPodcastFeed.bind(this))
-    this.router.post('/podcasts/opml', PodcastController.getFeedsFromOPMLText.bind(this))
+    this.router.post('/podcasts/opml/parse', PodcastController.getFeedsFromOPMLText.bind(this))
+    this.router.post('/podcasts/opml/create', PodcastController.bulkCreatePodcastsFromOpmlFeedUrls.bind(this))
    this.router.get('/podcasts/:id/checknew', PodcastController.middleware.bind(this), PodcastController.checkNewEpisodes.bind(this))
    this.router.get('/podcasts/:id/downloads', PodcastController.middleware.bind(this), PodcastController.getEpisodeDownloads.bind(this))
    this.router.get('/podcasts/:id/clear-queue', PodcastController.middleware.bind(this), PodcastController.clearEpisodeDownloadQueue.bind(this))
--- a/server/utils/parsers/parseOPML.js
+++ b/server/utils/parsers/parseOPML.js
@ -1,17 +1,21 @@
 const h = require('htmlparser2')
 const Logger = require('../../Logger')

+/**
+ *
+ * @param {string} opmlText
+ * @returns {Array<{title: string, feedUrl: string}>
+ */
 function parse(opmlText) {
  var feeds = []
  var parser = new h.Parser({
    onopentag: (name, attribs) => {
-      if (name === "outline" && attribs.type === 'rss') {
+      if (name === 'outline' && attribs.type === 'rss') {
        if (!attribs.xmlurl) {
          Logger.error('[parseOPML] Invalid opml outline tag has no xmlurl attribute')
        } else {
          feeds.push({
-            title: attribs.title || 'No Title',
-            text: attribs.text || '',
+            title: attribs.title || attribs.text || '',
            feedUrl: attribs.xmlurl
          })
        }