diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js
index 1e51cf52..f8e959bd 100644
--- a/server/scanner/BookScanner.js
+++ b/server/scanner/BookScanner.js
@@ -168,9 +168,7 @@ class BookScanner {
       hasMediaChanges = true
     }
 
-    // TODO: When metadata file is stored in /metadata/items/{libraryItemId}.[abs|json] we should load this
-    // TODO: store an additional array of metadata keys that the user has changed manually so we know what not to override
-    const bookMetadata = await this.getBookMetadataFromScanData(media.audioFiles, libraryItemData, libraryScan)
+    const bookMetadata = await this.getBookMetadataFromScanData(media.audioFiles, libraryItemData, libraryScan, existingLibraryItem.id)
     let authorsUpdated = false
     const bookAuthorsRemoved = []
     let seriesUpdated = false
@@ -550,9 +548,10 @@ class BookScanner {
    * @param {import('../models/Book').AudioFileObject[]} audioFiles 
    * @param {import('./LibraryItemScanData')} libraryItemData 
    * @param {LibraryScan} libraryScan 
+   * @param {string} [existingLibraryItemId]
    * @returns {Promise<BookMetadataObject>}
    */
-  async getBookMetadataFromScanData(audioFiles, libraryItemData, libraryScan) {
+  async getBookMetadataFromScanData(audioFiles, libraryItemData, libraryScan, existingLibraryItemId = null) {
     // First set book metadata from folder/file names
     const bookMetadata = {
       title: libraryItemData.mediaMetadata.title,
@@ -722,11 +721,31 @@ class BookScanner {
 
     // If metadata.json or metadata.abs use this for metadata
     const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile
-    const metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
+    let metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
+    let metadataFilePath = metadataLibraryFile?.metadata.path
+    let metadataFileFormat = libraryItemData.metadataJsonLibraryFile ? 'json' : 'abs'
+
+    // When metadata file is not stored with library item then check in the /metadata/items folder for it
+    if (!metadataText && existingLibraryItemId) {
+      let metadataPath = Path.join(global.MetadataPath, 'items', existingLibraryItemId)
+
+      let altFormat = global.ServerSettings.metadataFileFormat === 'json' ? 'abs' : 'json'
+      // First check the metadata format set in server settings, fallback to the alternate
+      metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`)
+      metadataFileFormat = global.ServerSettings.metadataFileFormat
+      if (await fsExtra.pathExists(metadataFilePath)) {
+        metadataText = await readTextFile(metadataFilePath)
+      } else if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.${altFormat}`))) {
+        metadataFilePath = Path.join(metadataPath, `metadata.${altFormat}`)
+        metadataFileFormat = altFormat
+        metadataText = await readTextFile(metadataFilePath)
+      }
+    }
+
     if (metadataText) {
-      libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataLibraryFile.metadata.path}" - preferring`)
+      libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataFilePath}" - preferring`)
       let abMetadata = null
-      if (!!libraryItemData.metadataJsonLibraryFile) {
+      if (metadataFileFormat === 'json') {
         abMetadata = abmetadataGenerator.parseJson(metadataText)
       } else {
         abMetadata = abmetadataGenerator.parse(metadataText, 'book')
diff --git a/server/scanner/PodcastScanner.js b/server/scanner/PodcastScanner.js
index d5a799cf..aa4a91a5 100644
--- a/server/scanner/PodcastScanner.js
+++ b/server/scanner/PodcastScanner.js
@@ -145,8 +145,7 @@ class PodcastScanner {
       hasMediaChanges = true
     }
 
-    // TODO: When metadata file is stored in /metadata/items/{libraryItemId}.[abs|json] we should load this
-    const podcastMetadata = await this.getPodcastMetadataFromScanData(existingPodcastEpisodes, libraryItemData, libraryScan)
+    const podcastMetadata = await this.getPodcastMetadataFromScanData(existingPodcastEpisodes, libraryItemData, libraryScan, existingLibraryItem.id)
 
     for (const key in podcastMetadata) {
       // Ignore unset metadata and empty arrays
@@ -312,9 +311,10 @@ class PodcastScanner {
    * @param {PodcastEpisode[]} podcastEpisodes Not the models for new podcasts
    * @param {import('./LibraryItemScanData')} libraryItemData 
    * @param {import('./LibraryScan')} libraryScan 
+   * @param {string} [existingLibraryItemId]
    * @returns {Promise<PodcastMetadataObject>}
    */
-  async getPodcastMetadataFromScanData(podcastEpisodes, libraryItemData, libraryScan) {
+  async getPodcastMetadataFromScanData(podcastEpisodes, libraryItemData, libraryScan, existingLibraryItemId = null) {
     const podcastMetadata = {
       title: libraryItemData.mediaMetadata.title,
       titleIgnorePrefix: getTitleIgnorePrefix(libraryItemData.mediaMetadata.title),
@@ -389,11 +389,31 @@ class PodcastScanner {
 
     // If metadata.json or metadata.abs use this for metadata
     const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile
-    const metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
+    let metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
+    let metadataFilePath = metadataLibraryFile?.metadata.path
+    let metadataFileFormat = libraryItemData.metadataJsonLibraryFile ? 'json' : 'abs'
+
+    // When metadata file is not stored with library item then check in the /metadata/items folder for it
+    if (!metadataText && existingLibraryItemId) {
+      let metadataPath = Path.join(global.MetadataPath, 'items', existingLibraryItemId)
+
+      let altFormat = global.ServerSettings.metadataFileFormat === 'json' ? 'abs' : 'json'
+      // First check the metadata format set in server settings, fallback to the alternate
+      metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`)
+      metadataFileFormat = global.ServerSettings.metadataFileFormat
+      if (await fsExtra.pathExists(metadataFilePath)) {
+        metadataText = await readTextFile(metadataFilePath)
+      } else if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.${altFormat}`))) {
+        metadataFilePath = Path.join(metadataPath, `metadata.${altFormat}`)
+        metadataFileFormat = altFormat
+        metadataText = await readTextFile(metadataFilePath)
+      }
+    }
+
     if (metadataText) {
-      libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataLibraryFile.metadata.path}" - preferring`)
+      libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataFilePath}" - preferring`)
       let abMetadata = null
-      if (!!libraryItemData.metadataJsonLibraryFile) {
+      if (metadataFileFormat === 'json') {
         abMetadata = abmetadataGenerator.parseJson(metadataText)
       } else {
         abMetadata = abmetadataGenerator.parse(metadataText, 'podcast')