New data model scanner update and change scan chunks to be based on total file size

server/objects/LibraryItem.js

@@ -141,6 +141,11 @@ class LibraryItem {
     this.libraryFiles.forEach((lf) => total += lf.metadata.size)
     return total
   }
+  get audioFileTotalSize() {
+    var total = 0
+    this.libraryFiles.filter(lf => lf.fileType == 'audio').forEach((lf) => total += lf.metadata.size)
+    return total
+  }
   get hasAudioFiles() {
     return this.libraryFiles.some(lf => lf.fileType === 'audio')
   }
@@ -347,7 +352,9 @@ class LibraryItem {
       return true
     })
     if (filesRemoved.length) {
-      this.media.checkUpdateMissingTracks()
+      if (this.media.audiobooks && this.media.audiobooks.length) {
+        this.media.audiobooks.forEach(ab => ab.checkUpdateMissingTracks())
+      }
       hasUpdated = true
     }
 

server/objects/entities/Audiobook.js

@@ -1,6 +1,6 @@
 const Path = require('path')
 const AudioFile = require('../files/AudioFile')
-const { areEquivalent, copyValue } = require('../../utils/index')
+const { areEquivalent, copyValue, getId } = require('../../utils/index')
 const AudioTrack = require('../files/AudioTrack')
 
 class Audiobook {
@@ -93,6 +93,14 @@ class Audiobook {
     return this.audioFiles.some(af => af.embeddedCoverArt)
   }
 
+  setData(name, index) {
+    this.id = getId('ab')
+    this.name = name
+    this.index = index
+    this.addedAt = Date.now()
+    this.updatedAt = Date.now()
+  }
+
   update(payload) {
     var json = this.toJSON()
     var hasUpdates = false

server/objects/entities/EBook.js

@@ -1,5 +1,5 @@
 const EBookFile = require('../files/EBookFile')
-const { areEquivalent, copyValue } = require('../../utils/index')
+const { areEquivalent, copyValue, getId } = require('../../utils/index')
 
 class EBook {
   constructor(ebook) {
@@ -64,6 +64,15 @@ class EBook {
     return this.ebookFile.metadata.size
   }
 
+  setData(ebookFile, index) {
+    this.id = getId('eb')
+    this.name = ebookFile.metadata.filename
+    this.index = index
+    this.ebookFile = ebookFile
+    this.addedAt = Date.now()
+    this.updatedAt = Date.now()
+  }
+
   findFileWithInode(inode) {
     return this.ebookFile.ino === inode
   }

server/objects/mediaTypes/Book.js

@@ -6,6 +6,7 @@ const { areEquivalent, copyValue } = require('../../utils/index')
 const { parseOpfMetadataXML } = require('../../utils/parseOpfMetadata')
 const { readTextFile } = require('../../utils/fileUtils')
 
+const EBookFile = require('../files/EBookFile')
 const Audiobook = require('../entities/Audiobook')
 const EBook = require('../entities/EBook')
 
@@ -267,9 +268,30 @@ class Book {
   }
 
   addEbookFile(libraryFile) {
-    // var newEbook = new EBookFile()
+    var ebookFile = new EBookFile()
-    // newEbook.setData(libraryFile)
+    ebookFile.setData(libraryFile)
-    // this.ebookFiles.push(newEbook)
+
+    var ebookIndex = this.ebooks.length + 1
+    var newEBook = new EBook()
+    newEBook.setData(ebookFile, ebookIndex)
+    this.ebooks.push(newEBook)
+  }
+
+  getCreateAudiobookVariant(variant) {
+    if (this.audiobooks.length) {
+      var ab = this.audiobooks.find(ab => ab.name == variantName)
+      if (ab) return ab
+    }
+    var abIndex = this.audiobooks.length + 1
+    var newAb = new Audiobook()
+    newAb.setData(variant, abIndex)
+    this.audiobooks.push(newAb)
+    return newAb
+  }
+
+  addAudioFileToAudiobook(audioFile, variant = 'default') { // Create if none
+    var audiobook = this.getCreateAudiobookVariant(variant)
+    audiobook.audioFiles.push(audioFile)
   }
 }
 module.exports = Book

server/scanner/AudioFileScanner.js

@@ -169,11 +169,12 @@ class AudioFileScanner {
       if (existingAF) {
         if (existingAF.updateFromScan) existingAF.updateFromScan(audioFiles[i])
       } else {
-        libraryItem.media.audioFiles.push(audioFiles[i])
+        libraryItem.media.addAudioFileToAudiobook(audioFiles[i])
       }
     }
   }
 
+  // TODO: support for multiple audiobooks in a book item will need to pass an audiobook variant name here
   async scanAudioFiles(audioLibraryFiles, scanData, libraryItem, preferAudioMetadata, libraryScan = null) {
     var hasUpdated = false
 
@@ -195,14 +196,13 @@ class AudioFileScanner {
           if (totalAudioFilesToInclude === 1) {
             var af = audioScanResult.audioFiles[0]
             af.index = 1
-            libraryItem.media.audioFiles.push(af)
+            libraryItem.media.addAudioFileToAudiobook(af)
             hasUpdated = true
           } else {
             this.runSmartTrackOrder(libraryItem, audioScanResult.audioFiles)
             hasUpdated = true
           }
         } else {
-          Logger.debug(`[AudioFileScanner] No audio track re-order required`)
           // Only update metadata not index
           audioScanResult.audioFiles.forEach((af) => {
             var existingAF = libraryItem.media.findFileWithInode(af.ino)
@@ -221,7 +221,12 @@ class AudioFileScanner {
         }
 
         if (hasUpdated) {
-          libraryItem.media.rebuildTracks()
+          if (!libraryItem.media.audiobooks.length) {
+            Logger.error(`[AudioFileScanner] Updates were made but library item has no audiobooks`, libraryItem)
+          } else {
+            var audiobook = libraryItem.media.audiobooks[0]
+            audiobook.rebuildTracks()
+          }
         }
       } // End Book media type
     }

server/scanner/Scanner.js

@@ -3,14 +3,12 @@ const Path = require('path')
 
 // Utils
 const Logger = require('../Logger')
-const { version } = require('../../package.json')
 const { groupFilesIntoLibraryItemPaths, getLibraryItemFileData, scanFolder } = require('../utils/scandir')
-const { comparePaths, getId } = require('../utils/index')
+const { comparePaths } = require('../utils/index')
 const { ScanResult, LogLevel } = require('../utils/constants')
 
 const AudioFileScanner = require('./AudioFileScanner')
 const BookFinder = require('../finders/BookFinder')
-const Audiobook = require('../objects/legacy/Audiobook')
 const LibraryItem = require('../objects/LibraryItem')
 const LibraryScan = require('./LibraryScan')
 const ScanOptions = require('./ScanOptions')
@@ -181,13 +179,14 @@ class Scanner {
     libraryItemDataFound = libraryItemDataFound.filter(lid => lid.ino)
     var libraryItemsInLibrary = this.db.libraryItems.filter(li => li.libraryId === libraryScan.libraryId)
 
-    const NumScansPerChunk = 25
+    const MaxSizePerChunk = 2.5e9
-    const itemsToUpdateChunks = []
     const itemDataToRescanChunks = []
     const newItemDataToScanChunks = []
     var itemsToUpdate = []
     var itemDataToRescan = []
+    var itemDataToRescanSize = 0
     var newItemDataToScan = []
+    var newItemDataToScanSize = 0
     var itemsToFindCovers = []
 
     // Check for existing & removed library items
@@ -200,40 +199,37 @@ class Scanner {
         libraryScan.resultsMissing++
         libraryItem.setMissing()
         itemsToUpdate.push(libraryItem)
-        if (itemsToUpdate.length === NumScansPerChunk) {
-          itemsToUpdateChunks.push(itemsToUpdate)
-          itemsToUpdate = []
-        }
       } else {
         var checkRes = libraryItem.checkScanData(dataFound)
         if (checkRes.newLibraryFiles.length || libraryScan.scanOptions.forceRescan) { // Item has new files
           checkRes.libraryItem = libraryItem
           checkRes.scanData = dataFound
-          itemDataToRescan.push(checkRes)
+
-          if (itemDataToRescan.length === NumScansPerChunk) {
+          // If this item will go over max size then push current chunk
+          if (libraryItem.audioFileTotalSize + itemDataToRescanSize > MaxSizePerChunk && itemDataToRescan.length > 0) {
             itemDataToRescanChunks.push(itemDataToRescan)
+            itemDataToRescanSize = 0
             itemDataToRescan = []
           }
-        } else if (libraryScan.findCovers && libraryItem.media.shouldSearchForCover) {
+
+          itemDataToRescan.push(checkRes)
+          itemDataToRescanSize += libraryItem.audioFileTotalSize
+          if (itemDataToRescanSize >= MaxSizePerChunk) {
+            itemDataToRescanChunks.push(itemDataToRescan)
+            itemDataToRescanSize = 0
+            itemDataToRescan = []
+          }
+        } else if (libraryScan.findCovers && libraryItem.media.shouldSearchForCover) { // Search cover
           libraryScan.resultsUpdated++
           itemsToFindCovers.push(libraryItem)
           itemsToUpdate.push(libraryItem)
-          if (itemsToUpdate.length === NumScansPerChunk) {
-            itemsToUpdateChunks.push(itemsToUpdate)
-            itemsToUpdate = []
-          }
         } else if (checkRes.updated) { // Updated but no scan required
           libraryScan.resultsUpdated++
           itemsToUpdate.push(libraryItem)
-          if (itemsToUpdate.length === NumScansPerChunk) {
-            itemsToUpdateChunks.push(itemsToUpdate)
-            itemsToUpdate = []
-          }
         }
         libraryItemDataFound = libraryItemDataFound.filter(lid => lid.ino !== dataFound.ino)
       }
     }
-    if (itemsToUpdate.length) itemsToUpdateChunks.push(itemsToUpdate)
     if (itemDataToRescan.length) itemDataToRescanChunks.push(itemDataToRescan)
 
     // Potential NEW Library Items
@@ -244,9 +240,21 @@ class Scanner {
       if (!hasMediaFile) {
         libraryScan.addLog(LogLevel.WARN, `Directory found "${libraryItemDataFound.path}" has no media files`)
       } else {
-        newItemDataToScan.push(dataFound)
+        var audioFileSize = 0
-        if (newItemDataToScan.length === NumScansPerChunk) {
+        dataFound.libraryFiles.filter(lf => lf.fileType == 'audio').forEach(lf => audioFileSize += lf.metadata.size)
+
+        // If this item will go over max size then push current chunk
+        if (audioFileSize + newItemDataToScanSize > MaxSizePerChunk && newItemDataToScan.length > 0) {
           newItemDataToScanChunks.push(newItemDataToScan)
+          newItemDataToScanSize = 0
+          newItemDataToScan = []
+        }
+
+        newItemDataToScan.push(dataFound)
+        newItemDataToScanSize += audioFileSize
+        if (newItemDataToScanSize >= MaxSizePerChunk) {
+          newItemDataToScanChunks.push(newItemDataToScan)
+          newItemDataToScanSize = 0
           newItemDataToScan = []
         }
       }
@@ -260,10 +268,9 @@ class Scanner {
       libraryItem.media.updateLastCoverSearch(updatedCover)
     }
 
-    for (let i = 0; i < itemsToUpdateChunks.length; i++) {
+    if (itemsToUpdate.length) {
-      await this.updateLibraryItemChunk(itemsToUpdateChunks[i])
+      await this.updateLibraryItemChunk(itemsToUpdate)
       if (this.cancelLibraryScan[libraryScan.libraryId]) return true
-      // console.log('Update chunk done', i, 'of', itemsToUpdateChunks.length)
     }
     for (let i = 0; i < itemDataToRescanChunks.length; i++) {
       await this.rescanLibraryItemDataChunk(itemDataToRescanChunks[i], libraryScan)