Cleanup scanner

This commit is contained in:
advplyr 2023-01-02 12:05:07 -06:00
parent 591d8a8ab1
commit 0807146aab
3 changed files with 46 additions and 33 deletions

View File

@ -107,7 +107,7 @@ class Db {
checkPreviousVersion() { checkPreviousVersion() {
return this.settingsDb.select(() => true).then((results) => { return this.settingsDb.select(() => true).then((results) => {
if (results.data && results.data.length) { if (results.data && results.data.length) {
var serverSettings = results.data.find(s => s.id === 'server-settings') const serverSettings = results.data.find(s => s.id === 'server-settings')
if (serverSettings && serverSettings.version && serverSettings.version !== version) { if (serverSettings && serverSettings.version && serverSettings.version !== version) {
return serverSettings.version return serverSettings.version
} }
@ -163,7 +163,7 @@ class Db {
const p4 = this.settingsDb.select(() => true).then(async (results) => { const p4 = this.settingsDb.select(() => true).then(async (results) => {
if (results.data && results.data.length) { if (results.data && results.data.length) {
this.settings = results.data this.settings = results.data
var serverSettings = this.settings.find(s => s.id === 'server-settings') const serverSettings = this.settings.find(s => s.id === 'server-settings')
if (serverSettings) { if (serverSettings) {
this.serverSettings = new ServerSettings(serverSettings) this.serverSettings = new ServerSettings(serverSettings)
@ -185,7 +185,7 @@ class Db {
} }
} }
var notificationSettings = this.settings.find(s => s.id === 'notification-settings') const notificationSettings = this.settings.find(s => s.id === 'notification-settings')
if (notificationSettings) { if (notificationSettings) {
this.notificationSettings = new NotificationSettings(notificationSettings) this.notificationSettings = new NotificationSettings(notificationSettings)
} }

View File

@ -110,14 +110,14 @@ class MediaFileScanner {
} }
// Returns array of { MediaFile, elapsed, averageScanDuration } from audio file scan objects // Returns array of { MediaFile, elapsed, averageScanDuration } from audio file scan objects
async executeMediaFileScans(libraryItem, mediaLibraryFiles, scanData) { async executeMediaFileScans(libraryItem, mediaLibraryFiles) {
const mediaType = libraryItem.mediaType const mediaType = libraryItem.mediaType
const scanStart = Date.now() const scanStart = Date.now()
const mediaMetadataFromScan = scanData.media.metadata || null const mediaMetadata = libraryItem.media.metadata || null
const proms = [] const proms = []
for (let i = 0; i < mediaLibraryFiles.length; i++) { for (let i = 0; i < mediaLibraryFiles.length; i++) {
proms.push(this.scan(mediaType, mediaLibraryFiles[i], mediaMetadataFromScan)) proms.push(this.scan(mediaType, mediaLibraryFiles[i], mediaMetadata))
} }
const results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr)) const results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr))
return { return {
@ -206,10 +206,21 @@ class MediaFileScanner {
} }
} }
async scanMediaFiles(mediaLibraryFiles, scanData, libraryItem, preferAudioMetadata, preferOverdriveMediaMarker, libraryScan = null) { /**
* Scans media files for a library item and adds them as audio tracks and sets library item metadata
* @async
* @param {Array<LibraryFile>} mediaLibraryFiles - Media files for this library item
* @param {LibraryItem} libraryItem
* @param {LibraryScan} [libraryScan=null] - Optional when doing a library scan to use LibraryScan config/logs
* @return {Promise<Boolean>} True if any updates were made
*/
async scanMediaFiles(mediaLibraryFiles, libraryItem, libraryScan = null) {
const preferAudioMetadata = libraryScan ? !!libraryScan.preferAudioMetadata : !!global.ServerSettings.scannerPreferAudioMetadata
const preferOverdriveMediaMarker = libraryScan ? !!libraryScan.preferOverdriveMediaMarker : !!global.ServerSettings.scannerPreferOverdriveMediaMarker
let hasUpdated = false let hasUpdated = false
const mediaScanResult = await this.executeMediaFileScans(libraryItem, mediaLibraryFiles, scanData) const mediaScanResult = await this.executeMediaFileScans(libraryItem, mediaLibraryFiles)
if (libraryItem.mediaType === 'video') { if (libraryItem.mediaType === 'video') {
if (mediaScanResult.videoFiles.length) { if (mediaScanResult.videoFiles.length) {
@ -219,9 +230,9 @@ class MediaFileScanner {
} }
} else if (mediaScanResult.audioFiles.length) { } else if (mediaScanResult.audioFiles.length) {
if (libraryScan) { if (libraryScan) {
libraryScan.addLog(LogLevel.DEBUG, `Library Item "${scanData.path}" Media file scan took ${mediaScanResult.elapsed}ms for ${mediaScanResult.audioFiles.length} with average time of ${mediaScanResult.averageScanDuration}ms per MB`) libraryScan.addLog(LogLevel.DEBUG, `Library Item "${libraryItem.path}" Media file scan took ${mediaScanResult.elapsed}ms for ${mediaScanResult.audioFiles.length} with average time of ${mediaScanResult.averageScanDuration}ms per MB`)
} }
Logger.debug(`Library Item "${scanData.path}" Media file scan took ${mediaScanResult.elapsed}ms with ${mediaScanResult.audioFiles.length} audio files averaging ${mediaScanResult.averageScanDuration}ms per MB`) Logger.debug(`Library Item "${libraryItem.path}" Media file scan took ${mediaScanResult.elapsed}ms with ${mediaScanResult.audioFiles.length} audio files averaging ${mediaScanResult.averageScanDuration}ms per MB`)
const newAudioFiles = mediaScanResult.audioFiles.filter(af => { const newAudioFiles = mediaScanResult.audioFiles.filter(af => {
return !libraryItem.media.findFileWithInode(af.ino) return !libraryItem.media.findFileWithInode(af.ino)

View File

@ -85,7 +85,7 @@ class Scanner {
// Scan all audio files // Scan all audio files
if (libraryItem.hasAudioFiles) { if (libraryItem.hasAudioFiles) {
const libraryAudioFiles = libraryItem.libraryFiles.filter(lf => lf.fileType === 'audio') const libraryAudioFiles = libraryItem.libraryFiles.filter(lf => lf.fileType === 'audio')
if (await MediaFileScanner.scanMediaFiles(libraryAudioFiles, libraryItemData, libraryItem, this.db.serverSettings.scannerPreferAudioMetadata, this.db.serverSettings.scannerPreferOverdriveMediaMarker)) { if (await MediaFileScanner.scanMediaFiles(libraryAudioFiles, libraryItem)) {
hasUpdated = true hasUpdated = true
} }
@ -168,7 +168,7 @@ class Scanner {
} }
async scanLibrary(libraryScan) { async scanLibrary(libraryScan) {
var libraryItemDataFound = [] let libraryItemDataFound = []
// Scan each library // Scan each library
for (let i = 0; i < libraryScan.folders.length; i++) { for (let i = 0; i < libraryScan.folders.length; i++) {
@ -182,17 +182,17 @@ class Scanner {
// Remove items with no inode // Remove items with no inode
libraryItemDataFound = libraryItemDataFound.filter(lid => lid.ino) libraryItemDataFound = libraryItemDataFound.filter(lid => lid.ino)
var libraryItemsInLibrary = this.db.libraryItems.filter(li => li.libraryId === libraryScan.libraryId) const libraryItemsInLibrary = this.db.libraryItems.filter(li => li.libraryId === libraryScan.libraryId)
const MaxSizePerChunk = 2.5e9 const MaxSizePerChunk = 2.5e9
const itemDataToRescanChunks = [] const itemDataToRescanChunks = []
const newItemDataToScanChunks = [] const newItemDataToScanChunks = []
var itemsToUpdate = [] let itemsToUpdate = []
var itemDataToRescan = [] let itemDataToRescan = []
var itemDataToRescanSize = 0 let itemDataToRescanSize = 0
var newItemDataToScan = [] let newItemDataToScan = []
var newItemDataToScanSize = 0 let newItemDataToScanSize = 0
var itemsToFindCovers = [] const itemsToFindCovers = []
// Check for existing & removed library items // Check for existing & removed library items
for (let i = 0; i < libraryItemsInLibrary.length; i++) { for (let i = 0; i < libraryItemsInLibrary.length; i++) {
@ -323,8 +323,8 @@ class Scanner {
} }
async scanNewLibraryItemDataChunk(newLibraryItemsData, libraryScan) { async scanNewLibraryItemDataChunk(newLibraryItemsData, libraryScan) {
var newLibraryItems = await Promise.all(newLibraryItemsData.map((lid) => { let newLibraryItems = await Promise.all(newLibraryItemsData.map((lid) => {
return this.scanNewLibraryItem(lid, libraryScan.libraryMediaType, libraryScan.preferAudioMetadata, libraryScan.preferOpfMetadata, libraryScan.findCovers, libraryScan.preferOverdriveMediaMarker, libraryScan) return this.scanNewLibraryItem(lid, libraryScan.libraryMediaType, libraryScan)
})) }))
newLibraryItems = newLibraryItems.filter(li => li) // Filter out nulls newLibraryItems = newLibraryItems.filter(li => li) // Filter out nulls
@ -341,7 +341,7 @@ class Scanner {
async rescanLibraryItem(libraryItemCheckData, libraryScan) { async rescanLibraryItem(libraryItemCheckData, libraryScan) {
const { newLibraryFiles, filesRemoved, existingLibraryFiles, libraryItem, scanData, updated } = libraryItemCheckData const { newLibraryFiles, filesRemoved, existingLibraryFiles, libraryItem, scanData, updated } = libraryItemCheckData
libraryScan.addLog(LogLevel.DEBUG, `Library "${libraryScan.libraryName}" Re-scanning "${libraryItem.path}"`) libraryScan.addLog(LogLevel.DEBUG, `Library "${libraryScan.libraryName}" Re-scanning "${libraryItem.path}"`)
var hasUpdated = updated let hasUpdated = updated
// Sync other files first to use local images as cover before extracting audio file cover // Sync other files first to use local images as cover before extracting audio file cover
if (await libraryItem.syncFiles(libraryScan.preferOpfMetadata)) { if (await libraryItem.syncFiles(libraryScan.preferOpfMetadata)) {
@ -349,24 +349,24 @@ class Scanner {
} }
// forceRescan all existing audio files - will probe and update ID3 tag metadata // forceRescan all existing audio files - will probe and update ID3 tag metadata
var existingAudioFiles = existingLibraryFiles.filter(lf => lf.fileType === 'audio') const existingAudioFiles = existingLibraryFiles.filter(lf => lf.fileType === 'audio')
if (libraryScan.scanOptions.forceRescan && existingAudioFiles.length) { if (libraryScan.scanOptions.forceRescan && existingAudioFiles.length) {
if (await MediaFileScanner.scanMediaFiles(existingAudioFiles, scanData, libraryItem, libraryScan.preferAudioMetadata, libraryScan.preferOverdriveMediaMarker, libraryScan)) { if (await MediaFileScanner.scanMediaFiles(existingAudioFiles, libraryItem, libraryScan)) {
hasUpdated = true hasUpdated = true
} }
} }
// Scan new audio files // Scan new audio files
var newAudioFiles = newLibraryFiles.filter(lf => lf.fileType === 'audio') const newAudioFiles = newLibraryFiles.filter(lf => lf.fileType === 'audio')
var removedAudioFiles = filesRemoved.filter(lf => lf.fileType === 'audio') const removedAudioFiles = filesRemoved.filter(lf => lf.fileType === 'audio')
if (newAudioFiles.length || removedAudioFiles.length) { if (newAudioFiles.length || removedAudioFiles.length) {
if (await MediaFileScanner.scanMediaFiles(newAudioFiles, scanData, libraryItem, libraryScan.preferAudioMetadata, libraryScan.preferOverdriveMediaMarker, libraryScan)) { if (await MediaFileScanner.scanMediaFiles(newAudioFiles, libraryItem, libraryScan)) {
hasUpdated = true hasUpdated = true
} }
} }
// If an audio file has embedded cover art and no cover is set yet, extract & use it // If an audio file has embedded cover art and no cover is set yet, extract & use it
if (newAudioFiles.length || libraryScan.scanOptions.forceRescan) { if (newAudioFiles.length || libraryScan.scanOptions.forceRescan) {
if (libraryItem.media.hasEmbeddedCoverArt && !libraryItem.media.coverPath) { if (libraryItem.media.hasEmbeddedCoverArt && !libraryItem.media.coverPath) {
var savedCoverPath = await this.coverManager.saveEmbeddedCoverArt(libraryItem) const savedCoverPath = await this.coverManager.saveEmbeddedCoverArt(libraryItem)
if (savedCoverPath) { if (savedCoverPath) {
hasUpdated = true hasUpdated = true
libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${savedCoverPath}"`) libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${savedCoverPath}"`)
@ -385,7 +385,7 @@ class Scanner {
// Scan for cover if enabled and has no cover (and author or title has changed OR has been 7 days since last lookup) // Scan for cover if enabled and has no cover (and author or title has changed OR has been 7 days since last lookup)
if (libraryScan.findCovers && !libraryItem.media.coverPath && libraryItem.media.shouldSearchForCover) { if (libraryScan.findCovers && !libraryItem.media.coverPath && libraryItem.media.shouldSearchForCover) {
var updatedCover = await this.searchForCover(libraryItem, libraryScan) const updatedCover = await this.searchForCover(libraryItem, libraryScan)
libraryItem.media.updateLastCoverSearch(updatedCover) libraryItem.media.updateLastCoverSearch(updatedCover)
hasUpdated = true hasUpdated = true
} }
@ -393,16 +393,19 @@ class Scanner {
return hasUpdated ? libraryItem : null return hasUpdated ? libraryItem : null
} }
async scanNewLibraryItem(libraryItemData, libraryMediaType, preferAudioMetadata, preferOpfMetadata, findCovers, preferOverdriveMediaMarker, libraryScan = null) { async scanNewLibraryItem(libraryItemData, libraryMediaType, libraryScan = null) {
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Scanning new library item "${libraryItemData.path}"`) if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Scanning new library item "${libraryItemData.path}"`)
else Logger.debug(`[Scanner] Scanning new item "${libraryItemData.path}"`) else Logger.debug(`[Scanner] Scanning new item "${libraryItemData.path}"`)
const preferOpfMetadata = libraryScan ? !!libraryScan.preferOpfMetadata : !!global.ServerSettings.scannerPreferOpfMetadata
const findCovers = libraryScan ? !!libraryScan.findCovers : !!global.ServerSettings.scannerFindCovers
const libraryItem = new LibraryItem() const libraryItem = new LibraryItem()
libraryItem.setData(libraryMediaType, libraryItemData) libraryItem.setData(libraryMediaType, libraryItemData)
const mediaFiles = libraryItemData.libraryFiles.filter(lf => lf.fileType === 'audio' || lf.fileType === 'video') const mediaFiles = libraryItemData.libraryFiles.filter(lf => lf.fileType === 'audio' || lf.fileType === 'video')
if (mediaFiles.length) { if (mediaFiles.length) {
await MediaFileScanner.scanMediaFiles(mediaFiles, libraryItemData, libraryItem, preferAudioMetadata, preferOverdriveMediaMarker, libraryScan) await MediaFileScanner.scanMediaFiles(mediaFiles, libraryItem, libraryScan)
} }
await libraryItem.syncFiles(preferOpfMetadata) await libraryItem.syncFiles(preferOpfMetadata)
@ -638,8 +641,7 @@ class Scanner {
async scanPotentialNewLibraryItem(libraryMediaType, folder, fullPath, isSingleMediaItem = false) { async scanPotentialNewLibraryItem(libraryMediaType, folder, fullPath, isSingleMediaItem = false) {
const libraryItemData = await getLibraryItemFileData(libraryMediaType, folder, fullPath, isSingleMediaItem, this.db.serverSettings) const libraryItemData = await getLibraryItemFileData(libraryMediaType, folder, fullPath, isSingleMediaItem, this.db.serverSettings)
if (!libraryItemData) return null if (!libraryItemData) return null
const serverSettings = this.db.serverSettings return this.scanNewLibraryItem(libraryItemData, libraryMediaType)
return this.scanNewLibraryItem(libraryItemData, libraryMediaType, serverSettings.scannerPreferAudioMetadata, serverSettings.scannerPreferOpfMetadata, serverSettings.scannerFindCovers, serverSettings.scannerPreferOverdriveMediaMarker)
} }
async searchForCover(libraryItem, libraryScan = null) { async searchForCover(libraryItem, libraryScan = null) {