Fix:Server crash when deleting library item #2031

This commit is contained in:
advplyr 2023-08-26 16:33:27 -05:00
parent a3899b68e1
commit a38e43213d
8 changed files with 437 additions and 20 deletions

View File

@ -34,9 +34,12 @@ class SocketAuthority {
return Object.values(this.clients).filter(c => c.user && c.user.id === userId)
}
// Emits event to all authorized clients
// optional filter function to only send event to specific users
// TODO: validate that filter is actually a function
/**
* Emits event to all authorized clients
* @param {string} evt
* @param {any} data
* @param {Function} [filter] optional filter function to only send event to specific users
*/
emitter(evt, data, filter = null) {
for (const socketId in this.clients) {
if (this.clients[socketId].user) {

View File

@ -0,0 +1,6 @@
class AudioFileScanner {
constructor() { }
}
module.exports = new AudioFileScanner()

View File

@ -0,0 +1,147 @@
const packageJson = require('../../package.json')
const { LogLevel } = require('../utils/constants')
const LibraryItem = require('../models/LibraryItem')
class LibraryItemScanData {
constructor(data) {
/** @type {string} */
this.libraryFolderId = data.libraryFolderId
/** @type {string} */
this.libraryId = data.libraryId
/** @type {string} */
this.ino = data.ino
/** @type {number} */
this.mtimeMs = data.mtimeMs
/** @type {number} */
this.ctimeMs = data.ctimeMs
/** @type {number} */
this.birthtimeMs = data.birthtimeMs
/** @type {string} */
this.path = data.path
/** @type {string} */
this.relPath = data.relPath
/** @type {boolean} */
this.isFile = data.isFile
/** @type {{title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} */
this.mediaMetadata = data.mediaMetadata
/** @type {import('../objects/files/LibraryFile')[]} */
this.libraryFiles = data.libraryFiles
// Set after check
/** @type {boolean} */
this.hasChanges
/** @type {boolean} */
this.hasPathChange
/** @type {LibraryItem.LibraryFileObject[]} */
this.libraryFilesRemoved
/** @type {LibraryItem.LibraryFileObject[]} */
this.libraryFilesAdded
/** @type {LibraryItem.LibraryFileObject[]} */
this.libraryFilesModified
}
/**
*
* @param {LibraryItem} existingLibraryItem
* @param {import('./LibraryScan')} libraryScan
*/
async checkLibraryItemData(existingLibraryItem, libraryScan) {
const keysToCompare = ['libraryFolderId', 'ino', 'mtimeMs', 'ctimeMs', 'birthtimeMs', 'path', 'relPath', 'isFile']
this.hasChanges = false
this.hasPathChange = false
for (const key of keysToCompare) {
if (existingLibraryItem[key] !== this[key]) {
libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "${key}" changed from "${existingLibraryItem[key]}" to "${this[key]}"`)
existingLibraryItem[key] = this[key]
this.hasChanges = true
if (key === 'relPath') {
this.hasPathChange = true
}
}
}
this.libraryFilesRemoved = []
this.libraryFilesModified = []
let libraryFilesAdded = this.libraryFiles.map(lf => lf)
for (const existingLibraryFile of existingLibraryItem.libraryFiles) {
// Find matching library file using path first and fallback to using inode value
let matchingLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === existingLibraryFile.metadata.path)
if (!matchingLibraryFile) {
matchingLibraryFile = this.libraryFiles.find(lf => lf.ino === existingLibraryFile.ino)
if (matchingLibraryFile) {
libraryScan.addLog(LogLevel.INFO, `Library file with path "${existingLibraryFile.metadata.path}" not found, but found file with matching inode value "${existingLibraryFile.ino}" at path "${matchingLibraryFile.metadata.path}"`)
}
}
if (!matchingLibraryFile) { // Library file removed
libraryScan.addLog(LogLevel.INFO, `Library file "${existingLibraryFile.metadata.path}" was removed from library item "${existingLibraryItem.path}"`)
this.libraryFilesRemoved.push(existingLibraryFile)
existingLibraryItem.libraryFiles = existingLibraryItem.libraryFiles.filter(lf => lf !== existingLibraryFile)
this.hasChanges = true
} else {
libraryFilesAdded = libraryFilesAdded.filter(lf => lf !== matchingLibraryFile)
if (this.compareUpdateLibraryFile(existingLibraryItem.path, existingLibraryFile, matchingLibraryFile, libraryScan)) {
this.libraryFilesModified.push(existingLibraryFile)
this.hasChanges = true
}
}
}
// Log new library files found
if (libraryFilesAdded.length) {
this.hasChanges = true
for (const libraryFile of libraryFilesAdded) {
libraryScan.addLog(LogLevel.INFO, `New library file found with path "${libraryFile.metadata.path}" for library item "${existingLibraryItem.path}"`)
existingLibraryItem.libraryFiles.push(libraryFile.toJSON())
}
}
if (this.hasChanges) {
existingLibraryItem.lastScan = Date.now()
existingLibraryItem.lastScanVersion = packageJson.version
await existingLibraryItem.save()
} else {
libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.path}" is up-to-date`)
}
this.libraryFilesAdded = libraryFilesAdded
}
/**
* Update existing library file with scanned in library file data
* @param {string} libraryItemPath
* @param {LibraryItem.LibraryFileObject} existingLibraryFile
* @param {import('../objects/files/LibraryFile')} scannedLibraryFile
* @param {import('./LibraryScan')} libraryScan
* @returns {boolean} false if no changes
*/
compareUpdateLibraryFile(libraryItemPath, existingLibraryFile, scannedLibraryFile, libraryScan) {
let hasChanges = false
if (existingLibraryFile.ino !== scannedLibraryFile.ino) {
existingLibraryFile.ino = scannedLibraryFile.ino
hasChanges = true
}
for (const key in existingLibraryFile.metadata) {
if (existingLibraryFile.metadata[key] !== scannedLibraryFile.metadata[key]) {
if (key !== 'path' && key !== 'relPath') {
libraryScan.addLog(LogLevel.DEBUG, `Library file "${existingLibraryFile.metadata.path}" for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`)
} else {
libraryScan.addLog(LogLevel.DEBUG, `Library file for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`)
}
existingLibraryFile.metadata[key] = scannedLibraryFile.metadata[key]
hasChanges = true
}
}
if (hasChanges) {
existingLibraryFile.updatedAt = Date.now()
}
return hasChanges
}
}
module.exports = LibraryItemScanData

View File

@ -0,0 +1,230 @@
const Path = require('path')
const packageJson = require('../../package.json')
const Logger = require('../Logger')
const SocketAuthority = require('../SocketAuthority')
const Database = require('../Database')
const fs = require('../libs/fsExtra')
const fileUtils = require('../utils/fileUtils')
const scanUtils = require('../utils/scandir')
const { ScanResult, LogLevel } = require('../utils/constants')
const LibraryItemScanData = require('./LibraryItemScanData')
class LibraryScanner {
constructor(coverManager, taskManager) {
this.coverManager = coverManager
this.taskManager = taskManager
this.cancelLibraryScan = {}
this.librariesScanning = []
}
/**
* @param {string} libraryId
* @returns {boolean}
*/
isLibraryScanning(libraryId) {
return this.librariesScanning.some(ls => ls.id === libraryId)
}
/**
*
* @param {import('../objects/Library')} library
* @param {*} options
*/
async scan(library, options = {}) {
if (this.isLibraryScanning(library.id)) {
Logger.error(`[Scanner] Already scanning ${library.id}`)
return
}
if (!library.folders.length) {
Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`)
return
}
const scanOptions = new ScanOptions()
scanOptions.setData(options, Database.serverSettings)
const libraryScan = new LibraryScan()
libraryScan.setData(library, scanOptions)
libraryScan.verbose = true
this.librariesScanning.push(libraryScan.getScanEmitData)
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
const canceled = await this.scanLibrary(libraryScan)
if (canceled) {
Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
delete this.cancelLibraryScan[libraryScan.libraryId]
}
libraryScan.setComplete()
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
if (canceled && !libraryScan.totalResults) {
const emitData = libraryScan.getScanEmitData
emitData.results = null
SocketAuthority.emitter('scan_complete', emitData)
return
}
SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
if (libraryScan.totalResults) {
libraryScan.saveLog()
}
}
/**
*
* @param {import('./LibraryScan')} libraryScan
*/
async scanLibrary(libraryScan) {
/** @type {LibraryItemScanData[]} */
let libraryItemDataFound = []
// Scan each library folder
for (let i = 0; i < libraryScan.folders.length; i++) {
const folder = libraryScan.folders[i]
const itemDataFoundInFolder = await this.scanFolder(libraryScan.library, folder)
libraryScan.addLog(LogLevel.INFO, `${itemDataFoundInFolder.length} item data found in folder "${folder.fullPath}"`)
libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder)
}
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
const existingLibraryItems = await Database.libraryItemModel.findAll({
where: {
libraryId: libraryScan.libraryId
},
attributes: ['id', 'mediaId', 'mediaType', 'path', 'relPath', 'ino', 'isMissing', 'mtime', 'ctime', 'birthtime', 'libraryFiles', 'libraryFolderId']
})
const libraryItemIdsMissing = []
for (const existingLibraryItem of existingLibraryItems) {
// First try to find matching library item with exact file path
let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path)
if (!libraryItemData) {
// Fallback to finding matching library item with matching inode value
libraryItemData = libraryItemDataFound.find(lid => lid.ino === existingLibraryItem.ino)
if (libraryItemData) {
libraryScan.addLog(LogLevel.INFO, `Library item with path "${existingLibraryItem.path}" was not found, but library item inode "${existingLibraryItem.ino}" was found at path "${libraryItemData.path}"`)
}
}
if (!libraryItemData) {
// Podcast folder can have no episodes and still be valid
if (libraryScan.libraryMediaType === 'podcast' && await fs.pathExists(existingLibraryItem.path)) {
libraryScan.addLog(LogLevel.INFO, `Library item "${existingLibraryItem.relPath}" folder exists but has no episodes`)
} else {
libraryScan.addLog(LogLevel.WARN, `Library Item "${existingLibraryItem.path}" (inode: ${existingLibraryItem.ino}) is missing`)
if (!existingLibraryItem.isMissing) {
libraryItemIdsMissing.push(existingLibraryItem.id)
}
}
} else {
await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)
if (libraryItemData.hasChanges) {
await this.rescanLibraryItem(existingLibraryItem, libraryItemData)
}
}
}
// Update missing library items
if (libraryItemIdsMissing.length) {
libraryScan.addLog(LogLevel.INFO, `Updating ${libraryItemIdsMissing.length} library items missing`)
await Database.libraryItemModel.update({
isMissing: true,
lastScan: Date.now(),
lastScanVersion: packageJson.version
}, {
where: {
id: libraryItemIdsMissing
}
})
}
}
/**
* Get scan data for library folder
* @param {import('../objects/Library')} library
* @param {import('../objects/Folder')} folder
* @returns {LibraryItemScanData[]}
*/
async scanFolder(library, folder) {
const folderPath = fileUtils.filePathToPOSIX(folder.fullPath)
const pathExists = await fs.pathExists(folderPath)
if (!pathExists) {
Logger.error(`[scandir] Invalid folder path does not exist "${folderPath}"`)
return []
}
const fileItems = await fileUtils.recurseFiles(folderPath)
const libraryItemGrouping = scanUtils.groupFileItemsIntoLibraryItemDirs(library.mediaType, fileItems, library.settings.audiobooksOnly)
if (!Object.keys(libraryItemGrouping).length) {
Logger.error(`Root path has no media folders: ${folderPath}`)
return []
}
const items = []
for (const libraryItemPath in libraryItemGrouping) {
let isFile = false // item is not in a folder
let libraryItemData = null
let fileObjs = []
if (libraryItemPath === libraryItemGrouping[libraryItemPath]) {
// Media file in root only get title
libraryItemData = {
mediaMetadata: {
title: Path.basename(libraryItemPath, Path.extname(libraryItemPath))
},
path: Path.posix.join(folderPath, libraryItemPath),
relPath: libraryItemPath
}
fileObjs = await scanUtils.buildLibraryFile(folderPath, [libraryItemPath])
isFile = true
} else {
libraryItemData = scanUtils.getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath)
fileObjs = await scanUtils.buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath])
}
const libraryItemFolderStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path)
if (!libraryItemFolderStats.ino) {
Logger.warn(`[LibraryScanner] Library item folder "${libraryItemData.path}" has no inode value`)
continue
}
items.push(new LibraryItemScanData({
libraryFolderId: folder.id,
libraryId: folder.libraryId,
ino: libraryItemFolderStats.ino,
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
isFile,
mediaMetadata: libraryItemData.mediaMetadata || null,
libraryFiles: fileObjs
}))
}
return items
}
/**
*
* @param {import('../models/LibraryItem')} existingLibraryItem
* @param {LibraryItemScanData} libraryItemData
*/
async rescanLibraryItem(existingLibraryItem, libraryItemData) {
}
}
module.exports = LibraryScanner

View File

@ -4,13 +4,18 @@ const AudioFile = require('../objects/files/AudioFile')
const VideoFile = require('../objects/files/VideoFile')
const prober = require('../utils/prober')
const toneProber = require('../utils/toneProber')
const Logger = require('../Logger')
const { LogLevel } = require('../utils/constants')
class MediaFileScanner {
constructor() { }
/**
* Get track and disc number from audio filename
* @param {{title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} mediaMetadataFromScan
* @param {import('../objects/files/LibraryFile')} audioLibraryFile
* @returns {{trackNumber:number, discNumber:number}}
*/
getTrackAndDiscNumberFromFilename(mediaMetadataFromScan, audioLibraryFile) {
const { title, author, series, publishedYear } = mediaMetadataFromScan
const { filename, path } = audioLibraryFile.metadata
@ -102,7 +107,12 @@ class MediaFileScanner {
}
}
// Returns array of { MediaFile, elapsed, averageScanDuration } from audio file scan objects
/**
* Returns array of { MediaFile, elapsed, averageScanDuration } from audio file scan objects
* @param {import('../objects/LibraryItem')} libraryItem
* @param {import('../objects/files/LibraryFile')[]} mediaLibraryFiles
* @returns {Promise<object>}
*/
async executeMediaFileScans(libraryItem, mediaLibraryFiles) {
const mediaType = libraryItem.mediaType
@ -206,11 +216,10 @@ class MediaFileScanner {
/**
* Scans media files for a library item and adds them as audio tracks and sets library item metadata
* @async
* @param {Array<LibraryFile>} mediaLibraryFiles - Media files for this library item
* @param {LibraryItem} libraryItem
* @param {LibraryScan} [libraryScan=null] - Optional when doing a library scan to use LibraryScan config/logs
* @return {Promise<Boolean>} True if any updates were made
* @param {import('../objects/files/LibraryFile')[]} mediaLibraryFiles
* @param {import('../objects/LibraryItem')} libraryItem
* @param {import('./LibraryScan')} [libraryScan=null] - Optional when doing a library scan to use LibraryScan config/logs
* @return {Promise<boolean>} True if any updates were made
*/
async scanMediaFiles(mediaLibraryFiles, libraryItem, libraryScan = null) {
const preferAudioMetadata = libraryScan ? !!libraryScan.preferAudioMetadata : !!global.ServerSettings.scannerPreferAudioMetadata

View File

@ -695,7 +695,7 @@ class Scanner {
Logger.debug(`[Scanner] Folder update for relative path "${itemDir}" is in library item "${existingLibraryItem.media.metadata.title}" - scan for updates`)
itemGroupingResults[itemDir] = await this.scanLibraryItem(library, folder, existingLibraryItem)
continue
} else if (library.settings.audiobooksOnly && !fileUpdateGroup[itemDir].some(checkFilepathIsAudioFile)) {
} else if (library.settings.audiobooksOnly && !fileUpdateGroup[itemDir].some?.(checkFilepathIsAudioFile)) {
Logger.debug(`[Scanner] Folder update for relative path "${itemDir}" has no audio files`)
continue
}

View File

@ -92,6 +92,12 @@ function bytesPretty(bytes, decimals = 0) {
}
module.exports.bytesPretty = bytesPretty
/**
* Get array of files inside dir
* @param {string} path
* @param {string} [relPathToReplace]
* @returns {{name:string, path:string, dirpath:string, reldirpath:string, fullpath:string, extension:string, deep:number}[]}
*/
async function recurseFiles(path, relPathToReplace = null) {
path = filePathToPOSIX(path)
if (!path.endsWith('/')) path = path + '/'

View File

@ -22,9 +22,12 @@ function checkFilepathIsAudioFile(filepath) {
}
module.exports.checkFilepathIsAudioFile = checkFilepathIsAudioFile
// TODO: Function needs to be re-done
// Input: array of relative file paths
// Output: map of files grouped into potential item dirs
/**
* TODO: Function needs to be re-done
* @param {string} mediaType
* @param {string[]} paths array of relative file paths
* @returns {Record<string,string[]>} map of files grouped into potential libarary item dirs
*/
function groupFilesIntoLibraryItemPaths(mediaType, paths) {
// Step 1: Clean path, Remove leading "/", Filter out non-media files in root dir
var nonMediaFilePaths = []
@ -97,8 +100,12 @@ function groupFilesIntoLibraryItemPaths(mediaType, paths) {
}
module.exports.groupFilesIntoLibraryItemPaths = groupFilesIntoLibraryItemPaths
// Input: array of relative file items (see recurseFiles)
// Output: map of files grouped into potential libarary item dirs
/**
* @param {string} mediaType
* @param {{name:string, path:string, dirpath:string, reldirpath:string, fullpath:string, extension:string, deep:number}[]} fileItems (see recurseFiles)
* @param {boolean} [audiobooksOnly=false]
* @returns {Record<string,string[]>} map of files grouped into potential libarary item dirs
*/
function groupFileItemsIntoLibraryItemDirs(mediaType, fileItems, audiobooksOnly = false) {
// Handle music where every audio file is a library item
if (mediaType === 'music') {
@ -173,8 +180,15 @@ function groupFileItemsIntoLibraryItemDirs(mediaType, fileItems, audiobooksOnly
})
return libraryItemGroup
}
module.exports.groupFileItemsIntoLibraryItemDirs = groupFileItemsIntoLibraryItemDirs
function cleanFileObjects(libraryItemPath, files) {
/**
* Get LibraryFile from filepath
* @param {string} libraryItemPath
* @param {string[]} files
* @returns {import('../objects/files/LibraryFile')}
*/
function buildLibraryFile(libraryItemPath, files) {
return Promise.all(files.map(async (file) => {
const filePath = Path.posix.join(libraryItemPath, file)
const newLibraryFile = new LibraryFile()
@ -182,6 +196,7 @@ function cleanFileObjects(libraryItemPath, files) {
return newLibraryFile
}))
}
module.exports.buildLibraryFile = buildLibraryFile
// Scan folder
async function scanFolder(library, folder) {
@ -211,7 +226,7 @@ async function scanFolder(library, folder) {
path: Path.posix.join(folderPath, libraryItemPath),
relPath: libraryItemPath
}
fileObjs = await cleanFileObjects(folderPath, [libraryItemPath])
fileObjs = await buildLibraryFile(folderPath, [libraryItemPath])
isFile = true
} else if (libraryItemPath === libraryItemGrouping[libraryItemPath]) {
// Media file in root only get title
@ -222,11 +237,11 @@ async function scanFolder(library, folder) {
path: Path.posix.join(folderPath, libraryItemPath),
relPath: libraryItemPath
}
fileObjs = await cleanFileObjects(folderPath, [libraryItemPath])
fileObjs = await buildLibraryFile(folderPath, [libraryItemPath])
isFile = true
} else {
libraryItemData = getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath)
fileObjs = await cleanFileObjects(libraryItemData.path, libraryItemGrouping[libraryItemPath])
fileObjs = await buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath])
}
const libraryItemFolderStats = await getFileTimestampsWithIno(libraryItemData.path)
@ -365,6 +380,7 @@ function getDataFromMediaDir(libraryMediaType, folderPath, relPath) {
return getPodcastDataFromDir(folderPath, relPath)
}
}
module.exports.getDataFromMediaDir = getDataFromMediaDir
// Called from Scanner.js
async function getLibraryItemFileData(libraryMediaType, folder, libraryItemPath, isSingleMediaItem) {