audiobookshelf/server/scanner/LibraryScanner.js
2023-09-02 17:49:28 -05:00

366 lines
14 KiB
JavaScript

const sequelize = require('sequelize')
const Path = require('path')
const packageJson = require('../../package.json')
const Logger = require('../Logger')
const SocketAuthority = require('../SocketAuthority')
const Database = require('../Database')
const fs = require('../libs/fsExtra')
const fileUtils = require('../utils/fileUtils')
const scanUtils = require('../utils/scandir')
const { ScanResult, LogLevel } = require('../utils/constants')
const libraryFilters = require('../utils/queries/libraryFilters')
const ScanOptions = require('./ScanOptions')
const LibraryScan = require('./LibraryScan')
const LibraryItemScanData = require('./LibraryItemScanData')
const BookScanner = require('./BookScanner')
class LibraryScanner {
constructor(coverManager, taskManager) {
this.coverManager = coverManager
this.taskManager = taskManager
this.cancelLibraryScan = {}
this.librariesScanning = []
}
/**
* @param {string} libraryId
* @returns {boolean}
*/
isLibraryScanning(libraryId) {
return this.librariesScanning.some(ls => ls.id === libraryId)
}
/**
*
* @param {import('../objects/Library')} library
* @param {*} options
*/
async scan(library, options = {}) {
if (this.isLibraryScanning(library.id)) {
Logger.error(`[Scanner] Already scanning ${library.id}`)
return
}
if (!library.folders.length) {
Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`)
return
}
const scanOptions = new ScanOptions()
scanOptions.setData(options, Database.serverSettings)
const libraryScan = new LibraryScan()
libraryScan.setData(library, scanOptions)
libraryScan.verbose = true
this.librariesScanning.push(libraryScan.getScanEmitData)
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
const canceled = await this.scanLibrary(libraryScan)
if (canceled) {
Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
delete this.cancelLibraryScan[libraryScan.libraryId]
}
libraryScan.setComplete()
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
if (canceled && !libraryScan.totalResults) {
const emitData = libraryScan.getScanEmitData
emitData.results = null
SocketAuthority.emitter('scan_complete', emitData)
return
}
SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
if (libraryScan.totalResults) {
libraryScan.saveLog()
}
}
/**
*
* @param {import('./LibraryScan')} libraryScan
* @returns {boolean} true if scan canceled
*/
async scanLibrary(libraryScan) {
// Make sure library filter data is set
// this is used to check for existing authors & series
await libraryFilters.getFilterData(libraryScan.library)
/** @type {LibraryItemScanData[]} */
let libraryItemDataFound = []
// Scan each library folder
for (let i = 0; i < libraryScan.folders.length; i++) {
const folder = libraryScan.folders[i]
const itemDataFoundInFolder = await this.scanFolder(libraryScan.library, folder)
libraryScan.addLog(LogLevel.INFO, `${itemDataFoundInFolder.length} item data found in folder "${folder.fullPath}"`)
libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder)
}
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
const existingLibraryItems = await Database.libraryItemModel.findAll({
where: {
libraryId: libraryScan.libraryId
}
})
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
const libraryItemIdsMissing = []
let oldLibraryItemsUpdated = []
for (const existingLibraryItem of existingLibraryItems) {
// First try to find matching library item with exact file path
let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path)
if (!libraryItemData) {
// Fallback to finding matching library item with matching inode value
libraryItemData = libraryItemDataFound.find(lid => lid.ino === existingLibraryItem.ino)
if (libraryItemData) {
libraryScan.addLog(LogLevel.INFO, `Library item with path "${existingLibraryItem.path}" was not found, but library item inode "${existingLibraryItem.ino}" was found at path "${libraryItemData.path}"`)
}
}
if (!libraryItemData) {
// Podcast folder can have no episodes and still be valid
if (libraryScan.libraryMediaType === 'podcast' && await fs.pathExists(existingLibraryItem.path)) {
libraryScan.addLog(LogLevel.INFO, `Library item "${existingLibraryItem.relPath}" folder exists but has no episodes`)
} else {
libraryScan.addLog(LogLevel.WARN, `Library Item "${existingLibraryItem.path}" (inode: ${existingLibraryItem.ino}) is missing`)
libraryScan.resultsMissing++
if (!existingLibraryItem.isMissing) {
libraryItemIdsMissing.push(existingLibraryItem.id)
// TODO: Temporary while using old model to socket emit
const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id)
oldLibraryItem.isMissing = true
oldLibraryItem.updatedAt = Date.now()
oldLibraryItemsUpdated.push(oldLibraryItem)
}
}
} else {
libraryItemDataFound = libraryItemDataFound.filter(lidf => lidf !== libraryItemData)
if (await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)) {
libraryScan.resultsUpdated++
if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
const libraryItem = await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
await oldLibraryItem.saveMetadata() // Save metadata.json
oldLibraryItemsUpdated.push(oldLibraryItem)
} else {
// TODO: Temporary while using old model to socket emit
const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id)
oldLibraryItemsUpdated.push(oldLibraryItem)
}
}
}
// Emit item updates in chunks of 10 to client
if (oldLibraryItemsUpdated.length === 10) {
// TODO: Should only emit to clients where library item is accessible
SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded()))
oldLibraryItemsUpdated = []
}
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
}
// Emit item updates to client
if (oldLibraryItemsUpdated.length) {
// TODO: Should only emit to clients where library item is accessible
SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded()))
}
// Check authors that were removed from a book and remove them if they no longer have any books
// keep authors without books that have a asin, description or imagePath
if (libraryScan.authorsRemovedFromBooks.length) {
const bookAuthorsToRemove = (await Database.authorModel.findAll({
where: [
{
id: libraryScan.authorsRemovedFromBooks,
asin: {
[sequelize.Op.or]: [null, ""]
},
description: {
[sequelize.Op.or]: [null, ""]
},
imagePath: {
[sequelize.Op.or]: [null, ""]
}
},
sequelize.where(sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 0)
],
attributes: ['id'],
raw: true
})).map(au => au.id)
if (bookAuthorsToRemove.length) {
await Database.authorModel.destroy({
where: {
id: bookAuthorsToRemove
}
})
bookAuthorsToRemove.forEach((authorId) => {
Database.removeAuthorFromFilterData(libraryScan.libraryId, authorId)
// TODO: Clients were expecting full author in payload but its unnecessary
SocketAuthority.emitter('author_removed', { id: authorId })
})
libraryScan.addLog(LogLevel.INFO, `Removed ${bookAuthorsToRemove.length} authors`)
}
}
// Update missing library items
if (libraryItemIdsMissing.length) {
libraryScan.addLog(LogLevel.INFO, `Updating ${libraryItemIdsMissing.length} library items missing`)
await Database.libraryItemModel.update({
isMissing: true,
lastScan: Date.now(),
lastScanVersion: packageJson.version
}, {
where: {
id: libraryItemIdsMissing
}
})
}
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
// Add new library items
if (libraryItemDataFound.length) {
let newOldLibraryItems = []
for (const libraryItemData of libraryItemDataFound) {
const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan)
if (newLibraryItem) {
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem)
await oldLibraryItem.saveMetadata() // Save metadata.json
newOldLibraryItems.push(oldLibraryItem)
libraryScan.resultsAdded++
}
// Emit new items in chunks of 10 to client
if (newOldLibraryItems.length === 10) {
// TODO: Should only emit to clients where library item is accessible
SocketAuthority.emitter('items_added', newOldLibraryItems.map(li => li.toJSONExpanded()))
newOldLibraryItems = []
}
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
}
// Emit new items to client
if (newOldLibraryItems.length) {
// TODO: Should only emit to clients where library item is accessible
SocketAuthority.emitter('items_added', newOldLibraryItems.map(li => li.toJSONExpanded()))
}
}
}
/**
* Get scan data for library folder
* @param {import('../objects/Library')} library
* @param {import('../objects/Folder')} folder
* @returns {LibraryItemScanData[]}
*/
async scanFolder(library, folder) {
const folderPath = fileUtils.filePathToPOSIX(folder.fullPath)
const pathExists = await fs.pathExists(folderPath)
if (!pathExists) {
Logger.error(`[scandir] Invalid folder path does not exist "${folderPath}"`)
return []
}
const fileItems = await fileUtils.recurseFiles(folderPath)
const libraryItemGrouping = scanUtils.groupFileItemsIntoLibraryItemDirs(library.mediaType, fileItems, library.settings.audiobooksOnly)
if (!Object.keys(libraryItemGrouping).length) {
Logger.error(`Root path has no media folders: ${folderPath}`)
return []
}
const items = []
for (const libraryItemPath in libraryItemGrouping) {
let isFile = false // item is not in a folder
let libraryItemData = null
let fileObjs = []
if (libraryItemPath === libraryItemGrouping[libraryItemPath]) {
// Media file in root only get title
libraryItemData = {
mediaMetadata: {
title: Path.basename(libraryItemPath, Path.extname(libraryItemPath))
},
path: Path.posix.join(folderPath, libraryItemPath),
relPath: libraryItemPath
}
fileObjs = await scanUtils.buildLibraryFile(folderPath, [libraryItemPath])
isFile = true
} else {
libraryItemData = scanUtils.getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath)
fileObjs = await scanUtils.buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath])
}
const libraryItemFolderStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path)
if (!libraryItemFolderStats.ino) {
Logger.warn(`[LibraryScanner] Library item folder "${libraryItemData.path}" has no inode value`)
continue
}
items.push(new LibraryItemScanData({
libraryFolderId: folder.id,
libraryId: folder.libraryId,
mediaType: library.mediaType,
ino: libraryItemFolderStats.ino,
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
isFile,
mediaMetadata: libraryItemData.mediaMetadata || null,
libraryFiles: fileObjs
}))
}
return items
}
/**
*
* @param {import('../models/LibraryItem')} existingLibraryItem
* @param {LibraryItemScanData} libraryItemData
* @param {LibraryScan} libraryScan
*/
async rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) {
if (existingLibraryItem.mediaType === 'book') {
const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
return libraryItem
} else {
// TODO: Scan updated podcast
}
}
/**
*
* @param {LibraryItemScanData} libraryItemData
* @param {LibraryScan} libraryScan
*/
async scanNewLibraryItem(libraryItemData, libraryScan) {
if (libraryScan.libraryMediaType === 'book') {
const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, libraryScan)
if (newLibraryItem) {
libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
}
return newLibraryItem
} else {
// TODO: Scan new podcast
return null
}
}
}
module.exports = LibraryScanner