mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2024-12-20 19:06:06 +01:00
366 lines
14 KiB
JavaScript
366 lines
14 KiB
JavaScript
const sequelize = require('sequelize')
|
|
const Path = require('path')
|
|
const packageJson = require('../../package.json')
|
|
const Logger = require('../Logger')
|
|
const SocketAuthority = require('../SocketAuthority')
|
|
const Database = require('../Database')
|
|
const fs = require('../libs/fsExtra')
|
|
const fileUtils = require('../utils/fileUtils')
|
|
const scanUtils = require('../utils/scandir')
|
|
const { ScanResult, LogLevel } = require('../utils/constants')
|
|
const libraryFilters = require('../utils/queries/libraryFilters')
|
|
const ScanOptions = require('./ScanOptions')
|
|
const LibraryScan = require('./LibraryScan')
|
|
const LibraryItemScanData = require('./LibraryItemScanData')
|
|
const BookScanner = require('./BookScanner')
|
|
|
|
class LibraryScanner {
|
|
constructor(coverManager, taskManager) {
|
|
this.coverManager = coverManager
|
|
this.taskManager = taskManager
|
|
|
|
this.cancelLibraryScan = {}
|
|
this.librariesScanning = []
|
|
}
|
|
|
|
/**
|
|
* @param {string} libraryId
|
|
* @returns {boolean}
|
|
*/
|
|
isLibraryScanning(libraryId) {
|
|
return this.librariesScanning.some(ls => ls.id === libraryId)
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param {import('../objects/Library')} library
|
|
* @param {*} options
|
|
*/
|
|
async scan(library, options = {}) {
|
|
if (this.isLibraryScanning(library.id)) {
|
|
Logger.error(`[Scanner] Already scanning ${library.id}`)
|
|
return
|
|
}
|
|
|
|
if (!library.folders.length) {
|
|
Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`)
|
|
return
|
|
}
|
|
|
|
const scanOptions = new ScanOptions()
|
|
scanOptions.setData(options, Database.serverSettings)
|
|
|
|
const libraryScan = new LibraryScan()
|
|
libraryScan.setData(library, scanOptions)
|
|
libraryScan.verbose = true
|
|
this.librariesScanning.push(libraryScan.getScanEmitData)
|
|
|
|
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
|
|
|
|
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
|
|
|
|
const canceled = await this.scanLibrary(libraryScan)
|
|
|
|
if (canceled) {
|
|
Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
|
|
delete this.cancelLibraryScan[libraryScan.libraryId]
|
|
}
|
|
|
|
libraryScan.setComplete()
|
|
|
|
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
|
|
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
|
|
|
|
if (canceled && !libraryScan.totalResults) {
|
|
const emitData = libraryScan.getScanEmitData
|
|
emitData.results = null
|
|
SocketAuthority.emitter('scan_complete', emitData)
|
|
return
|
|
}
|
|
|
|
SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
|
|
|
|
if (libraryScan.totalResults) {
|
|
libraryScan.saveLog()
|
|
}
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param {import('./LibraryScan')} libraryScan
|
|
* @returns {boolean} true if scan canceled
|
|
*/
|
|
async scanLibrary(libraryScan) {
|
|
// Make sure library filter data is set
|
|
// this is used to check for existing authors & series
|
|
await libraryFilters.getFilterData(libraryScan.library)
|
|
|
|
/** @type {LibraryItemScanData[]} */
|
|
let libraryItemDataFound = []
|
|
|
|
// Scan each library folder
|
|
for (let i = 0; i < libraryScan.folders.length; i++) {
|
|
const folder = libraryScan.folders[i]
|
|
const itemDataFoundInFolder = await this.scanFolder(libraryScan.library, folder)
|
|
libraryScan.addLog(LogLevel.INFO, `${itemDataFoundInFolder.length} item data found in folder "${folder.fullPath}"`)
|
|
libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder)
|
|
}
|
|
|
|
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
|
|
|
const existingLibraryItems = await Database.libraryItemModel.findAll({
|
|
where: {
|
|
libraryId: libraryScan.libraryId
|
|
}
|
|
})
|
|
|
|
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
|
|
|
const libraryItemIdsMissing = []
|
|
let oldLibraryItemsUpdated = []
|
|
for (const existingLibraryItem of existingLibraryItems) {
|
|
// First try to find matching library item with exact file path
|
|
let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path)
|
|
if (!libraryItemData) {
|
|
// Fallback to finding matching library item with matching inode value
|
|
libraryItemData = libraryItemDataFound.find(lid => lid.ino === existingLibraryItem.ino)
|
|
if (libraryItemData) {
|
|
libraryScan.addLog(LogLevel.INFO, `Library item with path "${existingLibraryItem.path}" was not found, but library item inode "${existingLibraryItem.ino}" was found at path "${libraryItemData.path}"`)
|
|
}
|
|
}
|
|
|
|
if (!libraryItemData) {
|
|
// Podcast folder can have no episodes and still be valid
|
|
if (libraryScan.libraryMediaType === 'podcast' && await fs.pathExists(existingLibraryItem.path)) {
|
|
libraryScan.addLog(LogLevel.INFO, `Library item "${existingLibraryItem.relPath}" folder exists but has no episodes`)
|
|
} else {
|
|
libraryScan.addLog(LogLevel.WARN, `Library Item "${existingLibraryItem.path}" (inode: ${existingLibraryItem.ino}) is missing`)
|
|
libraryScan.resultsMissing++
|
|
if (!existingLibraryItem.isMissing) {
|
|
libraryItemIdsMissing.push(existingLibraryItem.id)
|
|
|
|
// TODO: Temporary while using old model to socket emit
|
|
const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id)
|
|
oldLibraryItem.isMissing = true
|
|
oldLibraryItem.updatedAt = Date.now()
|
|
oldLibraryItemsUpdated.push(oldLibraryItem)
|
|
}
|
|
}
|
|
} else {
|
|
libraryItemDataFound = libraryItemDataFound.filter(lidf => lidf !== libraryItemData)
|
|
if (await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)) {
|
|
libraryScan.resultsUpdated++
|
|
if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
|
|
const libraryItem = await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
|
|
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
|
|
await oldLibraryItem.saveMetadata() // Save metadata.json
|
|
oldLibraryItemsUpdated.push(oldLibraryItem)
|
|
} else {
|
|
// TODO: Temporary while using old model to socket emit
|
|
const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id)
|
|
oldLibraryItemsUpdated.push(oldLibraryItem)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Emit item updates in chunks of 10 to client
|
|
if (oldLibraryItemsUpdated.length === 10) {
|
|
// TODO: Should only emit to clients where library item is accessible
|
|
SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded()))
|
|
oldLibraryItemsUpdated = []
|
|
}
|
|
|
|
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
|
}
|
|
// Emit item updates to client
|
|
if (oldLibraryItemsUpdated.length) {
|
|
// TODO: Should only emit to clients where library item is accessible
|
|
SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded()))
|
|
}
|
|
|
|
// Check authors that were removed from a book and remove them if they no longer have any books
|
|
// keep authors without books that have a asin, description or imagePath
|
|
if (libraryScan.authorsRemovedFromBooks.length) {
|
|
const bookAuthorsToRemove = (await Database.authorModel.findAll({
|
|
where: [
|
|
{
|
|
id: libraryScan.authorsRemovedFromBooks,
|
|
asin: {
|
|
[sequelize.Op.or]: [null, ""]
|
|
},
|
|
description: {
|
|
[sequelize.Op.or]: [null, ""]
|
|
},
|
|
imagePath: {
|
|
[sequelize.Op.or]: [null, ""]
|
|
}
|
|
},
|
|
sequelize.where(sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 0)
|
|
],
|
|
attributes: ['id'],
|
|
raw: true
|
|
})).map(au => au.id)
|
|
if (bookAuthorsToRemove.length) {
|
|
await Database.authorModel.destroy({
|
|
where: {
|
|
id: bookAuthorsToRemove
|
|
}
|
|
})
|
|
bookAuthorsToRemove.forEach((authorId) => {
|
|
Database.removeAuthorFromFilterData(libraryScan.libraryId, authorId)
|
|
// TODO: Clients were expecting full author in payload but its unnecessary
|
|
SocketAuthority.emitter('author_removed', { id: authorId })
|
|
})
|
|
libraryScan.addLog(LogLevel.INFO, `Removed ${bookAuthorsToRemove.length} authors`)
|
|
}
|
|
}
|
|
|
|
// Update missing library items
|
|
if (libraryItemIdsMissing.length) {
|
|
libraryScan.addLog(LogLevel.INFO, `Updating ${libraryItemIdsMissing.length} library items missing`)
|
|
await Database.libraryItemModel.update({
|
|
isMissing: true,
|
|
lastScan: Date.now(),
|
|
lastScanVersion: packageJson.version
|
|
}, {
|
|
where: {
|
|
id: libraryItemIdsMissing
|
|
}
|
|
})
|
|
}
|
|
|
|
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
|
|
|
// Add new library items
|
|
if (libraryItemDataFound.length) {
|
|
let newOldLibraryItems = []
|
|
for (const libraryItemData of libraryItemDataFound) {
|
|
const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan)
|
|
if (newLibraryItem) {
|
|
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem)
|
|
await oldLibraryItem.saveMetadata() // Save metadata.json
|
|
newOldLibraryItems.push(oldLibraryItem)
|
|
|
|
libraryScan.resultsAdded++
|
|
}
|
|
|
|
// Emit new items in chunks of 10 to client
|
|
if (newOldLibraryItems.length === 10) {
|
|
// TODO: Should only emit to clients where library item is accessible
|
|
SocketAuthority.emitter('items_added', newOldLibraryItems.map(li => li.toJSONExpanded()))
|
|
newOldLibraryItems = []
|
|
}
|
|
|
|
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
|
}
|
|
// Emit new items to client
|
|
if (newOldLibraryItems.length) {
|
|
// TODO: Should only emit to clients where library item is accessible
|
|
SocketAuthority.emitter('items_added', newOldLibraryItems.map(li => li.toJSONExpanded()))
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get scan data for library folder
|
|
* @param {import('../objects/Library')} library
|
|
* @param {import('../objects/Folder')} folder
|
|
* @returns {LibraryItemScanData[]}
|
|
*/
|
|
async scanFolder(library, folder) {
|
|
const folderPath = fileUtils.filePathToPOSIX(folder.fullPath)
|
|
|
|
const pathExists = await fs.pathExists(folderPath)
|
|
if (!pathExists) {
|
|
Logger.error(`[scandir] Invalid folder path does not exist "${folderPath}"`)
|
|
return []
|
|
}
|
|
|
|
const fileItems = await fileUtils.recurseFiles(folderPath)
|
|
const libraryItemGrouping = scanUtils.groupFileItemsIntoLibraryItemDirs(library.mediaType, fileItems, library.settings.audiobooksOnly)
|
|
|
|
if (!Object.keys(libraryItemGrouping).length) {
|
|
Logger.error(`Root path has no media folders: ${folderPath}`)
|
|
return []
|
|
}
|
|
|
|
const items = []
|
|
for (const libraryItemPath in libraryItemGrouping) {
|
|
let isFile = false // item is not in a folder
|
|
let libraryItemData = null
|
|
let fileObjs = []
|
|
if (libraryItemPath === libraryItemGrouping[libraryItemPath]) {
|
|
// Media file in root only get title
|
|
libraryItemData = {
|
|
mediaMetadata: {
|
|
title: Path.basename(libraryItemPath, Path.extname(libraryItemPath))
|
|
},
|
|
path: Path.posix.join(folderPath, libraryItemPath),
|
|
relPath: libraryItemPath
|
|
}
|
|
fileObjs = await scanUtils.buildLibraryFile(folderPath, [libraryItemPath])
|
|
isFile = true
|
|
} else {
|
|
libraryItemData = scanUtils.getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath)
|
|
fileObjs = await scanUtils.buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath])
|
|
}
|
|
|
|
const libraryItemFolderStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path)
|
|
|
|
if (!libraryItemFolderStats.ino) {
|
|
Logger.warn(`[LibraryScanner] Library item folder "${libraryItemData.path}" has no inode value`)
|
|
continue
|
|
}
|
|
|
|
items.push(new LibraryItemScanData({
|
|
libraryFolderId: folder.id,
|
|
libraryId: folder.libraryId,
|
|
mediaType: library.mediaType,
|
|
ino: libraryItemFolderStats.ino,
|
|
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
|
|
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
|
|
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
|
|
path: libraryItemData.path,
|
|
relPath: libraryItemData.relPath,
|
|
isFile,
|
|
mediaMetadata: libraryItemData.mediaMetadata || null,
|
|
libraryFiles: fileObjs
|
|
}))
|
|
}
|
|
return items
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param {import('../models/LibraryItem')} existingLibraryItem
|
|
* @param {LibraryItemScanData} libraryItemData
|
|
* @param {LibraryScan} libraryScan
|
|
*/
|
|
async rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) {
|
|
if (existingLibraryItem.mediaType === 'book') {
|
|
const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
|
|
return libraryItem
|
|
} else {
|
|
// TODO: Scan updated podcast
|
|
}
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param {LibraryItemScanData} libraryItemData
|
|
* @param {LibraryScan} libraryScan
|
|
*/
|
|
async scanNewLibraryItem(libraryItemData, libraryScan) {
|
|
if (libraryScan.libraryMediaType === 'book') {
|
|
const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, libraryScan)
|
|
if (newLibraryItem) {
|
|
libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
|
|
}
|
|
return newLibraryItem
|
|
} else {
|
|
// TODO: Scan new podcast
|
|
return null
|
|
}
|
|
}
|
|
}
|
|
module.exports = LibraryScanner |