Update match all books to load items from DB, remove library items loading to memory on init

This commit is contained in:
advplyr 2023-09-04 16:33:55 -05:00
parent 03115e5e53
commit 1dd1fe8994
19 changed files with 127 additions and 140 deletions

View File

@ -15,9 +15,6 @@ class Database {
this.isNew = false // New absdatabase.sqlite created this.isNew = false // New absdatabase.sqlite created
this.hasRootUser = false // Used to show initialization page in web ui this.hasRootUser = false // Used to show initialization page in web ui
// Temporarily using format of old DB
// TODO: below data should be loaded from the DB as needed
this.libraryItems = []
this.settings = [] this.settings = []
// Cached library filter data // Cached library filter data
@ -255,8 +252,6 @@ class Database {
await dbMigration.migrate(this.models) await dbMigration.migrate(this.models)
} }
const startTime = Date.now()
const settingsData = await this.models.setting.getOldSettings() const settingsData = await this.models.setting.getOldSettings()
this.settings = settingsData.settings this.settings = settingsData.settings
this.emailSettings = settingsData.emailSettings this.emailSettings = settingsData.emailSettings
@ -272,16 +267,9 @@ class Database {
await dbMigration.migrationPatch2(this) await dbMigration.migrationPatch2(this)
} }
Logger.info(`[Database] Loading db data...`)
this.libraryItems = await this.models.libraryItem.loadAllLibraryItems()
Logger.info(`[Database] Loaded ${this.libraryItems.length} library items`)
// Set if root user has been created // Set if root user has been created
this.hasRootUser = await this.models.user.getHasRootUser() this.hasRootUser = await this.models.user.getHasRootUser()
Logger.info(`[Database] Db data loaded in ${((Date.now() - startTime) / 1000).toFixed(2)}s`)
if (packageJson.version !== this.serverSettings.version) { if (packageJson.version !== this.serverSettings.version) {
Logger.info(`[Database] Server upgrade detected from ${this.serverSettings.version} to ${packageJson.version}`) Logger.info(`[Database] Server upgrade detected from ${this.serverSettings.version} to ${packageJson.version}`)
this.serverSettings.version = packageJson.version this.serverSettings.version = packageJson.version
@ -380,20 +368,10 @@ class Database {
return this.models.playlistMediaItem.bulkCreate(playlistMediaItems) return this.models.playlistMediaItem.bulkCreate(playlistMediaItems)
} }
getLibraryItem(libraryItemId) {
if (!this.sequelize || !libraryItemId) return false
// Temp support for old library item ids from mobile
if (libraryItemId.startsWith('li_')) return this.libraryItems.find(li => li.oldLibraryItemId === libraryItemId)
return this.libraryItems.find(li => li.id === libraryItemId)
}
async createLibraryItem(oldLibraryItem) { async createLibraryItem(oldLibraryItem) {
if (!this.sequelize) return false if (!this.sequelize) return false
await oldLibraryItem.saveMetadata() await oldLibraryItem.saveMetadata()
await this.models.libraryItem.fullCreateFromOld(oldLibraryItem) await this.models.libraryItem.fullCreateFromOld(oldLibraryItem)
this.libraryItems.push(oldLibraryItem)
} }
async updateLibraryItem(oldLibraryItem) { async updateLibraryItem(oldLibraryItem) {
@ -420,14 +398,12 @@ class Database {
for (const oldLibraryItem of oldLibraryItems) { for (const oldLibraryItem of oldLibraryItems) {
await oldLibraryItem.saveMetadata() await oldLibraryItem.saveMetadata()
await this.models.libraryItem.fullCreateFromOld(oldLibraryItem) await this.models.libraryItem.fullCreateFromOld(oldLibraryItem)
this.libraryItems.push(oldLibraryItem)
} }
} }
async removeLibraryItem(libraryItemId) { async removeLibraryItem(libraryItemId) {
if (!this.sequelize) return false if (!this.sequelize) return false
await this.models.libraryItem.removeById(libraryItemId) await this.models.libraryItem.removeById(libraryItemId)
this.libraryItems = this.libraryItems.filter(li => li.id !== libraryItemId)
} }
async createFeed(oldFeed) { async createFeed(oldFeed) {

View File

@ -76,7 +76,7 @@ class Server {
this.audioMetadataManager = new AudioMetadataMangaer(this.taskManager) this.audioMetadataManager = new AudioMetadataMangaer(this.taskManager)
this.rssFeedManager = new RssFeedManager() this.rssFeedManager = new RssFeedManager()
this.scanner = new Scanner(this.coverManager, this.taskManager) this.scanner = new Scanner(this.coverManager)
this.cronManager = new CronManager(this.podcastManager) this.cronManager = new CronManager(this.podcastManager)
// Routers // Routers

View File

@ -5,6 +5,7 @@ const { createNewSortInstance } = require('../libs/fastSort')
const Logger = require('../Logger') const Logger = require('../Logger')
const SocketAuthority = require('../SocketAuthority') const SocketAuthority = require('../SocketAuthority')
const Database = require('../Database') const Database = require('../Database')
const AuthorFinder = require('../finders/AuthorFinder')
const { reqSupportsWebp } = require('../utils/index') const { reqSupportsWebp } = require('../utils/index')
@ -70,7 +71,7 @@ class AuthorController {
await this.cacheManager.purgeImageCache(req.author.id) // Purge cache await this.cacheManager.purgeImageCache(req.author.id) // Purge cache
await this.coverManager.removeFile(req.author.imagePath) await this.coverManager.removeFile(req.author.imagePath)
} else if (payload.imagePath.startsWith('http')) { // Check if image path is a url } else if (payload.imagePath.startsWith('http')) { // Check if image path is a url
const imageData = await this.authorFinder.saveAuthorImage(req.author.id, payload.imagePath) const imageData = await AuthorFinder.saveAuthorImage(req.author.id, payload.imagePath)
if (imageData) { if (imageData) {
if (req.author.imagePath) { if (req.author.imagePath) {
await this.cacheManager.purgeImageCache(req.author.id) // Purge cache await this.cacheManager.purgeImageCache(req.author.id) // Purge cache
@ -168,9 +169,9 @@ class AuthorController {
let authorData = null let authorData = null
const region = req.body.region || 'us' const region = req.body.region || 'us'
if (req.body.asin) { if (req.body.asin) {
authorData = await this.authorFinder.findAuthorByASIN(req.body.asin, region) authorData = await AuthorFinder.findAuthorByASIN(req.body.asin, region)
} else { } else {
authorData = await this.authorFinder.findAuthorByName(req.body.q, region) authorData = await AuthorFinder.findAuthorByName(req.body.q, region)
} }
if (!authorData) { if (!authorData) {
return res.status(404).send('Author not found') return res.status(404).send('Author not found')
@ -187,7 +188,7 @@ class AuthorController {
if (authorData.image && (!req.author.imagePath || hasUpdates)) { if (authorData.image && (!req.author.imagePath || hasUpdates)) {
this.cacheManager.purgeImageCache(req.author.id) this.cacheManager.purgeImageCache(req.author.id)
const imageData = await this.authorFinder.saveAuthorImage(req.author.id, authorData.image) const imageData = await AuthorFinder.saveAuthorImage(req.author.id, authorData.image)
if (imageData) { if (imageData) {
req.author.imagePath = imageData.path req.author.imagePath = imageData.path
hasUpdates = true hasUpdates = true

View File

@ -54,7 +54,7 @@ class EmailController {
async sendEBookToDevice(req, res) { async sendEBookToDevice(req, res) {
Logger.debug(`[EmailController] Send ebook to device request for libraryItemId=${req.body.libraryItemId}, deviceName=${req.body.deviceName}`) Logger.debug(`[EmailController] Send ebook to device request for libraryItemId=${req.body.libraryItemId}, deviceName=${req.body.deviceName}`)
const libraryItem = Database.getLibraryItem(req.body.libraryItemId) const libraryItem = await Database.libraryItemModel.getOldById(req.body.libraryItemId)
if (!libraryItem) { if (!libraryItem) {
return res.status(404).send('Library item not found') return res.status(404).send('Library item not found')
} }

View File

@ -411,7 +411,9 @@ class LibraryItemController {
return res.sendStatus(400) return res.sendStatus(400)
} }
const libraryItems = req.body.libraryItemIds.map(lid => Database.getLibraryItem(lid)).filter(li => li) const libraryItems = await Database.libraryItemModel.getAllOldLibraryItems({
id: req.body.libraryItemIds
})
if (!libraryItems?.length) { if (!libraryItems?.length) {
return res.sendStatus(400) return res.sendStatus(400)
} }

View File

@ -193,7 +193,8 @@ class MeController {
Logger.error(`[MeController] syncLocalMediaProgress invalid local media progress object`, localProgress) Logger.error(`[MeController] syncLocalMediaProgress invalid local media progress object`, localProgress)
continue continue
} }
const libraryItem = Database.getLibraryItem(localProgress.libraryItemId)
const libraryItem = await Database.libraryItemModel.getOldById(localProgress.libraryItemId)
if (!libraryItem) { if (!libraryItem) {
Logger.error(`[MeController] syncLocalMediaProgress invalid local media progress object no library item`, localProgress) Logger.error(`[MeController] syncLocalMediaProgress invalid local media progress object no library item`, localProgress)
continue continue
@ -245,13 +246,15 @@ class MeController {
} }
// GET: api/me/items-in-progress // GET: api/me/items-in-progress
getAllLibraryItemsInProgress(req, res) { async getAllLibraryItemsInProgress(req, res) {
const limit = !isNaN(req.query.limit) ? Number(req.query.limit) || 25 : 25 const limit = !isNaN(req.query.limit) ? Number(req.query.limit) || 25 : 25
let itemsInProgress = [] let itemsInProgress = []
// TODO: More efficient to do this in a single query
for (const mediaProgress of req.user.mediaProgress) { for (const mediaProgress of req.user.mediaProgress) {
if (!mediaProgress.isFinished && (mediaProgress.progress > 0 || mediaProgress.ebookProgress > 0)) { if (!mediaProgress.isFinished && (mediaProgress.progress > 0 || mediaProgress.ebookProgress > 0)) {
const libraryItem = Database.getLibraryItem(mediaProgress.libraryItemId)
const libraryItem = await Database.libraryItemModel.getOldById(mediaProgress.libraryItemId)
if (libraryItem) { if (libraryItem) {
if (mediaProgress.episodeId && libraryItem.mediaType === 'podcast') { if (mediaProgress.episodeId && libraryItem.mediaType === 'podcast') {
const episode = libraryItem.media.episodes.find(ep => ep.id === mediaProgress.episodeId) const episode = libraryItem.media.episodes.find(ep => ep.id === mediaProgress.episodeId)

View File

@ -1,4 +1,8 @@
const Logger = require("../Logger") const Logger = require("../Logger")
const BookFinder = require('../finders/BookFinder')
const PodcastFinder = require('../finders/PodcastFinder')
const AuthorFinder = require('../finders/AuthorFinder')
const MusicFinder = require('../finders/MusicFinder')
class SearchController { class SearchController {
constructor() { } constructor() { }
@ -7,7 +11,7 @@ class SearchController {
const provider = req.query.provider || 'google' const provider = req.query.provider || 'google'
const title = req.query.title || '' const title = req.query.title || ''
const author = req.query.author || '' const author = req.query.author || ''
const results = await this.bookFinder.search(provider, title, author) const results = await BookFinder.search(provider, title, author)
res.json(results) res.json(results)
} }
@ -21,8 +25,8 @@ class SearchController {
} }
let results = null let results = null
if (podcast) results = await this.podcastFinder.findCovers(query.title) if (podcast) results = await PodcastFinder.findCovers(query.title)
else results = await this.bookFinder.findCovers(query.provider || 'google', query.title, query.author || null) else results = await BookFinder.findCovers(query.provider || 'google', query.title, query.author || null)
res.json({ res.json({
results results
}) })
@ -30,20 +34,20 @@ class SearchController {
async findPodcasts(req, res) { async findPodcasts(req, res) {
const term = req.query.term const term = req.query.term
const results = await this.podcastFinder.search(term) const results = await PodcastFinder.search(term)
res.json(results) res.json(results)
} }
async findAuthor(req, res) { async findAuthor(req, res) {
const query = req.query.q const query = req.query.q
const author = await this.authorFinder.findAuthorByName(query) const author = await AuthorFinder.findAuthorByName(query)
res.json(author) res.json(author)
} }
async findChapters(req, res) { async findChapters(req, res) {
const asin = req.query.asin const asin = req.query.asin
const region = (req.query.region || 'us').toLowerCase() const region = (req.query.region || 'us').toLowerCase()
const chapterData = await this.bookFinder.findChapters(asin, region) const chapterData = await BookFinder.findChapters(asin, region)
if (!chapterData) { if (!chapterData) {
return res.json({ error: 'Chapters not found' }) return res.json({ error: 'Chapters not found' })
} }
@ -51,7 +55,7 @@ class SearchController {
} }
async findMusicTrack(req, res) { async findMusicTrack(req, res) {
const tracks = await this.musicFinder.searchTrack(req.query || {}) const tracks = await MusicFinder.searchTrack(req.query || {})
res.json({ res.json({
tracks tracks
}) })

View File

@ -62,9 +62,9 @@ class SessionController {
}) })
} }
getOpenSession(req, res) { async getOpenSession(req, res) {
var libraryItem = Database.getLibraryItem(req.session.libraryItemId) const libraryItem = await Database.libraryItemModel.getOldById(req.session.libraryItemId)
var sessionForClient = req.session.toJSONForClient(libraryItem) const sessionForClient = req.session.toJSONForClient(libraryItem)
res.json(sessionForClient) res.json(sessionForClient)
} }

View File

@ -66,7 +66,7 @@ class ToolsController {
const libraryItems = [] const libraryItems = []
for (const libraryItemId of libraryItemIds) { for (const libraryItemId of libraryItemIds) {
const libraryItem = Database.getLibraryItem(libraryItemId) const libraryItem = await Database.libraryItemModel.getOldById(libraryItemId)
if (!libraryItem) { if (!libraryItem) {
Logger.error(`[ToolsController] Batch embed metadata library item (${libraryItemId}) not found`) Logger.error(`[ToolsController] Batch embed metadata library item (${libraryItemId}) not found`)
return res.sendStatus(404) return res.sendStatus(404)

View File

@ -8,8 +8,6 @@ const filePerms = require('../utils/filePerms')
class AuthorFinder { class AuthorFinder {
constructor() { constructor() {
this.AuthorPath = Path.join(global.MetadataPath, 'authors')
this.audnexus = new Audnexus() this.audnexus = new Audnexus()
} }
@ -37,7 +35,7 @@ class AuthorFinder {
} }
async saveAuthorImage(authorId, url) { async saveAuthorImage(authorId, url) {
var authorDir = this.AuthorPath var authorDir = Path.join(global.MetadataPath, 'authors')
var relAuthorDir = Path.posix.join('/metadata', 'authors') var relAuthorDir = Path.posix.join('/metadata', 'authors')
if (!await fs.pathExists(authorDir)) { if (!await fs.pathExists(authorDir)) {
@ -61,4 +59,4 @@ class AuthorFinder {
} }
} }
} }
module.exports = AuthorFinder module.exports = new AuthorFinder()

View File

@ -253,4 +253,4 @@ class BookFinder {
return this.audnexus.getChaptersByASIN(asin, region) return this.audnexus.getChaptersByASIN(asin, region)
} }
} }
module.exports = BookFinder module.exports = new BookFinder()

View File

@ -9,4 +9,4 @@ class MusicFinder {
return this.musicBrainz.searchTrack(options) return this.musicBrainz.searchTrack(options)
} }
} }
module.exports = MusicFinder module.exports = new MusicFinder()

View File

@ -22,4 +22,4 @@ class PodcastFinder {
return results.map(r => r.cover).filter(r => r) return results.map(r => r.cover).filter(r => r)
} }
} }
module.exports = PodcastFinder module.exports = new PodcastFinder()

View File

@ -93,7 +93,7 @@ class PlaybackSessionManager {
} }
async syncLocalSession(user, sessionJson, deviceInfo) { async syncLocalSession(user, sessionJson, deviceInfo) {
const libraryItem = Database.getLibraryItem(sessionJson.libraryItemId) const libraryItem = await Database.libraryItemModel.getOldById(sessionJson.libraryItemId)
const episode = (sessionJson.episodeId && libraryItem && libraryItem.isPodcast) ? libraryItem.media.getEpisode(sessionJson.episodeId) : null const episode = (sessionJson.episodeId && libraryItem && libraryItem.isPodcast) ? libraryItem.media.getEpisode(sessionJson.episodeId) : null
if (!libraryItem || (libraryItem.isPodcast && !episode)) { if (!libraryItem || (libraryItem.isPodcast && !episode)) {
Logger.error(`[PlaybackSessionManager] syncLocalSession: Media item not found for session "${sessionJson.displayTitle}" (${sessionJson.id})`) Logger.error(`[PlaybackSessionManager] syncLocalSession: Media item not found for session "${sessionJson.displayTitle}" (${sessionJson.id})`)

View File

@ -87,7 +87,7 @@ class RssFeedManager {
// Check if feed needs to be updated // Check if feed needs to be updated
if (feed.entityType === 'libraryItem') { if (feed.entityType === 'libraryItem') {
const libraryItem = Database.getLibraryItem(feed.entityId) const libraryItem = await Database.libraryItemModel.getOldById(feed.entityId)
let mostRecentlyUpdatedAt = libraryItem.updatedAt let mostRecentlyUpdatedAt = libraryItem.updatedAt
if (libraryItem.isPodcast) { if (libraryItem.isPodcast) {

View File

@ -118,12 +118,13 @@ class LibraryItem extends Model {
* @param {number} limit * @param {number} limit
* @returns {Promise<Model<LibraryItem>[]>} LibraryItem * @returns {Promise<Model<LibraryItem>[]>} LibraryItem
*/ */
static getLibraryItemsIncrement(offset, limit) { static getLibraryItemsIncrement(offset, limit, where = null) {
return this.findAll({ return this.findAll({
benchmark: true, benchmark: true,
logging: (sql, timeMs) => { logging: (sql, timeMs) => {
console.log(`[Query] Elapsed ${timeMs}ms.`) console.log(`[Query] Elapsed ${timeMs}ms.`)
}, },
where,
include: [ include: [
{ {
model: this.sequelize.models.book, model: this.sequelize.models.book,

View File

@ -29,11 +29,6 @@ const ToolsController = require('../controllers/ToolsController')
const RSSFeedController = require('../controllers/RSSFeedController') const RSSFeedController = require('../controllers/RSSFeedController')
const MiscController = require('../controllers/MiscController') const MiscController = require('../controllers/MiscController')
const BookFinder = require('../finders/BookFinder')
const AuthorFinder = require('../finders/AuthorFinder')
const PodcastFinder = require('../finders/PodcastFinder')
const MusicFinder = require('../finders/MusicFinder')
const Author = require('../objects/entities/Author') const Author = require('../objects/entities/Author')
const Series = require('../objects/entities/Series') const Series = require('../objects/entities/Series')
@ -55,11 +50,6 @@ class ApiRouter {
this.emailManager = Server.emailManager this.emailManager = Server.emailManager
this.taskManager = Server.taskManager this.taskManager = Server.taskManager
this.bookFinder = new BookFinder()
this.authorFinder = new AuthorFinder()
this.podcastFinder = new PodcastFinder()
this.musicFinder = new MusicFinder()
this.router = express() this.router = express()
this.router.disable('x-powered-by') this.router.disable('x-powered-by')
this.init() this.init()

View File

@ -16,6 +16,7 @@ const CoverManager = require('../managers/CoverManager')
const LibraryFile = require('../objects/files/LibraryFile') const LibraryFile = require('../objects/files/LibraryFile')
const SocketAuthority = require('../SocketAuthority') const SocketAuthority = require('../SocketAuthority')
const fsExtra = require("../libs/fsExtra") const fsExtra = require("../libs/fsExtra")
// const BookFinder = require('../finders/BookFinder')
/** /**
* Metadata for books pulled from files * Metadata for books pulled from files
@ -1049,5 +1050,32 @@ class BookScanner {
scanLogger.addLog(LogLevel.INFO, `Removed ${bookSeriesToRemove.length} series`) scanLogger.addLog(LogLevel.INFO, `Removed ${bookSeriesToRemove.length} series`)
} }
} }
// async searchForCover(libraryItem, libraryScan = null) {
// const options = {
// titleDistance: 2,
// authorDistance: 2
// }
// const scannerCoverProvider = Database.serverSettings.scannerCoverProvider
// const results = await BookFinder.findCovers(scannerCoverProvider, libraryItem.media.metadata.title, libraryItem.media.metadata.authorName, options)
// if (results.length) {
// if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Found best cover for "${libraryItem.media.metadata.title}"`)
// else Logger.debug(`[Scanner] Found best cover for "${libraryItem.media.metadata.title}"`)
// // If the first cover result fails, attempt to download the second
// for (let i = 0; i < results.length && i < 2; i++) {
// // Downloads and updates the book cover
// const result = await this.coverManager.downloadCoverFromUrl(libraryItem, results[i])
// if (result.error) {
// Logger.error(`[Scanner] Failed to download cover from url "${results[i]}" | Attempt ${i + 1}`, result.error)
// } else {
// return true
// }
// }
// }
// return false
// }
} }
module.exports = new BookScanner() module.exports = new BookScanner()

View File

@ -3,7 +3,6 @@ const SocketAuthority = require('../SocketAuthority')
const Database = require('../Database') const Database = require('../Database')
// Utils // Utils
const { LogLevel } = require('../utils/constants')
const { findMatchingEpisodesInFeed, getPodcastFeed } = require('../utils/podcastUtils') const { findMatchingEpisodesInFeed, getPodcastFeed } = require('../utils/podcastUtils')
const BookFinder = require('../finders/BookFinder') const BookFinder = require('../finders/BookFinder')
@ -11,44 +10,11 @@ const PodcastFinder = require('../finders/PodcastFinder')
const LibraryScan = require('./LibraryScan') const LibraryScan = require('./LibraryScan')
const Author = require('../objects/entities/Author') const Author = require('../objects/entities/Author')
const Series = require('../objects/entities/Series') const Series = require('../objects/entities/Series')
const LibraryScanner = require('./LibraryScanner')
class Scanner { class Scanner {
constructor(coverManager, taskManager) { constructor(coverManager) {
this.coverManager = coverManager this.coverManager = coverManager
this.taskManager = taskManager
this.cancelLibraryScan = {}
this.librariesScanning = []
this.bookFinder = new BookFinder()
this.podcastFinder = new PodcastFinder()
}
async searchForCover(libraryItem, libraryScan = null) {
const options = {
titleDistance: 2,
authorDistance: 2
}
const scannerCoverProvider = Database.serverSettings.scannerCoverProvider
const results = await this.bookFinder.findCovers(scannerCoverProvider, libraryItem.media.metadata.title, libraryItem.media.metadata.authorName, options)
if (results.length) {
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Found best cover for "${libraryItem.media.metadata.title}"`)
else Logger.debug(`[Scanner] Found best cover for "${libraryItem.media.metadata.title}"`)
// If the first cover result fails, attempt to download the second
for (let i = 0; i < results.length && i < 2; i++) {
// Downloads and updates the book cover
const result = await this.coverManager.downloadCoverFromUrl(libraryItem, results[i])
if (result.error) {
Logger.error(`[Scanner] Failed to download cover from url "${results[i]}" | Attempt ${i + 1}`, result.error)
} else {
return true
}
}
}
return false
} }
async quickMatchLibraryItem(libraryItem, options = {}) { async quickMatchLibraryItem(libraryItem, options = {}) {
@ -71,7 +37,7 @@ class Scanner {
var searchISBN = options.isbn || libraryItem.media.metadata.isbn var searchISBN = options.isbn || libraryItem.media.metadata.isbn
var searchASIN = options.asin || libraryItem.media.metadata.asin var searchASIN = options.asin || libraryItem.media.metadata.asin
var results = await this.bookFinder.search(provider, searchTitle, searchAuthor, searchISBN, searchASIN) var results = await BookFinder.search(provider, searchTitle, searchAuthor, searchISBN, searchASIN)
if (!results.length) { if (!results.length) {
return { return {
warning: `No ${provider} match found` warning: `No ${provider} match found`
@ -92,7 +58,7 @@ class Scanner {
updatePayload = await this.quickMatchBookBuildUpdatePayload(libraryItem, matchData, options) updatePayload = await this.quickMatchBookBuildUpdatePayload(libraryItem, matchData, options)
} else if (libraryItem.isPodcast) { // Podcast quick match } else if (libraryItem.isPodcast) { // Podcast quick match
var results = await this.podcastFinder.search(searchTitle) var results = await PodcastFinder.search(searchTitle)
if (!results.length) { if (!results.length) {
return { return {
warning: `No ${provider} match found` warning: `No ${provider} match found`
@ -315,62 +281,80 @@ class Scanner {
return false return false
} }
async matchLibraryItems(library) { async matchLibraryItemsChunk(library, libraryItems, libraryScan) {
if (library.mediaType === 'podcast') { for (let i = 0; i < libraryItems.length; i++) {
Logger.error(`[Scanner] matchLibraryItems: Match all not supported for podcasts yet`) const libraryItem = libraryItems[i]
return
}
const itemsInLibrary = Database.libraryItems.filter(li => li.libraryId === library.id)
if (!itemsInLibrary.length) {
Logger.error(`[Scanner] matchLibraryItems: Library has no items ${library.id}`)
return
}
const provider = library.provider
var libraryScan = new LibraryScan()
libraryScan.setData(library, null, 'match')
this.librariesScanning.push(libraryScan.getScanEmitData)
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
Logger.info(`[Scanner] matchLibraryItems: Starting library match scan ${libraryScan.id} for ${libraryScan.libraryName}`)
for (let i = 0; i < itemsInLibrary.length; i++) {
var libraryItem = itemsInLibrary[i]
if (libraryItem.media.metadata.asin && library.settings.skipMatchingMediaWithAsin) { if (libraryItem.media.metadata.asin && library.settings.skipMatchingMediaWithAsin) {
Logger.debug(`[Scanner] matchLibraryItems: Skipping "${libraryItem.media.metadata.title Logger.debug(`[Scanner] matchLibraryItems: Skipping "${libraryItem.media.metadata.title
}" because it already has an ASIN (${i + 1} of ${itemsInLibrary.length})`) }" because it already has an ASIN (${i + 1} of ${libraryItems.length})`)
continue; continue
} }
if (libraryItem.media.metadata.isbn && library.settings.skipMatchingMediaWithIsbn) { if (libraryItem.media.metadata.isbn && library.settings.skipMatchingMediaWithIsbn) {
Logger.debug(`[Scanner] matchLibraryItems: Skipping "${libraryItem.media.metadata.title Logger.debug(`[Scanner] matchLibraryItems: Skipping "${libraryItem.media.metadata.title
}" because it already has an ISBN (${i + 1} of ${itemsInLibrary.length})`) }" because it already has an ISBN (${i + 1} of ${libraryItems.length})`)
continue; continue
} }
Logger.debug(`[Scanner] matchLibraryItems: Quick matching "${libraryItem.media.metadata.title}" (${i + 1} of ${itemsInLibrary.length})`) Logger.debug(`[Scanner] matchLibraryItems: Quick matching "${libraryItem.media.metadata.title}" (${i + 1} of ${libraryItems.length})`)
var result = await this.quickMatchLibraryItem(libraryItem, { provider }) const result = await this.quickMatchLibraryItem(libraryItem, { provider: library.provider })
if (result.warning) { if (result.warning) {
Logger.warn(`[Scanner] matchLibraryItems: Match warning ${result.warning} for library item "${libraryItem.media.metadata.title}"`) Logger.warn(`[Scanner] matchLibraryItems: Match warning ${result.warning} for library item "${libraryItem.media.metadata.title}"`)
} else if (result.updated) { } else if (result.updated) {
libraryScan.resultsUpdated++ libraryScan.resultsUpdated++
} }
if (this.cancelLibraryScan[libraryScan.libraryId]) { if (LibraryScanner.cancelLibraryScan[libraryScan.libraryId]) {
Logger.info(`[Scanner] matchLibraryItems: Library match scan canceled for "${libraryScan.libraryName}"`) Logger.info(`[Scanner] matchLibraryItems: Library match scan canceled for "${libraryScan.libraryName}"`)
delete this.cancelLibraryScan[libraryScan.libraryId] return false
var scanData = libraryScan.getScanEmitData
scanData.results = null
SocketAuthority.emitter('scan_complete', scanData)
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
return
} }
} }
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id) return true
}
async matchLibraryItems(library) {
if (library.mediaType === 'podcast') {
Logger.error(`[Scanner] matchLibraryItems: Match all not supported for podcasts yet`)
return
}
if (LibraryScanner.isLibraryScanning(library.id)) {
Logger.error(`[Scanner] Library "${library.name}" is already scanning`)
return
}
const limit = 100
let offset = 0
const libraryScan = new LibraryScan()
libraryScan.setData(library, null, 'match')
LibraryScanner.librariesScanning.push(libraryScan.getScanEmitData)
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
Logger.info(`[Scanner] matchLibraryItems: Starting library match scan ${libraryScan.id} for ${libraryScan.libraryName}`)
let hasMoreChunks = true
while (hasMoreChunks) {
const libraryItems = await Database.libraryItemModel.getLibraryItemsIncrement(offset, limit, { libraryId: library.id })
if (!libraryItems.length) {
Logger.error(`[Scanner] matchLibraryItems: Library has no items ${library.id}`)
SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
return
}
offset += limit
hasMoreChunks = libraryItems.length < limit
let oldLibraryItems = libraryItems.map(li => Database.libraryItemModel.getOldLibraryItem(li))
const shouldContinue = await this.matchLibraryItemsChunk(library, oldLibraryItems, libraryScan)
if (!shouldContinue) {
break
}
}
delete LibraryScanner.cancelLibraryScan[libraryScan.libraryId]
LibraryScanner.librariesScanning = LibraryScanner.librariesScanning.filter(ls => ls.id !== library.id)
SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData) SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
} }
} }