Mirrors/audiobookshelf
Mirrors/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2025-03-05 00:18:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Update match all books to load items from DB, remove library items loading to memory on init

Browse Source
This commit is contained in:
advplyr 2023-09-04 16:33:55 -05:00
parent 03115e5e53
commit 1dd1fe8994
19 changed files with 127 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
server/Database.js
View File

@ -15,9 +15,6 @@ class Database {
this.isNew = false // New absdatabase.sqlite created
this.hasRootUser = false // Used to show initialization page in web ui
// Temporarily using format of old DB
// TODO: below data should be loaded from the DB as needed
this.libraryItems = []
this.settings = []
// Cached library filter data
@ -255,8 +252,6 @@ class Database {
await dbMigration.migrate(this.models)
}
const startTime = Date.now()
const settingsData = await this.models.setting.getOldSettings()
this.settings = settingsData.settings
this.emailSettings = settingsData.emailSettings
@ -272,16 +267,9 @@ class Database {
await dbMigration.migrationPatch2(this)
}
Logger.info(`[Database] Loading db data...`)
this.libraryItems = await this.models.libraryItem.loadAllLibraryItems()
Logger.info(`[Database] Loaded ${this.libraryItems.length} library items`)
// Set if root user has been created
this.hasRootUser = await this.models.user.getHasRootUser()
Logger.info(`[Database] Db data loaded in ${((Date.now() - startTime) / 1000).toFixed(2)}s`)
if (packageJson.version !== this.serverSettings.version) {
Logger.info(`[Database] Server upgrade detected from ${this.serverSettings.version} to ${packageJson.version}`)
this.serverSettings.version = packageJson.version
@ -380,20 +368,10 @@ class Database {
return this.models.playlistMediaItem.bulkCreate(playlistMediaItems)
}
getLibraryItem(libraryItemId) {
if (!this.sequelize || !libraryItemId) return false
// Temp support for old library item ids from mobile
if (libraryItemId.startsWith('li_')) return this.libraryItems.find(li => li.oldLibraryItemId === libraryItemId)
return this.libraryItems.find(li => li.id === libraryItemId)
}
async createLibraryItem(oldLibraryItem) {
if (!this.sequelize) return false
await oldLibraryItem.saveMetadata()
await this.models.libraryItem.fullCreateFromOld(oldLibraryItem)
this.libraryItems.push(oldLibraryItem)
}
async updateLibraryItem(oldLibraryItem) {
@ -420,14 +398,12 @@ class Database {
for (const oldLibraryItem of oldLibraryItems) {
await oldLibraryItem.saveMetadata()
await this.models.libraryItem.fullCreateFromOld(oldLibraryItem)
this.libraryItems.push(oldLibraryItem)
}
}
async removeLibraryItem(libraryItemId) {
if (!this.sequelize) return false
await this.models.libraryItem.removeById(libraryItemId)
this.libraryItems = this.libraryItems.filter(li => li.id !== libraryItemId)
}
async createFeed(oldFeed) {

2
server/Server.js
View File

@ -76,7 +76,7 @@ class Server {
this.audioMetadataManager = new AudioMetadataMangaer(this.taskManager)
this.rssFeedManager = new RssFeedManager()
this.scanner = new Scanner(this.coverManager, this.taskManager)
this.scanner = new Scanner(this.coverManager)
this.cronManager = new CronManager(this.podcastManager)
// Routers

9
server/controllers/AuthorController.js
View File

@ -5,6 +5,7 @@ const { createNewSortInstance } = require('../libs/fastSort')
const Logger = require('../Logger')
const SocketAuthority = require('../SocketAuthority')
const Database = require('../Database')
const AuthorFinder = require('../finders/AuthorFinder')
const { reqSupportsWebp } = require('../utils/index')
@ -70,7 +71,7 @@ class AuthorController {
await this.cacheManager.purgeImageCache(req.author.id) // Purge cache
await this.coverManager.removeFile(req.author.imagePath)
} else if (payload.imagePath.startsWith('http')) { // Check if image path is a url
const imageData = await this.authorFinder.saveAuthorImage(req.author.id, payload.imagePath)
const imageData = await AuthorFinder.saveAuthorImage(req.author.id, payload.imagePath)
if (imageData) {
if (req.author.imagePath) {
await this.cacheManager.purgeImageCache(req.author.id) // Purge cache
@ -168,9 +169,9 @@ class AuthorController {
let authorData = null
const region = req.body.region || 'us'
if (req.body.asin) {
authorData = await this.authorFinder.findAuthorByASIN(req.body.asin, region)
authorData = await AuthorFinder.findAuthorByASIN(req.body.asin, region)
} else {
authorData = await this.authorFinder.findAuthorByName(req.body.q, region)
authorData = await AuthorFinder.findAuthorByName(req.body.q, region)
}
if (!authorData) {
return res.status(404).send('Author not found')
@ -187,7 +188,7 @@ class AuthorController {
if (authorData.image && (!req.author.imagePath || hasUpdates)) {
this.cacheManager.purgeImageCache(req.author.id)
const imageData = await this.authorFinder.saveAuthorImage(req.author.id, authorData.image)
const imageData = await AuthorFinder.saveAuthorImage(req.author.id, authorData.image)
if (imageData) {
req.author.imagePath = imageData.path
hasUpdates = true

2
server/controllers/EmailController.js
View File

@ -54,7 +54,7 @@ class EmailController {
async sendEBookToDevice(req, res) {
Logger.debug(`[EmailController] Send ebook to device request for libraryItemId=${req.body.libraryItemId}, deviceName=${req.body.deviceName}`)
const libraryItem = Database.getLibraryItem(req.body.libraryItemId)
const libraryItem = await Database.libraryItemModel.getOldById(req.body.libraryItemId)
if (!libraryItem) {
return res.status(404).send('Library item not found')
}

4
server/controllers/LibraryItemController.js
View File

@ -411,7 +411,9 @@ class LibraryItemController {
return res.sendStatus(400)
}
const libraryItems = req.body.libraryItemIds.map(lid => Database.getLibraryItem(lid)).filter(li => li)
const libraryItems = await Database.libraryItemModel.getAllOldLibraryItems({
id: req.body.libraryItemIds
})
if (!libraryItems?.length) {
return res.sendStatus(400)
}

9
server/controllers/MeController.js
View File

@ -193,7 +193,8 @@ class MeController {
Logger.error(`[MeController] syncLocalMediaProgress invalid local media progress object`, localProgress)
continue
}
const libraryItem = Database.getLibraryItem(localProgress.libraryItemId)
const libraryItem = await Database.libraryItemModel.getOldById(localProgress.libraryItemId)
if (!libraryItem) {
Logger.error(`[MeController] syncLocalMediaProgress invalid local media progress object no library item`, localProgress)
continue
@ -245,13 +246,15 @@ class MeController {
}
// GET: api/me/items-in-progress
getAllLibraryItemsInProgress(req, res) {
async getAllLibraryItemsInProgress(req, res) {
const limit = !isNaN(req.query.limit) ? Number(req.query.limit) || 25 : 25
let itemsInProgress = []
// TODO: More efficient to do this in a single query
for (const mediaProgress of req.user.mediaProgress) {
if (!mediaProgress.isFinished && (mediaProgress.progress > 0 || mediaProgress.ebookProgress > 0)) {
const libraryItem = Database.getLibraryItem(mediaProgress.libraryItemId)
const libraryItem = await Database.libraryItemModel.getOldById(mediaProgress.libraryItemId)
if (libraryItem) {
if (mediaProgress.episodeId && libraryItem.mediaType === 'podcast') {
const episode = libraryItem.media.episodes.find(ep => ep.id === mediaProgress.episodeId)

18
server/controllers/SearchController.js
View File

@ -1,4 +1,8 @@
const Logger = require("../Logger")
const BookFinder = require('../finders/BookFinder')
const PodcastFinder = require('../finders/PodcastFinder')
const AuthorFinder = require('../finders/AuthorFinder')
const MusicFinder = require('../finders/MusicFinder')
class SearchController {
constructor() { }
@ -7,7 +11,7 @@ class SearchController {
const provider = req.query.provider || 'google'
const title = req.query.title || ''
const author = req.query.author || ''
const results = await this.bookFinder.search(provider, title, author)
const results = await BookFinder.search(provider, title, author)
res.json(results)
}
@ -21,8 +25,8 @@ class SearchController {
}
let results = null
if (podcast) results = await this.podcastFinder.findCovers(query.title)
else results = await this.bookFinder.findCovers(query.provider || 'google', query.title, query.author || null)
if (podcast) results = await PodcastFinder.findCovers(query.title)
else results = await BookFinder.findCovers(query.provider || 'google', query.title, query.author || null)
res.json({
results
})
@ -30,20 +34,20 @@ class SearchController {
async findPodcasts(req, res) {
const term = req.query.term
const results = await this.podcastFinder.search(term)
const results = await PodcastFinder.search(term)
res.json(results)
}
async findAuthor(req, res) {
const query = req.query.q
const author = await this.authorFinder.findAuthorByName(query)
const author = await AuthorFinder.findAuthorByName(query)
res.json(author)
}
async findChapters(req, res) {
const asin = req.query.asin
const region = (req.query.region || 'us').toLowerCase()
const chapterData = await this.bookFinder.findChapters(asin, region)
const chapterData = await BookFinder.findChapters(asin, region)
if (!chapterData) {
return res.json({ error: 'Chapters not found' })
}
@ -51,7 +55,7 @@ class SearchController {
}
async findMusicTrack(req, res) {
const tracks = await this.musicFinder.searchTrack(req.query || {})
const tracks = await MusicFinder.searchTrack(req.query || {})
res.json({
tracks
})

6
server/controllers/SessionController.js
View File

@ -62,9 +62,9 @@ class SessionController {
})
}
getOpenSession(req, res) {
var libraryItem = Database.getLibraryItem(req.session.libraryItemId)
var sessionForClient = req.session.toJSONForClient(libraryItem)
async getOpenSession(req, res) {
const libraryItem = await Database.libraryItemModel.getOldById(req.session.libraryItemId)
const sessionForClient = req.session.toJSONForClient(libraryItem)
res.json(sessionForClient)
}

2
server/controllers/ToolsController.js
View File

@ -66,7 +66,7 @@ class ToolsController {
const libraryItems = []
for (const libraryItemId of libraryItemIds) {
const libraryItem = Database.getLibraryItem(libraryItemId)
const libraryItem = await Database.libraryItemModel.getOldById(libraryItemId)
if (!libraryItem) {
Logger.error(`[ToolsController] Batch embed metadata library item (${libraryItemId}) not found`)
return res.sendStatus(404)

6
server/finders/AuthorFinder.js
View File

@ -8,8 +8,6 @@ const filePerms = require('../utils/filePerms')
class AuthorFinder {
constructor() {
this.AuthorPath = Path.join(global.MetadataPath, 'authors')
this.audnexus = new Audnexus()
}
@ -37,7 +35,7 @@ class AuthorFinder {
}
async saveAuthorImage(authorId, url) {
var authorDir = this.AuthorPath
var authorDir = Path.join(global.MetadataPath, 'authors')
var relAuthorDir = Path.posix.join('/metadata', 'authors')
if (!await fs.pathExists(authorDir)) {
@ -61,4 +59,4 @@ class AuthorFinder {
}
}
}
module.exports = AuthorFinder
module.exports = new AuthorFinder()

2
server/finders/BookFinder.js
View File

@ -253,4 +253,4 @@ class BookFinder {
return this.audnexus.getChaptersByASIN(asin, region)
}
}
module.exports = BookFinder
module.exports = new BookFinder()

2
server/finders/MusicFinder.js
View File

@ -9,4 +9,4 @@ class MusicFinder {
return this.musicBrainz.searchTrack(options)
}
}
module.exports = MusicFinder
module.exports = new MusicFinder()

2
server/finders/PodcastFinder.js
View File

@ -22,4 +22,4 @@ class PodcastFinder {
return results.map(r => r.cover).filter(r => r)
}
}
module.exports = PodcastFinder
module.exports = new PodcastFinder()

2
server/managers/PlaybackSessionManager.js
View File

@ -93,7 +93,7 @@ class PlaybackSessionManager {
}
async syncLocalSession(user, sessionJson, deviceInfo) {
const libraryItem = Database.getLibraryItem(sessionJson.libraryItemId)
const libraryItem = await Database.libraryItemModel.getOldById(sessionJson.libraryItemId)
const episode = (sessionJson.episodeId && libraryItem && libraryItem.isPodcast) ? libraryItem.media.getEpisode(sessionJson.episodeId) : null
if (!libraryItem || (libraryItem.isPodcast && !episode)) {
Logger.error(`[PlaybackSessionManager] syncLocalSession: Media item not found for session "${sessionJson.displayTitle}" (${sessionJson.id})`)

2
server/managers/RssFeedManager.js
View File

@ -87,7 +87,7 @@ class RssFeedManager {
// Check if feed needs to be updated
if (feed.entityType === 'libraryItem') {
const libraryItem = Database.getLibraryItem(feed.entityId)
const libraryItem = await Database.libraryItemModel.getOldById(feed.entityId)
let mostRecentlyUpdatedAt = libraryItem.updatedAt
if (libraryItem.isPodcast) {

3
server/models/LibraryItem.js
View File

@ -118,12 +118,13 @@ class LibraryItem extends Model {
* @param {number} limit
* @returns {Promise<Model<LibraryItem>[]>} LibraryItem
*/
static getLibraryItemsIncrement(offset, limit) {
static getLibraryItemsIncrement(offset, limit, where = null) {
return this.findAll({
benchmark: true,
logging: (sql, timeMs) => {
console.log(`[Query] Elapsed ${timeMs}ms.`)
},
where,
include: [
{
model: this.sequelize.models.book,

10
server/routers/ApiRouter.js
View File

@ -29,11 +29,6 @@ const ToolsController = require('../controllers/ToolsController')
const RSSFeedController = require('../controllers/RSSFeedController')
const MiscController = require('../controllers/MiscController')
const BookFinder = require('../finders/BookFinder')
const AuthorFinder = require('../finders/AuthorFinder')
const PodcastFinder = require('../finders/PodcastFinder')
const MusicFinder = require('../finders/MusicFinder')
const Author = require('../objects/entities/Author')
const Series = require('../objects/entities/Series')
@ -55,11 +50,6 @@ class ApiRouter {
this.emailManager = Server.emailManager
this.taskManager = Server.taskManager
this.bookFinder = new BookFinder()
this.authorFinder = new AuthorFinder()
this.podcastFinder = new PodcastFinder()
this.musicFinder = new MusicFinder()
this.router = express()
this.router.disable('x-powered-by')
this.init()

28
server/scanner/BookScanner.js
View File

@ -16,6 +16,7 @@ const CoverManager = require('../managers/CoverManager')
const LibraryFile = require('../objects/files/LibraryFile')
const SocketAuthority = require('../SocketAuthority')
const fsExtra = require("../libs/fsExtra")
// const BookFinder = require('../finders/BookFinder')
/**
* Metadata for books pulled from files
@ -1049,5 +1050,32 @@ class BookScanner {
scanLogger.addLog(LogLevel.INFO, `Removed ${bookSeriesToRemove.length} series`)
}
}
// async searchForCover(libraryItem, libraryScan = null) {
// const options = {
// titleDistance: 2,
// authorDistance: 2
// }
// const scannerCoverProvider = Database.serverSettings.scannerCoverProvider
// const results = await BookFinder.findCovers(scannerCoverProvider, libraryItem.media.metadata.title, libraryItem.media.metadata.authorName, options)
// if (results.length) {
// if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Found best cover for "${libraryItem.media.metadata.title}"`)
// else Logger.debug(`[Scanner] Found best cover for "${libraryItem.media.metadata.title}"`)
// // If the first cover result fails, attempt to download the second
// for (let i = 0; i < results.length && i < 2; i++) {
// // Downloads and updates the book cover
// const result = await this.coverManager.downloadCoverFromUrl(libraryItem, results[i])
// if (result.error) {
// Logger.error(`[Scanner] Failed to download cover from url "${results[i]}" | Attempt ${i + 1}`, result.error)
// } else {
// return true
// }
// }
// }
// return false
// }
}
module.exports = new BookScanner()

134
server/scanner/Scanner.js
View File

@ -3,7 +3,6 @@ const SocketAuthority = require('../SocketAuthority')
const Database = require('../Database')
// Utils
const { LogLevel } = require('../utils/constants')
const { findMatchingEpisodesInFeed, getPodcastFeed } = require('../utils/podcastUtils')
const BookFinder = require('../finders/BookFinder')
@ -11,44 +10,11 @@ const PodcastFinder = require('../finders/PodcastFinder')
const LibraryScan = require('./LibraryScan')
const Author = require('../objects/entities/Author')
const Series = require('../objects/entities/Series')
const LibraryScanner = require('./LibraryScanner')
class Scanner {
constructor(coverManager, taskManager) {
constructor(coverManager) {
this.coverManager = coverManager
this.taskManager = taskManager
this.cancelLibraryScan = {}
this.librariesScanning = []
this.bookFinder = new BookFinder()
this.podcastFinder = new PodcastFinder()
}
async searchForCover(libraryItem, libraryScan = null) {
const options = {
titleDistance: 2,
authorDistance: 2
}
const scannerCoverProvider = Database.serverSettings.scannerCoverProvider
const results = await this.bookFinder.findCovers(scannerCoverProvider, libraryItem.media.metadata.title, libraryItem.media.metadata.authorName, options)
if (results.length) {
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Found best cover for "${libraryItem.media.metadata.title}"`)
else Logger.debug(`[Scanner] Found best cover for "${libraryItem.media.metadata.title}"`)
// If the first cover result fails, attempt to download the second
for (let i = 0; i < results.length && i < 2; i++) {
// Downloads and updates the book cover
const result = await this.coverManager.downloadCoverFromUrl(libraryItem, results[i])
if (result.error) {
Logger.error(`[Scanner] Failed to download cover from url "${results[i]}" | Attempt ${i + 1}`, result.error)
} else {
return true
}
}
}
return false
}
async quickMatchLibraryItem(libraryItem, options = {}) {
@ -71,7 +37,7 @@ class Scanner {
var searchISBN = options.isbn || libraryItem.media.metadata.isbn
var searchASIN = options.asin || libraryItem.media.metadata.asin
var results = await this.bookFinder.search(provider, searchTitle, searchAuthor, searchISBN, searchASIN)
var results = await BookFinder.search(provider, searchTitle, searchAuthor, searchISBN, searchASIN)
if (!results.length) {
return {
warning: `No ${provider} match found`
@ -92,7 +58,7 @@ class Scanner {
updatePayload = await this.quickMatchBookBuildUpdatePayload(libraryItem, matchData, options)
} else if (libraryItem.isPodcast) { // Podcast quick match
var results = await this.podcastFinder.search(searchTitle)
var results = await PodcastFinder.search(searchTitle)
if (!results.length) {
return {
warning: `No ${provider} match found`
@ -315,62 +281,80 @@ class Scanner {
return false
}
async matchLibraryItems(library) {
if (library.mediaType === 'podcast') {
Logger.error(`[Scanner] matchLibraryItems: Match all not supported for podcasts yet`)
return
}
const itemsInLibrary = Database.libraryItems.filter(li => li.libraryId === library.id)
if (!itemsInLibrary.length) {
Logger.error(`[Scanner] matchLibraryItems: Library has no items ${library.id}`)
return
}
const provider = library.provider
var libraryScan = new LibraryScan()
libraryScan.setData(library, null, 'match')
this.librariesScanning.push(libraryScan.getScanEmitData)
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
Logger.info(`[Scanner] matchLibraryItems: Starting library match scan ${libraryScan.id} for ${libraryScan.libraryName}`)
for (let i = 0; i < itemsInLibrary.length; i++) {
var libraryItem = itemsInLibrary[i]
async matchLibraryItemsChunk(library, libraryItems, libraryScan) {
for (let i = 0; i < libraryItems.length; i++) {
const libraryItem = libraryItems[i]
if (libraryItem.media.metadata.asin && library.settings.skipMatchingMediaWithAsin) {
Logger.debug(`[Scanner] matchLibraryItems: Skipping "${libraryItem.media.metadata.title
}" because it already has an ASIN (${i + 1} of ${itemsInLibrary.length})`)
continue;
}" because it already has an ASIN (${i + 1} of ${libraryItems.length})`)
continue
}
if (libraryItem.media.metadata.isbn && library.settings.skipMatchingMediaWithIsbn) {
Logger.debug(`[Scanner] matchLibraryItems: Skipping "${libraryItem.media.metadata.title
}" because it already has an ISBN (${i + 1} of ${itemsInLibrary.length})`)
continue;
}" because it already has an ISBN (${i + 1} of ${libraryItems.length})`)
continue
}
Logger.debug(`[Scanner] matchLibraryItems: Quick matching "${libraryItem.media.metadata.title}" (${i + 1} of ${itemsInLibrary.length})`)
var result = await this.quickMatchLibraryItem(libraryItem, { provider })
Logger.debug(`[Scanner] matchLibraryItems: Quick matching "${libraryItem.media.metadata.title}" (${i + 1} of ${libraryItems.length})`)
const result = await this.quickMatchLibraryItem(libraryItem, { provider: library.provider })
if (result.warning) {
Logger.warn(`[Scanner] matchLibraryItems: Match warning ${result.warning} for library item "${libraryItem.media.metadata.title}"`)
} else if (result.updated) {
libraryScan.resultsUpdated++
}
if (this.cancelLibraryScan[libraryScan.libraryId]) {
if (LibraryScanner.cancelLibraryScan[libraryScan.libraryId]) {
Logger.info(`[Scanner] matchLibraryItems: Library match scan canceled for "${libraryScan.libraryName}"`)
delete this.cancelLibraryScan[libraryScan.libraryId]
var scanData = libraryScan.getScanEmitData
scanData.results = null
SocketAuthority.emitter('scan_complete', scanData)
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
return
return false
}
}
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
return true
}
async matchLibraryItems(library) {
if (library.mediaType === 'podcast') {
Logger.error(`[Scanner] matchLibraryItems: Match all not supported for podcasts yet`)
return
}
if (LibraryScanner.isLibraryScanning(library.id)) {
Logger.error(`[Scanner] Library "${library.name}" is already scanning`)
return
}
const limit = 100
let offset = 0
const libraryScan = new LibraryScan()
libraryScan.setData(library, null, 'match')
LibraryScanner.librariesScanning.push(libraryScan.getScanEmitData)
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
Logger.info(`[Scanner] matchLibraryItems: Starting library match scan ${libraryScan.id} for ${libraryScan.libraryName}`)
let hasMoreChunks = true
while (hasMoreChunks) {
const libraryItems = await Database.libraryItemModel.getLibraryItemsIncrement(offset, limit, { libraryId: library.id })
if (!libraryItems.length) {
Logger.error(`[Scanner] matchLibraryItems: Library has no items ${library.id}`)
SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
return
}
offset += limit
hasMoreChunks = libraryItems.length < limit
let oldLibraryItems = libraryItems.map(li => Database.libraryItemModel.getOldLibraryItem(li))
const shouldContinue = await this.matchLibraryItemsChunk(library, oldLibraryItems, libraryScan)
if (!shouldContinue) {
break
}
}
delete LibraryScanner.cancelLibraryScan[libraryScan.libraryId]
LibraryScanner.librariesScanning = LibraryScanner.librariesScanning.filter(ls => ls.id !== library.id)
SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
}
}