diff --git a/docs/LibraryItemModelDemo.js b/docs/LibraryItemModelDemo.js index da7428ae..b4d03827 100644 --- a/docs/LibraryItemModelDemo.js +++ b/docs/LibraryItemModelDemo.js @@ -68,6 +68,7 @@ new LibraryItem({ discNumFromFilename: 1, manuallyVerified: false, exclude: false, + invalid: false, format: "MP2/3 (MPEG audio layer 2/3)", duration: 2342342, bitRate: 324234, @@ -78,7 +79,7 @@ new LibraryItem({ channelLayout: "mono", chapters: [], embeddedCoverArt: 'jpeg', // Video stream codec ['mjpeg', 'jpeg', 'png'] or null - metatags: { // AudioMetatags.js : (Metatags/ID3 tags - only stores values that are found) + metaTags: { // AudioMetaTags.js tagAlbum: '', tagArtist: '', tagGenre: '', @@ -101,7 +102,7 @@ new LibraryItem({ tagASIN: '' }, addedAt: 1646784672127, - lastUpdate: 1646784672127 + updatedAt: 1646784672127 } ], ebookFiles: [ @@ -119,7 +120,7 @@ new LibraryItem({ }, ebookFormat: 'mobi', addedAt: 1646784672127, - lastUpdate: 1646784672127 + updatedAt: 1646784672127 } ], chapters: [ @@ -145,7 +146,7 @@ new LibraryItem({ size: 1197449516 }, addedAt: 1646784672127, - lastUpdate: 1646784672127 + updatedAt: 1646784672127 }, { // LibraryFile.js ino: "55450570412017066", @@ -160,7 +161,7 @@ new LibraryItem({ size: 1197449516 }, addedAt: 1646784672127, - lastUpdate: 1646784672127 + updatedAt: 1646784672127 } ] }) \ No newline at end of file diff --git a/server/Db.js b/server/Db.js index 035ccc9a..a1dcb97a 100644 --- a/server/Db.js +++ b/server/Db.js @@ -4,38 +4,43 @@ const fs = require('fs-extra') const jwt = require('jsonwebtoken') const Logger = require('./Logger') const { version } = require('../package.json') -const Audiobook = require('./objects/Audiobook') +// const Audiobook = require('./objects/Audiobook') +const LibraryItem = require('./objects/LibraryItem') const User = require('./objects/User') const UserCollection = require('./objects/UserCollection') const Library = require('./objects/Library') -const Author = require('./objects/Author') +const Author = require('./objects/entities/Author') +const Series = require('./objects/entities/Series') const ServerSettings = require('./objects/ServerSettings') class Db { constructor() { - this.AudiobooksPath = Path.join(global.ConfigPath, 'audiobooks') + this.LibraryItemsPath = Path.join(global.ConfigPath, 'libraryItems') this.UsersPath = Path.join(global.ConfigPath, 'users') this.SessionsPath = Path.join(global.ConfigPath, 'sessions') this.LibrariesPath = Path.join(global.ConfigPath, 'libraries') this.SettingsPath = Path.join(global.ConfigPath, 'settings') this.CollectionsPath = Path.join(global.ConfigPath, 'collections') this.AuthorsPath = Path.join(global.ConfigPath, 'authors') + this.SeriesPath = Path.join(global.ConfigPath, 'series') - this.audiobooksDb = new njodb.Database(this.AudiobooksPath) + this.libraryItemsDb = new njodb.Database(this.LibraryItemsPath) this.usersDb = new njodb.Database(this.UsersPath) this.sessionsDb = new njodb.Database(this.SessionsPath) this.librariesDb = new njodb.Database(this.LibrariesPath, { datastores: 2 }) this.settingsDb = new njodb.Database(this.SettingsPath, { datastores: 2 }) this.collectionsDb = new njodb.Database(this.CollectionsPath, { datastores: 2 }) this.authorsDb = new njodb.Database(this.AuthorsPath) + this.seriesDb = new njodb.Database(this.SeriesPath, { datastores: 2 }) + this.libraryItems = [] this.users = [] this.sessions = [] this.libraries = [] - this.audiobooks = [] this.settings = [] this.collections = [] this.authors = [] + this.series = [] this.serverSettings = null @@ -46,22 +51,24 @@ class Db { getEntityDb(entityName) { if (entityName === 'user') return this.usersDb else if (entityName === 'session') return this.sessionsDb - else if (entityName === 'audiobook') return this.audiobooksDb + else if (entityName === 'libraryItem') return this.libraryItemsDb else if (entityName === 'library') return this.librariesDb else if (entityName === 'settings') return this.settingsDb else if (entityName === 'collection') return this.collectionsDb else if (entityName === 'author') return this.authorsDb + else if (entityName === 'series') return this.seriesDb return null } getEntityArrayKey(entityName) { if (entityName === 'user') return 'users' else if (entityName === 'session') return 'sessions' - else if (entityName === 'audiobook') return 'audiobooks' + else if (entityName === 'libraryItem') return 'libraryItems' else if (entityName === 'library') return 'libraries' else if (entityName === 'settings') return 'settings' else if (entityName === 'collection') return 'collections' else if (entityName === 'author') return 'authors' + else if (entityName === 'series') return 'series' return null } @@ -93,13 +100,14 @@ class Db { } reinit() { - this.audiobooksDb = new njodb.Database(this.AudiobooksPath) + this.libraryItemsDb = new njodb.Database(this.LibraryItemsPath) this.usersDb = new njodb.Database(this.UsersPath) this.sessionsDb = new njodb.Database(this.SessionsPath) this.librariesDb = new njodb.Database(this.LibrariesPath, { datastores: 2 }) this.settingsDb = new njodb.Database(this.SettingsPath, { datastores: 2 }) this.collectionsDb = new njodb.Database(this.CollectionsPath, { datastores: 2 }) this.authorsDb = new njodb.Database(this.AuthorsPath) + this.seriesDb = new njodb.Database(this.SeriesPath, { datastores: 2 }) return this.init() } @@ -129,9 +137,9 @@ class Db { } async load() { - var p1 = this.audiobooksDb.select(() => true).then((results) => { - this.audiobooks = results.data.map(a => new Audiobook(a)) - Logger.info(`[DB] ${this.audiobooks.length} Audiobooks Loaded`) + var p1 = this.libraryItemsDb.select(() => true).then((results) => { + this.libraryItems = results.data.map(a => new LibraryItem(a)) + Logger.info(`[DB] ${this.libraryItems.length} Library Items Loaded`) }) var p2 = this.usersDb.select(() => true).then((results) => { this.users = results.data.map(u => new User(u)) @@ -163,7 +171,11 @@ class Db { this.authors = results.data.map(l => new Author(l)) Logger.info(`[DB] ${this.authors.length} Authors Loaded`) }) - await Promise.all([p1, p2, p3, p4, p5, p6]) + var p7 = this.seriesDb.select(() => true).then((results) => { + this.series = results.data.map(l => new Series(l)) + Logger.info(`[DB] ${this.series.length} Series Loaded`) + }) + await Promise.all([p1, p2, p3, p4, p5, p6, p7]) // Update server version in server settings if (this.previousVersion) { @@ -181,7 +193,7 @@ class Db { Logger.error(`[Db] Invalid audiobook object passed to updateAudiobook`, audiobook) } - return this.audiobooksDb.update((record) => record.id === audiobook.id, () => audiobook).then((results) => { + return this.libraryItemsDb.update((record) => record.id === audiobook.id, () => audiobook).then((results) => { Logger.debug(`[DB] Audiobook updated ${results.updated}`) return true }).catch((error) => { @@ -202,7 +214,7 @@ class Db { return null })) - return this.audiobooksDb.insert(audiobooks).then((results) => { + return this.libraryItemsDb.insert(audiobooks).then((results) => { Logger.debug(`[DB] Audiobooks inserted ${results.inserted}`) this.audiobooks = this.audiobooks.concat(audiobooks) return true @@ -321,14 +333,14 @@ class Db { }) } - recreateAudiobookDb() { - return this.audiobooksDb.drop().then((results) => { - Logger.info(`[DB] Dropped audiobook db`, results) - this.audiobooksDb = new njodb.Database(this.AudiobooksPath) - this.audiobooks = [] + recreateLibraryItemsDb() { + return this.libraryItemsDb.drop().then((results) => { + Logger.info(`[DB] Dropped library items db`, results) + this.libraryItemsDb = new njodb.Database(this.LibraryItemsPath) + this.libraryItems = [] return true }).catch((error) => { - Logger.error(`[DB] Failed to drop audiobook db`, error) + Logger.error(`[DB] Failed to drop library items db`, error) return false }) } diff --git a/server/Server.js b/server/Server.js index 2838f21e..44d6cb64 100644 --- a/server/Server.js +++ b/server/Server.js @@ -12,6 +12,7 @@ const { version } = require('../package.json') const { ScanResult } = require('./utils/constants') const filePerms = require('./utils/filePerms') const { secondsToTimestamp } = require('./utils/index') +const dbMigration = require('./utils/dbMigration') const Logger = require('./Logger') // Classes @@ -75,15 +76,6 @@ class Server { this.clients = {} } - get audiobooks() { - return this.db.audiobooks - } - get libraries() { - return this.db.libraries - } - get serverSettings() { - return this.db.serverSettings - } get usersOnline() { return Object.values(this.clients).filter(c => c.user).map(client => { return client.user.toJSONForPublic(this.streamManager.streams) @@ -121,11 +113,20 @@ class Server { await this.streamManager.removeOrphanStreams() await this.downloadManager.removeOrphanDownloads() + await this.db.init() + + if (version.localeCompare('1.7.3') < 0) { + await dbMigration(this.db) + // TODO: Eventually remove audiobooks db when stable + } + this.auth.init() - await this.checkUserAudiobookData() - await this.purgeMetadata() + // TODO: Implement method to remove old user auidobook data and book metadata folders + // await this.checkUserAudiobookData() + // await this.purgeMetadata() + await this.backupManager.init() await this.logManager.init() @@ -143,7 +144,7 @@ class Server { Logger.info(`[Server] Watcher is disabled`) this.watcher.disabled = true } else { - this.watcher.initWatcher(this.libraries) + this.watcher.initWatcher(this.db.libraries) this.watcher.on('files', this.filesChanged.bind(this)) } } @@ -180,7 +181,7 @@ class Server { // Static file routes app.get('/lib/:library/:folder/*', this.authMiddleware.bind(this), (req, res) => { - var library = this.libraries.find(lib => lib.id === req.params.library) + var library = this.db.libraries.find(lib => lib.id === req.params.library) if (!library) return res.sendStatus(404) var folder = library.folders.find(fol => fol.id === req.params.folder) if (!folder) return res.status(404).send('Folder not found') @@ -192,7 +193,7 @@ class Server { // Book static file routes app.get('/s/book/:id/*', this.authMiddleware.bind(this), (req, res) => { - var audiobook = this.audiobooks.find(ab => ab.id === req.params.id) + var audiobook = this.db.audiobooks.find(ab => ab.id === req.params.id) if (!audiobook) return res.status(404).send('Book not found with id ' + req.params.id) var remainingPath = req.params['0'] @@ -202,7 +203,7 @@ class Server { // EBook static file routes app.get('/ebook/:library/:folder/*', (req, res) => { - var library = this.libraries.find(lib => lib.id === req.params.library) + var library = this.db.libraries.find(lib => lib.id === req.params.library) if (!library) return res.sendStatus(404) var folder = library.folders.find(fol => fol.id === req.params.folder) if (!folder) return res.status(404).send('Folder not found') @@ -368,7 +369,7 @@ class Server { var purged = 0 await Promise.all(foldersInBooksMetadata.map(async foldername => { - var hasMatchingAudiobook = this.audiobooks.find(ab => ab.id === foldername) + var hasMatchingAudiobook = this.db.audiobooks.find(ab => ab.id === foldername) if (!hasMatchingAudiobook) { var folderPath = Path.join(booksMetadata, foldername) Logger.debug(`[Server] Purging unused metadata ${folderPath}`) @@ -633,7 +634,7 @@ class Server { await this.db.updateEntity('user', user) const initialPayload = { - serverSettings: this.serverSettings.toJSON(), + serverSettings: this.db.serverSettings.toJSON(), audiobookPath: global.AudiobookPath, metadataPath: global.MetadataPath, configPath: global.ConfigPath, diff --git a/server/controllers/BookController.js b/server/controllers/BookController.js index 0b05df04..eaa1c512 100644 --- a/server/controllers/BookController.js +++ b/server/controllers/BookController.js @@ -65,11 +65,11 @@ class BookController { // DELETE: api/books/all async deleteAll(req, res) { if (!req.user.isRoot) { - Logger.warn('User other than root attempted to delete all audiobooks', req.user) + Logger.warn('User other than root attempted to delete all library items', req.user) return res.sendStatus(403) } - Logger.info('Removing all Audiobooks') - var success = await this.db.recreateAudiobookDb() + Logger.info('Removing all Library Items') + var success = await this.db.recreateLibraryItemsDb() if (success) res.sendStatus(200) else res.sendStatus(500) } diff --git a/server/objects/AudioFile.js b/server/objects/AudioFile.js new file mode 100644 index 00000000..629a25af --- /dev/null +++ b/server/objects/AudioFile.js @@ -0,0 +1,243 @@ +const { isNullOrNaN } = require('../utils/index') +const AudioFileMetadata = require('./metadata/AudioMetaTags') + +class AudioFile { + constructor(data) { + this.index = null + this.ino = null + this.filename = null + this.ext = null + this.path = null + this.fullPath = null + this.mtimeMs = null + this.ctimeMs = null + this.birthtimeMs = null + this.addedAt = null + + this.trackNumFromMeta = null + this.discNumFromMeta = null + this.trackNumFromFilename = null + this.discNumFromFilename = null + + this.format = null + this.duration = null + this.size = null + this.bitRate = null + this.language = null + this.codec = null + this.timeBase = null + this.channels = null + this.channelLayout = null + this.chapters = [] + this.embeddedCoverArt = null + + // Tags scraped from the audio file + this.metadata = null + + this.manuallyVerified = false + this.invalid = false + this.exclude = false + this.error = null + + if (data) { + this.construct(data) + } + } + + toJSON() { + return { + index: this.index, + ino: this.ino, + filename: this.filename, + ext: this.ext, + path: this.path, + fullPath: this.fullPath, + mtimeMs: this.mtimeMs, + ctimeMs: this.ctimeMs, + birthtimeMs: this.birthtimeMs, + addedAt: this.addedAt, + trackNumFromMeta: this.trackNumFromMeta, + discNumFromMeta: this.discNumFromMeta, + trackNumFromFilename: this.trackNumFromFilename, + discNumFromFilename: this.discNumFromFilename, + manuallyVerified: !!this.manuallyVerified, + invalid: !!this.invalid, + exclude: !!this.exclude, + error: this.error || null, + format: this.format, + duration: this.duration, + size: this.size, + bitRate: this.bitRate, + language: this.language, + codec: this.codec, + timeBase: this.timeBase, + channels: this.channels, + channelLayout: this.channelLayout, + chapters: this.chapters, + embeddedCoverArt: this.embeddedCoverArt, + metadata: this.metadata ? this.metadata.toJSON() : {} + } + } + + construct(data) { + this.index = data.index + this.ino = data.ino + this.filename = data.filename + this.ext = data.ext + this.path = data.path + this.fullPath = data.fullPath + this.mtimeMs = data.mtimeMs || 0 + this.ctimeMs = data.ctimeMs || 0 + this.birthtimeMs = data.birthtimeMs || 0 + this.addedAt = data.addedAt + this.manuallyVerified = !!data.manuallyVerified + this.invalid = !!data.invalid + this.exclude = !!data.exclude + this.error = data.error || null + + this.trackNumFromMeta = data.trackNumFromMeta + this.discNumFromMeta = data.discNumFromMeta + this.trackNumFromFilename = data.trackNumFromFilename + + if (data.cdNumFromFilename !== undefined) this.discNumFromFilename = data.cdNumFromFilename // TEMP:Support old var name + else this.discNumFromFilename = data.discNumFromFilename + + this.format = data.format + this.duration = data.duration + this.size = data.size + this.bitRate = data.bitRate + this.language = data.language + this.codec = data.codec || null + this.timeBase = data.timeBase + this.channels = data.channels + this.channelLayout = data.channelLayout + this.chapters = data.chapters + this.embeddedCoverArt = data.embeddedCoverArt || null + + // Old version of AudioFile used `tagAlbum` etc. + var isOldVersion = Object.keys(data).find(key => key.startsWith('tag')) + if (isOldVersion) { + this.metadata = new AudioFileMetadata(data) + } else { + this.metadata = new AudioFileMetadata(data.metadata || {}) + } + } + + // New scanner creates AudioFile from AudioFileScanner + setDataFromProbe(fileData, probeData) { + this.index = fileData.index || null + this.ino = fileData.ino || null + this.filename = fileData.filename + this.ext = fileData.ext + this.path = fileData.path + this.fullPath = fileData.fullPath + this.mtimeMs = fileData.mtimeMs || 0 + this.ctimeMs = fileData.ctimeMs || 0 + this.birthtimeMs = fileData.birthtimeMs || 0 + this.addedAt = Date.now() + + this.trackNumFromMeta = fileData.trackNumFromMeta + this.discNumFromMeta = fileData.discNumFromMeta + this.trackNumFromFilename = fileData.trackNumFromFilename + this.discNumFromFilename = fileData.discNumFromFilename + + this.format = probeData.format + this.duration = probeData.duration + this.size = probeData.size + this.bitRate = probeData.bitRate || null + this.language = probeData.language + this.codec = probeData.codec || null + this.timeBase = probeData.timeBase + this.channels = probeData.channels + this.channelLayout = probeData.channelLayout + this.chapters = probeData.chapters || [] + this.metadata = probeData.audioFileMetadata + this.embeddedCoverArt = probeData.embeddedCoverArt + } + + validateTrackIndex() { + var numFromMeta = isNullOrNaN(this.trackNumFromMeta) ? null : Number(this.trackNumFromMeta) + var numFromFilename = isNullOrNaN(this.trackNumFromFilename) ? null : Number(this.trackNumFromFilename) + + if (numFromMeta !== null) return numFromMeta + if (numFromFilename !== null) return numFromFilename + + this.invalid = true + this.error = 'Failed to get track number' + return null + } + + setDuplicateTrackNumber(num) { + this.invalid = true + this.error = 'Duplicate track number "' + num + '"' + } + + syncChapters(updatedChapters) { + if (this.chapters.length !== updatedChapters.length) { + this.chapters = updatedChapters.map(ch => ({ ...ch })) + return true + } else if (updatedChapters.length === 0) { + if (this.chapters.length > 0) { + this.chapters = [] + return true + } + return false + } + + var hasUpdates = false + for (let i = 0; i < updatedChapters.length; i++) { + if (JSON.stringify(updatedChapters[i]) !== JSON.stringify(this.chapters[i])) { + hasUpdates = true + } + } + if (hasUpdates) { + this.chapters = updatedChapters.map(ch => ({ ...ch })) + } + return hasUpdates + } + + clone() { + return new AudioFile(this.toJSON()) + } + + // If the file or parent directory was renamed it is synced here + syncFile(newFile) { + var hasUpdates = false + var keysToSync = ['path', 'fullPath', 'ext', 'filename'] + keysToSync.forEach((key) => { + if (newFile[key] !== undefined && newFile[key] !== this[key]) { + hasUpdates = true + this[key] = newFile[key] + } + }) + return hasUpdates + } + + updateFromScan(scannedAudioFile) { + var hasUpdated = false + + var newjson = scannedAudioFile.toJSON() + if (this.manuallyVerified) newjson.manuallyVerified = true + if (this.exclude) newjson.exclude = true + newjson.addedAt = this.addedAt + + for (const key in newjson) { + if (key === 'metadata') { + if (!this.metadata || !this.metadata.isEqual(scannedAudioFile.metadata)) { + this.metadata = scannedAudioFile.metadata + hasUpdated = true + } + } else if (key === 'chapters') { + if (this.syncChapters(newjson.chapters || [])) { + hasUpdated = true + } + } else if (this[key] !== newjson[key]) { + // console.log(this.filename, 'key', key, 'updated', this[key], newjson[key]) + this[key] = newjson[key] + hasUpdated = true + } + } + return hasUpdated + } +} +module.exports = AudioFile \ No newline at end of file diff --git a/server/objects/metadata/AudioFileMetadata.js b/server/objects/AudioFileMetadata.js similarity index 100% rename from server/objects/metadata/AudioFileMetadata.js rename to server/objects/AudioFileMetadata.js diff --git a/server/objects/Audiobook.js b/server/objects/Audiobook.js index 799eaa2a..1cbe8cda 100644 --- a/server/objects/Audiobook.js +++ b/server/objects/Audiobook.js @@ -9,7 +9,7 @@ const abmetadataGenerator = require('../utils/abmetadataGenerator') const Logger = require('../Logger') const Book = require('./Book') const AudioTrack = require('./AudioTrack') -const AudioFile = require('./files/AudioFile') +const AudioFile = require('./AudioFile') const AudiobookFile = require('./AudiobookFile') class Audiobook { diff --git a/server/objects/entities/Author.js b/server/objects/entities/Author.js index 95ca9403..06d44e6e 100644 --- a/server/objects/entities/Author.js +++ b/server/objects/entities/Author.js @@ -1,10 +1,12 @@ +const { getId } = require('../../utils/index') + class Author { constructor(author) { this.id = null this.asin = null this.name = null this.imagePath = null - this.imageFullPath = null + this.relImagePath = null this.addedAt = null this.updatedAt = null @@ -18,7 +20,7 @@ class Author { this.asin = author.asin this.name = author.name this.imagePath = author.imagePath - this.imageFullPath = author.imageFullPath + this.relImagePath = author.relImagePath this.addedAt = author.addedAt this.updatedAt = author.updatedAt } @@ -29,9 +31,9 @@ class Author { asin: this.asin, name: this.name, imagePath: this.imagePath, - imageFullPath: this.imageFullPath, + relImagePath: this.relImagePath, addedAt: this.addedAt, - updatedAt: this.updatedAt + lastUpdate: this.updatedAt } } @@ -41,5 +43,15 @@ class Author { name: this.name } } + + setData(data) { + this.id = getId('aut') + this.name = data.name + this.asin = data.asin || null + this.imagePath = data.imagePath || null + this.relImagePath = data.relImagePath || null + this.addedAt = Date.now() + this.updatedAt = Date.now() + } } module.exports = Author \ No newline at end of file diff --git a/server/objects/entities/Book.js b/server/objects/entities/Book.js index 4779b651..37c497e1 100644 --- a/server/objects/entities/Book.js +++ b/server/objects/entities/Book.js @@ -1,39 +1,41 @@ const BookMetadata = require('../metadata/BookMetadata') const AudioFile = require('../files/AudioFile') const EBookFile = require('../files/EBookFile') -const AudioTrack = require('../AudioTrack') class Book { constructor(book) { this.metadata = null + this.coverPath = null + this.relCoverPath = null this.tags = [] this.audioFiles = [] this.ebookFiles = [] - this.audioTracks = [] this.chapters = [] - if (books) { + if (book) { this.construct(book) } } construct(book) { this.metadata = new BookMetadata(book.metadata) + this.coverPath = book.coverPath + this.relCoverPath = book.relCoverPath this.tags = [...book.tags] this.audioFiles = book.audioFiles.map(f => new AudioFile(f)) this.ebookFiles = book.ebookFiles.map(f => new EBookFile(f)) - this.audioTracks = book.audioTracks.map(a => new AudioTrack(a)) this.chapters = book.chapters.map(c => ({ ...c })) } toJSON() { return { metadata: this.metadata.toJSON(), + coverPath: this.coverPath, + relCoverPath: this.relCoverPath, tags: [...this.tags], audioFiles: this.audioFiles.map(f => f.toJSON()), ebookFiles: this.ebookFiles.map(f => f.toJSON()), - audioTracks: this.audioTracks.map(a => a.toJSON()), chapters: this.chapters.map(c => ({ ...c })) } } diff --git a/server/objects/entities/Series.js b/server/objects/entities/Series.js index b6941c1e..49db731b 100644 --- a/server/objects/entities/Series.js +++ b/server/objects/entities/Series.js @@ -1,8 +1,9 @@ +const { getId } = require('../../utils/index') + class Series { constructor(series) { this.id = null this.name = null - this.sequence = null this.addedAt = null this.updatedAt = null @@ -14,7 +15,6 @@ class Series { construct(series) { this.id = series.id this.name = series.name - this.sequence = series.sequence this.addedAt = series.addedAt this.updatedAt = series.updatedAt } @@ -23,18 +23,24 @@ class Series { return { id: this.id, name: this.name, - sequence: this.sequence, addedAt: this.addedAt, updatedAt: this.updatedAt } } - toJSONMinimal() { + toJSONMinimal(sequence) { return { id: this.id, name: this.name, - sequence: this.sequence + sequence } } + + setData(data) { + this.id = getId('ser') + this.name = data.name + this.addedAt = Date.now() + this.updatedAt = Date.now() + } } module.exports = Series \ No newline at end of file diff --git a/server/objects/files/AudioFile.js b/server/objects/files/AudioFile.js index aa3425ee..83327169 100644 --- a/server/objects/files/AudioFile.js +++ b/server/objects/files/AudioFile.js @@ -1,20 +1,14 @@ const { isNullOrNaN } = require('../../utils/index') - -const Logger = require('../../Logger') -const AudioFileMetadata = require('../metadata/AudioFileMetadata') +const AudioMetaTags = require('../metadata/AudioMetaTags') +const FileMetadata = require('../metadata/FileMetadata') class AudioFile { constructor(data) { this.index = null this.ino = null - this.filename = null - this.ext = null - this.path = null - this.fullPath = null - this.mtimeMs = null - this.ctimeMs = null - this.birthtimeMs = null + this.metadata = null this.addedAt = null + this.updatedAt = null this.trackNumFromMeta = null this.discNumFromMeta = null @@ -23,7 +17,6 @@ class AudioFile { this.format = null this.duration = null - this.size = null this.bitRate = null this.language = null this.codec = null @@ -34,7 +27,7 @@ class AudioFile { this.embeddedCoverArt = null // Tags scraped from the audio file - this.metadata = null + this.metaTags = null this.manuallyVerified = false this.invalid = false @@ -50,14 +43,9 @@ class AudioFile { return { index: this.index, ino: this.ino, - filename: this.filename, - ext: this.ext, - path: this.path, - fullPath: this.fullPath, - mtimeMs: this.mtimeMs, - ctimeMs: this.ctimeMs, - birthtimeMs: this.birthtimeMs, + metadata: this.metadata.toJSON(), addedAt: this.addedAt, + updatedAt: this.updatedAt, trackNumFromMeta: this.trackNumFromMeta, discNumFromMeta: this.discNumFromMeta, trackNumFromFilename: this.trackNumFromFilename, @@ -68,7 +56,6 @@ class AudioFile { error: this.error || null, format: this.format, duration: this.duration, - size: this.size, bitRate: this.bitRate, language: this.language, codec: this.codec, @@ -77,21 +64,16 @@ class AudioFile { channelLayout: this.channelLayout, chapters: this.chapters, embeddedCoverArt: this.embeddedCoverArt, - metadata: this.metadata ? this.metadata.toJSON() : {} + metaTags: this.metaTags ? this.metaTags.toJSON() : {} } } construct(data) { this.index = data.index this.ino = data.ino - this.filename = data.filename - this.ext = data.ext - this.path = data.path - this.fullPath = data.fullPath - this.mtimeMs = data.mtimeMs || 0 - this.ctimeMs = data.ctimeMs || 0 - this.birthtimeMs = data.birthtimeMs || 0 + this.metadata = new FileMetadata(data.metadata || {}) this.addedAt = data.addedAt + this.updatedAt = data.updatedAt this.manuallyVerified = !!data.manuallyVerified this.invalid = !!data.invalid this.exclude = !!data.exclude @@ -106,7 +88,6 @@ class AudioFile { this.format = data.format this.duration = data.duration - this.size = data.size this.bitRate = data.bitRate this.language = data.language this.codec = data.codec || null @@ -116,27 +97,17 @@ class AudioFile { this.chapters = data.chapters this.embeddedCoverArt = data.embeddedCoverArt || null - // Old version of AudioFile used `tagAlbum` etc. - var isOldVersion = Object.keys(data).find(key => key.startsWith('tag')) - if (isOldVersion) { - this.metadata = new AudioFileMetadata(data) - } else { - this.metadata = new AudioFileMetadata(data.metadata || {}) - } + this.metaTags = new AudioMetaTags(data.metaTags || {}) } // New scanner creates AudioFile from AudioFileScanner setDataFromProbe(fileData, probeData) { this.index = fileData.index || null this.ino = fileData.ino || null - this.filename = fileData.filename - this.ext = fileData.ext - this.path = fileData.path - this.fullPath = fileData.fullPath - this.mtimeMs = fileData.mtimeMs || 0 - this.ctimeMs = fileData.ctimeMs || 0 - this.birthtimeMs = fileData.birthtimeMs || 0 + + // TODO: Update file metadata for set data from probe this.addedAt = Date.now() + this.updatedAt = Date.now() this.trackNumFromMeta = fileData.trackNumFromMeta this.discNumFromMeta = fileData.discNumFromMeta @@ -145,7 +116,6 @@ class AudioFile { this.format = probeData.format this.duration = probeData.duration - this.size = probeData.size this.bitRate = probeData.bitRate || null this.language = probeData.language this.codec = probeData.codec || null @@ -153,7 +123,7 @@ class AudioFile { this.channels = probeData.channels this.channelLayout = probeData.channelLayout this.chapters = probeData.chapters || [] - this.metadata = probeData.audioFileMetadata + this.metaTags = probeData.audioFileMetadata this.embeddedCoverArt = probeData.embeddedCoverArt } @@ -204,15 +174,17 @@ class AudioFile { // If the file or parent directory was renamed it is synced here syncFile(newFile) { - var hasUpdates = false - var keysToSync = ['path', 'fullPath', 'ext', 'filename'] - keysToSync.forEach((key) => { - if (newFile[key] !== undefined && newFile[key] !== this[key]) { - hasUpdates = true - this[key] = newFile[key] - } - }) - return hasUpdates + // TODO: Sync file would update the file info if needed + return false + // var hasUpdates = false + // var keysToSync = ['path', 'relPath', 'ext', 'filename'] + // keysToSync.forEach((key) => { + // if (newFile[key] !== undefined && newFile[key] !== this[key]) { + // hasUpdates = true + // this[key] = newFile[key] + // } + // }) + // return hasUpdates } updateFromScan(scannedAudioFile) { @@ -224,9 +196,9 @@ class AudioFile { newjson.addedAt = this.addedAt for (const key in newjson) { - if (key === 'metadata') { - if (!this.metadata || !this.metadata.isEqual(scannedAudioFile.metadata)) { - this.metadata = scannedAudioFile.metadata + if (key === 'metaTags') { + if (!this.metaTags || !this.metaTags.isEqual(scannedAudioFile.metadata)) { + this.metaTags = scannedAudioFile.metadata hasUpdated = true } } else if (key === 'chapters') { diff --git a/server/objects/files/EBookFile.js b/server/objects/files/EBookFile.js index a964e72b..d5e81f01 100644 --- a/server/objects/files/EBookFile.js +++ b/server/objects/files/EBookFile.js @@ -6,7 +6,7 @@ class EBookFile { this.metadata = null this.ebookFormat = null this.addedAt = null - this.lastUpdate = null + this.updatedAt = null if (file) { this.construct(file) @@ -18,7 +18,7 @@ class EBookFile { this.metadata = new FileMetadata(file) this.ebookFormat = file.ebookFormat this.addedAt = file.addedAt - this.lastUpdate = file.lastUpdate + this.updatedAt = file.updatedAt } toJSON() { @@ -27,7 +27,7 @@ class EBookFile { metadata: this.metadata.toJSON(), ebookFormat: this.ebookFormat, addedAt: this.addedAt, - lastUpdate: this.lastUpdate + updatedAt: this.updatedAt } } } diff --git a/server/objects/metadata/AudioMetaTags.js b/server/objects/metadata/AudioMetaTags.js new file mode 100644 index 00000000..4b1d3891 --- /dev/null +++ b/server/objects/metadata/AudioMetaTags.js @@ -0,0 +1,129 @@ +class AudioMetaTags { + constructor(metadata) { + this.tagAlbum = null + this.tagArtist = null + this.tagGenre = null + this.tagTitle = null + this.tagSeries = null + this.tagSeriesPart = null + this.tagTrack = null + this.tagDisc = null + this.tagSubtitle = null + this.tagAlbumArtist = null + this.tagDate = null + this.tagComposer = null + this.tagPublisher = null + this.tagComment = null + this.tagDescription = null + this.tagEncoder = null + this.tagEncodedBy = null + this.tagIsbn = null + this.tagLanguage = null + this.tagASIN = null + + if (metadata) { + this.construct(metadata) + } + } + + toJSON() { + // Only return the tags that are actually set + var json = {} + for (const key in this) { + if (key.startsWith('tag') && this[key]) { + json[key] = this[key] + } + } + return json + } + + construct(metadata) { + this.tagAlbum = metadata.tagAlbum || null + this.tagArtist = metadata.tagArtist || null + this.tagGenre = metadata.tagGenre || null + this.tagTitle = metadata.tagTitle || null + this.tagSeries = metadata.tagSeries || null + this.tagSeriesPart = metadata.tagSeriesPart || null + this.tagTrack = metadata.tagTrack || null + this.tagDisc = metadata.tagDisc || null + this.tagSubtitle = metadata.tagSubtitle || null + this.tagAlbumArtist = metadata.tagAlbumArtist || null + this.tagDate = metadata.tagDate || null + this.tagComposer = metadata.tagComposer || null + this.tagPublisher = metadata.tagPublisher || null + this.tagComment = metadata.tagComment || null + this.tagDescription = metadata.tagDescription || null + this.tagEncoder = metadata.tagEncoder || null + this.tagEncodedBy = metadata.tagEncodedBy || null + this.tagIsbn = metadata.tagIsbn || null + this.tagLanguage = metadata.tagLanguage || null + this.tagASIN = metadata.tagASIN || null + } + + // Data parsed in prober.js + setData(payload) { + this.tagAlbum = payload.file_tag_album || null + this.tagArtist = payload.file_tag_artist || null + this.tagGenre = payload.file_tag_genre || null + this.tagTitle = payload.file_tag_title || null + this.tagSeries = payload.file_tag_series || null + this.tagSeriesPart = payload.file_tag_seriespart || null + this.tagTrack = payload.file_tag_track || null + this.tagDisc = payload.file_tag_disc || null + this.tagSubtitle = payload.file_tag_subtitle || null + this.tagAlbumArtist = payload.file_tag_albumartist || null + this.tagDate = payload.file_tag_date || null + this.tagComposer = payload.file_tag_composer || null + this.tagPublisher = payload.file_tag_publisher || null + this.tagComment = payload.file_tag_comment || null + this.tagDescription = payload.file_tag_description || null + this.tagEncoder = payload.file_tag_encoder || null + this.tagEncodedBy = payload.file_tag_encodedby || null + this.tagIsbn = payload.file_tag_isbn || null + this.tagLanguage = payload.file_tag_language || null + this.tagASIN = payload.file_tag_asin || null + } + + updateData(payload) { + const dataMap = { + tagAlbum: payload.file_tag_album || null, + tagArtist: payload.file_tag_artist || null, + tagGenre: payload.file_tag_genre || null, + tagTitle: payload.file_tag_title || null, + tagSeries: payload.file_tag_series || null, + tagSeriesPart: payload.file_tag_seriespart || null, + tagTrack: payload.file_tag_track || null, + tagDisc: payload.file_tag_disc || null, + tagSubtitle: payload.file_tag_subtitle || null, + tagAlbumArtist: payload.file_tag_albumartist || null, + tagDate: payload.file_tag_date || null, + tagComposer: payload.file_tag_composer || null, + tagPublisher: payload.file_tag_publisher || null, + tagComment: payload.file_tag_comment || null, + tagDescription: payload.file_tag_description || null, + tagEncoder: payload.file_tag_encoder || null, + tagEncodedBy: payload.file_tag_encodedby || null, + tagIsbn: payload.file_tag_isbn || null, + tagLanguage: payload.file_tag_language || null, + tagASIN: payload.file_tag_asin || null + } + + var hasUpdates = false + for (const key in dataMap) { + if (dataMap[key] !== this[key]) { + this[key] = dataMap[key] + hasUpdates = true + } + } + return hasUpdates + } + + isEqual(audioFileMetadata) { + if (!audioFileMetadata || !audioFileMetadata.toJSON) return false + for (const key in audioFileMetadata.toJSON()) { + if (audioFileMetadata[key] !== this[key]) return false + } + return true + } +} +module.exports = AudioMetaTags \ No newline at end of file diff --git a/server/objects/metadata/BookMetadata.js b/server/objects/metadata/BookMetadata.js index 1d32c82a..171e77ef 100644 --- a/server/objects/metadata/BookMetadata.js +++ b/server/objects/metadata/BookMetadata.js @@ -22,12 +22,12 @@ class BookMetadata { construct(metadata) { this.title = metadata.title this.subtitle = metadata.subtitle - this.authors = metadata.authors.map(a => ({ ...a })) - this.narrators = [...metadata.narrators] - this.series = metadata.series.map(s => ({ ...s })) - this.genres = [...metadata.genres] - this.publishedYear = metadata.publishedYear - this.publishedDate = metadata.publishedDate + this.authors = (metadata.authors && metadata.authors.map) ? metadata.authors.map(a => ({ ...a })) : [] + this.narrators = metadata.narrators ? [...metadata.narrators] : [] + this.series = (metadata.series && metadata.series.map) ? metadata.series.map(s => ({ ...s })) : [] + this.genres = metadata.genres ? [...metadata.genres] : [] + this.publishedYear = metadata.publishedYear || null + this.publishedDate = metadata.publishedDate || null this.publisher = metadata.publisher this.description = metadata.description this.isbn = metadata.isbn diff --git a/server/objects/metadata/FileMetadata.js b/server/objects/metadata/FileMetadata.js index f7bb8eb2..1cd55eb3 100644 --- a/server/objects/metadata/FileMetadata.js +++ b/server/objects/metadata/FileMetadata.js @@ -37,5 +37,9 @@ class FileMetadata { birthtimeMs: this.birthtimeMs } } + + clone() { + return new FileMetadata(this.toJSON()) + } } module.exports = FileMetadata \ No newline at end of file diff --git a/server/scanner/AudioProbeData.js b/server/scanner/AudioProbeData.js index 90c45e27..05e09891 100644 --- a/server/scanner/AudioProbeData.js +++ b/server/scanner/AudioProbeData.js @@ -1,4 +1,4 @@ -const AudioFileMetadata = require('../objects/metadata/AudioFileMetadata') +const AudioFileMetadata = require('../objects/AudioFileMetadata') class AudioProbeData { constructor() { diff --git a/server/utils/dbMigration.js b/server/utils/dbMigration.js new file mode 100644 index 00000000..72a39108 --- /dev/null +++ b/server/utils/dbMigration.js @@ -0,0 +1,210 @@ +const Path = require('path') +const fs = require('fs-extra') +const njodb = require("njodb") + +const { SupportedEbookTypes } = require('./globals') +const Audiobook = require('../objects/Audiobook') +const LibraryItem = require('../objects/LibraryItem') + +const Logger = require('../Logger') +const Book = require('../objects/entities/Book') +const BookMetadata = require('../objects/metadata/BookMetadata') +const Author = require('../objects/entities/Author') +const Series = require('../objects/entities/Series') +const AudioFile = require('../objects/files/AudioFile') +const EBookFile = require('../objects/files/EBookFile') +const LibraryFile = require('../objects/files/LibraryFile') +const FileMetadata = require('../objects/metadata/FileMetadata') +const AudioMetaTags = require('../objects/metadata/AudioMetaTags') + +var authorsToAdd = [] +var seriesToAdd = [] + +// Load old audiobooks +async function loadAudiobooks() { + var audiobookPath = Path.join(global.ConfigPath, 'audiobooks') + + var pathExists = await fs.pathExists(audiobookPath) + if (!pathExists) { + return [] + } + + var audiobooksDb = new njodb.Database(audiobookPath) + return audiobooksDb.select(() => true).then((results) => { + return results.data.map(a => new Audiobook(a)) + }) +} + +function makeAuthorsFromOldAb(authorsList) { + return authorsList.filter(a => !!a).map(authorName => { + var existingAuthor = authorsToAdd.find(a => a.name.toLowerCase() === authorName.toLowerCase()) + if (existingAuthor) { + return existingAuthor.toJSONMinimal() + } + + var newAuthor = new Author() + newAuthor.setData({ name: authorName }) + authorsToAdd.push(newAuthor) + Logger.info(`>>> Created new author named "${authorName}"`) + return newAuthor.toJSONMinimal() + }) +} + +function makeSeriesFromOldAb({ series, volumeNumber }) { + var existingSeries = seriesToAdd.find(s => s.name.toLowerCase() === series.toLowerCase()) + if (existingSeries) { + return [existingSeries.toJSONMinimal(volumeNumber)] + } + var newSeries = new Series() + newSeries.setData({ name: series }) + seriesToAdd.push(newSeries) + Logger.info(`>>> Created new series named "${series}"`) + return [newSeries.toJSONMinimal(volumeNumber)] +} + +function getRelativePath(srcPath, basePath) { + srcPath = srcPath.replace(/\\/g, '/') + basePath = basePath.replace(/\\/g, '/') + if (basePath.endsWith('/')) basePath = basePath.slice(0, -1) + return srcPath.replace(basePath, '') +} + +function makeFilesFromOldAb(audiobook) { + var libraryFiles = [] + var ebookFiles = [] + + var audioFiles = audiobook._audioFiles.map((af) => { + var fileMetadata = new FileMetadata(af) + fileMetadata.path = af.fullPath + fileMetadata.relPath = getRelativePath(af.fullPath, audiobook.fullPath) + + var newLibraryFile = new LibraryFile() + newLibraryFile.ino = af.ino + newLibraryFile.metadata = fileMetadata.clone() + newLibraryFile.addedAt = af.addedAt + newLibraryFile.updatedAt = Date.now() + libraryFiles.push(newLibraryFile) + + var audioMetaTags = new AudioMetaTags(af.metadata || {}) // Old metaTags was named metadata + delete af.metadata + + var newAudioFile = new AudioFile(af) + newAudioFile.metadata = fileMetadata + newAudioFile.metaTags = audioMetaTags + newAudioFile.updatedAt = Date.now() + return newAudioFile + }) + + audiobook._otherFiles.forEach((file) => { + var fileMetadata = new FileMetadata(file) + fileMetadata.path = file.fullPath + fileMetadata.relPath = getRelativePath(file.fullPath, audiobook.fullPath) + + var newLibraryFile = new LibraryFile() + newLibraryFile.ino = file.ino + newLibraryFile.metadata = fileMetadata.clone() + newLibraryFile.addedAt = file.addedAt + newLibraryFile.updatedAt = Date.now() + libraryFiles.push(newLibraryFile) + + var formatExt = (file.ext || '').slice(1) + if (SupportedEbookTypes.includes(formatExt)) { + var newEBookFile = new EBookFile() + newEBookFile.ino = file.ino + newEBookFile.metadata = fileMetadata + newEBookFile.ebookFormat = formatExt + newEBookFile.addedAt = file.addedAt + newEBookFile.updatedAt = Date.now() + ebookFiles.push(newEBookFile) + } + }) + + return { + libraryFiles, + ebookFiles, + audioFiles + } +} + +function makeLibraryItemFromOldAb(audiobook) { + var libraryItem = new LibraryItem() + libraryItem.id = audiobook.id + libraryItem.ino = audiobook.ino + libraryItem.libraryId = audiobook.libraryId + libraryItem.folderId = audiobook.folderId + libraryItem.path = audiobook.fullPath + libraryItem.relPath = audiobook.path + libraryItem.mtimeMs = audiobook.mtimeMs || 0 + libraryItem.ctimeMs = audiobook.ctimeMs || 0 + libraryItem.birthtimeMs = audiobook.birthtimeMs || 0 + libraryItem.addedAt = audiobook.addedAt + libraryItem.lastUpdate = audiobook.lastUpdate + libraryItem.lastScan = audiobook.lastScan + libraryItem.scanVersion = audiobook.scanVersion + libraryItem.isMissing = audiobook.isMissing + libraryItem.entityType = 'book' + + var bookEntity = new Book() + var bookMetadata = new BookMetadata(audiobook.book) + if (audiobook.book.narrator) { + bookMetadata.narrators = audiobook.book._narratorsList + } + // Returns array of json minimal authors + bookMetadata.authors = makeAuthorsFromOldAb(audiobook.book._authorsList) + + // Returns array of json minimal series + if (audiobook.book.series) { + bookMetadata.series = makeSeriesFromOldAb(audiobook.book) + } + + bookEntity.metadata = bookMetadata + bookEntity.coverPath = audiobook.book.coverFullPath + // Path relative to library item + bookEntity.relCoverPath = getRelativePath(audiobook.book.coverFullPath, audiobook.fullPath) + bookEntity.tags = [...audiobook.tags] + + var payload = makeFilesFromOldAb(audiobook) + bookEntity.audioFiles = payload.audioFiles + bookEntity.ebookFiles = payload.ebookFiles + + if (audiobook.chapters && audiobook.chapters.length) { + bookEntity.chapters = audiobook.chapters.map(c => ({ ...c })) + } + + libraryItem.entity = bookEntity + libraryItem.libraryFiles = payload.libraryFiles + return libraryItem +} + +async function migrateDb(db) { + Logger.info(`==== Starting DB Migration ====`) + + var audiobooks = await loadAudiobooks() + if (!audiobooks.length) { + Logger.info(`>>> No audiobooks in db, no migration necessary`) + return + } + + Logger.info(`>>> Loaded old audiobook data with ${audiobooks.length} records`) + + if (db.libraryItems.length) { + Logger.info(`>>> Some library items already loaded ${db.libraryItems.length} items | ${db.series.length} series | ${db.authors.length} authors`) + return + } + + var libraryItems = audiobooks.map((ab) => makeLibraryItemFromOldAb(ab)) + + Logger.info(`>>> ${libraryItems.length} Library Items made`) + await db.insertEntities('libraryItem', libraryItems) + if (authorsToAdd.length) { + Logger.info(`>>> ${authorsToAdd.length} Authors made`) + await db.insertEntities('author', authorsToAdd) + } + if (seriesToAdd.length) { + Logger.info(`>>> ${seriesToAdd.length} Series made`) + await db.insertEntities('series', seriesToAdd) + } + + Logger.info(`==== DB Migration Complete ====`) +} +module.exports = migrateDb \ No newline at end of file