const Path = require('path') const { Sequelize, Op } = require('sequelize') const packageJson = require('../package.json') const fs = require('./libs/fsExtra') const Logger = require('./Logger') const dbMigration = require('./utils/migrations/dbMigration') const Auth = require('./Auth') const MigrationManager = require('./managers/MigrationManager') class Database { constructor() { this.sequelize = null this.dbPath = null this.isNew = false // New absdatabase.sqlite created this.hasRootUser = false // Used to show initialization page in web ui this.settings = [] // Cached library filter data this.libraryFilterData = {} /** @type {import('./objects/settings/ServerSettings')} */ this.serverSettings = null /** @type {import('./objects/settings/NotificationSettings')} */ this.notificationSettings = null /** @type {import('./objects/settings/EmailSettings')} */ this.emailSettings = null this.supportsUnaccent = false this.supportsUnicodeFoldings = false } get models() { return this.sequelize?.models || {} } /** @type {typeof import('./models/User')} */ get userModel() { return this.models.user } /** @type {typeof import('./models/Library')} */ get libraryModel() { return this.models.library } /** @type {typeof import('./models/LibraryFolder')} */ get libraryFolderModel() { return this.models.libraryFolder } /** @type {typeof import('./models/Author')} */ get authorModel() { return this.models.author } /** @type {typeof import('./models/Series')} */ get seriesModel() { return this.models.series } /** @type {typeof import('./models/Book')} */ get bookModel() { return this.models.book } /** @type {typeof import('./models/BookSeries')} */ get bookSeriesModel() { return this.models.bookSeries } /** @type {typeof import('./models/BookAuthor')} */ get bookAuthorModel() { return this.models.bookAuthor } /** @type {typeof import('./models/Podcast')} */ get podcastModel() { return this.models.podcast } /** @type {typeof import('./models/PodcastEpisode')} */ get podcastEpisodeModel() { return this.models.podcastEpisode } /** @type {typeof import('./models/LibraryItem')} */ get libraryItemModel() { return this.models.libraryItem } /** @type {typeof import('./models/PodcastEpisode')} */ get podcastEpisodeModel() { return this.models.podcastEpisode } /** @type {typeof import('./models/MediaProgress')} */ get mediaProgressModel() { return this.models.mediaProgress } /** @type {typeof import('./models/Collection')} */ get collectionModel() { return this.models.collection } /** @type {typeof import('./models/CollectionBook')} */ get collectionBookModel() { return this.models.collectionBook } /** @type {typeof import('./models/Playlist')} */ get playlistModel() { return this.models.playlist } /** @type {typeof import('./models/PlaylistMediaItem')} */ get playlistMediaItemModel() { return this.models.playlistMediaItem } /** @type {typeof import('./models/Feed')} */ get feedModel() { return this.models.feed } /** @type {typeof import('./models/FeedEpisode')} */ get feedEpisodeModel() { return this.models.feedEpisode } /** @type {typeof import('./models/PlaybackSession')} */ get playbackSessionModel() { return this.models.playbackSession } /** @type {typeof import('./models/CustomMetadataProvider')} */ get customMetadataProviderModel() { return this.models.customMetadataProvider } /** @type {typeof import('./models/MediaItemShare')} */ get mediaItemShareModel() { return this.models.mediaItemShare } /** @type {typeof import('./models/Device')} */ get deviceModel() { return this.models.device } /** * Check if db file exists * @returns {boolean} */ async checkHasDb() { if (!(await fs.pathExists(this.dbPath))) { Logger.info(`[Database] absdatabase.sqlite not found at ${this.dbPath}`) return false } return true } /** * Connect to db, build models and run migrations * @param {boolean} [force=false] Used for testing, drops & re-creates all tables */ async init(force = false) { this.dbPath = Path.join(global.ConfigPath, 'absdatabase.sqlite') // First check if this is a new database this.isNew = !(await this.checkHasDb()) || force if (!(await this.connect())) { throw new Error('Database connection failed') } try { const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath) await migrationManager.init(packageJson.version) await migrationManager.runMigrations() } catch (error) { Logger.error(`[Database] Failed to run migrations`, error) throw new Error('Database migration failed') } await this.buildModels(force) Logger.info(`[Database] Db initialized with models:`, Object.keys(this.sequelize.models).join(', ')) await this.loadData() } /** * Connect to db * @returns {boolean} */ async connect() { Logger.info(`[Database] Initializing db at "${this.dbPath}"`) let logging = false let benchmark = false if (process.env.QUERY_LOGGING === 'log') { // Setting QUERY_LOGGING=log will log all Sequelize queries before they run Logger.info(`[Database] Query logging enabled`) logging = (query) => Logger.debug(`Running the following query:\n ${query}`) } else if (process.env.QUERY_LOGGING === 'benchmark') { // Setting QUERY_LOGGING=benchmark will log all Sequelize queries and their execution times, after they run Logger.info(`[Database] Query benchmarking enabled"`) logging = (query, time) => Logger.debug(`Ran the following query in ${time}ms:\n ${query}`) benchmark = true } this.sequelize = new Sequelize({ dialect: 'sqlite', storage: this.dbPath, logging: logging, benchmark: benchmark, transactionType: 'IMMEDIATE' }) // Helper function this.sequelize.uppercaseFirst = (str) => (str ? `${str[0].toUpperCase()}${str.substr(1)}` : '') try { await this.sequelize.authenticate() if (process.env.NUSQLITE3_PATH) { await this.loadExtension(process.env.NUSQLITE3_PATH) Logger.info(`[Database] Db supports unaccent and unicode foldings`) this.supportsUnaccent = true this.supportsUnicodeFoldings = true } Logger.info(`[Database] Db connection was successful`) return true } catch (error) { Logger.error(`[Database] Failed to connect to db`, error) return false } } /** * @param {string} extension paths to extension binary */ async loadExtension(extension) { // This is a hack to get the db connection for loading extensions. // The proper way would be to use the 'afterConnect' hook, but that hook is never called for sqlite due to a bug in sequelize. // See https://github.com/sequelize/sequelize/issues/12487 // This is not a public API and may break in the future. const db = await this.sequelize.dialect.connectionManager.getConnection() if (typeof db?.loadExtension !== 'function') throw new Error('Failed to get db connection for loading extensions') Logger.info(`[Database] Loading extension ${extension}`) await new Promise((resolve, reject) => { db.loadExtension(extension, (err) => { if (err) { Logger.error(`[Database] Failed to load extension ${extension}`, err) reject(err) return } Logger.info(`[Database] Successfully loaded extension ${extension}`) resolve() }) }) } /** * Disconnect from db */ async disconnect() { Logger.info(`[Database] Disconnecting sqlite db`) await this.sequelize.close() } /** * Reconnect to db and init */ async reconnect() { Logger.info(`[Database] Reconnecting sqlite db`) await this.init() } buildModels(force = false) { require('./models/User').init(this.sequelize) require('./models/Library').init(this.sequelize) require('./models/LibraryFolder').init(this.sequelize) require('./models/Book').init(this.sequelize) require('./models/Podcast').init(this.sequelize) require('./models/PodcastEpisode').init(this.sequelize) require('./models/LibraryItem').init(this.sequelize) require('./models/MediaProgress').init(this.sequelize) require('./models/Series').init(this.sequelize) require('./models/BookSeries').init(this.sequelize) require('./models/Author').init(this.sequelize) require('./models/BookAuthor').init(this.sequelize) require('./models/Collection').init(this.sequelize) require('./models/CollectionBook').init(this.sequelize) require('./models/Playlist').init(this.sequelize) require('./models/PlaylistMediaItem').init(this.sequelize) require('./models/Device').init(this.sequelize) require('./models/PlaybackSession').init(this.sequelize) require('./models/Feed').init(this.sequelize) require('./models/FeedEpisode').init(this.sequelize) require('./models/Setting').init(this.sequelize) require('./models/CustomMetadataProvider').init(this.sequelize) require('./models/MediaItemShare').init(this.sequelize) return this.sequelize.sync({ force, alter: false }) } /** * Compare two server versions * @param {string} v1 * @param {string} v2 * @returns {-1|0|1} 1 if v1 > v2 */ compareVersions(v1, v2) { if (!v1 || !v2) return 0 return v1.localeCompare(v2, undefined, { numeric: true, sensitivity: 'case', caseFirst: 'upper' }) } /** * Checks if migration to sqlite db is necessary & runs migration. * * Check if version was upgraded and run any version specific migrations. * * Loads most of the data from the database. This is a temporary solution. */ async loadData() { if (this.isNew && (await dbMigration.checkShouldMigrate())) { Logger.info(`[Database] New database was created and old database was detected - migrating old to new`) await dbMigration.migrate(this.models) } const settingsData = await this.models.setting.getOldSettings() this.settings = settingsData.settings this.emailSettings = settingsData.emailSettings this.serverSettings = settingsData.serverSettings this.notificationSettings = settingsData.notificationSettings global.ServerSettings = this.serverSettings.toJSON() // Version specific migrations if (packageJson.version !== this.serverSettings.version) { if (this.serverSettings.version === '2.3.0' && this.compareVersions(packageJson.version, '2.3.0') == 1) { await dbMigration.migrationPatch(this) } if (['2.3.0', '2.3.1', '2.3.2', '2.3.3'].includes(this.serverSettings.version) && this.compareVersions(packageJson.version, '2.3.3') >= 0) { await dbMigration.migrationPatch2(this) } } // Build migrations if (this.serverSettings.buildNumber <= 0) { await require('./utils/migrations/absMetadataMigration').migrate(this) } await this.cleanDatabase() // Set if root user has been created this.hasRootUser = await this.models.user.getHasRootUser() // Update server settings with version/build let updateServerSettings = false if (packageJson.version !== this.serverSettings.version) { Logger.info(`[Database] Server upgrade detected from ${this.serverSettings.version} to ${packageJson.version}`) this.serverSettings.version = packageJson.version this.serverSettings.buildNumber = packageJson.buildNumber updateServerSettings = true } else if (packageJson.buildNumber !== this.serverSettings.buildNumber) { Logger.info(`[Database] Server v${packageJson.version} build upgraded from ${this.serverSettings.buildNumber} to ${packageJson.buildNumber}`) this.serverSettings.buildNumber = packageJson.buildNumber updateServerSettings = true } if (updateServerSettings) { await this.updateServerSettings() } } /** * Create root user * @param {string} username * @param {string} pash * @param {Auth} auth * @returns {Promise} true if created */ async createRootUser(username, pash, auth) { if (!this.sequelize) return false await this.userModel.createRootUser(username, pash, auth) this.hasRootUser = true return true } updateServerSettings() { if (!this.sequelize) return false global.ServerSettings = this.serverSettings.toJSON() return this.updateSetting(this.serverSettings) } updateSetting(settings) { if (!this.sequelize) return false return this.models.setting.updateSettingObj(settings.toJSON()) } updateBulkBooks(oldBooks) { if (!this.sequelize) return false return Promise.all(oldBooks.map((oldBook) => this.models.book.saveFromOld(oldBook))) } removeLibrary(libraryId) { if (!this.sequelize) return false return this.models.library.removeById(libraryId) } createBulkCollectionBooks(collectionBooks) { if (!this.sequelize) return false return this.models.collectionBook.bulkCreate(collectionBooks) } createPlaylistMediaItem(playlistMediaItem) { if (!this.sequelize) return false return this.models.playlistMediaItem.create(playlistMediaItem) } createBulkPlaylistMediaItems(playlistMediaItems) { if (!this.sequelize) return false return this.models.playlistMediaItem.bulkCreate(playlistMediaItems) } async createLibraryItem(oldLibraryItem) { if (!this.sequelize) return false await oldLibraryItem.saveMetadata() await this.models.libraryItem.fullCreateFromOld(oldLibraryItem) } /** * Save metadata file and update library item * * @param {import('./objects/LibraryItem')} oldLibraryItem * @returns {Promise} */ async updateLibraryItem(oldLibraryItem) { if (!this.sequelize) return false await oldLibraryItem.saveMetadata() const updated = await this.models.libraryItem.fullUpdateFromOld(oldLibraryItem) // Clear library filter data cache if (updated) { delete this.libraryFilterData[oldLibraryItem.libraryId] } return updated } async createFeed(oldFeed) { if (!this.sequelize) return false await this.models.feed.fullCreateFromOld(oldFeed) } updateFeed(oldFeed) { if (!this.sequelize) return false return this.models.feed.fullUpdateFromOld(oldFeed) } async removeFeed(feedId) { if (!this.sequelize) return false await this.models.feed.removeById(feedId) } async createBulkBookAuthors(bookAuthors) { if (!this.sequelize) return false await this.models.bookAuthor.bulkCreate(bookAuthors) } async removeBulkBookAuthors(authorId = null, bookId = null) { if (!this.sequelize) return false if (!authorId && !bookId) return await this.models.bookAuthor.removeByIds(authorId, bookId) } getPlaybackSessions(where = null) { if (!this.sequelize) return false return this.models.playbackSession.getOldPlaybackSessions(where) } getPlaybackSession(sessionId) { if (!this.sequelize) return false return this.models.playbackSession.getById(sessionId) } createPlaybackSession(oldSession) { if (!this.sequelize) return false return this.models.playbackSession.createFromOld(oldSession) } updatePlaybackSession(oldSession) { if (!this.sequelize) return false return this.models.playbackSession.updateFromOld(oldSession) } removePlaybackSession(sessionId) { if (!this.sequelize) return false return this.models.playbackSession.removeById(sessionId) } replaceTagInFilterData(oldTag, newTag) { for (const libraryId in this.libraryFilterData) { const indexOf = this.libraryFilterData[libraryId].tags.findIndex((n) => n === oldTag) if (indexOf >= 0) { this.libraryFilterData[libraryId].tags.splice(indexOf, 1, newTag) } } } removeTagFromFilterData(tag) { for (const libraryId in this.libraryFilterData) { this.libraryFilterData[libraryId].tags = this.libraryFilterData[libraryId].tags.filter((t) => t !== tag) } } addTagsToFilterData(libraryId, tags) { if (!this.libraryFilterData[libraryId] || !tags?.length) return tags.forEach((t) => { if (!this.libraryFilterData[libraryId].tags.includes(t)) { this.libraryFilterData[libraryId].tags.push(t) } }) } replaceGenreInFilterData(oldGenre, newGenre) { for (const libraryId in this.libraryFilterData) { const indexOf = this.libraryFilterData[libraryId].genres.findIndex((n) => n === oldGenre) if (indexOf >= 0) { this.libraryFilterData[libraryId].genres.splice(indexOf, 1, newGenre) } } } removeGenreFromFilterData(genre) { for (const libraryId in this.libraryFilterData) { this.libraryFilterData[libraryId].genres = this.libraryFilterData[libraryId].genres.filter((g) => g !== genre) } } addGenresToFilterData(libraryId, genres) { if (!this.libraryFilterData[libraryId] || !genres?.length) return genres.forEach((g) => { if (!this.libraryFilterData[libraryId].genres.includes(g)) { this.libraryFilterData[libraryId].genres.push(g) } }) } replaceNarratorInFilterData(oldNarrator, newNarrator) { for (const libraryId in this.libraryFilterData) { const indexOf = this.libraryFilterData[libraryId].narrators.findIndex((n) => n === oldNarrator) if (indexOf >= 0) { this.libraryFilterData[libraryId].narrators.splice(indexOf, 1, newNarrator) } } } removeNarratorFromFilterData(narrator) { for (const libraryId in this.libraryFilterData) { this.libraryFilterData[libraryId].narrators = this.libraryFilterData[libraryId].narrators.filter((n) => n !== narrator) } } addNarratorsToFilterData(libraryId, narrators) { if (!this.libraryFilterData[libraryId] || !narrators?.length) return narrators.forEach((n) => { if (!this.libraryFilterData[libraryId].narrators.includes(n)) { this.libraryFilterData[libraryId].narrators.push(n) } }) } removeSeriesFromFilterData(libraryId, seriesId) { if (!this.libraryFilterData[libraryId]) return this.libraryFilterData[libraryId].series = this.libraryFilterData[libraryId].series.filter((se) => se.id !== seriesId) } addSeriesToFilterData(libraryId, seriesName, seriesId) { if (!this.libraryFilterData[libraryId]) return // Check if series is already added if (this.libraryFilterData[libraryId].series.some((se) => se.id === seriesId)) return this.libraryFilterData[libraryId].series.push({ id: seriesId, name: seriesName }) } removeAuthorFromFilterData(libraryId, authorId) { if (!this.libraryFilterData[libraryId]) return this.libraryFilterData[libraryId].authors = this.libraryFilterData[libraryId].authors.filter((au) => au.id !== authorId) } addAuthorToFilterData(libraryId, authorName, authorId) { if (!this.libraryFilterData[libraryId]) return // Check if author is already added if (this.libraryFilterData[libraryId].authors.some((au) => au.id === authorId)) return this.libraryFilterData[libraryId].authors.push({ id: authorId, name: authorName }) } addPublisherToFilterData(libraryId, publisher) { if (!this.libraryFilterData[libraryId] || !publisher || this.libraryFilterData[libraryId].publishers.includes(publisher)) return this.libraryFilterData[libraryId].publishers.push(publisher) } addPublishedDecadeToFilterData(libraryId, decade) { if (!this.libraryFilterData[libraryId] || !decade || this.libraryFilterData[libraryId].publishedDecades.includes(decade)) return this.libraryFilterData[libraryId].publishedDecades.push(decade) } addLanguageToFilterData(libraryId, language) { if (!this.libraryFilterData[libraryId] || !language || this.libraryFilterData[libraryId].languages.includes(language)) return this.libraryFilterData[libraryId].languages.push(language) } /** * Used when updating items to make sure author id exists * If library filter data is set then use that for check * otherwise lookup in db * @param {string} libraryId * @param {string} authorId * @returns {Promise} */ async checkAuthorExists(libraryId, authorId) { if (!this.libraryFilterData[libraryId]) { return this.authorModel.checkExistsById(authorId) } return this.libraryFilterData[libraryId].authors.some((au) => au.id === authorId) } /** * Used when updating items to make sure series id exists * If library filter data is set then use that for check * otherwise lookup in db * @param {string} libraryId * @param {string} seriesId * @returns {Promise} */ async checkSeriesExists(libraryId, seriesId) { if (!this.libraryFilterData[libraryId]) { return this.seriesModel.checkExistsById(seriesId) } return this.libraryFilterData[libraryId].series.some((se) => se.id === seriesId) } /** * Get author id for library by name. Uses library filter data if available * * @param {string} libraryId * @param {string} authorName * @returns {Promise} author id or null if not found */ async getAuthorIdByName(libraryId, authorName) { if (!this.libraryFilterData[libraryId]) { return (await this.authorModel.getByNameAndLibrary(authorName, libraryId))?.id || null } return this.libraryFilterData[libraryId].authors.find((au) => au.name === authorName)?.id || null } /** * Get series id for library by name. Uses library filter data if available * * @param {string} libraryId * @param {string} seriesName * @returns {Promise} series id or null if not found */ async getSeriesIdByName(libraryId, seriesName) { if (!this.libraryFilterData[libraryId]) { return (await this.seriesModel.getByNameAndLibrary(seriesName, libraryId))?.id || null } return this.libraryFilterData[libraryId].series.find((se) => se.name === seriesName)?.id || null } /** * Reset numIssues for library * @param {string} libraryId */ async resetLibraryIssuesFilterData(libraryId) { if (!this.libraryFilterData[libraryId]) return // Do nothing if filter data is not set this.libraryFilterData[libraryId].numIssues = await this.libraryItemModel.count({ where: { libraryId, [Sequelize.Op.or]: [ { isMissing: true }, { isInvalid: true } ] } }) } /** * Clean invalid records in database * Series should have atleast one Book * Book and Podcast must have an associated LibraryItem * Remove playback sessions that are 3 seconds or less */ async cleanDatabase() { // Remove invalid Podcast records const podcastsWithNoLibraryItem = await this.podcastModel.findAll({ include: { model: this.libraryItemModel, required: false }, where: { '$libraryItem.id$': null } }) for (const podcast of podcastsWithNoLibraryItem) { Logger.warn(`Found podcast "${podcast.title}" with no libraryItem - removing it`) await podcast.destroy() } // Remove invalid Book records const booksWithNoLibraryItem = await this.bookModel.findAll({ include: { model: this.libraryItemModel, required: false }, where: { '$libraryItem.id$': null } }) for (const book of booksWithNoLibraryItem) { Logger.warn(`Found book "${book.title}" with no libraryItem - removing it`) await book.destroy() } // Remove empty series const emptySeries = await this.seriesModel.findAll({ include: { model: this.bookSeriesModel, required: false }, where: { '$bookSeries.id$': null } }) for (const series of emptySeries) { Logger.warn(`Found series "${series.name}" with no books - removing it`) await series.destroy() } // Remove playback sessions that were 3 seconds or less const badSessionsRemoved = await this.playbackSessionModel.destroy({ where: { timeListening: { [Op.lte]: 3 } } }) if (badSessionsRemoved > 0) { Logger.warn(`Removed ${badSessionsRemoved} sessions that were 3 seconds or less`) } } async createTextSearchQuery(query) { const textQuery = new this.TextSearchQuery(this.sequelize, this.supportsUnaccent, query) await textQuery.init() return textQuery } TextSearchQuery = class { constructor(sequelize, supportsUnaccent, query) { this.sequelize = sequelize this.supportsUnaccent = supportsUnaccent this.query = query this.hasAccents = false } /** * Returns a normalized (accents-removed) expression for the specified value. * * @param {string} value * @returns {string} */ normalize(value) { return `unaccent(${value})` } /** * Initialize the text query. * */ async init() { if (!this.supportsUnaccent) return const escapedQuery = this.sequelize.escape(this.query) const normalizedQueryExpression = this.normalize(escapedQuery) const normalizedQueryResult = await this.sequelize.query(`SELECT ${normalizedQueryExpression} as normalized_query`) const normalizedQuery = normalizedQueryResult[0][0].normalized_query this.hasAccents = escapedQuery !== this.sequelize.escape(normalizedQuery) } /** * Get match expression for the specified column. * If the query contains accents, match against the column as-is (case-insensitive exact match). * otherwise match against a normalized column (case-insensitive match with accents removed). * * @param {string} column * @returns {string} */ matchExpression(column) { const pattern = this.sequelize.escape(`%${this.query}%`) if (!this.supportsUnaccent) return `${column} LIKE ${pattern}` const normalizedColumn = this.hasAccents ? column : this.normalize(column) return `${normalizedColumn} LIKE ${pattern}` } } } module.exports = new Database()