diff --git a/client/components/modals/AccountModal.vue b/client/components/modals/AccountModal.vue index df1f7cbf..247fd08d 100644 --- a/client/components/modals/AccountModal.vue +++ b/client/components/modals/AccountModal.vue @@ -351,7 +351,7 @@ export default { update: type === 'admin', delete: type === 'admin', upload: type === 'admin', - accessExplicitContent: true, + accessExplicitContent: type === 'admin', accessAllLibraries: true, accessAllTags: true, selectedTagsNotAccessible: false @@ -386,7 +386,7 @@ export default { upload: false, accessAllLibraries: true, accessAllTags: true, - accessExplicitContent: true, + accessExplicitContent: false, selectedTagsNotAccessible: false }, librariesAccessible: [], diff --git a/package.json b/package.json index da10e000..70cf40c2 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "pkg": { "assets": [ "client/dist/**/*", - "node_modules/sqlite3/lib/binding/**/*.node" + "node_modules/sqlite3/lib/binding/**/*.node", + "server/migrations/*.js" ], "scripts": [ "prod.js", diff --git a/server/Database.js b/server/Database.js index 9eedfe1c..e7bad49b 100644 --- a/server/Database.js +++ b/server/Database.js @@ -176,9 +176,9 @@ class Database { } try { - const migrationManager = new MigrationManager(this.sequelize, global.ConfigPath) + const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath) await migrationManager.init(packageJson.version) - if (!this.isNew) await migrationManager.runMigrations() + await migrationManager.runMigrations() } catch (error) { Logger.error(`[Database] Failed to run migrations`, error) throw new Error('Database migration failed') diff --git a/server/managers/MigrationManager.js b/server/managers/MigrationManager.js index 53db461b..706e359c 100644 --- a/server/managers/MigrationManager.js +++ b/server/managers/MigrationManager.js @@ -11,11 +11,13 @@ class MigrationManager { /** * @param {import('../Database').sequelize} sequelize + * @param {boolean} isDatabaseNew * @param {string} [configPath] */ - constructor(sequelize, configPath = global.configPath) { + constructor(sequelize, isDatabaseNew, configPath = global.configPath) { if (!sequelize || !(sequelize instanceof Sequelize)) throw new Error('Sequelize instance is required for MigrationManager.') this.sequelize = sequelize + this.isDatabaseNew = isDatabaseNew if (!configPath) throw new Error('Config path is required for MigrationManager.') this.configPath = configPath this.migrationsSourceDir = path.join(__dirname, '..', 'migrations') @@ -42,6 +44,7 @@ class MigrationManager { await this.fetchVersionsFromDatabase() if (!this.maxVersion || !this.databaseVersion) throw new Error('Failed to fetch versions from the database.') + Logger.debug(`[MigrationManager] Database version: ${this.databaseVersion}, Max version: ${this.maxVersion}, Server version: ${this.serverVersion}`) if (semver.gt(this.serverVersion, this.maxVersion)) { try { @@ -63,6 +66,11 @@ class MigrationManager { async runMigrations() { if (!this.initialized) throw new Error('MigrationManager is not initialized. Call init() first.') + if (this.isDatabaseNew) { + Logger.info('[MigrationManager] Database is new. Skipping migrations.') + return + } + const versionCompare = semver.compare(this.serverVersion, this.databaseVersion) if (versionCompare == 0) { Logger.info('[MigrationManager] Database is already up to date.') @@ -180,7 +188,15 @@ class MigrationManager { async checkOrCreateMigrationsMetaTable() { const queryInterface = this.sequelize.getQueryInterface() - if (!(await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE))) { + let migrationsMetaTableExists = await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE) + + if (this.isDatabaseNew && migrationsMetaTableExists) { + // This can happen if database was initialized with force: true + await queryInterface.dropTable(MigrationManager.MIGRATIONS_META_TABLE) + migrationsMetaTableExists = false + } + + if (!migrationsMetaTableExists) { await queryInterface.createTable(MigrationManager.MIGRATIONS_META_TABLE, { key: { type: DataTypes.STRING, @@ -192,9 +208,10 @@ class MigrationManager { } }) await this.sequelize.query("INSERT INTO :migrationsMeta (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')", { - replacements: { version: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, + replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0', migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE }, type: Sequelize.QueryTypes.INSERT }) + Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`) } } @@ -219,6 +236,7 @@ class MigrationManager { await fs.copy(sourceFile, targetFile) // Asynchronously copy the files }) ) + Logger.debug(`[MigrationManager] Copied migrations to the config directory: "${this.migrationsDir}"`) } /** diff --git a/server/models/Author.js b/server/models/Author.js index f3bbba57..40e7f75a 100644 --- a/server/models/Author.js +++ b/server/models/Author.js @@ -1,5 +1,6 @@ const { DataTypes, Model, where, fn, col } = require('sequelize') const parseNameString = require('../utils/parsers/parseNameString') +const { asciiOnlyToLowerCase } = require('../utils/index') class Author extends Model { constructor(values, options) { @@ -55,7 +56,7 @@ class Author extends Model { static async getByNameAndLibrary(authorName, libraryId) { return this.findOne({ where: [ - where(fn('lower', col('name')), authorName.toLowerCase()), + where(fn('lower', col('name')), asciiOnlyToLowerCase(authorName)), { libraryId } diff --git a/server/models/Series.js b/server/models/Series.js index 731908e9..65ba5c04 100644 --- a/server/models/Series.js +++ b/server/models/Series.js @@ -1,6 +1,7 @@ const { DataTypes, Model, where, fn, col } = require('sequelize') const { getTitlePrefixAtEnd } = require('../utils/index') +const { asciiOnlyToLowerCase } = require('../utils/index') class Series extends Model { constructor(values, options) { @@ -41,7 +42,7 @@ class Series extends Model { static async getByNameAndLibrary(seriesName, libraryId) { return this.findOne({ where: [ - where(fn('lower', col('name')), seriesName.toLowerCase()), + where(fn('lower', col('name')), asciiOnlyToLowerCase(seriesName)), { libraryId } diff --git a/server/models/User.js b/server/models/User.js index 2dd02b68..4333db88 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -107,7 +107,7 @@ class User extends Model { upload: type === 'root' || type === 'admin', accessAllLibraries: true, accessAllTags: true, - accessExplicitContent: true, + accessExplicitContent: type === 'root' || type === 'admin', selectedTagsNotAccessible: false, librariesAccessible: [], itemTagsSelected: [] diff --git a/server/scanner/LibraryScan.js b/server/scanner/LibraryScan.js index 5ae5c06a..8994aa23 100644 --- a/server/scanner/LibraryScan.js +++ b/server/scanner/LibraryScan.js @@ -75,13 +75,14 @@ class LibraryScan { return date.format(new Date(), 'YYYY-MM-DD') + '_' + this.id + '.txt' } get scanResultsString() { - if (this.error) return this.error const strs = [] if (this.resultsAdded) strs.push(`${this.resultsAdded} added`) if (this.resultsUpdated) strs.push(`${this.resultsUpdated} updated`) if (this.resultsMissing) strs.push(`${this.resultsMissing} missing`) - if (!strs.length) return `Everything was up to date (${elapsedPretty(this.elapsed / 1000)})` - return strs.join(', ') + ` (${elapsedPretty(this.elapsed / 1000)})` + const changesDetected = strs.length > 0 ? strs.join(', ') : 'No changes detected' + const timeElapsed = `(${elapsedPretty(this.elapsed / 1000)})` + const error = this.error ? `${this.error}. ` : '' + return `${error}${changesDetected} ${timeElapsed}` } toJSON() { diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index 75d18df0..76022415 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -79,43 +79,39 @@ class LibraryScanner { Logger.info(`[LibraryScanner] Starting${forceRescan ? ' (forced)' : ''} library scan ${libraryScan.id} for ${libraryScan.libraryName}`) - const canceled = await this.scanLibrary(libraryScan, forceRescan) + try { + const canceled = await this.scanLibrary(libraryScan, forceRescan) + libraryScan.setComplete() - if (canceled) { - Logger.info(`[LibraryScanner] Library scan canceled for "${libraryScan.libraryName}"`) - delete this.cancelLibraryScan[libraryScan.libraryId] + Logger.info(`[LibraryScanner] Library scan "${libraryScan.id}" ${canceled ? 'canceled after' : 'completed in'} ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`) + + if (!canceled) { + library.lastScan = Date.now() + library.lastScanVersion = packageJson.version + if (library.isBook) { + const newExtraData = library.extraData || {} + newExtraData.lastScanMetadataPrecedence = library.settings.metadataPrecedence + library.extraData = newExtraData + library.changed('extraData', true) + } + await library.save() + } + + task.setFinished(`${canceled ? 'Canceled' : 'Completed'}. ${libraryScan.scanResultsString}`) + } catch (err) { + libraryScan.setComplete(err) + + Logger.error(`[LibraryScanner] Library scan ${libraryScan.id} failed after ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}.`, err) + + task.setFailed(`Failed. ${libraryScan.scanResultsString}`) } - libraryScan.setComplete() - - Logger.info(`[LibraryScanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`) + if (this.cancelLibraryScan[libraryScan.libraryId]) delete this.cancelLibraryScan[libraryScan.libraryId] this.librariesScanning = this.librariesScanning.filter((ls) => ls.id !== library.id) - if (canceled && !libraryScan.totalResults) { - task.setFinished('Scan canceled') - TaskManager.taskFinished(task) - - const emitData = libraryScan.getScanEmitData - emitData.results = null - return - } - - library.lastScan = Date.now() - library.lastScanVersion = packageJson.version - if (library.isBook) { - const newExtraData = library.extraData || {} - newExtraData.lastScanMetadataPrecedence = library.settings.metadataPrecedence - library.extraData = newExtraData - library.changed('extraData', true) - } - await library.save() - - task.setFinished(libraryScan.scanResultsString) TaskManager.taskFinished(task) - if (libraryScan.totalResults) { - libraryScan.saveLog() - } + libraryScan.saveLog() } /** @@ -140,7 +136,7 @@ class LibraryScanner { libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder) } - if (this.cancelLibraryScan[libraryScan.libraryId]) return true + if (this.shouldCancelScan(libraryScan)) return true const existingLibraryItems = await Database.libraryItemModel.findAll({ where: { @@ -148,7 +144,7 @@ class LibraryScanner { } }) - if (this.cancelLibraryScan[libraryScan.libraryId]) return true + if (this.shouldCancelScan(libraryScan)) return true const libraryItemIdsMissing = [] let oldLibraryItemsUpdated = [] @@ -216,7 +212,7 @@ class LibraryScanner { oldLibraryItemsUpdated = [] } - if (this.cancelLibraryScan[libraryScan.libraryId]) return true + if (this.shouldCancelScan(libraryScan)) return true } // Emit item updates to client if (oldLibraryItemsUpdated.length) { @@ -247,7 +243,7 @@ class LibraryScanner { ) } - if (this.cancelLibraryScan[libraryScan.libraryId]) return true + if (this.shouldCancelScan(libraryScan)) return true // Add new library items if (libraryItemDataFound.length) { @@ -271,7 +267,7 @@ class LibraryScanner { newOldLibraryItems = [] } - if (this.cancelLibraryScan[libraryScan.libraryId]) return true + if (this.shouldCancelScan(libraryScan)) return true } // Emit new items to client if (newOldLibraryItems.length) { @@ -282,6 +278,17 @@ class LibraryScanner { ) } } + + libraryScan.addLog(LogLevel.INFO, `Scan completed. ${libraryScan.resultStats}`) + return false + } + + shouldCancelScan(libraryScan) { + if (this.cancelLibraryScan[libraryScan.libraryId]) { + libraryScan.addLog(LogLevel.INFO, `Scan canceled. ${libraryScan.resultStats}`) + return true + } + return false } /** diff --git a/test/server/managers/MigrationManager.test.js b/test/server/managers/MigrationManager.test.js index ae28c0d1..ae94cd75 100644 --- a/test/server/managers/MigrationManager.test.js +++ b/test/server/managers/MigrationManager.test.js @@ -31,7 +31,7 @@ describe('MigrationManager', () => { down: sinon.stub() } sequelizeStub.getQueryInterface.returns({}) - migrationManager = new MigrationManager(sequelizeStub, configPath) + migrationManager = new MigrationManager(sequelizeStub, false, configPath) migrationManager.fetchVersionsFromDatabase = sinon.stub().resolves() migrationManager.copyMigrationsToConfigDir = sinon.stub().resolves() migrationManager.updateMaxVersion = sinon.stub().resolves() @@ -131,6 +131,21 @@ describe('MigrationManager', () => { expect(loggerInfoStub.calledWith(sinon.match('Migrations successfully applied'))).to.be.true }) + it('should log that migrations will be skipped if database is new', async () => { + // Arrange + migrationManager.isDatabaseNew = true + migrationManager.initialized = true + + // Act + await migrationManager.runMigrations() + + // Assert + expect(loggerInfoStub.calledWith(sinon.match('Database is new. Skipping migrations.'))).to.be.true + expect(migrationManager.initUmzug.called).to.be.false + expect(umzugStub.up.called).to.be.false + expect(umzugStub.down.called).to.be.false + }) + it('should log that no migrations are needed if serverVersion equals databaseVersion', async () => { // Arrange migrationManager.serverVersion = '1.2.0' @@ -181,7 +196,7 @@ describe('MigrationManager', () => { // Create a migrationsMeta table and populate it with version and maxVersion await sequelize.query('CREATE TABLE migrationsMeta (key VARCHAR(255), value VARCHAR(255))') await sequelize.query("INSERT INTO migrationsMeta (key, value) VALUES ('version', '1.1.0'), ('maxVersion', '1.1.0')") - const migrationManager = new MigrationManager(sequelize, configPath) + const migrationManager = new MigrationManager(sequelize, false, configPath) migrationManager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves() // Act @@ -195,7 +210,26 @@ describe('MigrationManager', () => { it('should create the migrationsMeta table if it does not exist and fetch versions from it', async () => { // Arrange const sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) - const migrationManager = new MigrationManager(sequelize, configPath) + const migrationManager = new MigrationManager(sequelize, false, configPath) + migrationManager.serverVersion = serverVersion + + // Act + await migrationManager.fetchVersionsFromDatabase() + + // Assert + const tableDescription = await sequelize.getQueryInterface().describeTable('migrationsMeta') + expect(tableDescription).to.deep.equal({ + key: { type: 'VARCHAR(255)', allowNull: false, defaultValue: undefined, primaryKey: false, unique: false }, + value: { type: 'VARCHAR(255)', allowNull: false, defaultValue: undefined, primaryKey: false, unique: false } + }) + expect(migrationManager.maxVersion).to.equal('0.0.0') + expect(migrationManager.databaseVersion).to.equal('0.0.0') + }) + + it('should create the migrationsMeta with databaseVersion=serverVersion if database is new', async () => { + // Arrange + const sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + const migrationManager = new MigrationManager(sequelize, true, configPath) migrationManager.serverVersion = serverVersion // Act @@ -211,11 +245,28 @@ describe('MigrationManager', () => { expect(migrationManager.databaseVersion).to.equal(serverVersion) }) + it('should re-create the migrationsMeta table if it existed and database is new (Database force=true)', async () => { + // Arrange + const sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) + // Create a migrationsMeta table and populate it with version and maxVersion + await sequelize.query('CREATE TABLE migrationsMeta (key VARCHAR(255), value VARCHAR(255))') + await sequelize.query("INSERT INTO migrationsMeta (key, value) VALUES ('version', '1.1.0'), ('maxVersion', '1.1.0')") + const migrationManager = new MigrationManager(sequelize, true, configPath) + migrationManager.serverVersion = serverVersion + + // Act + await migrationManager.fetchVersionsFromDatabase() + + // Assert + expect(migrationManager.maxVersion).to.equal('0.0.0') + expect(migrationManager.databaseVersion).to.equal(serverVersion) + }) + it('should throw an error if the database query fails', async () => { // Arrange const sequelizeStub = sinon.createStubInstance(Sequelize) sequelizeStub.query.rejects(new Error('Database query failed')) - const migrationManager = new MigrationManager(sequelizeStub, configPath) + const migrationManager = new MigrationManager(sequelizeStub, false, configPath) migrationManager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves() // Act @@ -236,7 +287,7 @@ describe('MigrationManager', () => { // Create a migrationsMeta table and populate it with version and maxVersion await sequelize.query('CREATE TABLE migrationsMeta (key VARCHAR(255), value VARCHAR(255))') await sequelize.query("INSERT INTO migrationsMeta (key, value) VALUES ('version', '1.1.0'), ('maxVersion', '1.1.0')") - const migrationManager = new MigrationManager(sequelize, configPath) + const migrationManager = new MigrationManager(sequelize, false, configPath) migrationManager.serverVersion = '1.2.0' // Act @@ -253,7 +304,7 @@ describe('MigrationManager', () => { describe('extractVersionFromTag', () => { it('should return null if tag is not provided', () => { // Arrange - const migrationManager = new MigrationManager(sequelizeStub, configPath) + const migrationManager = new MigrationManager(sequelizeStub, false, configPath) // Act const result = migrationManager.extractVersionFromTag() @@ -264,7 +315,7 @@ describe('MigrationManager', () => { it('should return null if tag does not match the version format', () => { // Arrange - const migrationManager = new MigrationManager(sequelizeStub, configPath) + const migrationManager = new MigrationManager(sequelizeStub, false, configPath) const tag = 'invalid-tag' // Act @@ -276,7 +327,7 @@ describe('MigrationManager', () => { it('should extract the version from the tag', () => { // Arrange - const migrationManager = new MigrationManager(sequelizeStub, configPath) + const migrationManager = new MigrationManager(sequelizeStub, false, configPath) const tag = 'v1.2.3' // Act @@ -290,7 +341,7 @@ describe('MigrationManager', () => { describe('copyMigrationsToConfigDir', () => { it('should copy migrations to the config directory', async () => { // Arrange - const migrationManager = new MigrationManager(sequelizeStub, configPath) + const migrationManager = new MigrationManager(sequelizeStub, false, configPath) migrationManager.migrationsDir = path.join(configPath, 'migrations') const migrationsSourceDir = path.join(__dirname, '..', '..', '..', 'server', 'migrations') const targetDir = migrationManager.migrationsDir @@ -313,7 +364,7 @@ describe('MigrationManager', () => { it('should throw an error if copying the migrations fails', async () => { // Arrange - const migrationManager = new MigrationManager(sequelizeStub, configPath) + const migrationManager = new MigrationManager(sequelizeStub, false, configPath) migrationManager.migrationsDir = path.join(configPath, 'migrations') const migrationsSourceDir = path.join(__dirname, '..', '..', '..', 'server', 'migrations') const targetDir = migrationManager.migrationsDir @@ -484,7 +535,7 @@ describe('MigrationManager', () => { const readdirStub = sinon.stub(fs, 'readdir').resolves(['v1.0.0-migration.js', 'v1.10.0-migration.js', 'v1.2.0-migration.js', 'v1.1.0-migration.js']) const readFileSyncStub = sinon.stub(fs, 'readFileSync').returns('module.exports = { up: () => {}, down: () => {} }') const umzugStorage = memoryStorage() - migrationManager = new MigrationManager(sequelizeStub, configPath) + migrationManager = new MigrationManager(sequelizeStub, false, configPath) migrationManager.migrationsDir = path.join(configPath, 'migrations') const resolvedMigrationNames = ['v1.0.0-migration.js', 'v1.1.0-migration.js', 'v1.2.0-migration.js', 'v1.10.0-migration.js'] const resolvedMigrationPaths = resolvedMigrationNames.map((name) => path.resolve(path.join(migrationManager.migrationsDir, name)))