Merge branch 'master' into series_cleanup_2

client/components/modals/AccountModal.vue

@@ -351,7 +351,7 @@ export default {
         update: type === 'admin',
         delete: type === 'admin',
         upload: type === 'admin',
-        accessExplicitContent: true,
+        accessExplicitContent: type === 'admin',
         accessAllLibraries: true,
         accessAllTags: true,
         selectedTagsNotAccessible: false
@@ -386,7 +386,7 @@ export default {
             upload: false,
             accessAllLibraries: true,
             accessAllTags: true,
-            accessExplicitContent: true,
+            accessExplicitContent: false,
             selectedTagsNotAccessible: false
           },
           librariesAccessible: [],

package.json

@@ -22,7 +22,8 @@
   "pkg": {
     "assets": [
       "client/dist/**/*",
-      "node_modules/sqlite3/lib/binding/**/*.node"
+      "node_modules/sqlite3/lib/binding/**/*.node",
+      "server/migrations/*.js"
     ],
     "scripts": [
       "prod.js",

server/Database.js

@@ -176,9 +176,9 @@ class Database {
     }
 
     try {
-      const migrationManager = new MigrationManager(this.sequelize, global.ConfigPath)
+      const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath)
       await migrationManager.init(packageJson.version)
-      if (!this.isNew) await migrationManager.runMigrations()
+      await migrationManager.runMigrations()
     } catch (error) {
       Logger.error(`[Database] Failed to run migrations`, error)
       throw new Error('Database migration failed')

server/managers/MigrationManager.js

@@ -11,11 +11,13 @@ class MigrationManager {
 
   /**
    * @param {import('../Database').sequelize} sequelize
+   * @param {boolean} isDatabaseNew
    * @param {string} [configPath]
    */
-  constructor(sequelize, configPath = global.configPath) {
+  constructor(sequelize, isDatabaseNew, configPath = global.configPath) {
     if (!sequelize || !(sequelize instanceof Sequelize)) throw new Error('Sequelize instance is required for MigrationManager.')
     this.sequelize = sequelize
+    this.isDatabaseNew = isDatabaseNew
     if (!configPath) throw new Error('Config path is required for MigrationManager.')
     this.configPath = configPath
     this.migrationsSourceDir = path.join(__dirname, '..', 'migrations')
@@ -42,6 +44,7 @@ class MigrationManager {
 
     await this.fetchVersionsFromDatabase()
     if (!this.maxVersion || !this.databaseVersion) throw new Error('Failed to fetch versions from the database.')
+    Logger.debug(`[MigrationManager] Database version: ${this.databaseVersion}, Max version: ${this.maxVersion}, Server version: ${this.serverVersion}`)
 
     if (semver.gt(this.serverVersion, this.maxVersion)) {
       try {
@@ -63,6 +66,11 @@ class MigrationManager {
   async runMigrations() {
     if (!this.initialized) throw new Error('MigrationManager is not initialized. Call init() first.')
 
+    if (this.isDatabaseNew) {
+      Logger.info('[MigrationManager] Database is new. Skipping migrations.')
+      return
+    }
+
     const versionCompare = semver.compare(this.serverVersion, this.databaseVersion)
     if (versionCompare == 0) {
       Logger.info('[MigrationManager] Database is already up to date.')
@@ -180,7 +188,15 @@ class MigrationManager {
 
   async checkOrCreateMigrationsMetaTable() {
     const queryInterface = this.sequelize.getQueryInterface()
-    if (!(await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE))) {
+    let migrationsMetaTableExists = await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE)
+
+    if (this.isDatabaseNew && migrationsMetaTableExists) {
+      // This can happen if database was initialized with force: true
+      await queryInterface.dropTable(MigrationManager.MIGRATIONS_META_TABLE)
+      migrationsMetaTableExists = false
+    }
+
+    if (!migrationsMetaTableExists) {
       await queryInterface.createTable(MigrationManager.MIGRATIONS_META_TABLE, {
         key: {
           type: DataTypes.STRING,
@@ -192,9 +208,10 @@ class MigrationManager {
         }
       })
       await this.sequelize.query("INSERT INTO :migrationsMeta (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')", {
-        replacements: { version: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
+        replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0', migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
         type: Sequelize.QueryTypes.INSERT
       })
+      Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`)
     }
   }
 
@@ -219,6 +236,7 @@ class MigrationManager {
           await fs.copy(sourceFile, targetFile) // Asynchronously copy the files
         })
     )
+    Logger.debug(`[MigrationManager] Copied migrations to the config directory: "${this.migrationsDir}"`)
   }
 
   /**

server/models/Author.js

@@ -1,5 +1,6 @@
 const { DataTypes, Model, where, fn, col } = require('sequelize')
 const parseNameString = require('../utils/parsers/parseNameString')
+const { asciiOnlyToLowerCase } = require('../utils/index')
 
 class Author extends Model {
   constructor(values, options) {
@@ -55,7 +56,7 @@ class Author extends Model {
   static async getByNameAndLibrary(authorName, libraryId) {
     return this.findOne({
       where: [
-        where(fn('lower', col('name')), authorName.toLowerCase()),
+        where(fn('lower', col('name')), asciiOnlyToLowerCase(authorName)),
         {
           libraryId
         }

server/models/Series.js

@@ -1,6 +1,7 @@
 const { DataTypes, Model, where, fn, col } = require('sequelize')
 
 const { getTitlePrefixAtEnd } = require('../utils/index')
+const { asciiOnlyToLowerCase } = require('../utils/index')
 
 class Series extends Model {
   constructor(values, options) {
@@ -41,7 +42,7 @@ class Series extends Model {
   static async getByNameAndLibrary(seriesName, libraryId) {
     return this.findOne({
       where: [
-        where(fn('lower', col('name')), seriesName.toLowerCase()),
+        where(fn('lower', col('name')), asciiOnlyToLowerCase(seriesName)),
         {
           libraryId
         }

server/models/User.js

@@ -107,7 +107,7 @@ class User extends Model {
       upload: type === 'root' || type === 'admin',
       accessAllLibraries: true,
       accessAllTags: true,
-      accessExplicitContent: true,
+      accessExplicitContent: type === 'root' || type === 'admin',
       selectedTagsNotAccessible: false,
       librariesAccessible: [],
       itemTagsSelected: []

server/scanner/LibraryScan.js

@@ -75,13 +75,14 @@ class LibraryScan {
     return date.format(new Date(), 'YYYY-MM-DD') + '_' + this.id + '.txt'
   }
   get scanResultsString() {
-    if (this.error) return this.error
     const strs = []
     if (this.resultsAdded) strs.push(`${this.resultsAdded} added`)
     if (this.resultsUpdated) strs.push(`${this.resultsUpdated} updated`)
     if (this.resultsMissing) strs.push(`${this.resultsMissing} missing`)
-    if (!strs.length) return `Everything was up to date (${elapsedPretty(this.elapsed / 1000)})`
+    const changesDetected = strs.length > 0 ? strs.join(', ') : 'No changes detected'
-    return strs.join(', ') + ` (${elapsedPretty(this.elapsed / 1000)})`
+    const timeElapsed = `(${elapsedPretty(this.elapsed / 1000)})`
+    const error = this.error ? `${this.error}. ` : ''
+    return `${error}${changesDetected} ${timeElapsed}`
   }
 
   toJSON() {

server/scanner/LibraryScanner.js

@@ -79,43 +79,39 @@ class LibraryScanner {
 
     Logger.info(`[LibraryScanner] Starting${forceRescan ? ' (forced)' : ''} library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
 
-    const canceled = await this.scanLibrary(libraryScan, forceRescan)
+    try {
+      const canceled = await this.scanLibrary(libraryScan, forceRescan)
+      libraryScan.setComplete()
 
-    if (canceled) {
+      Logger.info(`[LibraryScanner] Library scan "${libraryScan.id}" ${canceled ? 'canceled after' : 'completed in'} ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
-      Logger.info(`[LibraryScanner] Library scan canceled for "${libraryScan.libraryName}"`)
+
-      delete this.cancelLibraryScan[libraryScan.libraryId]
+      if (!canceled) {
+        library.lastScan = Date.now()
+        library.lastScanVersion = packageJson.version
+        if (library.isBook) {
+          const newExtraData = library.extraData || {}
+          newExtraData.lastScanMetadataPrecedence = library.settings.metadataPrecedence
+          library.extraData = newExtraData
+          library.changed('extraData', true)
+        }
+        await library.save()
+      }
+
+      task.setFinished(`${canceled ? 'Canceled' : 'Completed'}. ${libraryScan.scanResultsString}`)
+    } catch (err) {
+      libraryScan.setComplete(err)
+
+      Logger.error(`[LibraryScanner] Library scan ${libraryScan.id} failed after ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}.`, err)
+
+      task.setFailed(`Failed. ${libraryScan.scanResultsString}`)
     }
 
-    libraryScan.setComplete()
+    if (this.cancelLibraryScan[libraryScan.libraryId]) delete this.cancelLibraryScan[libraryScan.libraryId]
-
-    Logger.info(`[LibraryScanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
     this.librariesScanning = this.librariesScanning.filter((ls) => ls.id !== library.id)
 
-    if (canceled && !libraryScan.totalResults) {
-      task.setFinished('Scan canceled')
-      TaskManager.taskFinished(task)
-
-      const emitData = libraryScan.getScanEmitData
-      emitData.results = null
-      return
-    }
-
-    library.lastScan = Date.now()
-    library.lastScanVersion = packageJson.version
-    if (library.isBook) {
-      const newExtraData = library.extraData || {}
-      newExtraData.lastScanMetadataPrecedence = library.settings.metadataPrecedence
-      library.extraData = newExtraData
-      library.changed('extraData', true)
-    }
-    await library.save()
-
-    task.setFinished(libraryScan.scanResultsString)
     TaskManager.taskFinished(task)
 
-    if (libraryScan.totalResults) {
+    libraryScan.saveLog()
-      libraryScan.saveLog()
-    }
   }
 
   /**
@@ -140,7 +136,7 @@ class LibraryScanner {
       libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder)
     }
 
-    if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+    if (this.shouldCancelScan(libraryScan)) return true
 
     const existingLibraryItems = await Database.libraryItemModel.findAll({
       where: {
@@ -148,7 +144,7 @@ class LibraryScanner {
       }
     })
 
-    if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+    if (this.shouldCancelScan(libraryScan)) return true
 
     const libraryItemIdsMissing = []
     let oldLibraryItemsUpdated = []
@@ -216,7 +212,7 @@ class LibraryScanner {
         oldLibraryItemsUpdated = []
       }
 
-      if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+      if (this.shouldCancelScan(libraryScan)) return true
     }
     // Emit item updates to client
     if (oldLibraryItemsUpdated.length) {
@@ -247,7 +243,7 @@ class LibraryScanner {
       )
     }
 
-    if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+    if (this.shouldCancelScan(libraryScan)) return true
 
     // Add new library items
     if (libraryItemDataFound.length) {
@@ -271,7 +267,7 @@ class LibraryScanner {
           newOldLibraryItems = []
         }
 
-        if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+        if (this.shouldCancelScan(libraryScan)) return true
       }
       // Emit new items to client
       if (newOldLibraryItems.length) {
@@ -282,6 +278,17 @@ class LibraryScanner {
         )
       }
     }
+
+    libraryScan.addLog(LogLevel.INFO, `Scan completed. ${libraryScan.resultStats}`)
+    return false
+  }
+
+  shouldCancelScan(libraryScan) {
+    if (this.cancelLibraryScan[libraryScan.libraryId]) {
+      libraryScan.addLog(LogLevel.INFO, `Scan canceled. ${libraryScan.resultStats}`)
+      return true
+    }
+    return false
   }
 
   /**

test/server/managers/MigrationManager.test.js

@@ -31,7 +31,7 @@ describe('MigrationManager', () => {
       down: sinon.stub()
     }
     sequelizeStub.getQueryInterface.returns({})
-    migrationManager = new MigrationManager(sequelizeStub, configPath)
+    migrationManager = new MigrationManager(sequelizeStub, false, configPath)
     migrationManager.fetchVersionsFromDatabase = sinon.stub().resolves()
     migrationManager.copyMigrationsToConfigDir = sinon.stub().resolves()
     migrationManager.updateMaxVersion = sinon.stub().resolves()
@@ -131,6 +131,21 @@ describe('MigrationManager', () => {
       expect(loggerInfoStub.calledWith(sinon.match('Migrations successfully applied'))).to.be.true
     })
 
+    it('should log that migrations will be skipped if database is new', async () => {
+      // Arrange
+      migrationManager.isDatabaseNew = true
+      migrationManager.initialized = true
+
+      // Act
+      await migrationManager.runMigrations()
+
+      // Assert
+      expect(loggerInfoStub.calledWith(sinon.match('Database is new. Skipping migrations.'))).to.be.true
+      expect(migrationManager.initUmzug.called).to.be.false
+      expect(umzugStub.up.called).to.be.false
+      expect(umzugStub.down.called).to.be.false
+    })
+
     it('should log that no migrations are needed if serverVersion equals databaseVersion', async () => {
       // Arrange
       migrationManager.serverVersion = '1.2.0'
@@ -181,7 +196,7 @@ describe('MigrationManager', () => {
       // Create a migrationsMeta table and populate it with version and maxVersion
       await sequelize.query('CREATE TABLE migrationsMeta (key VARCHAR(255), value VARCHAR(255))')
       await sequelize.query("INSERT INTO migrationsMeta (key, value) VALUES ('version', '1.1.0'), ('maxVersion', '1.1.0')")
-      const migrationManager = new MigrationManager(sequelize, configPath)
+      const migrationManager = new MigrationManager(sequelize, false, configPath)
       migrationManager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves()
 
       // Act
@@ -195,7 +210,26 @@ describe('MigrationManager', () => {
     it('should create the migrationsMeta table if it does not exist and fetch versions from it', async () => {
       // Arrange
       const sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
-      const migrationManager = new MigrationManager(sequelize, configPath)
+      const migrationManager = new MigrationManager(sequelize, false, configPath)
+      migrationManager.serverVersion = serverVersion
+
+      // Act
+      await migrationManager.fetchVersionsFromDatabase()
+
+      // Assert
+      const tableDescription = await sequelize.getQueryInterface().describeTable('migrationsMeta')
+      expect(tableDescription).to.deep.equal({
+        key: { type: 'VARCHAR(255)', allowNull: false, defaultValue: undefined, primaryKey: false, unique: false },
+        value: { type: 'VARCHAR(255)', allowNull: false, defaultValue: undefined, primaryKey: false, unique: false }
+      })
+      expect(migrationManager.maxVersion).to.equal('0.0.0')
+      expect(migrationManager.databaseVersion).to.equal('0.0.0')
+    })
+
+    it('should create the migrationsMeta with databaseVersion=serverVersion if database is new', async () => {
+      // Arrange
+      const sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
+      const migrationManager = new MigrationManager(sequelize, true, configPath)
       migrationManager.serverVersion = serverVersion
 
       // Act
@@ -211,11 +245,28 @@ describe('MigrationManager', () => {
       expect(migrationManager.databaseVersion).to.equal(serverVersion)
     })
 
+    it('should re-create the migrationsMeta table if it existed and database is new (Database force=true)', async () => {
+      // Arrange
+      const sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
+      // Create a migrationsMeta table and populate it with version and maxVersion
+      await sequelize.query('CREATE TABLE migrationsMeta (key VARCHAR(255), value VARCHAR(255))')
+      await sequelize.query("INSERT INTO migrationsMeta (key, value) VALUES ('version', '1.1.0'), ('maxVersion', '1.1.0')")
+      const migrationManager = new MigrationManager(sequelize, true, configPath)
+      migrationManager.serverVersion = serverVersion
+
+      // Act
+      await migrationManager.fetchVersionsFromDatabase()
+
+      // Assert
+      expect(migrationManager.maxVersion).to.equal('0.0.0')
+      expect(migrationManager.databaseVersion).to.equal(serverVersion)
+    })
+
     it('should throw an error if the database query fails', async () => {
       // Arrange
       const sequelizeStub = sinon.createStubInstance(Sequelize)
       sequelizeStub.query.rejects(new Error('Database query failed'))
-      const migrationManager = new MigrationManager(sequelizeStub, configPath)
+      const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
       migrationManager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves()
 
       // Act
@@ -236,7 +287,7 @@ describe('MigrationManager', () => {
       // Create a migrationsMeta table and populate it with version and maxVersion
       await sequelize.query('CREATE TABLE migrationsMeta (key VARCHAR(255), value VARCHAR(255))')
       await sequelize.query("INSERT INTO migrationsMeta (key, value) VALUES ('version', '1.1.0'), ('maxVersion', '1.1.0')")
-      const migrationManager = new MigrationManager(sequelize, configPath)
+      const migrationManager = new MigrationManager(sequelize, false, configPath)
       migrationManager.serverVersion = '1.2.0'
 
       // Act
@@ -253,7 +304,7 @@ describe('MigrationManager', () => {
   describe('extractVersionFromTag', () => {
     it('should return null if tag is not provided', () => {
       // Arrange
-      const migrationManager = new MigrationManager(sequelizeStub, configPath)
+      const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
 
       // Act
       const result = migrationManager.extractVersionFromTag()
@@ -264,7 +315,7 @@ describe('MigrationManager', () => {
 
     it('should return null if tag does not match the version format', () => {
       // Arrange
-      const migrationManager = new MigrationManager(sequelizeStub, configPath)
+      const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
       const tag = 'invalid-tag'
 
       // Act
@@ -276,7 +327,7 @@ describe('MigrationManager', () => {
 
     it('should extract the version from the tag', () => {
       // Arrange
-      const migrationManager = new MigrationManager(sequelizeStub, configPath)
+      const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
       const tag = 'v1.2.3'
 
       // Act
@@ -290,7 +341,7 @@ describe('MigrationManager', () => {
   describe('copyMigrationsToConfigDir', () => {
     it('should copy migrations to the config directory', async () => {
       // Arrange
-      const migrationManager = new MigrationManager(sequelizeStub, configPath)
+      const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
       migrationManager.migrationsDir = path.join(configPath, 'migrations')
       const migrationsSourceDir = path.join(__dirname, '..', '..', '..', 'server', 'migrations')
       const targetDir = migrationManager.migrationsDir
@@ -313,7 +364,7 @@ describe('MigrationManager', () => {
 
     it('should throw an error if copying the migrations fails', async () => {
       // Arrange
-      const migrationManager = new MigrationManager(sequelizeStub, configPath)
+      const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
       migrationManager.migrationsDir = path.join(configPath, 'migrations')
       const migrationsSourceDir = path.join(__dirname, '..', '..', '..', 'server', 'migrations')
       const targetDir = migrationManager.migrationsDir
@@ -484,7 +535,7 @@ describe('MigrationManager', () => {
       const readdirStub = sinon.stub(fs, 'readdir').resolves(['v1.0.0-migration.js', 'v1.10.0-migration.js', 'v1.2.0-migration.js', 'v1.1.0-migration.js'])
       const readFileSyncStub = sinon.stub(fs, 'readFileSync').returns('module.exports = { up: () => {}, down: () => {} }')
       const umzugStorage = memoryStorage()
-      migrationManager = new MigrationManager(sequelizeStub, configPath)
+      migrationManager = new MigrationManager(sequelizeStub, false, configPath)
       migrationManager.migrationsDir = path.join(configPath, 'migrations')
       const resolvedMigrationNames = ['v1.0.0-migration.js', 'v1.1.0-migration.js', 'v1.2.0-migration.js', 'v1.10.0-migration.js']
       const resolvedMigrationPaths = resolvedMigrationNames.map((name) => path.resolve(path.join(migrationManager.migrationsDir, name)))