Mirrors/audiobookshelf
Mirrors/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2024-12-20 19:06:06 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into series_cleanup_2

Browse Source
This commit is contained in:
advplyr 2024-09-14 10:11:16 -05:00
commit bedba39af9
10 changed files with 140 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
client/components/modals/AccountModal.vue
View File

@ -351,7 +351,7 @@ export default {
update: type === 'admin',
delete: type === 'admin',
upload: type === 'admin',
accessExplicitContent: true,
accessExplicitContent: type === 'admin',
accessAllLibraries: true,
accessAllTags: true,
selectedTagsNotAccessible: false
@ -386,7 +386,7 @@ export default {
upload: false,
accessAllLibraries: true,
accessAllTags: true,
accessExplicitContent: true,
accessExplicitContent: false,
selectedTagsNotAccessible: false
},
librariesAccessible: [],

3
package.json
View File

@ -22,7 +22,8 @@
"pkg": {
"assets": [
"client/dist/**/*",
"node_modules/sqlite3/lib/binding/**/*.node"
"node_modules/sqlite3/lib/binding/**/*.node",
"server/migrations/*.js"
],
"scripts": [
"prod.js",

4
server/Database.js
View File

@ -176,9 +176,9 @@ class Database {
}
try {
const migrationManager = new MigrationManager(this.sequelize, global.ConfigPath)
const migrationManager = new MigrationManager(this.sequelize, this.isNew, global.ConfigPath)
await migrationManager.init(packageJson.version)
if (!this.isNew) await migrationManager.runMigrations()
await migrationManager.runMigrations()
} catch (error) {
Logger.error(`[Database] Failed to run migrations`, error)
throw new Error('Database migration failed')

24
server/managers/MigrationManager.js
View File

@ -11,11 +11,13 @@ class MigrationManager {
/**
* @param {import('../Database').sequelize} sequelize
* @param {boolean} isDatabaseNew
* @param {string} [configPath]
*/
constructor(sequelize, configPath = global.configPath) {
constructor(sequelize, isDatabaseNew, configPath = global.configPath) {
if (!sequelize || !(sequelize instanceof Sequelize)) throw new Error('Sequelize instance is required for MigrationManager.')
this.sequelize = sequelize
this.isDatabaseNew = isDatabaseNew
if (!configPath) throw new Error('Config path is required for MigrationManager.')
this.configPath = configPath
this.migrationsSourceDir = path.join(__dirname, '..', 'migrations')
@ -42,6 +44,7 @@ class MigrationManager {
await this.fetchVersionsFromDatabase()
if (!this.maxVersion || !this.databaseVersion) throw new Error('Failed to fetch versions from the database.')
Logger.debug(`[MigrationManager] Database version: ${this.databaseVersion}, Max version: ${this.maxVersion}, Server version: ${this.serverVersion}`)
if (semver.gt(this.serverVersion, this.maxVersion)) {
try {
@ -63,6 +66,11 @@ class MigrationManager {
async runMigrations() {
if (!this.initialized) throw new Error('MigrationManager is not initialized. Call init() first.')
if (this.isDatabaseNew) {
Logger.info('[MigrationManager] Database is new. Skipping migrations.')
return
}
const versionCompare = semver.compare(this.serverVersion, this.databaseVersion)
if (versionCompare == 0) {
Logger.info('[MigrationManager] Database is already up to date.')
@ -180,7 +188,15 @@ class MigrationManager {
async checkOrCreateMigrationsMetaTable() {
const queryInterface = this.sequelize.getQueryInterface()
if (!(await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE))) {
let migrationsMetaTableExists = await queryInterface.tableExists(MigrationManager.MIGRATIONS_META_TABLE)
if (this.isDatabaseNew && migrationsMetaTableExists) {
// This can happen if database was initialized with force: true
await queryInterface.dropTable(MigrationManager.MIGRATIONS_META_TABLE)
migrationsMetaTableExists = false
}
if (!migrationsMetaTableExists) {
await queryInterface.createTable(MigrationManager.MIGRATIONS_META_TABLE, {
key: {
type: DataTypes.STRING,
@ -192,9 +208,10 @@ class MigrationManager {
}
})
await this.sequelize.query("INSERT INTO :migrationsMeta (key, value) VALUES ('version', :version), ('maxVersion', '0.0.0')", {
replacements: { version: this.serverVersion, migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
replacements: { version: this.isDatabaseNew ? this.serverVersion : '0.0.0', migrationsMeta: MigrationManager.MIGRATIONS_META_TABLE },
type: Sequelize.QueryTypes.INSERT
})
Logger.debug(`[MigrationManager] Created migrationsMeta table: "${MigrationManager.MIGRATIONS_META_TABLE}"`)
}
}
@ -219,6 +236,7 @@ class MigrationManager {
await fs.copy(sourceFile, targetFile) // Asynchronously copy the files
})
)
Logger.debug(`[MigrationManager] Copied migrations to the config directory: "${this.migrationsDir}"`)
}
/**

3
server/models/Author.js
View File

@ -1,5 +1,6 @@
const { DataTypes, Model, where, fn, col } = require('sequelize')
const parseNameString = require('../utils/parsers/parseNameString')
const { asciiOnlyToLowerCase } = require('../utils/index')
class Author extends Model {
constructor(values, options) {
@ -55,7 +56,7 @@ class Author extends Model {
static async getByNameAndLibrary(authorName, libraryId) {
return this.findOne({
where: [
where(fn('lower', col('name')), authorName.toLowerCase()),
where(fn('lower', col('name')), asciiOnlyToLowerCase(authorName)),
{
libraryId
}

3
server/models/Series.js
View File

@ -1,6 +1,7 @@
const { DataTypes, Model, where, fn, col } = require('sequelize')
const { getTitlePrefixAtEnd } = require('../utils/index')
const { asciiOnlyToLowerCase } = require('../utils/index')
class Series extends Model {
constructor(values, options) {
@ -41,7 +42,7 @@ class Series extends Model {
static async getByNameAndLibrary(seriesName, libraryId) {
return this.findOne({
where: [
where(fn('lower', col('name')), seriesName.toLowerCase()),
where(fn('lower', col('name')), asciiOnlyToLowerCase(seriesName)),
{
libraryId
}

2
server/models/User.js
View File

@ -107,7 +107,7 @@ class User extends Model {
upload: type === 'root' || type === 'admin',
accessAllLibraries: true,
accessAllTags: true,
accessExplicitContent: true,
accessExplicitContent: type === 'root' || type === 'admin',
selectedTagsNotAccessible: false,
librariesAccessible: [],
itemTagsSelected: []

7
server/scanner/LibraryScan.js
View File

@ -75,13 +75,14 @@ class LibraryScan {
return date.format(new Date(), 'YYYY-MM-DD') + '_' + this.id + '.txt'
}
get scanResultsString() {
if (this.error) return this.error
const strs = []
if (this.resultsAdded) strs.push(`${this.resultsAdded} added`)
if (this.resultsUpdated) strs.push(`${this.resultsUpdated} updated`)
if (this.resultsMissing) strs.push(`${this.resultsMissing} missing`)
if (!strs.length) return `Everything was up to date (${elapsedPretty(this.elapsed / 1000)})`
return strs.join(', ') + ` (${elapsedPretty(this.elapsed / 1000)})`
const changesDetected = strs.length > 0 ? strs.join(', ') : 'No changes detected'
const timeElapsed = `(${elapsedPretty(this.elapsed / 1000)})`
const error = this.error ? `${this.error}. ` : ''
return `${error}${changesDetected} ${timeElapsed}`
}
toJSON() {

77
server/scanner/LibraryScanner.js
View File

@ -79,43 +79,39 @@ class LibraryScanner {
Logger.info(`[LibraryScanner] Starting${forceRescan ? ' (forced)' : ''} library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
const canceled = await this.scanLibrary(libraryScan, forceRescan)
try {
const canceled = await this.scanLibrary(libraryScan, forceRescan)
libraryScan.setComplete()
if (canceled) {
Logger.info(`[LibraryScanner] Library scan canceled for "${libraryScan.libraryName}"`)
delete this.cancelLibraryScan[libraryScan.libraryId]
Logger.info(`[LibraryScanner] Library scan "${libraryScan.id}" ${canceled ? 'canceled after' : 'completed in'} ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
if (!canceled) {
library.lastScan = Date.now()
library.lastScanVersion = packageJson.version
if (library.isBook) {
const newExtraData = library.extraData || {}
newExtraData.lastScanMetadataPrecedence = library.settings.metadataPrecedence
library.extraData = newExtraData
library.changed('extraData', true)
}
await library.save()
}
task.setFinished(`${canceled ? 'Canceled' : 'Completed'}. ${libraryScan.scanResultsString}`)
} catch (err) {
libraryScan.setComplete(err)
Logger.error(`[LibraryScanner] Library scan ${libraryScan.id} failed after ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}.`, err)
task.setFailed(`Failed. ${libraryScan.scanResultsString}`)
}
libraryScan.setComplete()
Logger.info(`[LibraryScanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
if (this.cancelLibraryScan[libraryScan.libraryId]) delete this.cancelLibraryScan[libraryScan.libraryId]
this.librariesScanning = this.librariesScanning.filter((ls) => ls.id !== library.id)
if (canceled && !libraryScan.totalResults) {
task.setFinished('Scan canceled')
TaskManager.taskFinished(task)
const emitData = libraryScan.getScanEmitData
emitData.results = null
return
}
library.lastScan = Date.now()
library.lastScanVersion = packageJson.version
if (library.isBook) {
const newExtraData = library.extraData || {}
newExtraData.lastScanMetadataPrecedence = library.settings.metadataPrecedence
library.extraData = newExtraData
library.changed('extraData', true)
}
await library.save()
task.setFinished(libraryScan.scanResultsString)
TaskManager.taskFinished(task)
if (libraryScan.totalResults) {
libraryScan.saveLog()
}
libraryScan.saveLog()
}
/**
@ -140,7 +136,7 @@ class LibraryScanner {
libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder)
}
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
if (this.shouldCancelScan(libraryScan)) return true
const existingLibraryItems = await Database.libraryItemModel.findAll({
where: {
@ -148,7 +144,7 @@ class LibraryScanner {
}
})
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
if (this.shouldCancelScan(libraryScan)) return true
const libraryItemIdsMissing = []
let oldLibraryItemsUpdated = []
@ -216,7 +212,7 @@ class LibraryScanner {
oldLibraryItemsUpdated = []
}
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
if (this.shouldCancelScan(libraryScan)) return true
}
// Emit item updates to client
if (oldLibraryItemsUpdated.length) {
@ -247,7 +243,7 @@ class LibraryScanner {
)
}
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
if (this.shouldCancelScan(libraryScan)) return true
// Add new library items
if (libraryItemDataFound.length) {
@ -271,7 +267,7 @@ class LibraryScanner {
newOldLibraryItems = []
}
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
if (this.shouldCancelScan(libraryScan)) return true
}
// Emit new items to client
if (newOldLibraryItems.length) {
@ -282,6 +278,17 @@ class LibraryScanner {
)
}
}
libraryScan.addLog(LogLevel.INFO, `Scan completed. ${libraryScan.resultStats}`)
return false
}
shouldCancelScan(libraryScan) {
if (this.cancelLibraryScan[libraryScan.libraryId]) {
libraryScan.addLog(LogLevel.INFO, `Scan canceled. ${libraryScan.resultStats}`)
return true
}
return false
}
/**

73
test/server/managers/MigrationManager.test.js
View File

@ -31,7 +31,7 @@ describe('MigrationManager', () => {
down: sinon.stub()
}
sequelizeStub.getQueryInterface.returns({})
migrationManager = new MigrationManager(sequelizeStub, configPath)
migrationManager = new MigrationManager(sequelizeStub, false, configPath)
migrationManager.fetchVersionsFromDatabase = sinon.stub().resolves()
migrationManager.copyMigrationsToConfigDir = sinon.stub().resolves()
migrationManager.updateMaxVersion = sinon.stub().resolves()
@ -131,6 +131,21 @@ describe('MigrationManager', () => {
expect(loggerInfoStub.calledWith(sinon.match('Migrations successfully applied'))).to.be.true
})
it('should log that migrations will be skipped if database is new', async () => {
// Arrange
migrationManager.isDatabaseNew = true
migrationManager.initialized = true
// Act
await migrationManager.runMigrations()
// Assert
expect(loggerInfoStub.calledWith(sinon.match('Database is new. Skipping migrations.'))).to.be.true
expect(migrationManager.initUmzug.called).to.be.false
expect(umzugStub.up.called).to.be.false
expect(umzugStub.down.called).to.be.false
})
it('should log that no migrations are needed if serverVersion equals databaseVersion', async () => {
// Arrange
migrationManager.serverVersion = '1.2.0'
@ -181,7 +196,7 @@ describe('MigrationManager', () => {
// Create a migrationsMeta table and populate it with version and maxVersion
await sequelize.query('CREATE TABLE migrationsMeta (key VARCHAR(255), value VARCHAR(255))')
await sequelize.query("INSERT INTO migrationsMeta (key, value) VALUES ('version', '1.1.0'), ('maxVersion', '1.1.0')")
const migrationManager = new MigrationManager(sequelize, configPath)
const migrationManager = new MigrationManager(sequelize, false, configPath)
migrationManager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves()
// Act
@ -195,7 +210,26 @@ describe('MigrationManager', () => {
it('should create the migrationsMeta table if it does not exist and fetch versions from it', async () => {
// Arrange
const sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
const migrationManager = new MigrationManager(sequelize, configPath)
const migrationManager = new MigrationManager(sequelize, false, configPath)
migrationManager.serverVersion = serverVersion
// Act
await migrationManager.fetchVersionsFromDatabase()
// Assert
const tableDescription = await sequelize.getQueryInterface().describeTable('migrationsMeta')
expect(tableDescription).to.deep.equal({
key: { type: 'VARCHAR(255)', allowNull: false, defaultValue: undefined, primaryKey: false, unique: false },
value: { type: 'VARCHAR(255)', allowNull: false, defaultValue: undefined, primaryKey: false, unique: false }
})
expect(migrationManager.maxVersion).to.equal('0.0.0')
expect(migrationManager.databaseVersion).to.equal('0.0.0')
})
it('should create the migrationsMeta with databaseVersion=serverVersion if database is new', async () => {
// Arrange
const sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
const migrationManager = new MigrationManager(sequelize, true, configPath)
migrationManager.serverVersion = serverVersion
// Act
@ -211,11 +245,28 @@ describe('MigrationManager', () => {
expect(migrationManager.databaseVersion).to.equal(serverVersion)
})
it('should re-create the migrationsMeta table if it existed and database is new (Database force=true)', async () => {
// Arrange
const sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
// Create a migrationsMeta table and populate it with version and maxVersion
await sequelize.query('CREATE TABLE migrationsMeta (key VARCHAR(255), value VARCHAR(255))')
await sequelize.query("INSERT INTO migrationsMeta (key, value) VALUES ('version', '1.1.0'), ('maxVersion', '1.1.0')")
const migrationManager = new MigrationManager(sequelize, true, configPath)
migrationManager.serverVersion = serverVersion
// Act
await migrationManager.fetchVersionsFromDatabase()
// Assert
expect(migrationManager.maxVersion).to.equal('0.0.0')
expect(migrationManager.databaseVersion).to.equal(serverVersion)
})
it('should throw an error if the database query fails', async () => {
// Arrange
const sequelizeStub = sinon.createStubInstance(Sequelize)
sequelizeStub.query.rejects(new Error('Database query failed'))
const migrationManager = new MigrationManager(sequelizeStub, configPath)
const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
migrationManager.checkOrCreateMigrationsMetaTable = sinon.stub().resolves()
// Act
@ -236,7 +287,7 @@ describe('MigrationManager', () => {
// Create a migrationsMeta table and populate it with version and maxVersion
await sequelize.query('CREATE TABLE migrationsMeta (key VARCHAR(255), value VARCHAR(255))')
await sequelize.query("INSERT INTO migrationsMeta (key, value) VALUES ('version', '1.1.0'), ('maxVersion', '1.1.0')")
const migrationManager = new MigrationManager(sequelize, configPath)
const migrationManager = new MigrationManager(sequelize, false, configPath)
migrationManager.serverVersion = '1.2.0'
// Act
@ -253,7 +304,7 @@ describe('MigrationManager', () => {
describe('extractVersionFromTag', () => {
it('should return null if tag is not provided', () => {
// Arrange
const migrationManager = new MigrationManager(sequelizeStub, configPath)
const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
// Act
const result = migrationManager.extractVersionFromTag()
@ -264,7 +315,7 @@ describe('MigrationManager', () => {
it('should return null if tag does not match the version format', () => {
// Arrange
const migrationManager = new MigrationManager(sequelizeStub, configPath)
const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
const tag = 'invalid-tag'
// Act
@ -276,7 +327,7 @@ describe('MigrationManager', () => {
it('should extract the version from the tag', () => {
// Arrange
const migrationManager = new MigrationManager(sequelizeStub, configPath)
const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
const tag = 'v1.2.3'
// Act
@ -290,7 +341,7 @@ describe('MigrationManager', () => {
describe('copyMigrationsToConfigDir', () => {
it('should copy migrations to the config directory', async () => {
// Arrange
const migrationManager = new MigrationManager(sequelizeStub, configPath)
const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
migrationManager.migrationsDir = path.join(configPath, 'migrations')
const migrationsSourceDir = path.join(__dirname, '..', '..', '..', 'server', 'migrations')
const targetDir = migrationManager.migrationsDir
@ -313,7 +364,7 @@ describe('MigrationManager', () => {
it('should throw an error if copying the migrations fails', async () => {
// Arrange
const migrationManager = new MigrationManager(sequelizeStub, configPath)
const migrationManager = new MigrationManager(sequelizeStub, false, configPath)
migrationManager.migrationsDir = path.join(configPath, 'migrations')
const migrationsSourceDir = path.join(__dirname, '..', '..', '..', 'server', 'migrations')
const targetDir = migrationManager.migrationsDir
@ -484,7 +535,7 @@ describe('MigrationManager', () => {
const readdirStub = sinon.stub(fs, 'readdir').resolves(['v1.0.0-migration.js', 'v1.10.0-migration.js', 'v1.2.0-migration.js', 'v1.1.0-migration.js'])
const readFileSyncStub = sinon.stub(fs, 'readFileSync').returns('module.exports = { up: () => {}, down: () => {} }')
const umzugStorage = memoryStorage()
migrationManager = new MigrationManager(sequelizeStub, configPath)
migrationManager = new MigrationManager(sequelizeStub, false, configPath)
migrationManager.migrationsDir = path.join(configPath, 'migrations')
const resolvedMigrationNames = ['v1.0.0-migration.js', 'v1.1.0-migration.js', 'v1.2.0-migration.js', 'v1.10.0-migration.js']
const resolvedMigrationPaths = resolvedMigrationNames.map((name) => path.resolve(path.join(migrationManager.migrationsDir, name)))