Added deviceId sequelize migration and completed unit tests

This commit is contained in:
Jason Axley 2025-08-24 15:56:43 -07:00
parent 423f2d311e
commit 41a288bcdf
10 changed files with 451 additions and 29 deletions

35
.vscode/launch.json vendored
View File

@ -9,36 +9,33 @@
"request": "launch",
"name": "Debug server",
"runtimeExecutable": "npm",
"args": [
"run",
"dev"
],
"skipFiles": [
"<node_internals>/**"
]
"cwd": "${workspaceFolder}/client",
"args": ["run", "dev"],
"skipFiles": ["<node_internals>/**"]
},
{
"type": "node",
"request": "launch",
"name": "Prod server",
"runtimeExecutable": "npm",
"args": ["run", "prod"],
"skipFiles": ["<node_internals>/**"]
},
{
"type": "node",
"request": "launch",
"name": "Debug client (nuxt)",
"runtimeExecutable": "npm",
"args": [
"run",
"dev"
],
"args": ["run", "dev"],
"cwd": "${workspaceFolder}/client",
"skipFiles": [
"${workspaceFolder}/<node_internals>/**"
]
"skipFiles": ["${workspaceFolder}/<node_internals>/**"]
}
],
"compounds": [
{
"name": "Debug server and client (nuxt)",
"configurations": [
"Debug server",
"Debug client (nuxt)"
]
"configurations": ["Debug server", "Debug client (nuxt)"]
}
]
}
}

View File

@ -1,6 +1,6 @@
{
"name": "audiobookshelf",
"version": "2.28.0",
"version": "2.29.0",
"buildNumber": 1,
"description": "Self-hosted audiobook and podcast server",
"main": "index.js",

View File

@ -16,3 +16,4 @@ Please add a record of every database migration that you create to this file. Th
| v2.19.1 | v2.19.1-copy-title-to-library-items | Copies title and titleIgnorePrefix to the libraryItems table, creates update triggers and indices |
| v2.19.4 | v2.19.4-improve-podcast-queries | Adds numEpisodes to podcasts, adds podcastId to mediaProgresses, copies podcast title to libraryItems |
| v2.20.0 | v2.20.0-improve-author-sort-queries | Adds AuthorNames(FirstLast\|LastFirst) to libraryItems to improve author sort queries |
| v2.29.0 | v2.29.0-add-deviceId | Adds deviceId to libraryItems table to uniquely identify files in a filesystem |

View File

@ -0,0 +1,180 @@
const util = require('util')
const { Sequelize, DataTypes } = require('sequelize')
/**
* @typedef MigrationContext
* @property {import('sequelize').QueryInterface} queryInterface - a sequelize QueryInterface object.
* @property {import('../Logger')} logger - a Logger object.
*
* @typedef MigrationOptions
* @property {MigrationContext} context - an object containing the migration context.
*/
const migrationVersion = '2.29.0'
const migrationName = `${migrationVersion}-add-deviceId`
const loggerPrefix = `[${migrationVersion} migration]`
// Migration constants
const libraryItems = 'libraryItems'
const columns = [{ name: 'deviceId', spec: { type: DataTypes.STRING, allowNull: true } }]
const columnNames = columns.map((column) => column.name).join(', ')
/**
* This upward migration adds a deviceId column to the libraryItems table and populates it.
* It also creates an index on the ino, deviceId columns.
*
* @param {MigrationOptions} options - an object containing the migration context.
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
*/
async function up({ context: { queryInterface, logger } }) {
const helper = new MigrationHelper(queryInterface, logger)
// Upwards migration script
logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
// Add authorNames columns to libraryItems table
await helper.addColumns()
// Populate authorNames columns with the author names for each libraryItem
// TODO
// await helper.populateColumnsFromSource()
// Create indexes on the authorNames columns
await helper.addIndexes()
// Add index on ino and deviceId to the podcastEpisodes table
await helper.addIndex('libraryItems', ['ino', 'deviceId'])
logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
}
/**
* This downward migration removes a deviceId column to the libraryItems table, *
* It also removes the index on ino and deviceId from the libraryItems table.
*
* @param {MigrationOptions} options - an object containing the migration context.
* @returns {Promise<void>} - A promise that resolves when the migration is complete.
*/
async function down({ context: { queryInterface, logger } }) {
// Downward migration script
logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
const helper = new MigrationHelper(queryInterface, logger)
// Remove index on publishedAt from the podcastEpisodes table
await helper.removeIndex('libraryItems', ['ino', 'deviceId'])
// Remove indexes on the authorNames columns
await helper.removeIndexes()
// Remove authorNames columns from libraryItems table
await helper.removeColumns()
logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
}
class MigrationHelper {
constructor(queryInterface, logger) {
this.queryInterface = queryInterface
this.logger = logger
}
async addColumn(table, column, options) {
this.logger.info(`${loggerPrefix} adding column "${column}" to table "${table}"`)
const tableDescription = await this.queryInterface.describeTable(table)
if (!tableDescription[column]) {
await this.queryInterface.addColumn(table, column, options)
this.logger.info(`${loggerPrefix} added column "${column}" to table "${table}"`)
} else {
this.logger.info(`${loggerPrefix} column "${column}" already exists in table "${table}"`)
}
}
async addColumns() {
this.logger.info(`${loggerPrefix} adding ${columnNames} columns to ${libraryItems} table`)
for (const column of columns) {
await this.addColumn(libraryItems, column.name, column.spec)
}
this.logger.info(`${loggerPrefix} added ${columnNames} columns to ${libraryItems} table`)
}
async removeColumn(table, column) {
this.logger.info(`${loggerPrefix} removing column "${column}" from table "${table}"`)
const tableDescription = await this.queryInterface.describeTable(table)
if (tableDescription[column]) {
await this.queryInterface.sequelize.query(`ALTER TABLE ${table} DROP COLUMN ${column}`)
this.logger.info(`${loggerPrefix} removed column "${column}" from table "${table}"`)
} else {
this.logger.info(`${loggerPrefix} column "${column}" does not exist in table "${table}"`)
}
}
async removeColumns() {
this.logger.info(`${loggerPrefix} removing ${columnNames} columns from ${libraryItems} table`)
for (const column of columns) {
await this.removeColumn(libraryItems, column.name)
}
this.logger.info(`${loggerPrefix} removed ${columnNames} columns from ${libraryItems} table`)
}
/* TODO - populate from existing files on filesystem
async populateColumnsFromSource() {
this.logger.info(`${loggerPrefix} populating ${columnNames} columns in ${libraryItems} table`)
const authorNamesSubQuery = `
SELECT ${columnSourcesExpression}
FROM ${authorsJoin}
WHERE ${bookAuthors}.bookId = ${libraryItems}.mediaId
`
await this.queryInterface.sequelize.query(`
UPDATE ${libraryItems}
SET (${columnNames}) = (${authorNamesSubQuery})
WHERE mediaType = 'book';
`)
this.logger.info(`${loggerPrefix} populated ${columnNames} columns in ${libraryItems} table`)
}
*/
async addIndex(tableName, columns) {
const columnString = columns.map((column) => util.inspect(column)).join(', ')
const indexName = convertToSnakeCase(`${tableName}_${columns.map((column) => (typeof column === 'string' ? column : column.name)).join('_')}`)
try {
this.logger.info(`${loggerPrefix} adding index on [${columnString}] to table ${tableName}. index name: ${indexName}"`)
await this.queryInterface.addIndex(tableName, columns)
this.logger.info(`${loggerPrefix} added index on [${columnString}] to table ${tableName}. index name: ${indexName}"`)
} catch (error) {
if (error.name === 'SequelizeDatabaseError' && error.message.includes('already exists')) {
this.logger.info(`${loggerPrefix} index [${columnString}] for table "${tableName}" already exists`)
} else {
throw error
}
}
}
async addIndexes() {
for (const column of columns) {
await this.addIndex(libraryItems, ['libraryId', 'mediaType', { name: column.name, collate: 'NOCASE' }])
}
}
async removeIndex(tableName, columns) {
this.logger.info(`${loggerPrefix} removing index [${columns.join(', ')}] from table "${tableName}"`)
await this.queryInterface.removeIndex(tableName, columns)
this.logger.info(`${loggerPrefix} removed index [${columns.join(', ')}] from table "${tableName}"`)
}
async removeIndexes() {
for (const column of columns) {
await this.removeIndex(libraryItems, ['libraryId', 'mediaType', column.name])
}
}
}
/**
* Utility function to convert a string to snake case, e.g. "titleIgnorePrefix" -> "title_ignore_prefix"
*
* @param {string} str - the string to convert to snake case.
* @returns {string} - the string in snake case.
*/
function convertToSnakeCase(str) {
return str.replace(/([A-Z])/g, '_$1').toLowerCase()
}
module.exports = { up, down, migrationName }

View File

@ -698,6 +698,9 @@ class LibraryItem extends Model {
sequelize,
modelName: 'libraryItem',
indexes: [
{
fields: ['ino', 'deviceId']
},
{
fields: ['createdAt']
},

View File

@ -724,12 +724,25 @@ async function findLibraryItemByFileToItemInoMatch(libraryId, fullPath, isSingle
/** @type {import('../models/LibraryItem').LibraryItemExpanded | null} */
let existingLibraryItem = null
for (let item in itemFileInos) {
// TODO: BUGBUG - this query is broken. It's passing a whole object instead of just the ino. Change the query.
existingLibraryItem = await Database.libraryItemModel.findOneExpanded({
libraryId: libraryId,
ino: {
[sequelize.Op.in]: itemFileInos
}
[sequelize.Op.or]: itemFileInos
})
/* existingLibraryItem = await Database.libraryItemModel.findOneExpanded([
{
libraryId: libraryId,
[sequelize.Op.and]: {
ino: itemFileInos.map((f) => f.ino),
deviceId: itemFileInos.map((f) => f.deviceId)
}
}
/* ino: {
[sequelize.Op.in]: itemFileInos
} */
// ]) */
if (existingLibraryItem) {
break
}

View File

@ -243,6 +243,8 @@ module.exports.recurseFiles = async (path, relPathToReplace = null) => {
item.fullname = filePathToPOSIX(item.fullname)
item.path = filePathToPOSIX(item.path)
// BUGBUG: This is broken with symlinked directory /tmp -> /private/tmp. when library is in /tmp/testLibrary, it tries to replace /tmp/testLibrary with '' but in a canonical path (non-symlinked)
// TODO: find the commit that added relPathToReplace and figure out what it's trying to do and make it do that properly
const relpath = item.fullname.replace(relPathToReplace, '')
let reldirname = Path.dirname(relpath)
if (reldirname === '.') reldirname = ''

View File

@ -86,12 +86,12 @@ function buildBookLibraryItemParams(libraryFile, bookId, libraryId, libraryFolde
}
exports.buildBookLibraryItemParams = buildBookLibraryItemParams
function stubFileUtils() {
function stubFileUtils(mockFileInfo = getMockFileInfo()) {
let getInoStub, getDeviceIdStub, getFileTimestampsWithInoStub
getInoStub = sinon.stub(fileUtils, 'getIno')
getInoStub.callsFake((path) => {
const normalizedPath = fileUtils.filePathToPOSIX(path).replace(/\/$/, '')
const stats = getMockFileInfo().get(normalizedPath)
const stats = mockFileInfo.get(normalizedPath)
if (stats) {
return stats.ino
} else {
@ -102,7 +102,7 @@ function stubFileUtils() {
getDeviceIdStub = sinon.stub(fileUtils, 'getDeviceId')
getDeviceIdStub.callsFake(async (path) => {
const normalizedPath = fileUtils.filePathToPOSIX(path).replace(/\/$/, '')
const stats = getMockFileInfo().get(normalizedPath)
const stats = mockFileInfo.get(normalizedPath)
if (stats) {
return stats.dev
} else {
@ -113,7 +113,7 @@ function stubFileUtils() {
getFileTimestampsWithInoStub = sinon.stub(fileUtils, 'getFileTimestampsWithIno')
getFileTimestampsWithInoStub.callsFake(async (path) => {
const normalizedPath = fileUtils.filePathToPOSIX(path).replace(/\/$/, '')
const stats = getMockFileInfo().get(normalizedPath)
const stats = mockFileInfo.get(normalizedPath)
if (stats) {
return stats
} else {

View File

@ -0,0 +1,169 @@
const chai = require('chai')
const sinon = require('sinon')
const { expect } = chai
const { DataTypes, Sequelize } = require('sequelize')
const Logger = require('../../../server/Logger')
const { up, down, migrationName } = require('../../../server/migrations/v2.29.0-add-deviceId')
const normalizeWhitespaceAndBackticks = (str) => str.replace(/\s+/g, ' ').trim().replace(/`/g, '')
describe(`Migration ${migrationName}`, () => {
let sequelize
let queryInterface
let loggerInfoStub
beforeEach(async () => {
sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
queryInterface = sequelize.getQueryInterface()
loggerInfoStub = sinon.stub(Logger, 'info')
await queryInterface.createTable('libraryItems', {
id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true },
ino: { type: DataTypes.STRING },
mediaId: { type: DataTypes.INTEGER, allowNull: false },
mediaType: { type: DataTypes.STRING, allowNull: false },
libraryId: { type: DataTypes.INTEGER, allowNull: false }
})
await queryInterface.createTable('authors', {
id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true },
name: { type: DataTypes.STRING, allowNull: false },
lastFirst: { type: DataTypes.STRING, allowNull: false }
})
await queryInterface.createTable('bookAuthors', {
id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true },
bookId: { type: DataTypes.INTEGER, allowNull: false, references: { model: 'libraryItems', key: 'id', onDelete: 'CASCADE' } },
authorId: { type: DataTypes.INTEGER, allowNull: false, references: { model: 'authors', key: 'id', onDelete: 'CASCADE' } },
createdAt: { type: DataTypes.DATE, allowNull: false }
})
await queryInterface.createTable('podcastEpisodes', {
id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true },
publishedAt: { type: DataTypes.DATE, allowNull: true }
})
await queryInterface.bulkInsert('libraryItems', [
{ id: 1, mediaId: 1, mediaType: 'book', libraryId: 1, ino: '1' },
{ id: 2, mediaId: 2, mediaType: 'book', libraryId: 1, ino: '2' }
])
await queryInterface.bulkInsert('authors', [
{ id: 1, name: 'John Doe', lastFirst: 'Doe, John' },
{ id: 2, name: 'Jane Smith', lastFirst: 'Smith, Jane' },
{ id: 3, name: 'John Smith', lastFirst: 'Smith, John' }
])
await queryInterface.bulkInsert('bookAuthors', [
{ id: 1, bookId: 1, authorId: 1, createdAt: '2025-01-01 00:00:00.000 +00:00' },
{ id: 2, bookId: 2, authorId: 2, createdAt: '2025-01-02 00:00:00.000 +00:00' },
{ id: 3, bookId: 1, authorId: 3, createdAt: '2024-12-31 00:00:00.000 +00:00' }
])
await queryInterface.bulkInsert('podcastEpisodes', [
{ id: 1, publishedAt: '2025-01-01 00:00:00.000 +00:00' },
{ id: 2, publishedAt: '2025-01-02 00:00:00.000 +00:00' },
{ id: 3, publishedAt: '2025-01-03 00:00:00.000 +00:00' }
])
})
afterEach(() => {
sinon.restore()
})
describe('up', () => {
it('should add the deviceId column to the libraryItems table', async () => {
await up({ context: { queryInterface, logger: Logger } })
const libraryItems = await queryInterface.describeTable('libraryItems')
expect(libraryItems.deviceId).to.exist
})
/* TODO
it('should populate the deviceId columns from the filesystem for each libraryItem', async () => {
await up({ context: { queryInterface, logger: Logger } })
const [libraryItems] = await queryInterface.sequelize.query('SELECT * FROM libraryItems')
expect(libraryItems).to.deep.equal([
{ id: 1, mediaId: 1, mediaType: 'book', libraryId: 1, authorNamesFirstLast: 'John Smith, John Doe', authorNamesLastFirst: 'Smith, John, Doe, John' },
{ id: 2, mediaId: 2, mediaType: 'book', libraryId: 1, authorNamesFirstLast: 'Jane Smith', authorNamesLastFirst: 'Smith, Jane' }
])
})
*/
it('should add an index on ino and deviceId to the libraryItems table', async () => {
await up({ context: { queryInterface, logger: Logger } })
const indexes = await queryInterface.sequelize.query(`SELECT * FROM sqlite_master WHERE type='index'`)
const [[{ count }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='library_items_ino_device_id'`)
expect(count).to.equal(1)
const [[{ sql }]] = await queryInterface.sequelize.query(`SELECT sql FROM sqlite_master WHERE type='index' AND name='library_items_ino_device_id'`)
expect(normalizeWhitespaceAndBackticks(sql)).to.equal(
normalizeWhitespaceAndBackticks(`
CREATE INDEX library_items_ino_device_id ON libraryItems (ino, deviceId)
`)
)
})
it('should be idempotent', async () => {
await up({ context: { queryInterface, logger: Logger } })
await up({ context: { queryInterface, logger: Logger } })
const libraryItemsTable = await queryInterface.describeTable('libraryItems')
expect(libraryItemsTable.deviceId).to.exist
const [[{ count: count6 }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='library_items_ino_device_id'`)
expect(count6).to.equal(1)
const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`)
expect(libraryItems).to.deep.equal([
{ id: 1, ino: '1', deviceId: null, mediaId: 1, mediaType: 'book', libraryId: 1 },
{ id: 2, ino: '2', deviceId: null, mediaId: 2, mediaType: 'book', libraryId: 1 }
])
})
})
describe('down', () => {
it('should remove the deviceId from the libraryItems table', async () => {
await up({ context: { queryInterface, logger: Logger } })
await down({ context: { queryInterface, logger: Logger } })
const libraryItemsTable = await queryInterface.describeTable('libraryItems')
expect(libraryItemsTable.deviceId).to.not.exist
const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`)
expect(libraryItems).to.deep.equal([
{ id: 1, mediaId: 1, mediaType: 'book', libraryId: 1, ino: '1' },
{ id: 2, mediaId: 2, mediaType: 'book', libraryId: 1, ino: '2' }
])
})
it('should remove the index on ino, deviceId from the libraryItems table', async () => {
await up({ context: { queryInterface, logger: Logger } })
await down({ context: { queryInterface, logger: Logger } })
const [[{ count }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='library_items_ino_device_id'`)
expect(count).to.equal(0)
})
it('should be idempotent', async () => {
await up({ context: { queryInterface, logger: Logger } })
await down({ context: { queryInterface, logger: Logger } })
await down({ context: { queryInterface, logger: Logger } })
const libraryItemsTable = await queryInterface.describeTable('libraryItems')
expect(libraryItemsTable.libraryItems).to.not.exist
const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`)
expect(libraryItems).to.deep.equal([
{ id: 1, ino: '1', mediaId: 1, mediaType: 'book', libraryId: 1 },
{ id: 2, ino: '2', mediaId: 2, mediaType: 'book', libraryId: 1 }
])
const [[{ count: count6 }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='library_items_ino_device_id'`)
expect(count6).to.equal(0)
})
})
})

View File

@ -61,7 +61,6 @@ describe('LibraryScanner', () => {
it('findLibraryItemByItemToItemInoMatch', async function () {
this.timeout(0)
// findLibraryItemByItemToItemInoMatch(libraryId, fullPath)
// findLibraryItemByFileToItemInoMatch(libraryId, fullPath, isSingleMedia, itemFiles)
let findLibraryItemByItemToItemInoMatch = LibraryScanner.__get__('findLibraryItemByItemToItemInoMatch')
let fullPath = '/test/file.pdf'
@ -71,13 +70,71 @@ describe('LibraryScanner', () => {
const fileInfo = mockFileInfo.get(fullPath)
/** @type {Promise<import('../../../server/models/LibraryItem') | null>} */
/** @returns {Promise<import('../../../server/models/LibraryItem') | null>} */
const result = await findLibraryItemByItemToItemInoMatch(testLibrary.id, fullPath)
expect(result).to.not.be.null
expect(result.libraryFiles[0].metadata.path).to.equal(fullPath)
expect(result.libraryFiles[0].deviceId).to.equal(fileInfo.dev)
})
it('findLibraryItemByFileToItemInoMatch-matchesRenamedFileByInoAndDeviceId', async function () {
this.timeout(0)
let mockFileInfo = getMockBookFileInfo()
sinon.restore()
stubFileUtils(mockFileInfo)
testLibrary = await loadTestDatabase(mockFileInfo)
// findLibraryItemByFileToItemInoMatch(libraryId, fullPath, isSingleMedia, itemFiles)
let findLibraryItemByItemToItemInoMatch = LibraryScanner.__get__('findLibraryItemByFileToItemInoMatch')
let bookFolderPath = '/test/bookfolder'
/**
* @param {UUIDV4} libraryId
* @param {string} fullPath
* @param {boolean} isSingleMedia
* @param {string[]} itemFiles
* @returns {Promise<import('../models/LibraryItem').LibraryItemExpanded | null>} library item that matches
*/
const existingItem = await findLibraryItemByItemToItemInoMatch(testLibrary.id, bookFolderPath, false, ['file.epub', 'file-renamed.epub', 'file.opf'])
expect(existingItem).to.not.be.null
expect(existingItem.ino).to.equal('1')
expect(existingItem.deviceId).to.equal('100')
})
it('findLibraryItemByFileToItemInoMatch-DoesNotMatchByInoAndDifferentDeviceId', async function () {
this.timeout(0)
testLibrary = await loadTestDatabase()
// findLibraryItemByFileToItemInoMatch(libraryId, fullPath, isSingleMedia, itemFiles)
let findLibraryItemByItemToItemInoMatch = LibraryScanner.__get__('findLibraryItemByFileToItemInoMatch')
let bookFolderPath = '/test/bookfolder'
/**
* @param {UUIDV4} libraryId
* @param {string} fullPath
* @param {boolean} isSingleMedia
* @param {string[]} itemFiles
* @returns {Promise<import('../models/LibraryItem').LibraryItemExpanded | null>} library item that matches
*/
const existingItem = await findLibraryItemByItemToItemInoMatch(testLibrary.id, bookFolderPath, false, ['file.epub', 'different-file.epub', 'file.opf'])
expect(existingItem).to.be.null
})
/** @returns {Map<string, import('fs').Stats>} */
function getMockBookFileInfo() {
// @ts-ignore
return new Map([
['/test/bookfolder/file-renamed.epub', { path: '/test/bookfolder/file-renamed.epub', isDirectory: () => false, size: 1024, mtimeMs: Date.now(), ino: '1', dev: '100' }],
['/test/bookfolder/file.epub', { path: '/test/bookfolder/file.epub', isDirectory: () => false, size: 1024, mtimeMs: Date.now(), ino: '1', dev: '100' }],
['/test/bookfolder/different-file.epub', { path: '/test/bookfolder/different-file.epub', isDirectory: () => false, size: 1024, mtimeMs: Date.now(), ino: '1', dev: '200' }],
['/test/bookfolder/file.opf', { path: '/test/bookfolder/file.opf', isDirectory: () => false, size: 42, mtimeMs: Date.now(), ino: '2', dev: '100' }]
])
}
// ItemToFileInoMatch
it('ItemToFileInoMatch-ItemMatchesSelf', async function () {
this.timeout(0)