Mirrors/audiobookshelf
Mirrors/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2025-09-10 17:58:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

Populate deviceIds from disk

Browse Source
This commit is contained in:
Jason Axley 2025-08-27 12:14:51 -07:00
parent 41a288bcdf
commit 5582a68546
2 changed files with 58 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
server/migrations/v2.29.0-add-deviceId.js
View File

@ -1,5 +1,7 @@
const util = require('util') const util = require('util')
const { Sequelize, DataTypes } = require('sequelize') const { Sequelize, DataTypes } = require('sequelize')
const fileUtils = require('../../server/utils/fileUtils')
const LibraryItem = require('../models/LibraryItem')
/** /**
* @typedef MigrationContext * @typedef MigrationContext
@ -15,7 +17,7 @@ const migrationName = `${migrationVersion}-add-deviceId`
const loggerPrefix = `[${migrationVersion} migration]` const loggerPrefix = `[${migrationVersion} migration]`
// Migration constants // Migration constants
const libraryItems = 'libraryItems' const libraryItemsTableName = 'libraryItems'
const columns = [{ name: 'deviceId', spec: { type: DataTypes.STRING, allowNull: true } }] const columns = [{ name: 'deviceId', spec: { type: DataTypes.STRING, allowNull: true } }]
const columnNames = columns.map((column) => column.name).join(', ') const columnNames = columns.map((column) => column.name).join(', ')
@ -37,7 +39,7 @@ async function up({ context: { queryInterface, logger } }) {
// Populate authorNames columns with the author names for each libraryItem // Populate authorNames columns with the author names for each libraryItem
// TODO // TODO
// await helper.populateColumnsFromSource() await helper.populateColumnsFromSource()
// Create indexes on the authorNames columns // Create indexes on the authorNames columns
await helper.addIndexes() await helper.addIndexes()
@ -91,11 +93,11 @@ class MigrationHelper {
} }
async addColumns() { async addColumns() {
this.logger.info(`${loggerPrefix} adding ${columnNames} columns to ${libraryItems} table`) this.logger.info(`${loggerPrefix} adding ${columnNames} columns to ${libraryItemsTableName} table`)
for (const column of columns) { for (const column of columns) {
await this.addColumn(libraryItems, column.name, column.spec) await this.addColumn(libraryItemsTableName, column.name, column.spec)
} }
this.logger.info(`${loggerPrefix} added ${columnNames} columns to ${libraryItems} table`) this.logger.info(`${loggerPrefix} added ${columnNames} columns to ${libraryItemsTableName} table`)
} }
async removeColumn(table, column) { async removeColumn(table, column) {
@ -110,28 +112,39 @@ class MigrationHelper {
} }
async removeColumns() { async removeColumns() {
this.logger.info(`${loggerPrefix} removing ${columnNames} columns from ${libraryItems} table`) this.logger.info(`${loggerPrefix} removing ${columnNames} columns from ${libraryItemsTableName} table`)
for (const column of columns) { for (const column of columns) {
await this.removeColumn(libraryItems, column.name) await this.removeColumn(libraryItemsTableName, column.name)
} }
this.logger.info(`${loggerPrefix} removed ${columnNames} columns from ${libraryItems} table`) this.logger.info(`${loggerPrefix} removed ${columnNames} columns from ${libraryItemsTableName} table`)
} }
/* TODO - populate from existing files on filesystem // populate from existing files on filesystem
async populateColumnsFromSource() { async populateColumnsFromSource() {
this.logger.info(`${loggerPrefix} populating ${columnNames} columns in ${libraryItems} table`) this.logger.info(`${loggerPrefix} populating ${columnNames} columns in ${libraryItemsTableName} table`)
const authorNamesSubQuery = `
SELECT ${columnSourcesExpression} // list all libraryItems
FROM ${authorsJoin} /** @type {[[LibraryItem], any]} */
WHERE ${bookAuthors}.bookId = ${libraryItems}.mediaId const [libraryItems, metadata] = await this.queryInterface.sequelize.query('SELECT * FROM libraryItems')
` // load file stats for all libraryItems
await this.queryInterface.sequelize.query(` libraryItems.forEach(async (item) => {
UPDATE ${libraryItems} const deviceId = await fileUtils.getDeviceId(item.path)
SET (${columnNames}) = (${authorNamesSubQuery}) // set deviceId for each libraryItem
WHERE mediaType = 'book'; await this.queryInterface.sequelize.query(
`) `UPDATE :libraryItemsTableName
SET (deviceId) = (:deviceId)
WHERE id = :id`,
{
replacements: {
libraryItemsTableName: libraryItemsTableName,
deviceId: deviceId,
id: item.id
}
}
)
})
this.logger.info(`${loggerPrefix} populated ${columnNames} columns in ${libraryItems} table`) this.logger.info(`${loggerPrefix} populated ${columnNames} columns in ${libraryItems} table`)
} }
*/
async addIndex(tableName, columns) { async addIndex(tableName, columns) {
const columnString = columns.map((column) => util.inspect(column)).join(', ') const columnString = columns.map((column) => util.inspect(column)).join(', ')
@ -151,7 +164,7 @@ class MigrationHelper {
async addIndexes() { async addIndexes() {
for (const column of columns) { for (const column of columns) {
await this.addIndex(libraryItems, ['libraryId', 'mediaType', { name: column.name, collate: 'NOCASE' }]) await this.addIndex(libraryItemsTableName, ['libraryId', 'mediaType', { name: column.name, collate: 'NOCASE' }])
} }
} }
@ -163,7 +176,7 @@ class MigrationHelper {
async removeIndexes() { async removeIndexes() {
for (const column of columns) { for (const column of columns) {
await this.removeIndex(libraryItems, ['libraryId', 'mediaType', column.name]) await this.removeIndex(libraryItemsTableName, ['libraryId', 'mediaType', column.name])
} }
} }
} }

35
test/server/migrations/v2.29.0-add-deviceId.test.js
View File

@ -6,6 +6,7 @@ const { DataTypes, Sequelize } = require('sequelize')
const Logger = require('../../../server/Logger') const Logger = require('../../../server/Logger')
const { up, down, migrationName } = require('../../../server/migrations/v2.29.0-add-deviceId') const { up, down, migrationName } = require('../../../server/migrations/v2.29.0-add-deviceId')
const { stubFileUtils, getMockFileInfo } = require('../MockDatabase')
const normalizeWhitespaceAndBackticks = (str) => str.replace(/\s+/g, ' ').trim().replace(/`/g, '') const normalizeWhitespaceAndBackticks = (str) => str.replace(/\s+/g, ' ').trim().replace(/`/g, '')
@ -13,15 +14,23 @@ describe(`Migration ${migrationName}`, () => {
let sequelize let sequelize
let queryInterface let queryInterface
let loggerInfoStub let loggerInfoStub
let mockFileInfo, file1stats, file2stats
beforeEach(async () => { beforeEach(async () => {
sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
queryInterface = sequelize.getQueryInterface() queryInterface = sequelize.getQueryInterface()
loggerInfoStub = sinon.stub(Logger, 'info') loggerInfoStub = sinon.stub(Logger, 'info')
mockFileInfo = getMockFileInfo()
file1stats = mockFileInfo.get('/test/file.pdf')
file2stats = mockFileInfo.get('/mnt/drive/file-same-ino-different-dev.pdf')
stubFileUtils(mockFileInfo)
await queryInterface.createTable('libraryItems', { await queryInterface.createTable('libraryItems', {
id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true }, id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true },
ino: { type: DataTypes.STRING }, ino: { type: DataTypes.STRING },
path: { type: DataTypes.STRING },
mediaId: { type: DataTypes.INTEGER, allowNull: false }, mediaId: { type: DataTypes.INTEGER, allowNull: false },
mediaType: { type: DataTypes.STRING, allowNull: false }, mediaType: { type: DataTypes.STRING, allowNull: false },
libraryId: { type: DataTypes.INTEGER, allowNull: false } libraryId: { type: DataTypes.INTEGER, allowNull: false }
@ -46,8 +55,8 @@ describe(`Migration ${migrationName}`, () => {
}) })
await queryInterface.bulkInsert('libraryItems', [ await queryInterface.bulkInsert('libraryItems', [
{ id: 1, mediaId: 1, mediaType: 'book', libraryId: 1, ino: '1' }, { id: 1, ino: file1stats.ino, mediaId: 1, path: file1stats.path, mediaType: 'book', libraryId: 1 },
{ id: 2, mediaId: 2, mediaType: 'book', libraryId: 1, ino: '2' } { id: 2, ino: file2stats.ino, mediaId: 2, path: file2stats.path, mediaType: 'book', libraryId: 1 }
]) ])
await queryInterface.bulkInsert('authors', [ await queryInterface.bulkInsert('authors', [
@ -80,17 +89,17 @@ describe(`Migration ${migrationName}`, () => {
const libraryItems = await queryInterface.describeTable('libraryItems') const libraryItems = await queryInterface.describeTable('libraryItems')
expect(libraryItems.deviceId).to.exist expect(libraryItems.deviceId).to.exist
}) })
/* TODO
it('should populate the deviceId columns from the filesystem for each libraryItem', async () => { it('should populate the deviceId columns from the filesystem for each libraryItem', async function () {
this.timeout(0)
await up({ context: { queryInterface, logger: Logger } }) await up({ context: { queryInterface, logger: Logger } })
const [libraryItems] = await queryInterface.sequelize.query('SELECT * FROM libraryItems') const [libraryItems] = await queryInterface.sequelize.query('SELECT * FROM libraryItems')
expect(libraryItems).to.deep.equal([ expect(libraryItems).to.deep.equal([
{ id: 1, mediaId: 1, mediaType: 'book', libraryId: 1, authorNamesFirstLast: 'John Smith, John Doe', authorNamesLastFirst: 'Smith, John, Doe, John' }, { id: 1, ino: file1stats.ino, deviceId: file1stats.dev, mediaId: 1, path: file1stats.path, mediaType: 'book', libraryId: 1 },
{ id: 2, mediaId: 2, mediaType: 'book', libraryId: 1, authorNamesFirstLast: 'Jane Smith', authorNamesLastFirst: 'Smith, Jane' } { id: 2, ino: file2stats.ino, deviceId: file2stats.dev, mediaId: 2, path: file2stats.path, mediaType: 'book', libraryId: 1 }
]) ])
}) })
*/
it('should add an index on ino and deviceId to the libraryItems table', async () => { it('should add an index on ino and deviceId to the libraryItems table', async () => {
await up({ context: { queryInterface, logger: Logger } }) await up({ context: { queryInterface, logger: Logger } })
@ -119,8 +128,8 @@ describe(`Migration ${migrationName}`, () => {
const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`) const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`)
expect(libraryItems).to.deep.equal([ expect(libraryItems).to.deep.equal([
{ id: 1, ino: '1', deviceId: null, mediaId: 1, mediaType: 'book', libraryId: 1 }, { id: 1, ino: file1stats.ino, deviceId: file1stats.dev, path: file1stats.path, mediaId: 1, mediaType: 'book', libraryId: 1 },
{ id: 2, ino: '2', deviceId: null, mediaId: 2, mediaType: 'book', libraryId: 1 } { id: 2, ino: file2stats.ino, deviceId: file2stats.dev, path: file2stats.path, mediaId: 2, mediaType: 'book', libraryId: 1 }
]) ])
}) })
}) })
@ -135,8 +144,8 @@ describe(`Migration ${migrationName}`, () => {
const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`) const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`)
expect(libraryItems).to.deep.equal([ expect(libraryItems).to.deep.equal([
{ id: 1, mediaId: 1, mediaType: 'book', libraryId: 1, ino: '1' }, { id: 1, ino: file1stats.ino, mediaId: 1, path: file1stats.path, mediaType: 'book', libraryId: 1 },
{ id: 2, mediaId: 2, mediaType: 'book', libraryId: 1, ino: '2' } { id: 2, ino: file2stats.ino, mediaId: 2, path: file2stats.path, mediaType: 'book', libraryId: 1 }
]) ])
}) })
@ -158,8 +167,8 @@ describe(`Migration ${migrationName}`, () => {
const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`) const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`)
expect(libraryItems).to.deep.equal([ expect(libraryItems).to.deep.equal([
{ id: 1, ino: '1', mediaId: 1, mediaType: 'book', libraryId: 1 }, { id: 1, ino: file1stats.ino, path: file1stats.path, mediaId: 1, mediaType: 'book', libraryId: 1 },
{ id: 2, ino: '2', mediaId: 2, mediaType: 'book', libraryId: 1 } { id: 2, ino: file2stats.ino, path: file2stats.path, mediaId: 2, mediaType: 'book', libraryId: 1 }
]) ])
const [[{ count: count6 }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='library_items_ino_device_id'`) const [[{ count: count6 }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='library_items_ino_device_id'`)