From 5582a685460dc8b32a1d475971a72023f3d0146b Mon Sep 17 00:00:00 2001 From: Jason Axley Date: Wed, 27 Aug 2025 12:14:51 -0700 Subject: [PATCH] Populate deviceIds from disk --- server/migrations/v2.29.0-add-deviceId.js | 59 +++++++++++-------- .../migrations/v2.29.0-add-deviceId.test.js | 35 +++++++---- 2 files changed, 58 insertions(+), 36 deletions(-) diff --git a/server/migrations/v2.29.0-add-deviceId.js b/server/migrations/v2.29.0-add-deviceId.js index 603a5c7ca..fdda4e297 100644 --- a/server/migrations/v2.29.0-add-deviceId.js +++ b/server/migrations/v2.29.0-add-deviceId.js @@ -1,5 +1,7 @@ const util = require('util') const { Sequelize, DataTypes } = require('sequelize') +const fileUtils = require('../../server/utils/fileUtils') +const LibraryItem = require('../models/LibraryItem') /** * @typedef MigrationContext @@ -15,7 +17,7 @@ const migrationName = `${migrationVersion}-add-deviceId` const loggerPrefix = `[${migrationVersion} migration]` // Migration constants -const libraryItems = 'libraryItems' +const libraryItemsTableName = 'libraryItems' const columns = [{ name: 'deviceId', spec: { type: DataTypes.STRING, allowNull: true } }] const columnNames = columns.map((column) => column.name).join(', ') @@ -37,7 +39,7 @@ async function up({ context: { queryInterface, logger } }) { // Populate authorNames columns with the author names for each libraryItem // TODO - // await helper.populateColumnsFromSource() + await helper.populateColumnsFromSource() // Create indexes on the authorNames columns await helper.addIndexes() @@ -91,11 +93,11 @@ class MigrationHelper { } async addColumns() { - this.logger.info(`${loggerPrefix} adding ${columnNames} columns to ${libraryItems} table`) + this.logger.info(`${loggerPrefix} adding ${columnNames} columns to ${libraryItemsTableName} table`) for (const column of columns) { - await this.addColumn(libraryItems, column.name, column.spec) + await this.addColumn(libraryItemsTableName, column.name, column.spec) } - this.logger.info(`${loggerPrefix} added ${columnNames} columns to ${libraryItems} table`) + this.logger.info(`${loggerPrefix} added ${columnNames} columns to ${libraryItemsTableName} table`) } async removeColumn(table, column) { @@ -110,28 +112,39 @@ class MigrationHelper { } async removeColumns() { - this.logger.info(`${loggerPrefix} removing ${columnNames} columns from ${libraryItems} table`) + this.logger.info(`${loggerPrefix} removing ${columnNames} columns from ${libraryItemsTableName} table`) for (const column of columns) { - await this.removeColumn(libraryItems, column.name) + await this.removeColumn(libraryItemsTableName, column.name) } - this.logger.info(`${loggerPrefix} removed ${columnNames} columns from ${libraryItems} table`) + this.logger.info(`${loggerPrefix} removed ${columnNames} columns from ${libraryItemsTableName} table`) } - /* TODO - populate from existing files on filesystem + // populate from existing files on filesystem async populateColumnsFromSource() { - this.logger.info(`${loggerPrefix} populating ${columnNames} columns in ${libraryItems} table`) - const authorNamesSubQuery = ` - SELECT ${columnSourcesExpression} - FROM ${authorsJoin} - WHERE ${bookAuthors}.bookId = ${libraryItems}.mediaId - ` - await this.queryInterface.sequelize.query(` - UPDATE ${libraryItems} - SET (${columnNames}) = (${authorNamesSubQuery}) - WHERE mediaType = 'book'; - `) + this.logger.info(`${loggerPrefix} populating ${columnNames} columns in ${libraryItemsTableName} table`) + + // list all libraryItems + /** @type {[[LibraryItem], any]} */ + const [libraryItems, metadata] = await this.queryInterface.sequelize.query('SELECT * FROM libraryItems') + // load file stats for all libraryItems + libraryItems.forEach(async (item) => { + const deviceId = await fileUtils.getDeviceId(item.path) + // set deviceId for each libraryItem + await this.queryInterface.sequelize.query( + `UPDATE :libraryItemsTableName + SET (deviceId) = (:deviceId) + WHERE id = :id`, + { + replacements: { + libraryItemsTableName: libraryItemsTableName, + deviceId: deviceId, + id: item.id + } + } + ) + }) + this.logger.info(`${loggerPrefix} populated ${columnNames} columns in ${libraryItems} table`) } - */ async addIndex(tableName, columns) { const columnString = columns.map((column) => util.inspect(column)).join(', ') @@ -151,7 +164,7 @@ class MigrationHelper { async addIndexes() { for (const column of columns) { - await this.addIndex(libraryItems, ['libraryId', 'mediaType', { name: column.name, collate: 'NOCASE' }]) + await this.addIndex(libraryItemsTableName, ['libraryId', 'mediaType', { name: column.name, collate: 'NOCASE' }]) } } @@ -163,7 +176,7 @@ class MigrationHelper { async removeIndexes() { for (const column of columns) { - await this.removeIndex(libraryItems, ['libraryId', 'mediaType', column.name]) + await this.removeIndex(libraryItemsTableName, ['libraryId', 'mediaType', column.name]) } } } diff --git a/test/server/migrations/v2.29.0-add-deviceId.test.js b/test/server/migrations/v2.29.0-add-deviceId.test.js index a20bb5f04..cdc70ea75 100644 --- a/test/server/migrations/v2.29.0-add-deviceId.test.js +++ b/test/server/migrations/v2.29.0-add-deviceId.test.js @@ -6,6 +6,7 @@ const { DataTypes, Sequelize } = require('sequelize') const Logger = require('../../../server/Logger') const { up, down, migrationName } = require('../../../server/migrations/v2.29.0-add-deviceId') +const { stubFileUtils, getMockFileInfo } = require('../MockDatabase') const normalizeWhitespaceAndBackticks = (str) => str.replace(/\s+/g, ' ').trim().replace(/`/g, '') @@ -13,15 +14,23 @@ describe(`Migration ${migrationName}`, () => { let sequelize let queryInterface let loggerInfoStub + let mockFileInfo, file1stats, file2stats beforeEach(async () => { sequelize = new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false }) queryInterface = sequelize.getQueryInterface() loggerInfoStub = sinon.stub(Logger, 'info') + mockFileInfo = getMockFileInfo() + file1stats = mockFileInfo.get('/test/file.pdf') + file2stats = mockFileInfo.get('/mnt/drive/file-same-ino-different-dev.pdf') + + stubFileUtils(mockFileInfo) + await queryInterface.createTable('libraryItems', { id: { type: DataTypes.INTEGER, allowNull: false, primaryKey: true, unique: true }, ino: { type: DataTypes.STRING }, + path: { type: DataTypes.STRING }, mediaId: { type: DataTypes.INTEGER, allowNull: false }, mediaType: { type: DataTypes.STRING, allowNull: false }, libraryId: { type: DataTypes.INTEGER, allowNull: false } @@ -46,8 +55,8 @@ describe(`Migration ${migrationName}`, () => { }) await queryInterface.bulkInsert('libraryItems', [ - { id: 1, mediaId: 1, mediaType: 'book', libraryId: 1, ino: '1' }, - { id: 2, mediaId: 2, mediaType: 'book', libraryId: 1, ino: '2' } + { id: 1, ino: file1stats.ino, mediaId: 1, path: file1stats.path, mediaType: 'book', libraryId: 1 }, + { id: 2, ino: file2stats.ino, mediaId: 2, path: file2stats.path, mediaType: 'book', libraryId: 1 } ]) await queryInterface.bulkInsert('authors', [ @@ -80,17 +89,17 @@ describe(`Migration ${migrationName}`, () => { const libraryItems = await queryInterface.describeTable('libraryItems') expect(libraryItems.deviceId).to.exist }) - /* TODO - it('should populate the deviceId columns from the filesystem for each libraryItem', async () => { + + it('should populate the deviceId columns from the filesystem for each libraryItem', async function () { + this.timeout(0) await up({ context: { queryInterface, logger: Logger } }) const [libraryItems] = await queryInterface.sequelize.query('SELECT * FROM libraryItems') expect(libraryItems).to.deep.equal([ - { id: 1, mediaId: 1, mediaType: 'book', libraryId: 1, authorNamesFirstLast: 'John Smith, John Doe', authorNamesLastFirst: 'Smith, John, Doe, John' }, - { id: 2, mediaId: 2, mediaType: 'book', libraryId: 1, authorNamesFirstLast: 'Jane Smith', authorNamesLastFirst: 'Smith, Jane' } + { id: 1, ino: file1stats.ino, deviceId: file1stats.dev, mediaId: 1, path: file1stats.path, mediaType: 'book', libraryId: 1 }, + { id: 2, ino: file2stats.ino, deviceId: file2stats.dev, mediaId: 2, path: file2stats.path, mediaType: 'book', libraryId: 1 } ]) }) -*/ it('should add an index on ino and deviceId to the libraryItems table', async () => { await up({ context: { queryInterface, logger: Logger } }) @@ -119,8 +128,8 @@ describe(`Migration ${migrationName}`, () => { const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`) expect(libraryItems).to.deep.equal([ - { id: 1, ino: '1', deviceId: null, mediaId: 1, mediaType: 'book', libraryId: 1 }, - { id: 2, ino: '2', deviceId: null, mediaId: 2, mediaType: 'book', libraryId: 1 } + { id: 1, ino: file1stats.ino, deviceId: file1stats.dev, path: file1stats.path, mediaId: 1, mediaType: 'book', libraryId: 1 }, + { id: 2, ino: file2stats.ino, deviceId: file2stats.dev, path: file2stats.path, mediaId: 2, mediaType: 'book', libraryId: 1 } ]) }) }) @@ -135,8 +144,8 @@ describe(`Migration ${migrationName}`, () => { const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`) expect(libraryItems).to.deep.equal([ - { id: 1, mediaId: 1, mediaType: 'book', libraryId: 1, ino: '1' }, - { id: 2, mediaId: 2, mediaType: 'book', libraryId: 1, ino: '2' } + { id: 1, ino: file1stats.ino, mediaId: 1, path: file1stats.path, mediaType: 'book', libraryId: 1 }, + { id: 2, ino: file2stats.ino, mediaId: 2, path: file2stats.path, mediaType: 'book', libraryId: 1 } ]) }) @@ -158,8 +167,8 @@ describe(`Migration ${migrationName}`, () => { const [libraryItems] = await queryInterface.sequelize.query(`SELECT * FROM libraryItems`) expect(libraryItems).to.deep.equal([ - { id: 1, ino: '1', mediaId: 1, mediaType: 'book', libraryId: 1 }, - { id: 2, ino: '2', mediaId: 2, mediaType: 'book', libraryId: 1 } + { id: 1, ino: file1stats.ino, path: file1stats.path, mediaId: 1, mediaType: 'book', libraryId: 1 }, + { id: 2, ino: file2stats.ino, path: file2stats.path, mediaId: 2, mediaType: 'book', libraryId: 1 } ]) const [[{ count: count6 }]] = await queryInterface.sequelize.query(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='index' AND name='library_items_ino_device_id'`)