Added support for deviceId

This commit is contained in:
Jason Axley 2025-08-22 11:37:04 -07:00
parent aae808544e
commit 423f2d311e
9 changed files with 369 additions and 222 deletions

View File

@ -8,10 +8,11 @@ const libraryItemsBookFilters = require('../utils/queries/libraryItemsBookFilter
/**
* @typedef EBookFileObject
* @property {string} ino
* @property {string} deviceId
* @property {string} ebookFormat
* @property {number} addedAt
* @property {number} updatedAt
* @property {{filename:string, ext:string, path:string, relPath:strFing, size:number, mtimeMs:number, ctimeMs:number, birthtimeMs:number}} metadata
* @property {{filename:string, ext:string, path:string, relPath:string, size:number, mtimeMs:number, ctimeMs:number, birthtimeMs:number}} metadata
*/
/**

View File

@ -2,6 +2,9 @@ const packageJson = require('../../package.json')
const { LogLevel } = require('../utils/constants')
const LibraryItem = require('../models/LibraryItem')
const globals = require('../utils/globals')
const LibraryFile = require('../objects/files/LibraryFile')
const LibraryScan = require('./LibraryScan')
const ScanLogger = require('./ScanLogger')
class LibraryItemScanData {
/**
@ -226,13 +229,7 @@ class LibraryItemScanData {
for (const existingLibraryFile of existingLibraryItem.libraryFiles) {
// Find matching library file using path first and fallback to using inode value
let matchingLibraryFile = this.libraryFiles.find((lf) => lf.metadata.path === existingLibraryFile.metadata.path)
if (!matchingLibraryFile) {
matchingLibraryFile = this.libraryFiles.find((lf) => lf.ino === existingLibraryFile.ino)
if (matchingLibraryFile) {
libraryScan.addLog(LogLevel.INFO, `Library file with path "${existingLibraryFile.metadata.path}" not found, but found file with matching inode value "${existingLibraryFile.ino}" at path "${matchingLibraryFile.metadata.path}"`)
}
}
let matchingLibraryFile = this.findMatchingLibraryFileByPathOrInodeAndDeviceId(existingLibraryFile, libraryScan)
if (!matchingLibraryFile) {
// Library file removed
@ -278,10 +275,9 @@ class LibraryItemScanData {
existingLibraryItem.changed('libraryFiles', true)
}
await existingLibraryItem.save()
return true
}
return false
return this.hasChanges
}
/**
@ -320,6 +316,23 @@ class LibraryItemScanData {
return hasChanges
}
/**
* @returns {LibraryFile | undefined} if [existingLibraryFile] matches an existing libraryFile
* @param {LibraryItem.LibraryFileObject} [existingLibraryFile]
* @param {LibraryScan | ScanLogger} [libraryScan]
*/
findMatchingLibraryFileByPathOrInodeAndDeviceId(existingLibraryFile, libraryScan) {
if (!existingLibraryFile) return
let matchingLibraryFile = this.libraryFiles.find((lf) => lf.metadata.path === existingLibraryFile.metadata.path)
if (!matchingLibraryFile) {
matchingLibraryFile = this.libraryFiles.find((lf) => lf.ino === existingLibraryFile.ino && lf.deviceId === existingLibraryFile.deviceId)
if (matchingLibraryFile) {
libraryScan && libraryScan.addLog(LogLevel.INFO, `Library file with path "${existingLibraryFile.metadata.path}" not found, but found file with matching inode value "${existingLibraryFile.ino}" at path "${matchingLibraryFile.metadata.path}"`)
}
}
return matchingLibraryFile
}
/**
* Check if existing audio file on Book was removed
* @param {import('../models/Book').AudioFileObject} existingAudioFile
@ -341,13 +354,13 @@ class LibraryItemScanData {
* @returns {boolean} true if ebook file was removed
*/
checkEbookFileRemoved(ebookFile) {
if (!this.ebookLibraryFiles.length) return true
if (!this.ebookLibraryFilesRemoved.length) return false
if (this.ebookLibraryFiles.some((lf) => lf.metadata.path === ebookFile.metadata.path)) {
return false
if (this.ebookLibraryFilesRemoved.some((lf) => lf.metadata.path === ebookFile.metadata.path)) {
return true
}
return !this.ebookLibraryFiles.some((lf) => lf.ino === ebookFile.ino)
return this.ebookLibraryFilesRemoved.some((lf) => lf.ino === ebookFile.ino && lf.deviceId === ebookFile.deviceId)
}
/**

View File

@ -344,23 +344,7 @@ class LibraryScanner {
continue
}
items.push(
new LibraryItemScanData({
libraryFolderId: folder.id,
libraryId: folder.libraryId,
mediaType: library.mediaType,
ino: libraryItemFolderStats.ino,
deviceId: libraryItemFolderStats.dev,
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
isFile,
mediaMetadata: libraryItemData.mediaMetadata || null,
libraryFiles: fileObjs
})
)
items.push(createLibraryItemScanData(folder, library, libraryItemFolderStats, libraryItemData, isFile, fileObjs))
}
return items
}
@ -754,3 +738,30 @@ async function findLibraryItemByFileToItemInoMatch(libraryId, fullPath, isSingle
if (existingLibraryItem) Logger.debug(`[LibraryScanner] Found library item with inode matching one of "${itemFileInos.join(',')}" at path "${existingLibraryItem.path}"`)
return existingLibraryItem
}
/**
* @param {{ id: any; libraryId: any; }} folder
* @param {{ mediaType: any; }} library
* @param {{ ino: any; dev: any; mtimeMs: any; ctimeMs: any; birthtimeMs: any; }} libraryItemFolderStats
* @param {{ path: any; relPath: any; mediaMetadata: any; }} libraryItemData
* @param {any} isFile
* @param {any} fileObjs
* @returns {LibraryItemScanData} new object
*/
function createLibraryItemScanData(folder, library, libraryItemFolderStats, libraryItemData, isFile, fileObjs) {
return new LibraryItemScanData({
libraryFolderId: folder.id,
libraryId: folder.libraryId,
mediaType: library.mediaType,
ino: libraryItemFolderStats.ino,
deviceId: libraryItemFolderStats.dev,
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
isFile,
mediaMetadata: libraryItemData.mediaMetadata || null,
libraryFiles: fileObjs
})
}

View File

@ -366,7 +366,7 @@ class PodcastScanner {
* @param {PodcastEpisode[]} podcastEpisodes Not the models for new podcasts
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {import('./LibraryScan')} libraryScan
* @param {string} [existingLibraryItemId]
* @param {string | null} [existingLibraryItemId]
* @returns {Promise<PodcastMetadataObject>}
*/
async getPodcastMetadataFromScanData(podcastEpisodes, libraryItemData, libraryScan, existingLibraryItemId = null) {

View File

@ -124,7 +124,7 @@ function stubFileUtils() {
exports.stubFileUtils = stubFileUtils
/** @returns {{ libraryFolderId: any; libraryId: any; mediaType: any; ino: any; deviceId: any; mtimeMs: any; ctimeMs: any; birthtimeMs: any; path: any; relPath: any; isFile: any; mediaMetadata: any; libraryFiles: any; }} */
function buildFileProperties(path = '/tmp/foo.epub', ino = '12345', deviceId = '9876') {
function buildFileProperties(path = '/tmp/foo.epub', ino = '12345', deviceId = '9876', libraryFiles = []) {
const metadata = new FileMetadata()
metadata.filename = Path.basename(path)
metadata.path = path
@ -137,7 +137,35 @@ function buildFileProperties(path = '/tmp/foo.epub', ino = '12345', deviceId = '
metadata: metadata,
isSupplementary: false,
addedAt: Date.now(),
updatedAt: Date.now()
updatedAt: Date.now(),
libraryFiles: [...libraryFiles.map((lf) => lf.toJSON())]
}
}
exports.buildFileProperties = buildFileProperties
/**
* @returns {import('../../server/models/LibraryItem').LibraryFileObject}
* @param {string} [path]
* @param {string} [ino]
* @param {string} [deviceId]
*/
function buildLibraryFileProperties(path, ino, deviceId) {
return {
ino: ino,
deviceId: deviceId,
isSupplementary: false,
addedAt: 0,
updatedAt: 0,
metadata: {
filename: Path.basename(path),
ext: Path.extname(path),
path: path,
relPath: path,
size: 0,
mtimeMs: 0,
ctimeMs: 0,
birthtimeMs: 0
}
}
}
exports.buildLibraryFileProperties = buildLibraryFileProperties

View File

@ -1,20 +1,18 @@
const chai = require('chai')
const expect = chai.expect
const sinon = require('sinon')
const rewire = require('rewire')
const Path = require('path')
const { stubFileUtils, getMockFileInfo, loadTestDatabase, buildFileProperties } = require('../MockDatabase')
const { buildFileProperties, buildLibraryFileProperties } = require('../MockDatabase')
const LibraryItemScanData = require('../../../server/scanner/LibraryItemScanData')
const LibraryFile = require('../../../server/objects/files/LibraryFile')
const LibraryScan = require('../../../server/scanner/LibraryScan')
// TODO - need to check
// compareUpdateLibraryFile - returns false if no changes; true if changes
describe('compareUpdateLibraryFileWithDeviceId', () => {
const ScanLogger = require('../../../server/scanner/ScanLogger')
describe('LibraryItemScanData', () => {
// compareUpdateLibraryFile - returns false if no changes; true if changes
describe('compareUpdateLibraryFileWithDeviceId', () => {
it('fileChangeDetectedWhenInodeAndDeviceIdPairDiffers', () => {
const existing_lf = buildLibraryFileObject('/tmp/file.pdf', '4432', '300')
const existing_lf = buildLibraryFileProperties('/tmp/file.pdf', '4432', '300')
const scanned_lf = new LibraryFile({
ino: '1',
deviceId: '100'
@ -28,21 +26,50 @@ describe('compareUpdateLibraryFileWithDeviceId', () => {
it('fileChangeNotDetectedWhenInodeSameButDeviceIdDiffers', () => {
// Same inode on different deviceId does NOT mean these are the same file
const existing_lf = buildLibraryFileObject('/tmp/file.pdf', '4432', '300')
const scanned_lf = new LibraryFile(buildLibraryFileObject('/tmp/file.pdf', '4432', '100'))
const existing_lf = buildLibraryFileProperties('/tmp/file.pdf', '4432', '300')
const scanned_lf = new LibraryFile(buildLibraryFileProperties('/tmp/file.pdf', '4432', '100'))
expect(existing_lf.ino).to.equal(scanned_lf.ino)
expect(existing_lf.deviceId).to.not.equal(scanned_lf.deviceId)
const changeDetected = LibraryItemScanData.compareUpdateLibraryFile('/file/path.pdf', existing_lf, scanned_lf, new LibraryScan())
expect(changeDetected).to.be.false
})
})
})
describe('checkAudioFileRemoved', function () {
describe('findMatchingLibraryFileByPathOrInodeAndDeviceId', () => {
it('isMatchWhenInodeAndDeviceIdPairIsSame', () => {
const lisd = new LibraryItemScanData(buildFileProperties('/library/book/file.epub', '1', '1000', [new LibraryFile(buildLibraryFileProperties('/library/book/file.epub', '1', '1000'))]))
const scanned_lf_properties = buildLibraryFileProperties('/tmp/file.epub', '1', '1000')
const matchingFile = lisd.findMatchingLibraryFileByPathOrInodeAndDeviceId(scanned_lf_properties, new ScanLogger())
// don't want match based on filename
expect(lisd.path).to.not.equal(scanned_lf_properties.metadata.path)
expect(matchingFile).to.not.be.undefined
expect(matchingFile?.ino).to.equal(lisd.ino)
expect(matchingFile?.deviceId).to.equal(lisd.deviceId)
})
it('isNotMatchWhenInodeSameButDeviceIdDiffers', () => {
const lisd = new LibraryItemScanData(buildFileProperties('/library/book/file.epub', '1', '1000', [new LibraryFile(buildLibraryFileProperties('/library/book/file.epub', '1', '1000'))]))
const scanned_lf_properties = buildLibraryFileProperties('/tmp/file.epub', '1', '500')
// don't want match based on filename
expect(lisd.path).to.not.equal(scanned_lf_properties.metadata.path)
expect(lisd.deviceId).to.not.equal(scanned_lf_properties.ino)
const matchingFile = lisd.findMatchingLibraryFileByPathOrInodeAndDeviceId(scanned_lf_properties, new ScanLogger())
expect(matchingFile).to.be.undefined
})
})
describe('checkAudioFileRemoved', function () {
this.timeout(0)
it('doesNotDetectFileRemovedWhenInodeIsSameButDeviceIdDiffers', () => {
const lisd = new LibraryItemScanData(buildFileProperties('/library/book/file.mp3', '1', '1000'))
lisd.libraryFilesRemoved.push(buildLibraryFileObject('/library/book/file.mp3', '1', '1000'))
lisd.libraryFilesRemoved.push(buildLibraryFileProperties('/library/book/file.mp3', '1', '1000'))
const af_obj = buildAudioFileObject('/library/someotherbook/chapter1.mp3', '1', '200')
const fileRemoved = lisd.checkAudioFileRemoved(af_obj)
@ -52,7 +79,7 @@ describe('checkAudioFileRemoved', function () {
it('detectsFileRemovedWhenNameDoesNotMatchButInodeAndDeviceIdMatch', () => {
const lisd = new LibraryItemScanData(buildFileProperties('/library/book/file.mp3', '1', '1000'))
lisd.libraryFilesRemoved.push(buildLibraryFileObject('/library/book/file.mp3', '1', '1000'))
lisd.libraryFilesRemoved.push(buildLibraryFileProperties('/library/book/file.mp3', '1', '1000'))
const af_obj = buildAudioFileObject('/library/someotherbook/chapter1.mp3', '1', '1000')
expect(lisd.path).to.not.equal(af_obj.metadata.path)
@ -60,55 +87,42 @@ describe('checkAudioFileRemoved', function () {
expect(fileRemoved).to.be.true
})
})
// checkEbookFileRemoved
// libraryItemObject()
/*
new LibraryItemScanData({
libraryFolderId: folder.id,
libraryId: library.id,
mediaType: library.mediaType,
ino: libraryItemStats.ino,
deviceId: libraryItemStats.dev,
mtimeMs: libraryItemStats.mtimeMs || 0,
ctimeMs: libraryItemStats.ctimeMs || 0,
birthtimeMs: libraryItemStats.birthtimeMs || 0,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
isFile: isSingleMediaItem,
mediaMetadata: libraryItemData.mediaMetadata || null,
libraryFiles
})
*/
// checkEbookFileRemoved
describe('checkEbookFileRemoved', () => {
it('doesNotDetectFileRemovedWhenInodeIsSameButDeviceIdDiffers', () => {
const lisd = new LibraryItemScanData(buildFileProperties('/library/book/file.epub', '1', '1000', [new LibraryFile(buildLibraryFileProperties('/library/book/file.epub', '1', '1000'))]))
lisd.libraryFilesRemoved.push(buildLibraryFileProperties('/library/book/file.epub', '1', '1000')) // This is the file that was removed
const ebook_obj = buildEbookFileObject('/library/someotherbook/chapter1.epub', '1', '200') // this file was NOT removed
/**
* @returns {import('../../../server/models/LibraryItem').LibraryFileObject}
* @param {string} [path]
* @param {string} [ino]
* @param {string} [deviceId]
*/
function buildLibraryFileObject(path, ino, deviceId) {
return {
ino: ino,
deviceId: deviceId,
isSupplementary: false,
addedAt: 0,
updatedAt: 0,
metadata: {
filename: Path.basename(path),
ext: Path.extname(path),
path: path,
relPath: path,
size: 0,
mtimeMs: 0,
ctimeMs: 0,
birthtimeMs: 0
}
}
}
expect(lisd.path).to.not.equal(ebook_obj.metadata.path)
const fileRemoved = lisd.checkEbookFileRemoved(ebook_obj)
expect(fileRemoved).to.be.false
})
it('detectsFileRemovedWhenInodeAndDeviceIdIsSame', () => {
const lisd = new LibraryItemScanData(buildFileProperties('/library/book/file.epub', '1', '1000', [new LibraryFile(buildLibraryFileProperties('/library/book/file.epub', '1', '1000'))]))
lisd.libraryFilesRemoved.push(buildLibraryFileProperties('/library/book/file.epub', '1', '1000')) // This is the file that was removed
const ebook_obj = buildEbookFileObject('/library/someotherbook/chapter1.epub', '1', '1000') // this file was removed
expect(lisd.path).to.not.equal(ebook_obj.metadata.path)
const fileRemoved = lisd.checkEbookFileRemoved(ebook_obj)
expect(fileRemoved).to.be.true
})
})
// libraryItemObject()
describe('libraryItemObject', () => {
it('setsDeviceIdOnLibraryObject', () => {
const lisd = new LibraryItemScanData(buildFileProperties('/library/book/file.epub', '1', '1000', [new LibraryFile(buildLibraryFileProperties('/library/book/file.epub', '1', '1000'))]))
expect(lisd.libraryItemObject.ino).to.equal(lisd.ino)
expect(lisd.libraryItemObject.deviceId).to.equal(lisd.deviceId)
})
})
})
/** @returns {import('../../../server/models/Book').AudioFileObject} */
function buildAudioFileObject(path = '/library/somebook/file.mp3', ino = '1', deviceId = '1000') {
@ -146,3 +160,24 @@ function buildAudioFileObject(path = '/library/somebook/file.mp3', ino = '1', de
mimeType: ''
}
}
/** @returns {import('../../../server/models/Book').EBookFileObject} */
function buildEbookFileObject(path = '/library/somebook/file.epub', ino = '100', deviceId = '1000') {
return {
ino: ino,
deviceId: deviceId,
ebookFormat: Path.extname(path),
addedAt: 0,
updatedAt: 0,
metadata: {
filename: Path.basename(path),
ext: Path.extname(path),
path: path,
relPath: path,
size: 0,
mtimeMs: 0,
ctimeMs: 0,
birthtimeMs: 0
}
}
}

View File

@ -17,7 +17,8 @@ const lf = new LibraryFile(fileProperties)
const ebf = new EBookFile(fileProperties)
const af = new AudioFile(fileProperties)
describe('ObjectSetsDeviceIdWhenConstructed', function () {
describe('SimilarLibraryFileObjects', () => {
describe('ObjectSetsDeviceIdWhenConstructed', function () {
this.timeout(0)
beforeEach(async () => {
stubFileUtils()
@ -63,9 +64,9 @@ describe('ObjectSetsDeviceIdWhenConstructed', function () {
expect(lf_json).to.not.be.null
expect(lf_json?.deviceId).to.equal(mockFileInfo?.dev)
})
})
})
describe('ObjectSetsDeviceIdWhenSerialized', () => {
describe('ObjectSetsDeviceIdWhenSerialized', () => {
const objects = [lf, ebf, af]
objects.forEach((obj) => {
it(`${obj.constructor.name}SetsDeviceIdWhenSerialized`, () => {
@ -74,6 +75,7 @@ describe('ObjectSetsDeviceIdWhenSerialized', () => {
expect(obj_json.deviceId).to.equal(fileProperties.deviceId)
})
})
})
})
function buildLibraryItemProperties(fileProperties) {

View File

@ -10,7 +10,8 @@ const LibraryFile = require('../../../server/objects/files/LibraryFile')
const FileMetadata = require('../../../server/objects/metadata/FileMetadata')
const LibraryFolder = require('../../../server/models/LibraryFolder')
describe('buildLibraryItemScanData', () => {
describe('LibraryItemScanner', () => {
describe('buildLibraryItemScanData', () => {
let testLibrary = null
beforeEach(async () => {
stubFileUtils()
@ -62,6 +63,7 @@ describe('buildLibraryItemScanData', () => {
expect(scanData).to.not.be.null
expect(scanData.deviceId).to.equal(mockFileInfo?.dev)
})
})
})
/** @return {import("../../../server/models/LibraryFolder")} folder */

View File

@ -8,7 +8,9 @@ const LibraryItem = require('../../../server/models/LibraryItem')
const FileMetadata = require('../../../server/objects/metadata/FileMetadata')
const Path = require('path')
const Database = require('../../../server/Database')
const { stubFileUtils, loadTestDatabase, getMockFileInfo, getRenamedMockFileInfo, buildBookLibraryItemParams } = require('../MockDatabase')
const { stubFileUtils, loadTestDatabase, getMockFileInfo, getRenamedMockFileInfo, buildBookLibraryItemParams, buildFileProperties, buildLibraryFileProperties } = require('../MockDatabase')
const libraryScannerInstance = require('../../../server/scanner/LibraryScanner')
const LibraryScan = require('../../../server/scanner/LibraryScan')
describe('LibraryScanner', () => {
let LibraryScanner, testLibrary
@ -210,7 +212,60 @@ describe('LibraryScanner', () => {
expect(ItemToItemInoMatch(item1, item2)).to.be.false
})
it('ItemToItemInoMatch-RenamedFileShouldMatch', () => {
it('ItemToItemInoMatch-RenamedFileShouldMatch', async () => {
let ItemToItemInoMatch = LibraryScanner.__get__('ItemToItemInoMatch')
let mockFileInfo = getMockFileInfo()
testLibrary = await loadTestDatabase(mockFileInfo)
// this compares the inode from the first library item to the second library item's library file inode
const original = await Database.libraryItemModel.findOneExpanded({
libraryId: testLibrary.id,
path: '/test/file.pdf'
})
const renamedMockFileInfo = getRenamedMockFileInfo().get('/test/file-renamed.pdf')
const renamedFile = new LibraryFile()
var fileMetadata = new FileMetadata()
fileMetadata.setData(renamedMockFileInfo)
fileMetadata.filename = Path.basename(renamedMockFileInfo.path)
fileMetadata.path = fileUtils.filePathToPOSIX(renamedMockFileInfo.path)
fileMetadata.relPath = fileUtils.filePathToPOSIX(renamedMockFileInfo.path)
fileMetadata.ext = Path.extname(renamedMockFileInfo.path)
renamedFile.ino = renamedMockFileInfo.ino
renamedFile.deviceId = renamedMockFileInfo.dev
renamedFile.metadata = fileMetadata
renamedFile.addedAt = Date.now()
renamedFile.updatedAt = Date.now()
renamedFile.metadata = fileMetadata
const renamedItem = new LibraryItem(buildBookLibraryItemParams(renamedFile, null, testLibrary.id, null))
expect(ItemToItemInoMatch(original, renamedItem)).to.be.true
})
describe('createLibraryItemScanData', () => {
it('createLibraryItemScanDataSetsDeviceId', async () => {
/**
* @param {{ id: any; libraryId: any; }} folder
* @param {{ mediaType: any; }} library
* @param {{ ino: any; dev: any; mtimeMs: any; ctimeMs: any; birthtimeMs: any; }} libraryItemFolderStats
* @param {{ path: any; relPath: any; mediaMetadata: any; }} libraryItemData
* @param {any} isFile
* @param {any} fileObjs
* @returns {LibraryItemScanData} new object
*/
const createLibraryItemScanData = LibraryScanner.__get__('createLibraryItemScanData')
const liFolderStats = { path: '/library/book/file.epub', isDirectory: () => false, size: 1024, mtimeMs: Date.now(), ino: '1', dev: '1000' }
const lf_properties = buildLibraryFileProperties('/library/book/file.epub', '1', '1000')
const libraryFile = new LibraryFile(lf_properties)
const lisd = createLibraryItemScanData({ id: 'foo', libraryId: 'bar' }, { mediaType: 'ebook' }, liFolderStats, lf_properties, true, [libraryFile.toJSON()])
expect(lisd).to.not.be.null
expect(lisd.ino).to.equal(liFolderStats.ino)
expect(lisd.deviceId).to.equal(liFolderStats.dev)
})
})
})