diff --git a/server/Db.js b/server/Db.js index efcc77b2..b43594b8 100644 --- a/server/Db.js +++ b/server/Db.js @@ -316,6 +316,39 @@ class Db { }) } + async bulkInsertEntities(entityName, entities, batchSize = 500) { + // Group entities in batches of size batchSize + var entityBatches = [] + var batch = [] + var index = 0 + entities.forEach((ent) => { + batch.push(ent) + index++ + if (index >= batchSize) { + entityBatches.push(batch) + index = 0 + batch = [] + } + }) + if (batch.length) entityBatches.push(batch) + + Logger.info(`[Db] bulkInsertEntities: ${entities.length} ${entityName} to ${entityBatches.length} batches of max size ${batchSize}`) + + // Start inserting batches + var batchIndex = 1 + for (const entityBatch of entityBatches) { + Logger.info(`[Db] bulkInsertEntities: Start inserting batch ${batchIndex} of ${entityBatch.length} for ${entityName}`) + var success = await this.insertEntities(entityName, entityBatch) + if (success) { + Logger.info(`[Db] bulkInsertEntities: Success inserting batch ${batchIndex} for ${entityName}`) + } else { + Logger.info(`[Db] bulkInsertEntities: Failed inserting batch ${batchIndex} for ${entityName}`) + } + batchIndex++ + } + return true + } + updateEntities(entityName, entities) { var entityDb = this.getEntityDb(entityName) diff --git a/server/utils/dbMigration.js b/server/utils/dbMigration.js index bcb77722..0a7b86a6 100644 --- a/server/utils/dbMigration.js +++ b/server/utils/dbMigration.js @@ -242,10 +242,10 @@ async function migrateLibraryItems(db) { var libraryItems = audiobooks.map((ab) => makeLibraryItemFromOldAb(ab)) Logger.info(`>>> ${libraryItems.length} Library Items made`) - await db.insertEntities('libraryItem', libraryItems) + await db.bulkInsertEntities('libraryItem', libraryItems) if (authorsToAdd.length) { Logger.info(`>>> ${authorsToAdd.length} Authors made`) - await db.insertEntities('author', authorsToAdd) + await db.bulkInsertEntities('author', authorsToAdd) } if (seriesToAdd.length) { Logger.info(`>>> ${seriesToAdd.length} Series made`)