Mirrors/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/Frooodle/Stirling-PDF.git synced 2026-03-04 02:20:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

Feature/v2/file handling improvements (#4222)

Browse Source 
# Description of Changes

A new universal file context rather than the splintered ones for the
main views, tools and manager we had before (manager still has its own
but its better integreated with the core context)
File context has been split it into a handful of different files
managing various file related issues separately to reduce the monolith -
FileReducer.ts - State management
  fileActions.ts - File operations
  fileSelectors.ts - Data access patterns
  lifecycle.ts - Resource cleanup and memory management
  fileHooks.ts - React hooks interface
  contexts.ts - Context providers
Improved thumbnail generation
Improved indexxedb handling
Stopped handling files as blobs were not necessary to improve
performance
A new library handling drag and drop
https://github.com/atlassian/pragmatic-drag-and-drop (Out of scope yes
but I broke the old one with the new filecontext and it needed doing so
it was a might as well)
A new library handling virtualisation on page editor
@tanstack/react-virtual, as above.
Quickly ripped out the last remnants of the old URL params stuff and
replaced with the beginnings of what will later become the new URL
navigation system (for now it just restores the tool name in url
behavior)
Fixed selected file not regestered when opening a tool
Fixed png thumbnails
Closes #(issue_number)

---

## Checklist

### General

- [ ] I have read the [Contribution
Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md)
- [ ] I have read the [Stirling-PDF Developer
Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/DeveloperGuide.md)
(if applicable)
- [ ] I have read the [How to add new languages to
Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md)
(if applicable)
- [ ] I have performed a self-review of my own code
- [ ] My changes generate no new warnings

### Documentation

- [ ] I have updated relevant docs on [Stirling-PDF's doc
repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/)
(if functionality has heavily changed)
- [ ] I have read the section [Add New Translation
Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags)
(for new translation tags only)

### UI Changes (if applicable)

- [ ] Screenshots or videos demonstrating the UI changes are attached
(e.g., as comments or direct attachments in the PR)

### Testing (if applicable)

- [ ] I have tested my changes locally. Refer to the [Testing
Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/DeveloperGuide.md#6-testing)
for more details.

---------

Co-authored-by: Reece Browne <you@example.com>
This commit is contained in:
Reece Browne
2025-08-21 17:30:26 +01:00
committed by GitHub
parent a33e51351b
commit 949ffa01ad
90 changed files with 5416 additions and 4164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
frontend/src/services/enhancedPDFProcessingService.ts
View File

@@ -1,12 +1,10 @@
import { getDocument, GlobalWorkerOptions } from 'pdfjs-dist';
import * as pdfjsLib from 'pdfjs-dist';
import { ProcessedFile, ProcessingState, PDFPage, ProcessingStrategy, ProcessingConfig, ProcessingMetrics } from '../types/processing';
import { ProcessingCache } from './processingCache';
import { FileHasher } from '../utils/fileHash';
import { FileAnalyzer } from './fileAnalyzer';
import { ProcessingErrorHandler } from './processingErrorHandler';
// Set up PDF.js worker
GlobalWorkerOptions.workerSrc = '/pdf.worker.js';
import { pdfWorkerManager } from './pdfWorkerManager';
export class EnhancedPDFProcessingService {
private static instance: EnhancedPDFProcessingService;
@@ -183,43 +181,45 @@ export class EnhancedPDFProcessingService {
state: ProcessingState
): Promise<ProcessedFile> {
const arrayBuffer = await file.arrayBuffer();
const pdf = await getDocument({ data: arrayBuffer }).promise;
const totalPages = pdf.numPages;
const pdf = await pdfWorkerManager.createDocument(arrayBuffer);
try {
const totalPages = pdf.numPages;
state.progress = 10;
this.notifyListeners();
state.progress = 10;
this.notifyListeners();
const pages: PDFPage[] = [];
const pages: PDFPage[] = [];
for (let i = 1; i <= totalPages; i++) {
// Check for cancellation
if (state.cancellationToken?.signal.aborted) {
pdf.destroy();
throw new Error('Processing cancelled');
for (let i = 1; i <= totalPages; i++) {
// Check for cancellation
if (state.cancellationToken?.signal.aborted) {
throw new Error('Processing cancelled');
}
const page = await pdf.getPage(i);
const thumbnail = await this.renderPageThumbnail(page, config.thumbnailQuality);
pages.push({
id: `${file.name}-page-${i}`,
pageNumber: i,
thumbnail,
rotation: 0,
selected: false
});
// Update progress
state.progress = 10 + (i / totalPages) * 85;
state.currentPage = i;
this.notifyListeners();
}
const page = await pdf.getPage(i);
const thumbnail = await this.renderPageThumbnail(page, config.thumbnailQuality);
pages.push({
id: `${file.name}-page-${i}`,
pageNumber: i,
thumbnail,
rotation: 0,
selected: false
});
// Update progress
state.progress = 10 + (i / totalPages) * 85;
state.currentPage = i;
return this.createProcessedFile(file, pages, totalPages);
} finally {
pdfWorkerManager.destroyDocument(pdf);
state.progress = 100;
this.notifyListeners();
}
pdf.destroy();
state.progress = 100;
this.notifyListeners();
return this.createProcessedFile(file, pages, totalPages);
}
/**
@@ -231,7 +231,7 @@ export class EnhancedPDFProcessingService {
state: ProcessingState
): Promise<ProcessedFile> {
const arrayBuffer = await file.arrayBuffer();
const pdf = await getDocument({ data: arrayBuffer }).promise;
const pdf = await pdfWorkerManager.createDocument(arrayBuffer);
const totalPages = pdf.numPages;
state.progress = 10;
@@ -243,7 +243,7 @@ export class EnhancedPDFProcessingService {
// Process priority pages first
for (let i = 1; i <= priorityCount; i++) {
if (state.cancellationToken?.signal.aborted) {
pdf.destroy();
pdfWorkerManager.destroyDocument(pdf);
throw new Error('Processing cancelled');
}
@@ -274,7 +274,7 @@ export class EnhancedPDFProcessingService {
});
}
pdf.destroy();
pdfWorkerManager.destroyDocument(pdf);
state.progress = 100;
this.notifyListeners();
@@ -290,7 +290,7 @@ export class EnhancedPDFProcessingService {
state: ProcessingState
): Promise<ProcessedFile> {
const arrayBuffer = await file.arrayBuffer();
const pdf = await getDocument({ data: arrayBuffer }).promise;
const pdf = await pdfWorkerManager.createDocument(arrayBuffer);
const totalPages = pdf.numPages;
state.progress = 10;
@@ -305,7 +305,7 @@ export class EnhancedPDFProcessingService {
for (let i = 1; i <= firstChunkEnd; i++) {
if (state.cancellationToken?.signal.aborted) {
pdf.destroy();
pdfWorkerManager.destroyDocument(pdf);
throw new Error('Processing cancelled');
}
@@ -342,7 +342,7 @@ export class EnhancedPDFProcessingService {
});
}
pdf.destroy();
pdfWorkerManager.destroyDocument(pdf);
state.progress = 100;
this.notifyListeners();
@@ -358,7 +358,7 @@ export class EnhancedPDFProcessingService {
state: ProcessingState
): Promise<ProcessedFile> {
const arrayBuffer = await file.arrayBuffer();
const pdf = await getDocument({ data: arrayBuffer }).promise;
const pdf = await pdfWorkerManager.createDocument(arrayBuffer);
const totalPages = pdf.numPages;
state.progress = 50;
@@ -376,7 +376,7 @@ export class EnhancedPDFProcessingService {
});
}
pdf.destroy();
pdfWorkerManager.destroyDocument(pdf);
state.progress = 100;
this.notifyListeners();
@@ -519,7 +519,10 @@ export class EnhancedPDFProcessingService {
this.notifyListeners();
// Force memory cleanup hint
setTimeout(() => window?.gc?.(), 100);
if (typeof window !== 'undefined' && window.gc) {
let gc = window.gc;
setTimeout(() => gc(), 100);
}
}
/**
@@ -537,6 +540,15 @@ export class EnhancedPDFProcessingService {
this.processing.clear();
this.notifyListeners();
}
/**
* Emergency cleanup - destroy all PDF workers
*/
emergencyCleanup(): void {
this.clearAllProcessing();
this.clearAll();
pdfWorkerManager.destroyAllDocuments();
}
}
// Export singleton instance

11
frontend/src/services/fileAnalyzer.ts
View File

@@ -1,5 +1,5 @@
import { getDocument } from 'pdfjs-dist';
import { FileAnalysis, ProcessingStrategy } from '../types/processing';
import { pdfWorkerManager } from './pdfWorkerManager';
export class FileAnalyzer {
private static readonly SIZE_THRESHOLDS = {
@@ -66,17 +66,16 @@ export class FileAnalyzer {
// For large files, try the whole file first (PDF.js needs the complete structure)
const arrayBuffer = await file.arrayBuffer();
const pdf = await getDocument({
data: arrayBuffer,
const pdf = await pdfWorkerManager.createDocument(arrayBuffer, {
stopAtErrors: false, // Don't stop at minor errors
verbosity: 0 // Suppress PDF.js warnings
}).promise;
});
const pageCount = pdf.numPages;
const isEncrypted = (pdf as any).isEncrypted;
// Clean up
pdf.destroy();
// Clean up using worker manager
pdfWorkerManager.destroyDocument(pdf);
return {
pageCount,

194
frontend/src/services/fileOperationsService.ts
View File

@@ -1,194 +0,0 @@
import { FileWithUrl } from "../types/file";
import { fileStorage, StorageStats } from "./fileStorage";
import { loadFilesFromIndexedDB, createEnhancedFileFromStored, cleanupFileUrls } from "../utils/fileUtils";
import { generateThumbnailForFile } from "../utils/thumbnailUtils";
import { updateStorageStatsIncremental } from "../utils/storageUtils";
/**
* Service for file storage operations
* Contains all IndexedDB operations and file management logic
*/
export const fileOperationsService = {
/**
* Load storage statistics
*/
async loadStorageStats(): Promise<StorageStats | null> {
try {
return await fileStorage.getStorageStats();
} catch (error) {
console.error('Failed to load storage stats:', error);
return null;
}
},
/**
* Force reload files from IndexedDB
*/
async forceReloadFiles(): Promise<FileWithUrl[]> {
try {
return await loadFilesFromIndexedDB();
} catch (error) {
console.error('Failed to force reload files:', error);
return [];
}
},
/**
* Load existing files from IndexedDB if not already loaded
*/
async loadExistingFiles(
filesLoaded: boolean,
currentFiles: FileWithUrl[]
): Promise<FileWithUrl[]> {
if (filesLoaded && currentFiles.length > 0) {
return currentFiles;
}
try {
await fileStorage.init();
const storedFiles = await fileStorage.getAllFileMetadata();
// Detect if IndexedDB was purged by comparing with current UI state
if (currentFiles.length > 0 && storedFiles.length === 0) {
console.warn('IndexedDB appears to have been purged - clearing UI state');
return [];
}
return await loadFilesFromIndexedDB();
} catch (error) {
console.error('Failed to load existing files:', error);
return [];
}
},
/**
* Upload files to IndexedDB with thumbnail generation
*/
async uploadFiles(
uploadedFiles: File[],
useIndexedDB: boolean
): Promise<FileWithUrl[]> {
const newFiles: FileWithUrl[] = [];
for (const file of uploadedFiles) {
if (useIndexedDB) {
try {
console.log('Storing file in IndexedDB:', file.name);
// Generate thumbnail only during upload
const thumbnail = await generateThumbnailForFile(file);
const storedFile = await fileStorage.storeFile(file, thumbnail);
console.log('File stored with ID:', storedFile.id);
const baseFile = fileStorage.createFileFromStored(storedFile);
const enhancedFile = createEnhancedFileFromStored(storedFile, thumbnail);
// Copy File interface methods from baseFile
enhancedFile.arrayBuffer = baseFile.arrayBuffer.bind(baseFile);
enhancedFile.slice = baseFile.slice.bind(baseFile);
enhancedFile.stream = baseFile.stream.bind(baseFile);
enhancedFile.text = baseFile.text.bind(baseFile);
newFiles.push(enhancedFile);
} catch (error) {
console.error('Failed to store file in IndexedDB:', error);
// Fallback to RAM storage
const enhancedFile: FileWithUrl = Object.assign(file, {
url: URL.createObjectURL(file),
storedInIndexedDB: false
});
newFiles.push(enhancedFile);
}
} else {
// IndexedDB disabled - use RAM
const enhancedFile: FileWithUrl = Object.assign(file, {
url: URL.createObjectURL(file),
storedInIndexedDB: false
});
newFiles.push(enhancedFile);
}
}
return newFiles;
},
/**
* Remove a file from storage
*/
async removeFile(file: FileWithUrl): Promise<void> {
// Clean up blob URL
if (file.url && !file.url.startsWith('indexeddb:')) {
URL.revokeObjectURL(file.url);
}
// Remove from IndexedDB if stored there
if (file.storedInIndexedDB && file.id) {
try {
await fileStorage.deleteFile(file.id);
} catch (error) {
console.error('Failed to delete file from IndexedDB:', error);
}
}
},
/**
* Clear all files from storage
*/
async clearAllFiles(files: FileWithUrl[]): Promise<void> {
// Clean up all blob URLs
cleanupFileUrls(files);
// Clear IndexedDB
try {
await fileStorage.clearAll();
} catch (error) {
console.error('Failed to clear IndexedDB:', error);
}
},
/**
* Create blob URL for file viewing
*/
async createBlobUrlForFile(file: FileWithUrl): Promise<string> {
// For large files, use IndexedDB direct access to avoid memory issues
const FILE_SIZE_LIMIT = 100 * 1024 * 1024; // 100MB
if (file.size > FILE_SIZE_LIMIT) {
console.warn(`File ${file.name} is too large for blob URL. Use direct IndexedDB access.`);
return `indexeddb:${file.id}`;
}
// For all files, avoid persistent blob URLs
if (file.storedInIndexedDB && file.id) {
const storedFile = await fileStorage.getFile(file.id);
if (storedFile) {
return fileStorage.createBlobUrl(storedFile);
}
}
// Fallback for files not in IndexedDB
return URL.createObjectURL(file);
},
/**
* Check for IndexedDB purge
*/
async checkForPurge(currentFiles: FileWithUrl[]): Promise<boolean> {
if (currentFiles.length === 0) return false;
try {
await fileStorage.init();
const storedFiles = await fileStorage.getAllFileMetadata();
return storedFiles.length === 0; // Purge detected if no files in storage but UI shows files
} catch (error) {
console.error('Error checking for purge:', error);
return true; // Assume purged if can't access IndexedDB
}
},
/**
* Update storage stats incrementally (re-export utility for convenience)
*/
updateStorageStatsIncremental
};

209
frontend/src/services/fileProcessingService.ts Normal file
View File

@@ -0,0 +1,209 @@
/**
* Centralized file processing service
* Handles metadata discovery, page counting, and thumbnail generation
* Called when files are added to FileContext, before any view sees them
*/
import * as pdfjsLib from 'pdfjs-dist';
import { generateThumbnailForFile } from '../utils/thumbnailUtils';
import { pdfWorkerManager } from './pdfWorkerManager';
export interface ProcessedFileMetadata {
totalPages: number;
pages: Array<{
pageNumber: number;
thumbnail?: string;
rotation: number;
splitBefore: boolean;
}>;
thumbnailUrl?: string; // Page 1 thumbnail for FileEditor
lastProcessed: number;
}
export interface FileProcessingResult {
success: boolean;
metadata?: ProcessedFileMetadata;
error?: string;
}
interface ProcessingOperation {
promise: Promise<FileProcessingResult>;
abortController: AbortController;
}
class FileProcessingService {
private processingCache = new Map<string, ProcessingOperation>();
/**
* Process a file to extract metadata, page count, and generate thumbnails
* This is the single source of truth for file processing
*/
async processFile(file: File, fileId: string): Promise<FileProcessingResult> {
// Check if we're already processing this file
const existingOperation = this.processingCache.get(fileId);
if (existingOperation) {
console.log(`📁 FileProcessingService: Using cached processing for ${file.name}`);
return existingOperation.promise;
}
// Create abort controller for this operation
const abortController = new AbortController();
// Create processing promise
const processingPromise = this.performProcessing(file, fileId, abortController);
// Store operation with abort controller
const operation: ProcessingOperation = {
promise: processingPromise,
abortController
};
this.processingCache.set(fileId, operation);
// Clean up cache after completion
processingPromise.finally(() => {
this.processingCache.delete(fileId);
});
return processingPromise;
}
private async performProcessing(file: File, fileId: string, abortController: AbortController): Promise<FileProcessingResult> {
console.log(`📁 FileProcessingService: Starting processing for ${file.name} (${fileId})`);
try {
// Check for cancellation at start
if (abortController.signal.aborted) {
throw new Error('Processing cancelled');
}
let totalPages = 1;
let thumbnailUrl: string | undefined;
// Handle PDF files
if (file.type === 'application/pdf') {
// Read arrayBuffer once and reuse for both PDF.js and fallback
const arrayBuffer = await file.arrayBuffer();
// Check for cancellation after async operation
if (abortController.signal.aborted) {
throw new Error('Processing cancelled');
}
// Discover page count using PDF.js (most accurate)
try {
const pdfDoc = await pdfWorkerManager.createDocument(arrayBuffer, {
disableAutoFetch: true,
disableStream: true
});
totalPages = pdfDoc.numPages;
console.log(`📁 FileProcessingService: PDF.js discovered ${totalPages} pages for ${file.name}`);
// Clean up immediately
pdfWorkerManager.destroyDocument(pdfDoc);
// Check for cancellation after PDF.js processing
if (abortController.signal.aborted) {
throw new Error('Processing cancelled');
}
} catch (pdfError) {
console.warn(`📁 FileProcessingService: PDF.js failed for ${file.name}, setting pages to 0:`, pdfError);
totalPages = 0; // Unknown page count - UI will hide page count display
}
}
// Generate page 1 thumbnail
try {
thumbnailUrl = await generateThumbnailForFile(file);
console.log(`📁 FileProcessingService: Generated thumbnail for ${file.name}`);
// Check for cancellation after thumbnail generation
if (abortController.signal.aborted) {
throw new Error('Processing cancelled');
}
} catch (thumbError) {
console.warn(`📁 FileProcessingService: Thumbnail generation failed for ${file.name}:`, thumbError);
}
// Create page structure
const pages = Array.from({ length: totalPages }, (_, index) => ({
pageNumber: index + 1,
thumbnail: index === 0 ? thumbnailUrl : undefined, // Only page 1 gets thumbnail initially
rotation: 0,
splitBefore: false
}));
const metadata: ProcessedFileMetadata = {
totalPages,
pages,
thumbnailUrl, // For FileEditor display
lastProcessed: Date.now()
};
console.log(`📁 FileProcessingService: Processing complete for ${file.name} - ${totalPages} pages`);
return {
success: true,
metadata
};
} catch (error) {
console.error(`📁 FileProcessingService: Processing failed for ${file.name}:`, error);
return {
success: false,
error: error instanceof Error ? error.message : 'Unknown processing error'
};
}
}
/**
* Clear all processing caches
*/
clearCache(): void {
this.processingCache.clear();
}
/**
* Check if a file is currently being processed
*/
isProcessing(fileId: string): boolean {
return this.processingCache.has(fileId);
}
/**
* Cancel processing for a specific file
*/
cancelProcessing(fileId: string): boolean {
const operation = this.processingCache.get(fileId);
if (operation) {
operation.abortController.abort();
console.log(`📁 FileProcessingService: Cancelled processing for ${fileId}`);
return true;
}
return false;
}
/**
* Cancel all ongoing processing operations
*/
cancelAllProcessing(): void {
this.processingCache.forEach((operation, fileId) => {
operation.abortController.abort();
console.log(`📁 FileProcessingService: Cancelled processing for ${fileId}`);
});
console.log(`📁 FileProcessingService: Cancelled ${this.processingCache.size} processing operations`);
}
/**
* Emergency cleanup - cancel all processing and destroy workers
*/
emergencyCleanup(): void {
this.cancelAllProcessing();
this.clearCache();
pdfWorkerManager.destroyAllDocuments();
}
}
// Export singleton instance
export const fileProcessingService = new FileProcessingService();

144
frontend/src/services/fileStorage.ts
View File

@@ -1,8 +1,11 @@
/**
* IndexedDB File Storage Service
* Provides high-capacity file storage for PDF processing
* Now uses centralized IndexedDB manager
*/
import { indexedDBManager, DATABASE_CONFIGS } from './indexedDBManager';
export interface StoredFile {
id: string;
name: string;
@@ -22,75 +25,26 @@ export interface StorageStats {
}
class FileStorageService {
private dbName = 'stirling-pdf-files';
private dbVersion = 2; // Increment version to force schema update
private storeName = 'files';
private db: IDBDatabase | null = null;
private initPromise: Promise<void> | null = null;
private readonly dbConfig = DATABASE_CONFIGS.FILES;
private readonly storeName = 'files';
/**
* Initialize the IndexedDB database (singleton pattern)
* Get database connection using centralized manager
*/
async init(): Promise<void> {
if (this.db) {
return Promise.resolve();
}
if (this.initPromise) {
return this.initPromise;
}
this.initPromise = new Promise((resolve, reject) => {
const request = indexedDB.open(this.dbName, this.dbVersion);
request.onerror = () => {
this.initPromise = null;
reject(request.error);
};
request.onsuccess = () => {
this.db = request.result;
console.log('IndexedDB connection established');
resolve();
};
request.onupgradeneeded = (event) => {
const db = (event.target as IDBOpenDBRequest).result;
const oldVersion = (event as any).oldVersion;
console.log('IndexedDB upgrade needed from version', oldVersion, 'to', this.dbVersion);
// Only recreate object store if it doesn't exist or if upgrading from version < 2
if (!db.objectStoreNames.contains(this.storeName)) {
const store = db.createObjectStore(this.storeName, { keyPath: 'id' });
store.createIndex('name', 'name', { unique: false });
store.createIndex('lastModified', 'lastModified', { unique: false });
console.log('IndexedDB object store created with keyPath: id');
} else if (oldVersion < 2) {
// Only delete and recreate if upgrading from version 1 to 2
db.deleteObjectStore(this.storeName);
const store = db.createObjectStore(this.storeName, { keyPath: 'id' });
store.createIndex('name', 'name', { unique: false });
store.createIndex('lastModified', 'lastModified', { unique: false });
console.log('IndexedDB object store recreated with keyPath: id (version upgrade)');
}
};
});
return this.initPromise;
private async getDatabase(): Promise<IDBDatabase> {
return indexedDBManager.openDatabase(this.dbConfig);
}
/**
* Store a file in IndexedDB
* Store a file in IndexedDB with external UUID
*/
async storeFile(file: File, thumbnail?: string): Promise<StoredFile> {
if (!this.db) await this.init();
async storeFile(file: File, fileId: string, thumbnail?: string): Promise<StoredFile> {
const db = await this.getDatabase();
const id = `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
const arrayBuffer = await file.arrayBuffer();
const storedFile: StoredFile = {
id,
id: fileId, // Use provided UUID
name: file.name,
type: file.type,
size: file.size,
@@ -101,13 +55,13 @@ class FileStorageService {
return new Promise((resolve, reject) => {
try {
const transaction = this.db!.transaction([this.storeName], 'readwrite');
const transaction = db.transaction([this.storeName], 'readwrite');
const store = transaction.objectStore(this.storeName);
// Debug logging
console.log('Object store keyPath:', store.keyPath);
console.log('Storing file:', {
id: storedFile.id,
console.log('Storing file with UUID:', {
id: storedFile.id, // Now a UUID from FileContext
name: storedFile.name,
hasData: !!storedFile.data,
dataSize: storedFile.data.byteLength
@@ -135,10 +89,10 @@ class FileStorageService {
* Retrieve a file from IndexedDB
*/
async getFile(id: string): Promise<StoredFile | null> {
if (!this.db) await this.init();
const db = await this.getDatabase();
return new Promise((resolve, reject) => {
const transaction = this.db!.transaction([this.storeName], 'readonly');
const transaction = db.transaction([this.storeName], 'readonly');
const store = transaction.objectStore(this.storeName);
const request = store.get(id);
@@ -151,10 +105,10 @@ class FileStorageService {
* Get all stored files (WARNING: loads all data into memory)
*/
async getAllFiles(): Promise<StoredFile[]> {
if (!this.db) await this.init();
const db = await this.getDatabase();
return new Promise((resolve, reject) => {
const transaction = this.db!.transaction([this.storeName], 'readonly');
const transaction = db.transaction([this.storeName], 'readonly');
const store = transaction.objectStore(this.storeName);
const request = store.getAll();
@@ -176,10 +130,10 @@ class FileStorageService {
* Get metadata of all stored files (without loading data into memory)
*/
async getAllFileMetadata(): Promise<Omit<StoredFile, 'data'>[]> {
if (!this.db) await this.init();
const db = await this.getDatabase();
return new Promise((resolve, reject) => {
const transaction = this.db!.transaction([this.storeName], 'readonly');
const transaction = db.transaction([this.storeName], 'readonly');
const store = transaction.objectStore(this.storeName);
const request = store.openCursor();
const files: Omit<StoredFile, 'data'>[] = [];
@@ -202,7 +156,7 @@ class FileStorageService {
}
cursor.continue();
} else {
console.log('Loaded metadata for', files.length, 'files without loading data');
// Metadata loaded efficiently without file data
resolve(files);
}
};
@@ -213,10 +167,10 @@ class FileStorageService {
* Delete a file from IndexedDB
*/
async deleteFile(id: string): Promise<void> {
if (!this.db) await this.init();
const db = await this.getDatabase();
return new Promise((resolve, reject) => {
const transaction = this.db!.transaction([this.storeName], 'readwrite');
const transaction = db.transaction([this.storeName], 'readwrite');
const store = transaction.objectStore(this.storeName);
const request = store.delete(id);
@@ -229,9 +183,9 @@ class FileStorageService {
* Update the lastModified timestamp of a file (for most recently used sorting)
*/
async touchFile(id: string): Promise<boolean> {
if (!this.db) await this.init();
const db = await this.getDatabase();
return new Promise((resolve, reject) => {
const transaction = this.db!.transaction([this.storeName], 'readwrite');
const transaction = db.transaction([this.storeName], 'readwrite');
const store = transaction.objectStore(this.storeName);
const getRequest = store.get(id);
@@ -255,10 +209,10 @@ class FileStorageService {
* Clear all stored files
*/
async clearAll(): Promise<void> {
if (!this.db) await this.init();
const db = await this.getDatabase();
return new Promise((resolve, reject) => {
const transaction = this.db!.transaction([this.storeName], 'readwrite');
const transaction = db.transaction([this.storeName], 'readwrite');
const store = transaction.objectStore(this.storeName);
const request = store.clear();
@@ -271,8 +225,6 @@ class FileStorageService {
* Get storage statistics (only our IndexedDB usage)
*/
async getStorageStats(): Promise<StorageStats> {
if (!this.db) await this.init();
let used = 0;
let available = 0;
let quota: number | undefined;
@@ -315,10 +267,10 @@ class FileStorageService {
* Get file count quickly without loading metadata
*/
async getFileCount(): Promise<number> {
if (!this.db) await this.init();
const db = await this.getDatabase();
return new Promise((resolve, reject) => {
const transaction = this.db!.transaction([this.storeName], 'readonly');
const transaction = db.transaction([this.storeName], 'readonly');
const store = transaction.objectStore(this.storeName);
const request = store.count();
@@ -365,9 +317,9 @@ class FileStorageService {
// Also check our specific database with different versions
for (let version = 1; version <= 3; version++) {
try {
console.log(`Trying to open ${this.dbName} version ${version}...`);
console.log(`Trying to open ${this.dbConfig.name} version ${version}...`);
const db = await new Promise<IDBDatabase>((resolve, reject) => {
const request = indexedDB.open(this.dbName, version);
const request = indexedDB.open(this.dbConfig.name, version);
request.onsuccess = () => resolve(request.result);
request.onerror = () => reject(request.error);
request.onupgradeneeded = () => {
@@ -400,10 +352,10 @@ class FileStorageService {
* Debug method to check what's actually in the database
*/
async debugDatabaseContents(): Promise<void> {
if (!this.db) await this.init();
const db = await this.getDatabase();
return new Promise((resolve, reject) => {
const transaction = this.db!.transaction([this.storeName], 'readonly');
const transaction = db.transaction([this.storeName], 'readonly');
const store = transaction.objectStore(this.storeName);
// First try getAll to see if there's anything
@@ -460,7 +412,8 @@ class FileStorageService {
}
/**
* Convert StoredFile back to File object for compatibility
* Convert StoredFile back to pure File object without mutations
* Returns a clean File object - use FileContext.addStoredFiles() for proper metadata handling
*/
createFileFromStored(storedFile: StoredFile): File {
if (!storedFile || !storedFile.data) {
@@ -477,13 +430,26 @@ class FileStorageService {
lastModified: storedFile.lastModified
});
// Add custom properties for compatibility
Object.defineProperty(file, 'id', { value: storedFile.id, writable: false });
Object.defineProperty(file, 'thumbnail', { value: storedFile.thumbnail, writable: false });
// Use FileContext.addStoredFiles() to properly associate with metadata
return file;
}
/**
* Convert StoredFile to the format expected by FileContext.addStoredFiles()
* This is the recommended way to load stored files into FileContext
*/
createFileWithMetadata(storedFile: StoredFile): { file: File; originalId: string; metadata: { thumbnail?: string } } {
const file = this.createFileFromStored(storedFile);
return {
file,
originalId: storedFile.id,
metadata: {
thumbnail: storedFile.thumbnail
}
};
}
/**
* Create blob URL for stored file
*/
@@ -527,11 +493,11 @@ class FileStorageService {
* Update thumbnail for an existing file
*/
async updateThumbnail(id: string, thumbnail: string): Promise<boolean> {
if (!this.db) await this.init();
const db = await this.getDatabase();
return new Promise((resolve, reject) => {
try {
const transaction = this.db!.transaction([this.storeName], 'readwrite');
const transaction = db.transaction([this.storeName], 'readwrite');
const store = transaction.objectStore(this.storeName);
const getRequest = store.get(id);

227
frontend/src/services/indexedDBManager.ts Normal file
View File

@@ -0,0 +1,227 @@
/**
* Centralized IndexedDB Manager
* Handles all database initialization, schema management, and migrations
* Prevents race conditions and duplicate schema upgrades
*/
export interface DatabaseConfig {
name: string;
version: number;
stores: {
name: string;
keyPath?: string | string[];
autoIncrement?: boolean;
indexes?: {
name: string;
keyPath: string | string[];
unique: boolean;
}[];
}[];
}
class IndexedDBManager {
private static instance: IndexedDBManager;
private databases = new Map<string, IDBDatabase>();
private initPromises = new Map<string, Promise<IDBDatabase>>();
private constructor() {}
static getInstance(): IndexedDBManager {
if (!IndexedDBManager.instance) {
IndexedDBManager.instance = new IndexedDBManager();
}
return IndexedDBManager.instance;
}
/**
* Open or get existing database connection
*/
async openDatabase(config: DatabaseConfig): Promise<IDBDatabase> {
const existingDb = this.databases.get(config.name);
if (existingDb) {
return existingDb;
}
const existingPromise = this.initPromises.get(config.name);
if (existingPromise) {
return existingPromise;
}
const initPromise = this.performDatabaseInit(config);
this.initPromises.set(config.name, initPromise);
try {
const db = await initPromise;
this.databases.set(config.name, db);
return db;
} catch (error) {
this.initPromises.delete(config.name);
throw error;
}
}
private performDatabaseInit(config: DatabaseConfig): Promise<IDBDatabase> {
return new Promise((resolve, reject) => {
console.log(`Opening IndexedDB: ${config.name} v${config.version}`);
const request = indexedDB.open(config.name, config.version);
request.onerror = () => {
console.error(`Failed to open ${config.name}:`, request.error);
reject(request.error);
};
request.onsuccess = () => {
const db = request.result;
console.log(`Successfully opened ${config.name}`);
// Set up close handler to clean up our references
db.onclose = () => {
console.log(`Database ${config.name} closed`);
this.databases.delete(config.name);
this.initPromises.delete(config.name);
};
resolve(db);
};
request.onupgradeneeded = (event) => {
const db = request.result;
const oldVersion = event.oldVersion;
console.log(`Upgrading ${config.name} from v${oldVersion} to v${config.version}`);
// Create or update object stores
config.stores.forEach(storeConfig => {
let store: IDBObjectStore;
if (db.objectStoreNames.contains(storeConfig.name)) {
// Store exists - for now, just continue (could add migration logic here)
console.log(`Object store '${storeConfig.name}' already exists`);
return;
}
// Create new object store
const options: IDBObjectStoreParameters = {};
if (storeConfig.keyPath) {
options.keyPath = storeConfig.keyPath;
}
if (storeConfig.autoIncrement) {
options.autoIncrement = storeConfig.autoIncrement;
}
store = db.createObjectStore(storeConfig.name, options);
console.log(`Created object store '${storeConfig.name}'`);
// Create indexes
if (storeConfig.indexes) {
storeConfig.indexes.forEach(indexConfig => {
store.createIndex(
indexConfig.name,
indexConfig.keyPath,
{ unique: indexConfig.unique }
);
console.log(`Created index '${indexConfig.name}' on '${storeConfig.name}'`);
});
}
});
};
});
}
/**
* Get database connection (must be already opened)
*/
getDatabase(name: string): IDBDatabase | null {
return this.databases.get(name) || null;
}
/**
* Close database connection
*/
closeDatabase(name: string): void {
const db = this.databases.get(name);
if (db) {
db.close();
this.databases.delete(name);
this.initPromises.delete(name);
}
}
/**
* Close all database connections
*/
closeAllDatabases(): void {
this.databases.forEach((db, name) => {
console.log(`Closing database: ${name}`);
db.close();
});
this.databases.clear();
this.initPromises.clear();
}
/**
* Delete database completely
*/
async deleteDatabase(name: string): Promise<void> {
// Close connection if open
this.closeDatabase(name);
return new Promise((resolve, reject) => {
const deleteRequest = indexedDB.deleteDatabase(name);
deleteRequest.onerror = () => reject(deleteRequest.error);
deleteRequest.onsuccess = () => {
console.log(`Deleted database: ${name}`);
resolve();
};
});
}
/**
* Check if a database exists and what version it is
*/
async getDatabaseVersion(name: string): Promise<number | null> {
return new Promise((resolve) => {
const request = indexedDB.open(name);
request.onsuccess = () => {
const db = request.result;
const version = db.version;
db.close();
resolve(version);
};
request.onerror = () => resolve(null);
request.onupgradeneeded = () => {
// Cancel the upgrade
request.transaction?.abort();
resolve(null);
};
});
}
}
// Pre-defined database configurations
export const DATABASE_CONFIGS = {
FILES: {
name: 'stirling-pdf-files',
version: 2,
stores: [{
name: 'files',
keyPath: 'id',
indexes: [
{ name: 'name', keyPath: 'name', unique: false },
{ name: 'lastModified', keyPath: 'lastModified', unique: false }
]
}]
} as DatabaseConfig,
DRAFTS: {
name: 'stirling-pdf-drafts',
version: 1,
stores: [{
name: 'drafts',
keyPath: 'id'
}]
} as DatabaseConfig
} as const;
export const indexedDBManager = IndexedDBManager.getInstance();

9
frontend/src/services/pdfProcessingService.ts
View File

@@ -1,9 +1,6 @@
import { getDocument, GlobalWorkerOptions } from 'pdfjs-dist';
import { ProcessedFile, ProcessingState, PDFPage } from '../types/processing';
import { ProcessingCache } from './processingCache';
// Set up PDF.js worker
GlobalWorkerOptions.workerSrc = '/pdf.worker.js';
import { pdfWorkerManager } from './pdfWorkerManager';
export class PDFProcessingService {
private static instance: PDFProcessingService;
@@ -96,7 +93,7 @@ export class PDFProcessingService {
onProgress: (progress: number) => void
): Promise<ProcessedFile> {
const arrayBuffer = await file.arrayBuffer();
const pdf = await getDocument({ data: arrayBuffer }).promise;
const pdf = await pdfWorkerManager.createDocument(arrayBuffer);
const totalPages = pdf.numPages;
onProgress(10); // PDF loaded
@@ -129,7 +126,7 @@ export class PDFProcessingService {
onProgress(progress);
}
pdf.destroy();
pdfWorkerManager.destroyDocument(pdf);
onProgress(100);
return {

203
frontend/src/services/pdfWorkerManager.ts Normal file
View File

@@ -0,0 +1,203 @@
/**
* PDF.js Worker Manager - Centralized worker lifecycle management
*
* Prevents infinite worker creation by managing PDF.js workers globally
* and ensuring proper cleanup when operations complete.
*/
import * as pdfjsLib from 'pdfjs-dist';
const { getDocument, GlobalWorkerOptions } = pdfjsLib;
class PDFWorkerManager {
private static instance: PDFWorkerManager;
private activeDocuments = new Set<any>();
private workerCount = 0;
private maxWorkers = 3; // Limit concurrent workers
private isInitialized = false;
private constructor() {
this.initializeWorker();
}
static getInstance(): PDFWorkerManager {
if (!PDFWorkerManager.instance) {
PDFWorkerManager.instance = new PDFWorkerManager();
}
return PDFWorkerManager.instance;
}
/**
* Initialize PDF.js worker once globally
*/
private initializeWorker(): void {
if (!this.isInitialized) {
GlobalWorkerOptions.workerSrc = '/pdf.worker.js';
this.isInitialized = true;
console.log('🏭 PDF.js worker initialized');
}
}
/**
* Create a PDF document with proper lifecycle management
* Supports ArrayBuffer, Uint8Array, URL string, or {data: ArrayBuffer} object
*/
async createDocument(
data: ArrayBuffer | Uint8Array | string | { data: ArrayBuffer },
options: {
disableAutoFetch?: boolean;
disableStream?: boolean;
stopAtErrors?: boolean;
verbosity?: number;
} = {}
): Promise<any> {
// Wait if we've hit the worker limit
if (this.activeDocuments.size >= this.maxWorkers) {
console.warn(`🏭 PDF Worker limit reached (${this.maxWorkers}), waiting for available worker...`);
await this.waitForAvailableWorker();
}
// Normalize input data to PDF.js format
let pdfData: any;
if (data instanceof ArrayBuffer || data instanceof Uint8Array) {
pdfData = { data };
} else if (typeof data === 'string') {
pdfData = data; // URL string
} else if (data && typeof data === 'object' && 'data' in data) {
pdfData = data; // Already in {data: ArrayBuffer} format
} else {
pdfData = data; // Pass through as-is
}
const loadingTask = getDocument(
typeof pdfData === 'string' ? {
url: pdfData,
disableAutoFetch: options.disableAutoFetch ?? true,
disableStream: options.disableStream ?? true,
stopAtErrors: options.stopAtErrors ?? false,
verbosity: options.verbosity ?? 0
} : {
...pdfData,
disableAutoFetch: options.disableAutoFetch ?? true,
disableStream: options.disableStream ?? true,
stopAtErrors: options.stopAtErrors ?? false,
verbosity: options.verbosity ?? 0
}
);
try {
const pdf = await loadingTask.promise;
this.activeDocuments.add(pdf);
this.workerCount++;
console.log(`🏭 PDF document created (active: ${this.activeDocuments.size}/${this.maxWorkers})`);
return pdf;
} catch (error) {
// If document creation fails, make sure to clean up the loading task
if (loadingTask) {
try {
loadingTask.destroy();
} catch (destroyError) {
console.warn('🏭 Error destroying failed loading task:', destroyError);
}
}
throw error;
}
}
/**
* Properly destroy a PDF document and clean up resources
*/
destroyDocument(pdf: any): void {
if (this.activeDocuments.has(pdf)) {
try {
pdf.destroy();
this.activeDocuments.delete(pdf);
this.workerCount = Math.max(0, this.workerCount - 1);
console.log(`🏭 PDF document destroyed (active: ${this.activeDocuments.size}/${this.maxWorkers})`);
} catch (error) {
console.warn('🏭 Error destroying PDF document:', error);
// Still remove from tracking even if destroy failed
this.activeDocuments.delete(pdf);
this.workerCount = Math.max(0, this.workerCount - 1);
}
}
}
/**
* Destroy all active PDF documents
*/
destroyAllDocuments(): void {
console.log(`🏭 Destroying all PDF documents (${this.activeDocuments.size} active)`);
const documentsToDestroy = Array.from(this.activeDocuments);
documentsToDestroy.forEach(pdf => {
this.destroyDocument(pdf);
});
this.activeDocuments.clear();
this.workerCount = 0;
console.log('🏭 All PDF documents destroyed');
}
/**
* Wait for a worker to become available
*/
private async waitForAvailableWorker(): Promise<void> {
return new Promise((resolve) => {
const checkAvailability = () => {
if (this.activeDocuments.size < this.maxWorkers) {
resolve();
} else {
setTimeout(checkAvailability, 100);
}
};
checkAvailability();
});
}
/**
* Get current worker statistics
*/
getWorkerStats() {
return {
active: this.activeDocuments.size,
max: this.maxWorkers,
total: this.workerCount
};
}
/**
* Force cleanup of all workers (emergency cleanup)
*/
emergencyCleanup(): void {
console.warn('🏭 Emergency PDF worker cleanup initiated');
// Force destroy all documents
this.activeDocuments.forEach(pdf => {
try {
pdf.destroy();
} catch (error) {
console.warn('🏭 Emergency cleanup - error destroying document:', error);
}
});
this.activeDocuments.clear();
this.workerCount = 0;
console.warn('🏭 Emergency cleanup completed');
}
/**
* Set maximum concurrent workers
*/
setMaxWorkers(max: number): void {
this.maxWorkers = Math.max(1, Math.min(max, 10)); // Between 1-10 workers
console.log(`🏭 Max workers set to ${this.maxWorkers}`);
}
}
// Export singleton instance
export const pdfWorkerManager = PDFWorkerManager.getInstance();

477
frontend/src/services/thumbnailGenerationService.ts
View File

@@ -1,7 +1,9 @@
/**
* High-performance thumbnail generation service using Web Workers
* High-performance thumbnail generation service using main thread processing
*/
import { pdfWorkerManager } from './pdfWorkerManager';
interface ThumbnailResult {
pageNumber: number;
thumbnail: string;
@@ -22,245 +24,136 @@ interface CachedThumbnail {
sizeBytes: number;
}
interface CachedPDFDocument {
pdf: any; // PDFDocumentProxy from pdfjs-dist
lastUsed: number;
refCount: number;
}
export class ThumbnailGenerationService {
private workers: Worker[] = [];
private activeJobs = new Map<string, { resolve: Function; reject: Function; onProgress?: Function }>();
private jobCounter = 0;
private isGenerating = false;
// Session-based thumbnail cache
private thumbnailCache = new Map<string, CachedThumbnail>();
private maxCacheSizeBytes = 1024 * 1024 * 1024; // 1GB cache limit
private currentCacheSize = 0;
constructor(private maxWorkers: number = 3) {
this.initializeWorkers();
}
// PDF document cache to reuse PDF instances and avoid creating multiple workers
private pdfDocumentCache = new Map<string, CachedPDFDocument>();
private maxPdfCacheSize = 10; // Keep up to 10 PDF documents cached
private initializeWorkers(): void {
const workerPromises: Promise<Worker | null>[] = [];
for (let i = 0; i < this.maxWorkers; i++) {
const workerPromise = new Promise<Worker | null>((resolve) => {
try {
console.log(`Attempting to create worker ${i}...`);
const worker = new Worker('/thumbnailWorker.js');
let workerReady = false;
let pingTimeout: NodeJS.Timeout;
worker.onmessage = (e) => {
const { type, data, jobId } = e.data;
// Handle PONG response to confirm worker is ready
if (type === 'PONG') {
workerReady = true;
clearTimeout(pingTimeout);
console.log(`✓ Worker ${i} is ready and responsive`);
resolve(worker);
return;
}
const job = this.activeJobs.get(jobId);
if (!job) return;
switch (type) {
case 'PROGRESS':
if (job.onProgress) {
job.onProgress(data);
}
break;
case 'COMPLETE':
job.resolve(data.thumbnails);
this.activeJobs.delete(jobId);
break;
case 'ERROR':
job.reject(new Error(data.error));
this.activeJobs.delete(jobId);
break;
}
};
worker.onerror = (error) => {
console.error(`✗ Worker ${i} failed with error:`, error);
clearTimeout(pingTimeout);
worker.terminate();
resolve(null);
};
// Test worker with timeout
pingTimeout = setTimeout(() => {
if (!workerReady) {
console.warn(`✗ Worker ${i} timed out (no PONG response)`);
worker.terminate();
resolve(null);
}
}, 3000); // Reduced timeout for faster feedback
// Send PING to test worker
try {
worker.postMessage({ type: 'PING' });
} catch (pingError) {
console.error(`✗ Failed to send PING to worker ${i}:`, pingError);
clearTimeout(pingTimeout);
worker.terminate();
resolve(null);
}
} catch (error) {
console.error(`✗ Failed to create worker ${i}:`, error);
resolve(null);
}
});
workerPromises.push(workerPromise);
}
// Wait for all workers to initialize or fail
Promise.all(workerPromises).then((workers) => {
this.workers = workers.filter((w): w is Worker => w !== null);
const successCount = this.workers.length;
const failCount = this.maxWorkers - successCount;
console.log(`🔧 Worker initialization complete: ${successCount}/${this.maxWorkers} workers ready`);
if (failCount > 0) {
console.warn(`⚠️ ${failCount} workers failed to initialize - will use main thread fallback`);
}
if (successCount === 0) {
console.warn('🚨 No Web Workers available - all thumbnail generation will use main thread');
}
});
constructor(private maxWorkers: number = 3) {
// PDF rendering requires DOM access, so we use optimized main thread processing
}
/**
* Generate thumbnails for multiple pages using Web Workers
* Get or create a cached PDF document
*/
private async getCachedPDFDocument(fileId: string, pdfArrayBuffer: ArrayBuffer): Promise<any> {
const cached = this.pdfDocumentCache.get(fileId);
if (cached) {
cached.lastUsed = Date.now();
cached.refCount++;
return cached.pdf;
}
// Evict old PDFs if cache is full
while (this.pdfDocumentCache.size >= this.maxPdfCacheSize) {
this.evictLeastRecentlyUsedPDF();
}
// Use centralized worker manager instead of direct getDocument
const pdf = await pdfWorkerManager.createDocument(pdfArrayBuffer, {
disableAutoFetch: true,
disableStream: true,
stopAtErrors: false
});
this.pdfDocumentCache.set(fileId, {
pdf,
lastUsed: Date.now(),
refCount: 1
});
return pdf;
}
/**
* Release a reference to a cached PDF document
*/
private releasePDFDocument(fileId: string): void {
const cached = this.pdfDocumentCache.get(fileId);
if (cached) {
cached.refCount--;
// Don't destroy immediately - keep in cache for potential reuse
}
}
/**
* Evict the least recently used PDF document
*/
private evictLeastRecentlyUsedPDF(): void {
let oldestEntry: [string, CachedPDFDocument] | null = null;
let oldestTime = Date.now();
for (const [key, value] of this.pdfDocumentCache.entries()) {
if (value.lastUsed < oldestTime && value.refCount === 0) {
oldestTime = value.lastUsed;
oldestEntry = [key, value];
}
}
if (oldestEntry) {
pdfWorkerManager.destroyDocument(oldestEntry[1].pdf); // Use worker manager for cleanup
this.pdfDocumentCache.delete(oldestEntry[0]);
}
}
/**
* Generate thumbnails for multiple pages using main thread processing
*/
async generateThumbnails(
fileId: string,
pdfArrayBuffer: ArrayBuffer,
pageNumbers: number[],
options: ThumbnailGenerationOptions = {},
onProgress?: (progress: { completed: number; total: number; thumbnails: ThumbnailResult[] }) => void
): Promise<ThumbnailResult[]> {
if (this.isGenerating) {
console.warn('🚨 ThumbnailService: Thumbnail generation already in progress, rejecting new request');
throw new Error('Thumbnail generation already in progress');
// Input validation
if (!fileId || typeof fileId !== 'string' || fileId.trim() === '') {
throw new Error('generateThumbnails: fileId must be a non-empty string');
}
if (!pdfArrayBuffer || pdfArrayBuffer.byteLength === 0) {
throw new Error('generateThumbnails: pdfArrayBuffer must not be empty');
}
if (!pageNumbers || pageNumbers.length === 0) {
throw new Error('generateThumbnails: pageNumbers must not be empty');
}
console.log(`🎬 ThumbnailService: Starting thumbnail generation for ${pageNumbers.length} pages`);
this.isGenerating = true;
const {
scale = 0.2,
quality = 0.8,
batchSize = 20, // Pages per worker
parallelBatches = this.maxWorkers
quality = 0.8
} = options;
try {
// Check if workers are available, fallback to main thread if not
if (this.workers.length === 0) {
console.warn('No Web Workers available, falling back to main thread processing');
return await this.generateThumbnailsMainThread(pdfArrayBuffer, pageNumbers, scale, quality, onProgress);
}
// Split pages across workers
const workerBatches = this.distributeWork(pageNumbers, this.workers.length);
console.log(`🔧 ThumbnailService: Distributing ${pageNumbers.length} pages across ${this.workers.length} workers:`, workerBatches.map(batch => batch.length));
const jobPromises: Promise<ThumbnailResult[]>[] = [];
for (let i = 0; i < workerBatches.length; i++) {
const batch = workerBatches[i];
if (batch.length === 0) continue;
const worker = this.workers[i % this.workers.length];
const jobId = `job-${++this.jobCounter}`;
console.log(`🔧 ThumbnailService: Sending job ${jobId} with ${batch.length} pages to worker ${i}:`, batch);
const promise = new Promise<ThumbnailResult[]>((resolve, reject) => {
// Add timeout for worker jobs
const timeout = setTimeout(() => {
console.error(`⏰ ThumbnailService: Worker job ${jobId} timed out`);
this.activeJobs.delete(jobId);
reject(new Error(`Worker job ${jobId} timed out`));
}, 60000); // 1 minute timeout
// Create job with timeout handling
this.activeJobs.set(jobId, {
resolve: (result: any) => {
console.log(`✅ ThumbnailService: Job ${jobId} completed with ${result.length} thumbnails`);
clearTimeout(timeout);
resolve(result);
},
reject: (error: any) => {
console.error(`❌ ThumbnailService: Job ${jobId} failed:`, error);
clearTimeout(timeout);
reject(error);
},
onProgress: onProgress ? (progressData: any) => {
console.log(`📊 ThumbnailService: Job ${jobId} progress - ${progressData.completed}/${progressData.total} (${progressData.thumbnails.length} new)`);
onProgress(progressData);
} : undefined
});
worker.postMessage({
type: 'GENERATE_THUMBNAILS',
jobId,
data: {
pdfArrayBuffer,
pageNumbers: batch,
scale,
quality
}
});
});
jobPromises.push(promise);
}
// Wait for all workers to complete
const results = await Promise.all(jobPromises);
// Flatten and sort results by page number
const allThumbnails = results.flat().sort((a, b) => a.pageNumber - b.pageNumber);
console.log(`🎯 ThumbnailService: All workers completed, returning ${allThumbnails.length} thumbnails`);
return allThumbnails;
} catch (error) {
console.error('Web Worker thumbnail generation failed, falling back to main thread:', error);
return await this.generateThumbnailsMainThread(pdfArrayBuffer, pageNumbers, scale, quality, onProgress);
} finally {
console.log('🔄 ThumbnailService: Resetting isGenerating flag');
this.isGenerating = false;
}
return await this.generateThumbnailsMainThread(fileId, pdfArrayBuffer, pageNumbers, scale, quality, onProgress);
}
/**
* Fallback thumbnail generation on main thread
* Main thread thumbnail generation with batching for UI responsiveness
*/
private async generateThumbnailsMainThread(
fileId: string,
pdfArrayBuffer: ArrayBuffer,
pageNumbers: number[],
scale: number,
quality: number,
onProgress?: (progress: { completed: number; total: number; thumbnails: ThumbnailResult[] }) => void
): Promise<ThumbnailResult[]> {
console.log(`🔧 ThumbnailService: Fallback to main thread for ${pageNumbers.length} pages`);
// Import PDF.js dynamically for main thread
const { getDocument } = await import('pdfjs-dist');
// Load PDF once
const pdf = await getDocument({ data: pdfArrayBuffer }).promise;
console.log(`✓ ThumbnailService: PDF loaded on main thread`);
const pdf = await this.getCachedPDFDocument(fileId, pdfArrayBuffer);
const allResults: ThumbnailResult[] = [];
let completed = 0;
const batchSize = 5; // Small batches for UI responsiveness
const batchSize = 3; // Smaller batches for better UI responsiveness
// Process pages in small batches
for (let i = 0; i < pageNumbers.length; i += batchSize) {
@@ -308,143 +201,99 @@ export class ThumbnailGenerationService {
});
}
// Small delay to keep UI responsive
if (i + batchSize < pageNumbers.length) {
await new Promise(resolve => setTimeout(resolve, 10));
}
// Yield control to prevent UI blocking
await new Promise(resolve => setTimeout(resolve, 1));
}
// Clean up
pdf.destroy();
return allResults.filter(r => r.success);
// Release reference to PDF document (don't destroy - keep in cache)
this.releasePDFDocument(fileId);
return allResults;
}
/**
* Distribute work evenly across workers
*/
private distributeWork(pageNumbers: number[], numWorkers: number): number[][] {
const batches: number[][] = Array(numWorkers).fill(null).map(() => []);
pageNumbers.forEach((pageNum, index) => {
const workerIndex = index % numWorkers;
batches[workerIndex].push(pageNum);
});
return batches;
}
/**
* Generate a single thumbnail (fallback for individual pages)
*/
async generateSingleThumbnail(
pdfArrayBuffer: ArrayBuffer,
pageNumber: number,
options: ThumbnailGenerationOptions = {}
): Promise<string> {
const results = await this.generateThumbnails(pdfArrayBuffer, [pageNumber], options);
if (results.length === 0 || !results[0].success) {
throw new Error(`Failed to generate thumbnail for page ${pageNumber}`);
}
return results[0].thumbnail;
}
/**
* Add thumbnail to cache with size management
*/
addThumbnailToCache(pageId: string, thumbnail: string): void {
const thumbnailSizeBytes = thumbnail.length * 0.75; // Rough base64 size estimate
const now = Date.now();
// Add new thumbnail
this.thumbnailCache.set(pageId, {
thumbnail,
lastUsed: now,
sizeBytes: thumbnailSizeBytes
});
this.currentCacheSize += thumbnailSizeBytes;
// If we exceed 1GB, trigger cleanup
if (this.currentCacheSize > this.maxCacheSizeBytes) {
this.cleanupThumbnailCache();
}
}
/**
* Get thumbnail from cache and update last used timestamp
* Cache management
*/
getThumbnailFromCache(pageId: string): string | null {
const cached = this.thumbnailCache.get(pageId);
if (!cached) return null;
// Update last used timestamp
cached.lastUsed = Date.now();
return cached.thumbnail;
if (cached) {
cached.lastUsed = Date.now();
return cached.thumbnail;
}
return null;
}
/**
* Clean up cache using LRU eviction
*/
private cleanupThumbnailCache(): void {
const entries = Array.from(this.thumbnailCache.entries());
addThumbnailToCache(pageId: string, thumbnail: string): void {
const sizeBytes = thumbnail.length * 2; // Rough estimate for base64 string
// Sort by last used (oldest first)
entries.sort(([, a], [, b]) => a.lastUsed - b.lastUsed);
// Enforce cache size limits
while (this.currentCacheSize + sizeBytes > this.maxCacheSizeBytes && this.thumbnailCache.size > 0) {
this.evictLeastRecentlyUsed();
}
this.thumbnailCache.set(pageId, {
thumbnail,
lastUsed: Date.now(),
sizeBytes
});
this.thumbnailCache.clear();
this.currentCacheSize = 0;
const targetSize = this.maxCacheSizeBytes * 0.8; // Clean to 80% of limit
// Keep most recently used entries until we hit target size
for (let i = entries.length - 1; i >= 0 && this.currentCacheSize < targetSize; i--) {
const [key, value] = entries[i];
this.thumbnailCache.set(key, value);
this.currentCacheSize += value.sizeBytes;
this.currentCacheSize += sizeBytes;
}
private evictLeastRecentlyUsed(): void {
let oldestEntry: [string, CachedThumbnail] | null = null;
let oldestTime = Date.now();
for (const [key, value] of this.thumbnailCache.entries()) {
if (value.lastUsed < oldestTime) {
oldestTime = value.lastUsed;
oldestEntry = [key, value];
}
}
if (oldestEntry) {
this.thumbnailCache.delete(oldestEntry[0]);
this.currentCacheSize -= oldestEntry[1].sizeBytes;
}
}
/**
* Clear all cached thumbnails
*/
clearThumbnailCache(): void {
this.thumbnailCache.clear();
this.currentCacheSize = 0;
}
/**
* Get cache statistics
*/
getCacheStats() {
return {
entries: this.thumbnailCache.size,
totalSizeBytes: this.currentCacheSize,
size: this.thumbnailCache.size,
sizeBytes: this.currentCacheSize,
maxSizeBytes: this.maxCacheSizeBytes
};
}
/**
* Stop generation but keep cache and workers alive
*/
stopGeneration(): void {
this.activeJobs.clear();
this.isGenerating = false;
// No-op since we removed workers
}
clearCache(): void {
this.thumbnailCache.clear();
this.currentCacheSize = 0;
}
clearPDFCache(): void {
// Destroy all cached PDF documents using worker manager
for (const [, cached] of this.pdfDocumentCache) {
pdfWorkerManager.destroyDocument(cached.pdf);
}
this.pdfDocumentCache.clear();
}
clearPDFCacheForFile(fileId: string): void {
const cached = this.pdfDocumentCache.get(fileId);
if (cached) {
pdfWorkerManager.destroyDocument(cached.pdf);
this.pdfDocumentCache.delete(fileId);
}
}
/**
* Terminate all workers and clear cache (only on explicit cleanup)
*/
destroy(): void {
this.workers.forEach(worker => worker.terminate());
this.workers = [];
this.activeJobs.clear();
this.isGenerating = false;
this.clearThumbnailCache();
this.clearCache();
this.clearPDFCache();
}
}
// Export singleton instance
// Global singleton instance
export const thumbnailGenerationService = new ThumbnailGenerationService();