Clean up of thumbnail generation

This commit is contained in:
Reece 2025-06-27 20:30:47 +01:00
parent 3730429153
commit 61699a08a5
7 changed files with 232 additions and 225 deletions

22
frontend/public/pdf.js Normal file

File diff suppressed because one or more lines are too long

View File

@ -5,24 +5,20 @@ let pdfJsLoaded = false;
// Import PDF.js properly for worker context // Import PDF.js properly for worker context
try { try {
console.log('📦 Attempting to load PDF.js from CDN...'); console.log('📦 Loading PDF.js locally...');
importScripts('https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.min.js'); importScripts('/pdf.js');
if (self.pdfjsLib) { if (self.pdfjsLib) {
// Set up PDF.js worker // Set up PDF.js worker
self.pdfjsLib.GlobalWorkerOptions.workerSrc = 'https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.worker.min.js'; self.pdfjsLib.GlobalWorkerOptions.workerSrc = '/pdf.worker.js';
pdfJsLoaded = true; pdfJsLoaded = true;
console.log('✓ PDF.js loaded successfully from CDN'); console.log('✓ PDF.js loaded successfully from local files');
} else { } else {
throw new Error('pdfjsLib not available after import'); throw new Error('pdfjsLib not available after import');
} }
} catch (error) { } catch (error) {
console.warn('⚠️ Failed to load PDF.js from CDN:', error); console.error('✗ Failed to load local PDF.js:', error);
console.error('✗ PDF.js CDN loading failed - worker will not be available');
pdfJsLoaded = false; pdfJsLoaded = false;
// Note: Local PDF.js fallback removed as pdf.js file is not available
// The main thread fallback will handle thumbnail generation instead
} }
// Log the final status // Log the final status
@ -38,12 +34,10 @@ self.onmessage = async function(e) {
try { try {
// Handle PING for worker health check // Handle PING for worker health check
if (type === 'PING') { if (type === 'PING') {
console.log('🏓 Worker received PING, sending PONG...');
// Check if PDF.js is loaded before responding // Check if PDF.js is loaded before responding
if (pdfJsLoaded && self.pdfjsLib) { if (pdfJsLoaded && self.pdfjsLib) {
self.postMessage({ type: 'PONG', jobId }); self.postMessage({ type: 'PONG', jobId });
console.log('✓ PONG sent - worker is ready for thumbnail generation');
} else { } else {
console.error('✗ PDF.js not loaded - worker not ready'); console.error('✗ PDF.js not loaded - worker not ready');
self.postMessage({ self.postMessage({
@ -56,7 +50,6 @@ self.onmessage = async function(e) {
} }
if (type === 'GENERATE_THUMBNAILS') { if (type === 'GENERATE_THUMBNAILS') {
console.log(`🖼️ Worker starting thumbnail generation for ${data.pageNumbers?.length || 0} pages`);
if (!pdfJsLoaded || !self.pdfjsLib) { if (!pdfJsLoaded || !self.pdfjsLib) {
throw new Error('PDF.js not available in worker'); throw new Error('PDF.js not available in worker');

View File

@ -17,7 +17,7 @@ import {
ToggleSplitCommand ToggleSplitCommand
} from "../../commands/pageCommands"; } from "../../commands/pageCommands";
import { pdfExportService } from "../../services/pdfExportService"; import { pdfExportService } from "../../services/pdfExportService";
import { thumbnailGenerationService } from "../../services/thumbnailGenerationService"; import { useThumbnailGeneration } from "../../hooks/useThumbnailGeneration";
import './pageEditor.module.css'; import './pageEditor.module.css';
import PageThumbnail from './PageThumbnail'; import PageThumbnail from './PageThumbnail';
import BulkSelectionPanel from './BulkSelectionPanel'; import BulkSelectionPanel from './BulkSelectionPanel';
@ -198,90 +198,14 @@ const PageEditor = ({
const [sharedPdfInstance, setSharedPdfInstance] = useState<any>(null); const [sharedPdfInstance, setSharedPdfInstance] = useState<any>(null);
const [thumbnailGenerationStarted, setThumbnailGenerationStarted] = useState(false); const [thumbnailGenerationStarted, setThumbnailGenerationStarted] = useState(false);
// Session-based thumbnail cache with 1GB limit // Thumbnail generation (opt-in for visual tools)
const [thumbnailCache, setThumbnailCache] = useState<Map<string, { thumbnail: string; lastUsed: number; sizeBytes: number }>>(new Map()); const {
const maxCacheSizeBytes = 1024 * 1024 * 1024; // 1GB cache limit generateThumbnails,
const [currentCacheSize, setCurrentCacheSize] = useState(0); addThumbnailToCache,
getThumbnailFromCache,
// Cache management functions stopGeneration,
const addThumbnailToCache = useCallback((pageId: string, thumbnail: string) => { destroyThumbnails
const thumbnailSizeBytes = thumbnail.length * 0.75; // Rough base64 size estimate } = useThumbnailGeneration();
setThumbnailCache(prev => {
const newCache = new Map(prev);
const now = Date.now();
// Add new thumbnail
newCache.set(pageId, {
thumbnail,
lastUsed: now,
sizeBytes: thumbnailSizeBytes
});
return newCache;
});
setCurrentCacheSize(prev => {
const newSize = prev + thumbnailSizeBytes;
// If we exceed 1GB, trigger cleanup
if (newSize > maxCacheSizeBytes) {
setTimeout(() => cleanupThumbnailCache(), 0);
}
return newSize;
});
console.log(`Cached thumbnail for ${pageId} (${Math.round(thumbnailSizeBytes / 1024)}KB)`);
}, [maxCacheSizeBytes]);
const getThumbnailFromCache = useCallback((pageId: string): string | null => {
const cached = thumbnailCache.get(pageId);
if (!cached) return null;
// Update last used timestamp
setThumbnailCache(prev => {
const newCache = new Map(prev);
const entry = newCache.get(pageId);
if (entry) {
entry.lastUsed = Date.now();
}
return newCache;
});
return cached.thumbnail;
}, [thumbnailCache]);
const cleanupThumbnailCache = useCallback(() => {
setThumbnailCache(prev => {
const entries = Array.from(prev.entries());
// Sort by last used (oldest first)
entries.sort(([, a], [, b]) => a.lastUsed - b.lastUsed);
const newCache = new Map();
let newSize = 0;
const targetSize = maxCacheSizeBytes * 0.8; // Clean to 80% of limit
// Keep most recently used entries until we hit target size
for (let i = entries.length - 1; i >= 0 && newSize < targetSize; i--) {
const [key, value] = entries[i];
newCache.set(key, value);
newSize += value.sizeBytes;
}
setCurrentCacheSize(newSize);
console.log(`Cleaned thumbnail cache: ${prev.size}${newCache.size} entries (${Math.round(newSize / 1024 / 1024)}MB)`);
return newCache;
});
}, [maxCacheSizeBytes]);
const clearThumbnailCache = useCallback(() => {
setThumbnailCache(new Map());
setCurrentCacheSize(0);
console.log('Cleared thumbnail cache');
}, []);
// Start thumbnail generation process (separate from document loading) // Start thumbnail generation process (separate from document loading)
const startThumbnailGeneration = useCallback(() => { const startThumbnailGeneration = useCallback(() => {
@ -290,24 +214,19 @@ const PageEditor = ({
const file = activeFiles[0]; const file = activeFiles[0];
const totalPages = mergedPdfDocument.totalPages; const totalPages = mergedPdfDocument.totalPages;
console.log(`Starting Web Worker thumbnail generation for ${totalPages} pages`);
setThumbnailGenerationStarted(true); setThumbnailGenerationStarted(true);
// Run everything asynchronously to avoid blocking the main thread // Run everything asynchronously to avoid blocking the main thread
setTimeout(async () => { setTimeout(async () => {
try { try {
console.log('📖 Loading PDF array buffer...');
// Load PDF array buffer for Web Workers // Load PDF array buffer for Web Workers
const arrayBuffer = await file.arrayBuffer(); const arrayBuffer = await file.arrayBuffer();
console.log('✅ PDF array buffer loaded, starting Web Workers...');
// Generate all page numbers // Generate all page numbers
const pageNumbers = Array.from({ length: totalPages }, (_, i) => i + 1); const pageNumbers = Array.from({ length: totalPages }, (_, i) => i + 1);
// Start parallel thumbnail generation WITHOUT blocking the main thread // Start parallel thumbnail generation WITHOUT blocking the main thread
thumbnailGenerationService.generateThumbnails( generateThumbnails(
arrayBuffer, arrayBuffer,
pageNumbers, pageNumbers,
{ {
@ -318,11 +237,6 @@ const PageEditor = ({
}, },
// Progress callback (throttled for better performance) // Progress callback (throttled for better performance)
(progress) => { (progress) => {
// Reduce console spam - only log every 10 completions
if (progress.completed % 10 === 0) {
console.log(`Thumbnail progress: ${progress.completed}/${progress.total} completed`);
}
// Batch process thumbnails to reduce main thread work // Batch process thumbnails to reduce main thread work
requestAnimationFrame(() => { requestAnimationFrame(() => {
progress.thumbnails.forEach(({ pageNumber, thumbnail }) => { progress.thumbnails.forEach(({ pageNumber, thumbnail }) => {
@ -341,10 +255,8 @@ const PageEditor = ({
}); });
}); });
} }
).then(thumbnails => { ).catch(error => {
console.log(`🎉 Web Worker thumbnail generation completed: ${thumbnails.length} thumbnails generated`); console.error('Web Worker thumbnail generation failed:', error);
}).catch(error => {
console.error('❌ Web Worker thumbnail generation failed:', error);
setThumbnailGenerationStarted(false); setThumbnailGenerationStarted(false);
}); });
@ -353,8 +265,6 @@ const PageEditor = ({
setThumbnailGenerationStarted(false); setThumbnailGenerationStarted(false);
} }
}, 0); // setTimeout with 0ms to defer to next tick }, 0); // setTimeout with 0ms to defer to next tick
console.log('🚀 Thumbnail generation queued - UI remains responsive');
}, [mergedPdfDocument, activeFiles, thumbnailGenerationStarted, getThumbnailFromCache, addThumbnailToCache]); }, [mergedPdfDocument, activeFiles, thumbnailGenerationStarted, getThumbnailFromCache, addThumbnailToCache]);
// Start thumbnail generation after document loads and UI settles // Start thumbnail generation after document loads and UI settles
@ -366,7 +276,7 @@ const PageEditor = ({
} }
}, [mergedPdfDocument, startThumbnailGeneration, thumbnailGenerationStarted]); }, [mergedPdfDocument, startThumbnailGeneration, thumbnailGenerationStarted]);
// Cleanup shared PDF instance, workers, and cache when component unmounts or files change // Cleanup shared PDF instance when files change (but keep thumbnails cached)
useEffect(() => { useEffect(() => {
return () => { return () => {
if (sharedPdfInstance) { if (sharedPdfInstance) {
@ -374,12 +284,10 @@ const PageEditor = ({
setSharedPdfInstance(null); setSharedPdfInstance(null);
} }
setThumbnailGenerationStarted(false); setThumbnailGenerationStarted(false);
clearThumbnailCache(); // Clear cache when leaving/changing documents // Stop generation but keep cache and workers alive for cross-tool persistence
stopGeneration();
// Cancel any ongoing Web Worker operations
thumbnailGenerationService.destroy();
}; };
}, [activeFiles, clearThumbnailCache]); }, [activeFiles, stopGeneration]);
// Clear selections when files change // Clear selections when files change
useEffect(() => { useEffect(() => {
@ -816,7 +724,10 @@ const PageEditor = ({
setActiveFiles([]); setActiveFiles([]);
setMergedPdfDocument(null); setMergedPdfDocument(null);
setSelectedPages([]); setSelectedPages([]);
}, [setActiveFiles]);
// Only destroy thumbnails and workers on explicit PDF close
destroyThumbnails();
}, [setActiveFiles, destroyThumbnails]);
// PageEditorControls needs onExportSelected and onExportAll // PageEditorControls needs onExportSelected and onExportAll
const onExportSelected = useCallback(() => showExportPreview(true), [showExportPreview]); const onExportSelected = useCallback(() => showExportPreview(true), [showExportPreview]);

View File

@ -7,7 +7,9 @@ import RotateRightIcon from '@mui/icons-material/RotateRight';
import DeleteIcon from '@mui/icons-material/Delete'; import DeleteIcon from '@mui/icons-material/Delete';
import ContentCutIcon from '@mui/icons-material/ContentCut'; import ContentCutIcon from '@mui/icons-material/ContentCut';
import DragIndicatorIcon from '@mui/icons-material/DragIndicator'; import DragIndicatorIcon from '@mui/icons-material/DragIndicator';
import { PDFPage } from '../../../types/pageEditor'; import { PDFPage, PDFDocument } from '../../../types/pageEditor';
import { RotatePagesCommand, DeletePagesCommand, ToggleSplitCommand } from '../../../commands/pageCommands';
import { Command } from '../../../hooks/useUndoRedo';
import styles from './PageEditor.module.css'; import styles from './PageEditor.module.css';
import { getDocument, GlobalWorkerOptions } from 'pdfjs-dist'; import { getDocument, GlobalWorkerOptions } from 'pdfjs-dist';
@ -36,14 +38,14 @@ interface PageThumbnailProps {
onDrop: (e: React.DragEvent, pageId: string) => void; onDrop: (e: React.DragEvent, pageId: string) => void;
onTogglePage: (pageId: string) => void; onTogglePage: (pageId: string) => void;
onAnimateReorder: (pageId: string, targetIndex: number) => void; onAnimateReorder: (pageId: string, targetIndex: number) => void;
onExecuteCommand: (command: any) => void; onExecuteCommand: (command: Command) => void;
onSetStatus: (status: string) => void; onSetStatus: (status: string) => void;
onSetMovingPage: (pageId: string | null) => void; onSetMovingPage: (pageId: string | null) => void;
RotatePagesCommand: any; RotatePagesCommand: typeof RotatePagesCommand;
DeletePagesCommand: any; DeletePagesCommand: typeof DeletePagesCommand;
ToggleSplitCommand: any; ToggleSplitCommand: typeof ToggleSplitCommand;
pdfDocument: any; pdfDocument: PDFDocument;
setPdfDocument: any; setPdfDocument: (doc: PDFDocument) => void;
} }
const PageThumbnail = React.memo(({ const PageThumbnail = React.memo(({
@ -83,10 +85,6 @@ const PageThumbnail = React.memo(({
const handleThumbnailReady = (event: CustomEvent) => { const handleThumbnailReady = (event: CustomEvent) => {
const { pageNumber, thumbnail, pageId } = event.detail; const { pageNumber, thumbnail, pageId } = event.detail;
if (pageNumber === page.pageNumber && pageId === page.id && !thumbnailUrl) { if (pageNumber === page.pageNumber && pageId === page.id && !thumbnailUrl) {
// Reduce console spam during scrolling
if (pageNumber % 20 === 0) {
console.log(`Received Web Worker thumbnail for page ${page.pageNumber}`);
}
setThumbnailUrl(thumbnail); setThumbnailUrl(thumbnail);
} }
}; };
@ -95,47 +93,6 @@ const PageThumbnail = React.memo(({
return () => window.removeEventListener('thumbnailReady', handleThumbnailReady as EventListener); return () => window.removeEventListener('thumbnailReady', handleThumbnailReady as EventListener);
}, [page.pageNumber, page.id, thumbnailUrl]); }, [page.pageNumber, page.id, thumbnailUrl]);
const loadThumbnailFromSharedPdf = async (sharedPdf: any, addThumbnailToCache?: (pageId: string, thumbnail: string) => void) => {
if (isLoadingThumbnail || thumbnailUrl) return;
setIsLoadingThumbnail(true);
try {
const thumbnail = await generateThumbnailFromPdf(sharedPdf);
// Cache the generated thumbnail
if (addThumbnailToCache) {
addThumbnailToCache(page.id, thumbnail);
}
} catch (error) {
console.error(`Failed to load thumbnail for page ${page.pageNumber}:`, error);
} finally {
setIsLoadingThumbnail(false);
}
};
const generateThumbnailFromPdf = async (pdf: any): Promise<string> => {
const pdfPage = await pdf.getPage(page.pageNumber);
const scale = 0.2; // Low quality for page editor
const viewport = pdfPage.getViewport({ scale });
const canvas = document.createElement('canvas');
canvas.width = viewport.width;
canvas.height = viewport.height;
const context = canvas.getContext('2d');
if (!context) {
throw new Error('Could not get canvas context');
}
await pdfPage.render({ canvasContext: context, viewport }).promise;
const thumbnail = canvas.toDataURL('image/jpeg', 0.8);
setThumbnailUrl(thumbnail);
console.log(`Thumbnail generated for page ${page.pageNumber}`);
return thumbnail;
};
// Register this component with pageRefs for animations // Register this component with pageRefs for animations
const pageElementRef = useCallback((element: HTMLDivElement | null) => { const pageElementRef = useCallback((element: HTMLDivElement | null) => {

View File

@ -0,0 +1,56 @@
import { useCallback } from 'react';
import { thumbnailGenerationService } from '../services/thumbnailGenerationService';
/**
* Hook for tools that want to use thumbnail generation
* Tools can choose whether to include visual features
*/
export function useThumbnailGeneration() {
const generateThumbnails = useCallback(async (
pdfArrayBuffer: ArrayBuffer,
pageNumbers: number[],
options: {
scale?: number;
quality?: number;
batchSize?: number;
parallelBatches?: number;
} = {},
onProgress?: (progress: { completed: number; total: number; thumbnails: any[] }) => void
) => {
return thumbnailGenerationService.generateThumbnails(
pdfArrayBuffer,
pageNumbers,
options,
onProgress
);
}, []);
const addThumbnailToCache = useCallback((pageId: string, thumbnail: string) => {
thumbnailGenerationService.addThumbnailToCache(pageId, thumbnail);
}, []);
const getThumbnailFromCache = useCallback((pageId: string): string | null => {
return thumbnailGenerationService.getThumbnailFromCache(pageId);
}, []);
const getCacheStats = useCallback(() => {
return thumbnailGenerationService.getCacheStats();
}, []);
const stopGeneration = useCallback(() => {
thumbnailGenerationService.stopGeneration();
}, []);
const destroyThumbnails = useCallback(() => {
thumbnailGenerationService.destroy();
}, []);
return {
generateThumbnails,
addThumbnailToCache,
getThumbnailFromCache,
getCacheStats,
stopGeneration,
destroyThumbnails
};
}

View File

@ -50,14 +50,12 @@ export class EnhancedPDFProcessingService {
// Check cache first // Check cache first
const cached = this.cache.get(fileKey); const cached = this.cache.get(fileKey);
if (cached) { if (cached) {
console.log('Cache hit for:', file.name);
this.updateMetrics('cacheHit'); this.updateMetrics('cacheHit');
return cached; return cached;
} }
// Check if already processing // Check if already processing
if (this.processing.has(fileKey)) { if (this.processing.has(fileKey)) {
console.log('Already processing:', file.name);
return null; return null;
} }
@ -281,11 +279,6 @@ export class EnhancedPDFProcessingService {
state.progress = 100; state.progress = 100;
this.notifyListeners(); this.notifyListeners();
// Queue background processing for remaining pages (only if there are any)
if (priorityCount < totalPages) {
this.queueBackgroundProcessing(file, priorityCount + 1, totalPages);
}
return this.createProcessedFile(file, pages, totalPages); return this.createProcessedFile(file, pages, totalPages);
} }
@ -354,11 +347,6 @@ export class EnhancedPDFProcessingService {
state.progress = 100; state.progress = 100;
this.notifyListeners(); this.notifyListeners();
// Queue remaining chunks for background processing (only if there are any)
if (firstChunkEnd < totalPages) {
this.queueChunkedBackgroundProcessing(file, firstChunkEnd + 1, totalPages, chunkSize);
}
return this.createProcessedFile(file, pages, totalPages); return this.createProcessedFile(file, pages, totalPages);
} }
@ -433,21 +421,6 @@ export class EnhancedPDFProcessingService {
}; };
} }
/**
* Queue background processing for remaining pages
*/
private queueBackgroundProcessing(file: File, startPage: number, endPage: number): void {
// TODO: Implement background processing queue
console.log(`Queued background processing for ${file.name} pages ${startPage}-${endPage}`);
}
/**
* Queue chunked background processing
*/
private queueChunkedBackgroundProcessing(file: File, startPage: number, endPage: number, chunkSize: number): void {
// TODO: Implement chunked background processing
console.log(`Queued chunked background processing for ${file.name} pages ${startPage}-${endPage} in chunks of ${chunkSize}`);
}
/** /**
* Generate a unique, collision-resistant cache key * Generate a unique, collision-resistant cache key

View File

@ -16,12 +16,23 @@ interface ThumbnailGenerationOptions {
parallelBatches?: number; parallelBatches?: number;
} }
interface CachedThumbnail {
thumbnail: string;
lastUsed: number;
sizeBytes: number;
}
export class ThumbnailGenerationService { export class ThumbnailGenerationService {
private workers: Worker[] = []; private workers: Worker[] = [];
private activeJobs = new Map<string, { resolve: Function; reject: Function; onProgress?: Function }>(); private activeJobs = new Map<string, { resolve: Function; reject: Function; onProgress?: Function }>();
private jobCounter = 0; private jobCounter = 0;
private isGenerating = false; private isGenerating = false;
// Session-based thumbnail cache
private thumbnailCache = new Map<string, CachedThumbnail>();
private maxCacheSizeBytes = 1024 * 1024 * 1024; // 1GB cache limit
private currentCacheSize = 0;
constructor(private maxWorkers: number = 3) { constructor(private maxWorkers: number = 3) {
this.initializeWorkers(); this.initializeWorkers();
} }
@ -323,13 +334,97 @@ export class ThumbnailGenerationService {
} }
/** /**
* Terminate all workers and stop generation * Add thumbnail to cache with size management
*/
addThumbnailToCache(pageId: string, thumbnail: string): void {
const thumbnailSizeBytes = thumbnail.length * 0.75; // Rough base64 size estimate
const now = Date.now();
// Add new thumbnail
this.thumbnailCache.set(pageId, {
thumbnail,
lastUsed: now,
sizeBytes: thumbnailSizeBytes
});
this.currentCacheSize += thumbnailSizeBytes;
// If we exceed 1GB, trigger cleanup
if (this.currentCacheSize > this.maxCacheSizeBytes) {
this.cleanupThumbnailCache();
}
}
/**
* Get thumbnail from cache and update last used timestamp
*/
getThumbnailFromCache(pageId: string): string | null {
const cached = this.thumbnailCache.get(pageId);
if (!cached) return null;
// Update last used timestamp
cached.lastUsed = Date.now();
return cached.thumbnail;
}
/**
* Clean up cache using LRU eviction
*/
private cleanupThumbnailCache(): void {
const entries = Array.from(this.thumbnailCache.entries());
// Sort by last used (oldest first)
entries.sort(([, a], [, b]) => a.lastUsed - b.lastUsed);
this.thumbnailCache.clear();
this.currentCacheSize = 0;
const targetSize = this.maxCacheSizeBytes * 0.8; // Clean to 80% of limit
// Keep most recently used entries until we hit target size
for (let i = entries.length - 1; i >= 0 && this.currentCacheSize < targetSize; i--) {
const [key, value] = entries[i];
this.thumbnailCache.set(key, value);
this.currentCacheSize += value.sizeBytes;
}
}
/**
* Clear all cached thumbnails
*/
clearThumbnailCache(): void {
this.thumbnailCache.clear();
this.currentCacheSize = 0;
}
/**
* Get cache statistics
*/
getCacheStats() {
return {
entries: this.thumbnailCache.size,
totalSizeBytes: this.currentCacheSize,
maxSizeBytes: this.maxCacheSizeBytes
};
}
/**
* Stop generation but keep cache and workers alive
*/
stopGeneration(): void {
this.activeJobs.clear();
this.isGenerating = false;
}
/**
* Terminate all workers and clear cache (only on explicit cleanup)
*/ */
destroy(): void { destroy(): void {
this.workers.forEach(worker => worker.terminate()); this.workers.forEach(worker => worker.terminate());
this.workers = []; this.workers = [];
this.activeJobs.clear(); this.activeJobs.clear();
this.isGenerating = false; this.isGenerating = false;
this.clearThumbnailCache();
} }
} }