From c158664d2cad1bf7da2e33049da06d0a381b2095 Mon Sep 17 00:00:00 2001 From: Reece Browne <74901996+reecebrowne@users.noreply.github.com> Date: Tue, 6 Jan 2026 11:54:14 -0500 Subject: [PATCH] Chore/v2/improve uploads (#5351) Co-authored-by: James Brunton --- .../core/components/fileEditor/FileEditor.tsx | 4 +- .../fileEditor/FileEditorThumbnail.tsx | 11 +- .../src/core/components/shared/FileCard.tsx | 29 +-- .../tools/automate/AutomationRun.tsx | 13 +- .../src/core/contexts/file/fileActions.ts | 204 +++++++++++------- frontend/src/core/contexts/file/fileHooks.ts | 15 +- 6 files changed, 160 insertions(+), 116 deletions(-) diff --git a/frontend/src/core/components/fileEditor/FileEditor.tsx b/frontend/src/core/components/fileEditor/FileEditor.tsx index 76e6bb1a2..cc825d367 100644 --- a/frontend/src/core/components/fileEditor/FileEditor.tsx +++ b/frontend/src/core/components/fileEditor/FileEditor.tsx @@ -43,7 +43,7 @@ const FileEditor = ({ const { clearAllFileErrors } = fileContextActions; // Extract needed values from state (memoized to prevent infinite loops) - const activeStirlingFileStubs = useMemo(() => selectors.getStirlingFileStubs(), [selectors.getFilesSignature()]); + const activeStirlingFileStubs = useMemo(() => selectors.getStirlingFileStubs(), [state.files.byId, state.files.ids]); const selectedFileIds = state.ui.selectedFileIds; const totalItems = state.files.ids.length; const selectedCount = selectedFileIds.length; @@ -365,7 +365,7 @@ const FileEditor = ({ activateOnDrag={true} > - + diff --git a/frontend/src/core/components/fileEditor/FileEditorThumbnail.tsx b/frontend/src/core/components/fileEditor/FileEditorThumbnail.tsx index 953bd018f..f6339be5e 100644 --- a/frontend/src/core/components/fileEditor/FileEditorThumbnail.tsx +++ b/frontend/src/core/components/fileEditor/FileEditorThumbnail.tsx @@ -1,5 +1,5 @@ import React, { useState, useCallback, useRef, useMemo } from 'react'; -import { Text, ActionIcon, CheckboxIndicator, Tooltip, Modal, Button, Group, Stack } from '@mantine/core'; +import { Text, ActionIcon, CheckboxIndicator, Tooltip, Modal, Button, Group, Stack, Loader } from '@mantine/core'; import { useIsMobile } from '@app/hooks/useIsMobile'; import { alert } from '@app/components/toast'; import { useTranslation } from 'react-i18next'; @@ -389,7 +389,7 @@ const FileEditorThumbnail = ({ style={isSupported || hasError ? undefined : { filter: 'grayscale(80%)', opacity: 0.6 }} >
- {file.thumbnailUrl && ( + {file.thumbnailUrl ? ( - )} + ) : file.type?.startsWith('application/pdf') ? ( + + + Loading thumbnail... + + ) : null}
{/* Drag handle (span wrapper so we can attach a ref reliably) */} diff --git a/frontend/src/core/components/shared/FileCard.tsx b/frontend/src/core/components/shared/FileCard.tsx index 6eff26db9..dda479175 100644 --- a/frontend/src/core/components/shared/FileCard.tsx +++ b/frontend/src/core/components/shared/FileCard.tsx @@ -1,5 +1,5 @@ import { useState } from "react"; -import { Card, Stack, Text, Group, Badge, Button, Box, Image, ThemeIcon, ActionIcon, Tooltip } from "@mantine/core"; +import { Card, Stack, Text, Group, Badge, Button, Box, Image, ThemeIcon, ActionIcon, Tooltip, Loader } from "@mantine/core"; import { useTranslation } from "react-i18next"; import PictureAsPdfIcon from "@mui/icons-material/PictureAsPdf"; import StorageIcon from "@mui/icons-material/Storage"; @@ -29,6 +29,10 @@ const FileCard = ({ file, fileStub, onRemove, onDoubleClick, onView, onEdit, isS const thumb = fileStub?.thumbnailUrl || indexedDBThumb; const [isHovered, setIsHovered] = useState(false); + // Show loading state during hydration: PDF file without thumbnail yet + const isPdf = file.type === 'application/pdf'; + const isHydrating = isPdf && !thumb && !isGenerating; + return ( - ) : isGenerating ? ( -
-
- Generating... -
+ ) : (isGenerating || isHydrating) ? ( + + + Loading... + ) : (
✕; case EXECUTION_STATUS.RUNNING: - return
; + return ; default: return
Promise> = []; + +const scheduleMetadataHydration = (task: () => Promise): void => { + hydrationQueue.push(task); + // Defer start to next tick to ensure initial ADD_FILES dispatch completes + Promise.resolve().then(drainHydrationQueue); +}; + +const drainHydrationQueue = (): void => { + if (activeHydrations >= HYDRATION_CONCURRENCY) return; + const nextTask = hydrationQueue.shift(); + if (!nextTask) return; + + activeHydrations++; + nextTask() + .catch(() => { + // Silently handle hydration failures + }) + .finally(() => { + activeHydrations--; + drainHydrationQueue(); + }); +}; /** * Simple mutex to prevent race conditions in addFiles @@ -210,12 +235,12 @@ export async function addFiles( try { const stirlingFileStubs: StirlingFileStub[] = []; const stirlingFiles: StirlingFile[] = []; + // Hydration tasks are scheduled per-file to update thumbnails/metadata without blocking add flow // Build quickKey lookup from existing files for deduplication const existingQuickKeys = buildQuickKeySet(stateRef.current.files.byId); const { files = [] } = options; - if (DEBUG) console.log(`📄 addFiles(raw): Adding ${files.length} files with immediate thumbnail generation`); // ZIP pre-processing: Extract ZIP files with configurable behavior // - File uploads: skipAutoUnzip=true → always extract (except HTML) @@ -275,43 +300,14 @@ export async function addFiles( // Soft deduplication: Check if file already exists by metadata if (existingQuickKeys.has(quickKey)) { - if (DEBUG) console.log(`📄 Skipping duplicate file: ${file.name} (quickKey: ${quickKey})`); continue; } - if (DEBUG) console.log(`📄 Adding new file: ${file.name} (quickKey: ${quickKey})`); const fileId = createFileId(); filesRef.current.set(fileId, file); - // Generate processedFile metadata using centralized function - const processedFileMetadata = await generateProcessedFileMetadata(file); - - // Extract thumbnail for non-PDF files or use from processedFile for PDFs - let thumbnail: string | undefined; - if (processedFileMetadata) { - // PDF file - use thumbnail from processedFile metadata - thumbnail = processedFileMetadata.thumbnailUrl; - if (DEBUG) console.log(`📄 Generated PDF metadata for ${file.name}: ${processedFileMetadata.totalPages} pages, thumbnail: SUCCESS`); - } else if (!file.type.startsWith('application/pdf')) { - // Non-PDF files: simple thumbnail generation, no processedFile metadata - try { - if (DEBUG) console.log(`📄 Generating simple thumbnail for non-PDF file ${file.name}`); - const { generateThumbnailForFile } = await import('@app/utils/thumbnailUtils'); - thumbnail = await generateThumbnailForFile(file); - if (DEBUG) console.log(`📄 Generated simple thumbnail for ${file.name}: no page count, thumbnail: SUCCESS`); - } catch (error) { - if (DEBUG) console.warn(`📄 Failed to generate simple thumbnail for ${file.name}:`, error); - } - } - - // Create new filestub with processedFile metadata - const fileStub = createNewStirlingFileStub(file, fileId, thumbnail, processedFileMetadata); - if (thumbnail) { - // Track blob URLs for cleanup (images return blob URLs that need revocation) - if (thumbnail.startsWith('blob:')) { - lifecycleManager.trackBlobUrl(thumbnail); - } - } + // Create new filestub with minimal metadata; hydrate thumbnails/processedFile asynchronously + const fileStub = createNewStirlingFileStub(file, fileId); // Store insertion position if provided if (options.insertAfterPageId !== undefined) { @@ -321,9 +317,53 @@ export async function addFiles( existingQuickKeys.add(quickKey); stirlingFileStubs.push(fileStub); + // Dispatch immediately so each file appears as soon as it is processed + dispatch({ type: 'ADD_FILES', payload: { stirlingFileStubs: [fileStub] } }); + // Create StirlingFile directly const stirlingFile = createStirlingFile(file, fileId); stirlingFiles.push(stirlingFile); + + // Queue background hydration so add flow doesn't block on thumbnail/metadata work + scheduleMetadataHydration(async () => { + const targetFile = filesRef.current.get(fileId); + if (!targetFile) { + return; + } + + let processedFileMetadata: ProcessedFileMetadata | undefined; + let thumbnail: string | undefined; + + if (targetFile.type.startsWith('application/pdf')) { + processedFileMetadata = await generateProcessedFileMetadata(targetFile); + thumbnail = processedFileMetadata?.thumbnailUrl; + } else { + try { + const { generateThumbnailForFile } = await import('@app/utils/thumbnailUtils'); + thumbnail = await generateThumbnailForFile(targetFile); + } catch { + // Silently handle thumbnail generation failures + } + } + + const updates: Partial = {}; + const primaryThumbnail = thumbnail || processedFileMetadata?.thumbnailUrl || processedFileMetadata?.pages?.[0]?.thumbnail; + + if (processedFileMetadata) { + updates.processedFile = processedFileMetadata; + updates.thumbnailUrl = primaryThumbnail; + } else if (thumbnail) { + updates.thumbnailUrl = primaryThumbnail; + } + + if (primaryThumbnail && primaryThumbnail.startsWith('blob:')) { + lifecycleManager.trackBlobUrl(primaryThumbnail); + } + + if (Object.keys(updates).length > 0) { + lifecycleManager.updateStirlingFileStub(fileId, updates, stateRef); + } + }); } // Persist to storage if enabled using fileStorage service @@ -343,11 +383,6 @@ export async function addFiles( })); } - // Dispatch ADD_FILES action if we have new files - if (stirlingFileStubs.length > 0) { - dispatch({ type: 'ADD_FILES', payload: { stirlingFileStubs } }); - } - return stirlingFiles; } finally { // Always release mutex even if error occurs @@ -557,16 +592,21 @@ export async function addStirlingFileStubs( stateRef: React.MutableRefObject, filesRef: React.MutableRefObject>, dispatch: React.Dispatch, - _lifecycleManager: FileLifecycleManager + lifecycleManager: FileLifecycleManager ): Promise { await addFilesMutex.lock(); try { + // Show loading indicator while preparing files from storage + if (stirlingFileStubs.length > 0) { + dispatch({ type: 'SET_PROCESSING', payload: { isProcessing: true, progress: 0 } }); + } const existingQuickKeys = buildQuickKeySet(stateRef.current.files.byId); - const validStubs: StirlingFileStub[] = []; const loadedFiles: StirlingFile[] = []; + let firstFileDispatched = false; + // Process and dispatch files one by one for progressive UI updates for (const stub of stirlingFileStubs) { // Check for duplicates using quickKey if (existingQuickKeys.has(stub.quickKey || '')) { @@ -574,16 +614,6 @@ export async function addStirlingFileStubs( continue; } - // Load the actual StirlingFile from storage - const stirlingFile = await fileStorage.getStirlingFile(stub.id); - if (!stirlingFile) { - console.warn(`📄 Failed to load StirlingFile for stub: ${stub.name} (${stub.id})`); - continue; - } - - // Store the loaded file in filesRef - filesRef.current.set(stub.id, stirlingFile); - // Use the original stub (preserves thumbnails, history, metadata!) const record = { ...stub }; @@ -592,38 +622,60 @@ export async function addStirlingFileStubs( record.insertAfterPageId = options.insertAfterPageId; } - // Check if processedFile data needs regeneration for proper Page Editor support - if (stirlingFile.type.startsWith('application/pdf')) { - const needsProcessing = !record.processedFile || - !record.processedFile.pages || - record.processedFile.pages.length === 0 || - record.processedFile.totalPages !== record.processedFile.pages.length; + existingQuickKeys.add(stub.quickKey || ''); - if (needsProcessing) { + // Dispatch each file immediately as we process it (progressive loading) + dispatch({ type: 'ADD_FILES', payload: { stirlingFileStubs: [record] } }); - // Use centralized metadata generation function - const processedFileMetadata = await generateProcessedFileMetadata(stirlingFile); - if (processedFileMetadata) { - record.processedFile = processedFileMetadata; - record.thumbnailUrl = processedFileMetadata.thumbnailUrl; // Update thumbnail if needed - } else { - // Fallback for files that couldn't be processed - if (DEBUG) console.warn(`📄 addStirlingFileStubs: Failed to regenerate processedFile for ${record.name}`); - if (!record.processedFile) { - record.processedFile = createProcessedFile(1); // Fallback to 1 page + // Clear loading indicator after first file appears + if (!firstFileDispatched) { + firstFileDispatched = true; + dispatch({ type: 'SET_PROCESSING', payload: { isProcessing: false, progress: 0 } }); + } + + // Load File object and hydrate metadata in background (non-blocking) + const fileId = stub.id; + + // Load File object from IndexedDB asynchronously + scheduleMetadataHydration(async () => { + const stirlingFile = await fileStorage.getStirlingFile(fileId); + if (!stirlingFile) { + console.warn(`📄 Failed to load StirlingFile for stub: ${stub.name} (${fileId})`); + return; + } + + // Store the loaded file in filesRef + filesRef.current.set(fileId, stirlingFile); + + // Check if processedFile data needs regeneration + if (stirlingFile.type.startsWith('application/pdf')) { + const needsProcessing = !stub.processedFile || + !stub.processedFile.pages || + stub.processedFile.pages.length === 0 || + stub.processedFile.totalPages !== stub.processedFile.pages.length; + + if (needsProcessing) { + // Regenerate metadata + const processedFileMetadata = await generateProcessedFileMetadata(stirlingFile); + if (processedFileMetadata) { + const updates: Partial = { + processedFile: processedFileMetadata + }; + + // Update thumbnail only if current stub doesn't have one + const currentStub = stateRef.current.files.byId[fileId]; + if (!currentStub?.thumbnailUrl && processedFileMetadata.thumbnailUrl) { + updates.thumbnailUrl = processedFileMetadata.thumbnailUrl; + if (processedFileMetadata.thumbnailUrl.startsWith('blob:')) { + lifecycleManager.trackBlobUrl(processedFileMetadata.thumbnailUrl); + } + } + + lifecycleManager.updateStirlingFileStub(fileId, updates, stateRef); } } } - } - - existingQuickKeys.add(stub.quickKey || ''); - validStubs.push(record); - loadedFiles.push(stirlingFile); - } - - // Dispatch ADD_FILES action if we have new files - if (validStubs.length > 0) { - dispatch({ type: 'ADD_FILES', payload: { stirlingFileStubs: validStubs } }); + }); } return loadedFiles; diff --git a/frontend/src/core/contexts/file/fileHooks.ts b/frontend/src/core/contexts/file/fileHooks.ts index 47995c746..6b5614c11 100644 --- a/frontend/src/core/contexts/file/fileHooks.ts +++ b/frontend/src/core/contexts/file/fileHooks.ts @@ -42,10 +42,12 @@ export function useCurrentFile(): { file?: File; record?: StirlingFileStub } { const { state, selectors } = useFileState(); const primaryFileId = state.files.ids[0]; + const primaryFileRecord = primaryFileId ? state.files.byId[primaryFileId] : undefined; + return useMemo(() => ({ file: primaryFileId ? selectors.getFile(primaryFileId) : undefined, record: primaryFileId ? selectors.getStirlingFileStub(primaryFileId) : undefined - }), [primaryFileId, selectors]); + }), [primaryFileId, primaryFileRecord, selectors]); } /** @@ -58,7 +60,7 @@ export function useFileSelection() { // Memoize selected files to avoid recreating arrays const selectedFiles = useMemo(() => { return selectors.getSelectedFiles(); - }, [state.ui.selectedFileIds, selectors]); + }, [state.ui.selectedFileIds, state.files.byId, selectors]); return useMemo(() => ({ selectedFiles, @@ -112,12 +114,13 @@ export function useFileUI() { * Hook for specific file by ID (optimized for individual file access) */ export function useStirlingFileStub(fileId: FileId): { file?: File; record?: StirlingFileStub } { - const { selectors } = useFileState(); + const { state, selectors } = useFileState(); + const fileRecord = state.files.byId[fileId]; return useMemo(() => ({ file: selectors.getFile(fileId), record: selectors.getStirlingFileStub(fileId) - }), [fileId, selectors]); + }), [fileId, fileRecord, selectors]); } /** @@ -130,7 +133,7 @@ export function useAllFiles(): { files: StirlingFile[]; fileStubs: StirlingFileS files: selectors.getFiles(), fileStubs: selectors.getStirlingFileStubs(), fileIds: state.files.ids - }), [state.files.ids, selectors]); + }), [state.files.ids, state.files.byId, selectors]); } /** @@ -143,7 +146,7 @@ export function useSelectedFiles(): { selectedFiles: StirlingFile[]; selectedFil selectedFiles: selectors.getSelectedFiles(), selectedFileStubs: selectors.getSelectedStirlingFileStubs(), selectedFileIds: state.ui.selectedFileIds - }), [state.ui.selectedFileIds, selectors]); + }), [state.ui.selectedFileIds, state.files.byId, selectors]); } // Navigation management removed - moved to NavigationContext