diff --git a/build.gradle b/build.gradle index 9aa86d6e46..a0cdec19ef 100644 --- a/build.gradle +++ b/build.gradle @@ -78,7 +78,7 @@ springBoot { allprojects { group = 'stirling.software' - version = '2.7.2' + version = '2.7.3' configurations.configureEach { exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat" diff --git a/frontend/public/locales/en-GB/translation.toml b/frontend/public/locales/en-GB/translation.toml index 7de7327dec..810c52d03d 100644 --- a/frontend/public/locales/en-GB/translation.toml +++ b/frontend/public/locales/en-GB/translation.toml @@ -4465,6 +4465,10 @@ toggleSidebar = "Toggle Sidebar" toggleBookmarks = "Toggle Bookmarks" print = "Print PDF" ruler = "Ruler / Measure" +readAloud = "Read Aloud" +readAloudSpeed = "Speed" +readAloudLanguage = "Language" +selectLanguage = "Select language" draw = "Draw" redact = "Redact" exitRedaction = "Exit Redaction Mode" diff --git a/frontend/src-tauri/tauri.conf.json b/frontend/src-tauri/tauri.conf.json index 1578a571d0..798a820d0a 100644 --- a/frontend/src-tauri/tauri.conf.json +++ b/frontend/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "../node_modules/@tauri-apps/cli/config.schema.json", "productName": "Stirling-PDF", - "version": "2.7.1", + "version": "2.7.3", "identifier": "stirling.pdf.dev", "build": { "frontendDist": "../dist", diff --git a/frontend/src/core/components/viewer/PdfViewerToolbar.tsx b/frontend/src/core/components/viewer/PdfViewerToolbar.tsx index 258b8ba430..c81049d6bf 100644 --- a/frontend/src/core/components/viewer/PdfViewerToolbar.tsx +++ b/frontend/src/core/components/viewer/PdfViewerToolbar.tsx @@ -259,6 +259,7 @@ export function PdfViewerToolbar({ > + + ); diff --git a/frontend/src/core/components/viewer/readAloudHighlight.test.ts b/frontend/src/core/components/viewer/readAloudHighlight.test.ts new file mode 100644 index 0000000000..6f1cd540fd --- /dev/null +++ b/frontend/src/core/components/viewer/readAloudHighlight.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest'; +import { computeReadAloudHighlightRect } from '@app/components/viewer/readAloudHighlight'; + +describe('computeReadAloudHighlightRect', () => { + it('returns page-relative coordinates for an unrotated text item', () => { + expect( + computeReadAloudHighlightRect({ + viewportTransform: [2, 0, 0, -2, 0, 1000], + textTransform: [1, 0, 0, 1, 120, 300], + itemWidth: 40, + itemHeight: 10, + }) + ).toEqual({ + left: 240, + top: 380, + width: 80, + height: 20, + }); + }); + + it('enforces minimum visible highlight dimensions', () => { + expect( + computeReadAloudHighlightRect({ + viewportTransform: [1, 0, 0, -1, 0, 600], + textTransform: [1, 0, 0, 1, 12, 40], + itemWidth: 2, + itemHeight: 4, + }) + ).toEqual({ + left: 12, + top: 548, + width: 8, + height: 12, + }); + }); + + it('returns null for invalid geometry', () => { + expect( + computeReadAloudHighlightRect({ + viewportTransform: [1, 0, 0, -1, 0, 600], + textTransform: [1, 0, 0, 1, 0, 0], + itemWidth: 10, + itemHeight: 0, + }) + ).toBeNull(); + }); + + it('returns the correct box for a 90 degree rotated viewport', () => { + expect( + computeReadAloudHighlightRect({ + viewportTransform: [0, 2, 2, 0, 0, 0], + textTransform: [1, 0, 0, 1, 120, 300], + itemWidth: 40, + itemHeight: 10, + }) + ).toEqual({ + left: 600, + top: 220, + width: 80, + height: 20, + }); + }); +}); diff --git a/frontend/src/core/components/viewer/readAloudHighlight.ts b/frontend/src/core/components/viewer/readAloudHighlight.ts new file mode 100644 index 0000000000..5a5db8f765 --- /dev/null +++ b/frontend/src/core/components/viewer/readAloudHighlight.ts @@ -0,0 +1,52 @@ +interface ReadAloudHighlightParams { + viewportTransform: number[]; + textTransform: number[]; + itemWidth: number; + itemHeight: number; +} + +export interface ReadAloudHighlightRect { + left: number; + top: number; + width: number; + height: number; +} + +function multiplyTransforms(left: number[], right: number[]): number[] { + const [a1, b1, c1, d1, e1, f1] = left; + const [a2, b2, c2, d2, e2, f2] = right; + + return [ + a1 * a2 + c1 * b2, + b1 * a2 + d1 * b2, + a1 * c2 + c1 * d2, + b1 * c2 + d1 * d2, + a1 * e2 + c1 * f2 + e1, + b1 * e2 + d1 * f2 + f1, + ]; +} + +export function computeReadAloudHighlightRect({ + viewportTransform, + textTransform, + itemWidth, + itemHeight, +}: ReadAloudHighlightParams): ReadAloudHighlightRect | null { + if (viewportTransform.length < 6 || textTransform.length < 6 || itemWidth <= 0 || itemHeight <= 0) { + return null; + } + + const [, , , , e, f] = multiplyTransforms(viewportTransform, textTransform); + const [va, vb, vc, vd] = viewportTransform; + const viewportScaleX = Math.hypot(va, vb); + const viewportScaleY = Math.hypot(vc, vd); + const width = Math.max(itemWidth * viewportScaleX, 8); + const height = Math.max(itemHeight * viewportScaleY, 12); + + return { + left: e, + top: f - height, + width, + height, + }; +} diff --git a/frontend/src/core/components/viewer/useStopReadAloudOnNavigation.ts b/frontend/src/core/components/viewer/useStopReadAloudOnNavigation.ts new file mode 100644 index 0000000000..5faa82964e --- /dev/null +++ b/frontend/src/core/components/viewer/useStopReadAloudOnNavigation.ts @@ -0,0 +1,52 @@ +import { useEffect, useRef } from 'react'; +import { useNavigationState } from '@app/contexts/NavigationContext'; +import { useViewer } from '@app/contexts/ViewerContext'; + +/** + * Hook that automatically stops read-aloud when navigating away from the viewer. + * Monitors: workbench changes and active file changes. + */ +export function useStopReadAloudOnNavigation( + isReadingAloud: boolean, + onStop: () => void +) { + const { workbench } = useNavigationState(); + const viewer = useViewer(); + + const previousStateRef = useRef({ + workbench, + activeFileIndex: viewer.activeFileIndex, + }); + + // Monitor workbench and file changes + useEffect(() => { + // Stop on workbench change + if (isReadingAloud && previousStateRef.current.workbench !== workbench) { + onStop(); + previousStateRef.current.workbench = workbench; + return; + } + + // Stop on active file change + if (isReadingAloud && previousStateRef.current.activeFileIndex !== viewer.activeFileIndex) { + onStop(); + previousStateRef.current.activeFileIndex = viewer.activeFileIndex; + return; + } + + previousStateRef.current.workbench = workbench; + previousStateRef.current.activeFileIndex = viewer.activeFileIndex; + }, [workbench, viewer.activeFileIndex, isReadingAloud, onStop]); + + // Stop on page unload (F5, navigation, close) + useEffect(() => { + if (!isReadingAloud) return; + + const handleBeforeUnload = () => { + onStop(); + }; + + window.addEventListener('beforeunload', handleBeforeUnload); + return () => window.removeEventListener('beforeunload', handleBeforeUnload); + }, [isReadingAloud, onStop]); +} diff --git a/frontend/src/core/components/viewer/useViewerReadAloud.ts b/frontend/src/core/components/viewer/useViewerReadAloud.ts new file mode 100644 index 0000000000..cf8eb3ed31 --- /dev/null +++ b/frontend/src/core/components/viewer/useViewerReadAloud.ts @@ -0,0 +1,667 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { computeReadAloudHighlightRect } from '@app/components/viewer/readAloudHighlight'; +import { useFileState } from '@app/contexts/FileContext'; +import { useViewer } from '@app/contexts/ViewerContext'; +import { useStopReadAloudOnNavigation } from '@app/components/viewer/useStopReadAloudOnNavigation'; +import { pdfWorkerManager } from '@app/services/pdfWorkerManager'; +import { StirlingFile } from '@app/types/fileContext'; +import { ZINDEX } from '@app/constants/zIndex'; + +interface TextItemWithGeometry { + str: string; + transform: number[]; + width: number; + height: number; + viewportTransform: number[]; +} + +function isTextItem(value: unknown): value is { + str: string; + transform: number[]; + width: number; + height: number; +} { + if (!value || typeof value !== 'object') return false; + const item = value as Record; + return ( + typeof item.str === 'string' && + Array.isArray(item.transform) && + typeof item.width === 'number' && + typeof item.height === 'number' + ); +} + +function createHighlightElement(item: TextItemWithGeometry, pageEl: HTMLElement): HTMLElement | null { + const highlightRect = computeReadAloudHighlightRect({ + viewportTransform: item.viewportTransform, + textTransform: item.transform, + itemWidth: item.width, + itemHeight: item.height, + }); + if (!highlightRect) return null; + + const highlight = document.createElement('div'); + highlight.style.position = 'absolute'; + highlight.style.left = `${highlightRect.left}px`; + highlight.style.top = `${highlightRect.top}px`; + highlight.style.width = `${highlightRect.width}px`; + highlight.style.height = `${highlightRect.height}px`; + highlight.style.backgroundColor = 'rgba(255, 193, 7, 0.6)'; + highlight.style.pointerEvents = 'none'; + highlight.style.zIndex = String(ZINDEX.VIEWER_HIGHLIGHT); + highlight.style.borderRadius = '2px'; + pageEl.appendChild(highlight); + + return highlight; +} + +export function useViewerReadAloud(defaultLanguage?: string) { + const viewer = useViewer(); + const { selectors } = useFileState(); + + const [isReadingAloud, setIsReadingAloud] = useState(false); + const [speechRate, setSpeechRate] = useState(1); + const [speechLanguage, setSpeechLanguage] = useState(defaultLanguage || 'en-US'); + const [speechVoice, setSpeechVoice] = useState(null); + const [supportedLanguageCodes, setSupportedLanguageCodes] = useState>(new Set()); + + const utteranceRef = useRef(null); + const highlightedElementsRef = useRef([]); + const textItemsRef = useRef([]); + const speechTextRef = useRef(''); + const speechWordsRef = useRef([]); + const speechCharIndexRef = useRef(0); + const currentWordIndexRef = useRef(0); + const currentPageNumberRef = useRef(1); + const restartingSpeechRef = useRef(false); + const restartTimeoutRef = useRef(null); + const pageAdvanceTimeoutRef = useRef(null); + const currentFileRef = useRef(null); + const totalPagesRef = useRef(0); + const speechRateRef = useRef(1); // Keep track of current rate without recreating dependent functions + const speechLanguageRef = useRef(defaultLanguage || 'en-US'); + + // Cache parsed PDF document and page text items to avoid reparsing on every zoom/scroll + const cachedPdfDocRef = useRef> | null>(null); + const cachedPageNumberRef = useRef(null); + const cachedTextItemsRef = useRef(null); + + // Helper to find best voice for language + const findVoiceForLanguage = useCallback((languageCode: string): SpeechSynthesisVoice | null => { + if (typeof window === 'undefined' || !window.speechSynthesis) return null; + + const voices = window.speechSynthesis.getVoices(); + if (!voices || voices.length === 0) return null; + + // Try exact match first + const exactMatch = voices.find(v => v.lang === languageCode); + if (exactMatch) return exactMatch; + + // Try matching just the language part (e.g., 'es' from 'es-ES') + const baseLang = languageCode.split('-')[0]; + const baseMatch = voices.find(v => v.lang.startsWith(baseLang)); + if (baseMatch) { + return baseMatch; + } + + // Fallback to any English voice if requested language not found + const englishMatch = voices.find(v => v.lang.startsWith('en')); + if (englishMatch) { + return englishMatch; + } + + // Last resort: use any available voice + return voices[0] || null; + }, []); + + // Sync speechRate state to ref so page advance callbacks always have current rate + useEffect(() => { + speechRateRef.current = speechRate; + }, [speechRate]); + + // Sync speechLanguage state to ref + useEffect(() => { + speechLanguageRef.current = speechLanguage; + }, [speechLanguage]); + + // Helper to get supported language codes from available voices + const getSupportedLanguageCodes = useCallback((): Set => { + if (typeof window === 'undefined' || !window.speechSynthesis) return new Set(); + + const voices = window.speechSynthesis.getVoices(); + const supportedCodes = new Set(); + + // For each voice, add its language code and base language code + voices.forEach(voice => { + supportedCodes.add(voice.lang); + const baseLang = voice.lang.split('-')[0]; + supportedCodes.add(baseLang); + }); + + // Also add English as fallback + supportedCodes.add('en'); + supportedCodes.add('en-GB'); + supportedCodes.add('en-US'); + + return supportedCodes; + }, []); + + // Wait for voices to load, then check if default language has a voice + useEffect(() => { + if (typeof window === 'undefined' || !window.speechSynthesis) return; + + const handleVoicesChanged = () => { + const supportedCodes = getSupportedLanguageCodes(); + setSupportedLanguageCodes(supportedCodes); + // Once voices are loaded, update the voice for current language + const voice = findVoiceForLanguage(speechLanguage); + setSpeechVoice(voice); + }; + + // Check if voices are already loaded + if (window.speechSynthesis.getVoices().length > 0) { + const supportedCodes = getSupportedLanguageCodes(); + setSupportedLanguageCodes(supportedCodes); + const voice = findVoiceForLanguage(speechLanguage); + setSpeechVoice(voice); + } else { + // Voices not loaded yet, listen for voiceschanged event + window.speechSynthesis.addEventListener('voiceschanged', handleVoicesChanged); + return () => { + window.speechSynthesis.removeEventListener('voiceschanged', handleVoicesChanged); + }; + } + }, [speechLanguage, findVoiceForLanguage, getSupportedLanguageCodes]); + + const clearHighlights = useCallback(() => { + highlightedElementsRef.current.forEach((el) => el.remove()); + highlightedElementsRef.current = []; + }, []); + + const cleanupReadingSession = useCallback(() => { + // Destroy the cached PDF document to free memory + if (cachedPdfDocRef.current) { + pdfWorkerManager.destroyDocument(cachedPdfDocRef.current); + cachedPdfDocRef.current = null; + cachedPageNumberRef.current = null; + cachedTextItemsRef.current = null; + } + }, []); + + const stopReadingAloud = useCallback(() => { + if (typeof window !== 'undefined' && window.speechSynthesis) { + window.speechSynthesis.cancel(); + } + if (pageAdvanceTimeoutRef.current !== null) { + window.clearTimeout(pageAdvanceTimeoutRef.current); + pageAdvanceTimeoutRef.current = null; + } + clearHighlights(); + setIsReadingAloud(false); + currentFileRef.current = null; + utteranceRef.current = null; + cleanupReadingSession(); + }, [clearHighlights, cleanupReadingSession]); + + // Stop reading when navigating away (workbench, file, or window change) + useStopReadAloudOnNavigation(isReadingAloud, stopReadingAloud); + + const highlightWord = useCallback((wordIndex: number, words: string[], pageNumber: number) => { + clearHighlights(); + if (wordIndex < 0 || wordIndex >= words.length) return; + + const wordToFind = words[wordIndex]; + if (!wordToFind) return; + + try { + let currentWordCount = 0; + const currentPageIndex = pageNumber - 1; + const pageEl = document.querySelector(`[data-page-index="${currentPageIndex}"]`) as HTMLElement | null; + if (!pageEl) return; + + for (const item of textItemsRef.current) { + const itemText = item.str.trim(); + if (!itemText) continue; + + const subWords = itemText.split(/\s+/); + for (let i = 0; i < subWords.length; i++) { + if (currentWordCount === wordIndex && subWords[i].toLowerCase() === wordToFind.toLowerCase()) { + const highlight = createHighlightElement(item, pageEl); + if (highlight) { + highlightedElementsRef.current.push(highlight); + } + return; + } + currentWordCount++; + } + } + } catch { + // Highlighting is best-effort only. + } + }, [clearHighlights]); + + const readPage = useCallback(async ( + currentFile: StirlingFile | File, + pageNumber: number, + options?: { + preserveSpeechState?: boolean; + highlightWordIndex?: number; + } + ) => { + let pdfDoc: Awaited> | null = null; + + try { + const zoom = (viewer.getZoomState().zoomPercent || 100) / 100; + + // If we have a cached document for the same file, reuse it instead of recreating + if (cachedPdfDocRef.current) { + pdfDoc = cachedPdfDocRef.current; + } else { + pdfDoc = await pdfWorkerManager.createDocument(await currentFile.arrayBuffer()); + cachedPdfDocRef.current = pdfDoc; + } + + const page = await pdfDoc.getPage(pageNumber); + const textContent = await page.getTextContent(); + // The highlight is rendered inside the page element, so we keep geometry in + // page-local coordinates and let the viewer's own rotation transform it. + const viewportTransform = page.getViewport({ scale: zoom }).transform; + + const textItems: TextItemWithGeometry[] = []; + for (const item of textContent.items) { + if (!isTextItem(item)) continue; + textItems.push({ + ...item, + viewportTransform, + }); + } + + // Sort text items by visual position (top-to-bottom, then left-to-right) + // to preserve reading order instead of PDF internal order + const sortedItems = [...textItems].sort((a, b) => { + // transform array is [a, b, c, d, e, f] where e=x, f=y (translation components) + const yA = a.transform[5] ?? 0; // y position + const yB = b.transform[5] ?? 0; + const xA = a.transform[4] ?? 0; // x position + const xB = b.transform[4] ?? 0; + + // Sort top-to-bottom (higher y first in PDF coordinates), then left-to-right + // 5px threshold for "same line" to group text on same horizontal line + if (Math.abs(yA - yB) > 5) { + return yB - yA; // Top to bottom + } + return xA - xB; // Left to right + }); + + // Merge adjacent text items on same line, using PDF spaces as word boundaries + // This fixes PDFs where characters/syllables are individual text items + const mergedItems: TextItemWithGeometry[] = []; + const CHAR_MERGE_THRESHOLD = 5; // px - merge adjacent chars/syllables closer than this + + for (const item of sortedItems) { + const itemText = item.str; + const isSpace = itemText.trim() === ''; + + // Spaces mark word boundaries - always push them separately + if (isSpace) { + mergedItems.push(item); + continue; + } + + const lastItem = mergedItems[mergedItems.length - 1]; + + // Only merge if last item exists, is not a space, and items are on same line + if (lastItem && lastItem.str.trim()) { + const yDiff = Math.abs((lastItem.transform[5] ?? 0) - (item.transform[5] ?? 0)); + const xGap = (item.transform[4] ?? 0) - ((lastItem.transform[4] ?? 0) + (lastItem.width ?? 0)); + + // Same line and very close horizontally? + if (yDiff < 5 && xGap < CHAR_MERGE_THRESHOLD) { + lastItem.str += itemText; + // Update width: add the new item's width plus any gap between them + lastItem.width = (lastItem.width ?? 0) + Math.max(0, xGap) + (item.width ?? 0); + continue; + } + } + mergedItems.push({ ...item, str: itemText }); + } + + // Use merged items for both highlighting and caching + // This ensures word counting in highlightWord matches the spoken text order + textItemsRef.current = mergedItems; + cachedTextItemsRef.current = mergedItems; + cachedPageNumberRef.current = pageNumber; + + const spokenText = mergedItems + .map((item) => item.str) + .join(' ') + .replace(/\s+/g, ' ') + .trim(); + + if (!spokenText) { + return null; + } + + const words = spokenText.split(/\s+/).filter(Boolean); + + if (!options?.preserveSpeechState) { + speechTextRef.current = spokenText; + speechWordsRef.current = words; + speechCharIndexRef.current = 0; + currentWordIndexRef.current = 0; + } + + const highlightIndex = Math.max(0, Math.min(options?.highlightWordIndex ?? 0, Math.max(words.length - 1, 0))); + highlightWord(highlightIndex, words, pageNumber); + return { spokenText, words }; + } catch (error) { + // Clear cache on error to avoid stale state + if (pdfDoc && pdfDoc === cachedPdfDocRef.current) { + cachedPdfDocRef.current = null; + } + throw error; + } + }, [highlightWord, viewer]); + + const speakFromCharIndex = useCallback(( + spokenText: string, + words: string[], + startCharIndex: number, + pageNumber: number, + rateOverride?: number, + languageOverride?: string, + ) => { + if (typeof window === 'undefined' || !window.speechSynthesis) { + return; + } + + const clampedStart = Math.max(0, Math.min(startCharIndex, spokenText.length)); + const remainingText = spokenText.slice(clampedStart).trimStart(); + const trimmedDelta = spokenText.slice(clampedStart).length - remainingText.length; + const baseCharIndex = clampedStart + trimmedDelta; + + if (!remainingText) { + clearHighlights(); + setIsReadingAloud(false); + currentFileRef.current = null; + currentWordIndexRef.current = 0; + utteranceRef.current = null; + cleanupReadingSession(); + return; + } + + const utterance = new SpeechSynthesisUtterance(remainingText); + utterance.rate = rateOverride ?? speechRateRef.current; + const currentLang = languageOverride ?? speechLanguageRef.current; + utterance.lang = currentLang; + + // Set specific voice if available + const voice = findVoiceForLanguage(currentLang); + if (voice) { + utterance.voice = voice; + } + + utterance.onstart = () => setIsReadingAloud(true); + utterance.onend = () => { + utteranceRef.current = null; + if (restartingSpeechRef.current) { + return; + } + if (currentFileRef.current && pageNumber < totalPagesRef.current) { + clearHighlights(); + speechCharIndexRef.current = 0; + currentWordIndexRef.current = 0; + currentPageNumberRef.current = pageNumber + 1; + viewer.scrollActions.scrollToPage(pageNumber + 1, 'smooth'); + if (pageAdvanceTimeoutRef.current !== null) { + window.clearTimeout(pageAdvanceTimeoutRef.current); + } + pageAdvanceTimeoutRef.current = window.setTimeout(async () => { + pageAdvanceTimeoutRef.current = null; + try { + if (!currentFileRef.current) { + currentFileRef.current = null; + setIsReadingAloud(false); + cleanupReadingSession(); + return; + } + const nextPageData = await readPage(currentFileRef.current, pageNumber + 1); + if (!nextPageData) { + currentFileRef.current = null; + setIsReadingAloud(false); + cleanupReadingSession(); + return; + } + speakFromCharIndex(nextPageData.spokenText, nextPageData.words, 0, pageNumber + 1, speechRateRef.current, speechLanguageRef.current); + } catch (error) { + console.error('Read aloud page advance failed', error); + currentFileRef.current = null; + clearHighlights(); + setIsReadingAloud(false); + cleanupReadingSession(); + } + }, 250); + return; + } + clearHighlights(); + setIsReadingAloud(false); + speechCharIndexRef.current = spokenText.length; + currentFileRef.current = null; + currentWordIndexRef.current = 0; + cleanupReadingSession(); + }; + utterance.onerror = () => { + utteranceRef.current = null; + if (restartingSpeechRef.current) { + return; + } + restartingSpeechRef.current = false; + currentFileRef.current = null; + currentWordIndexRef.current = 0; + clearHighlights(); + setIsReadingAloud(false); + cleanupReadingSession(); + }; + utterance.onboundary = (event: SpeechSynthesisEvent) => { + if (event.name !== 'word') return; + + const absoluteCharIndex = baseCharIndex + event.charIndex; + speechCharIndexRef.current = absoluteCharIndex; + + let charCount = 0; + for (let i = 0; i < words.length; i++) { + const wordStart = charCount; + const wordEnd = charCount + words[i].length; + if (absoluteCharIndex >= wordStart && absoluteCharIndex < wordEnd) { + currentWordIndexRef.current = i; + currentPageNumberRef.current = pageNumber; + highlightWord(i, words, pageNumber); + break; + } + charCount = wordEnd + 1; + } + }; + + utteranceRef.current = utterance; + window.speechSynthesis.speak(utterance); + }, [clearHighlights, cleanupReadingSession, readPage, viewer.scrollActions]); + + const refreshActiveHighlight = useCallback(() => { + if (!isReadingAloud || !currentFileRef.current || !cachedTextItemsRef.current) { + return; + } + + // Use cached text items to refresh highlights without reparsing the PDF document. + // This is critical for performance during zoom/scroll updates while audio is playing. + const words = speechWordsRef.current; + const pageNumber = currentPageNumberRef.current; + const wordIndex = currentWordIndexRef.current; + + highlightWord(wordIndex, words, pageNumber); + }, [isReadingAloud, highlightWord]); + + const handleReadAloud = useCallback(async () => { + if (typeof window === 'undefined' || !window.speechSynthesis) { + return; + } + + if (isReadingAloud) { + restartingSpeechRef.current = false; + if (pageAdvanceTimeoutRef.current !== null) { + window.clearTimeout(pageAdvanceTimeoutRef.current); + pageAdvanceTimeoutRef.current = null; + } + window.speechSynthesis.cancel(); + clearHighlights(); + setIsReadingAloud(false); + currentFileRef.current = null; + utteranceRef.current = null; + cleanupReadingSession(); + return; + } + + try { + const selectedFiles = selectors.getSelectedFiles(); + const currentFile = selectedFiles[viewer.activeFileIndex] ?? selectedFiles[0]; + if (!currentFile) return; + currentFileRef.current = currentFile; + totalPagesRef.current = viewer.getScrollState().totalPages || 0; + + setIsReadingAloud(true); + try { + const currentPage = viewer.getScrollState().currentPage || 1; + currentPageNumberRef.current = currentPage; + const pageData = await readPage(currentFile, currentPage); + if (!pageData) { + currentFileRef.current = null; + setIsReadingAloud(false); + cleanupReadingSession(); + return; + } + + window.speechSynthesis.cancel(); + speakFromCharIndex(pageData.spokenText, pageData.words, 0, currentPage, speechRateRef.current, speechLanguageRef.current); + } finally { + // readPage handles pdf worker cleanup + } + } catch (error) { + console.error('Read aloud failed', error); + currentFileRef.current = null; + clearHighlights(); + setIsReadingAloud(false); + cleanupReadingSession(); + } + }, [clearHighlights, cleanupReadingSession, highlightWord, isReadingAloud, selectors, speakFromCharIndex, viewer]); + + const handleSpeechRateChange = useCallback((nextRate: number) => { + setSpeechRate(nextRate); + + if (!isReadingAloud || !utteranceRef.current || !speechTextRef.current || typeof window === 'undefined' || !window.speechSynthesis) { + return; + } + + restartingSpeechRef.current = true; + setIsReadingAloud(true); + window.speechSynthesis.cancel(); + utteranceRef.current = null; + if (restartTimeoutRef.current !== null) { + window.clearTimeout(restartTimeoutRef.current); + } + restartTimeoutRef.current = window.setTimeout(() => { + restartTimeoutRef.current = null; + if (!restartingSpeechRef.current) { + return; + } + restartingSpeechRef.current = false; + speakFromCharIndex( + speechTextRef.current, + speechWordsRef.current, + speechCharIndexRef.current, + currentPageNumberRef.current, + nextRate, + speechLanguageRef.current + ); + }, 80); + }, [isReadingAloud, speakFromCharIndex, viewer]); + + useEffect(() => { + return viewer.registerImmediateZoomUpdate(() => { + requestAnimationFrame(() => { + refreshActiveHighlight(); + }); + }); + }, [refreshActiveHighlight, viewer]); + + useEffect(() => { + return viewer.registerImmediateScrollUpdate(() => { + requestAnimationFrame(() => { + refreshActiveHighlight(); + }); + }); + }, [refreshActiveHighlight, viewer]); + + useEffect(() => { + return () => { + if (restartTimeoutRef.current !== null) { + window.clearTimeout(restartTimeoutRef.current); + } + if (pageAdvanceTimeoutRef.current !== null) { + window.clearTimeout(pageAdvanceTimeoutRef.current); + } + if (typeof window !== 'undefined' && window.speechSynthesis) { + window.speechSynthesis.cancel(); + } + restartingSpeechRef.current = false; + clearHighlights(); + currentFileRef.current = null; + currentWordIndexRef.current = 0; + utteranceRef.current = null; + cleanupReadingSession(); + }; + }, [clearHighlights, cleanupReadingSession]); + + const handleSpeechLanguageChange = useCallback((nextLanguage: string) => { + setSpeechLanguage(nextLanguage); + speechLanguageRef.current = nextLanguage; + const voice = findVoiceForLanguage(nextLanguage); + setSpeechVoice(voice); + + if (!isReadingAloud || !utteranceRef.current || !speechTextRef.current || typeof window === 'undefined' || !window.speechSynthesis) { + return; + } + + restartingSpeechRef.current = true; + setIsReadingAloud(true); + window.speechSynthesis.cancel(); + utteranceRef.current = null; + if (restartTimeoutRef.current !== null) { + window.clearTimeout(restartTimeoutRef.current); + } + restartTimeoutRef.current = window.setTimeout(() => { + restartTimeoutRef.current = null; + if (!restartingSpeechRef.current) { + return; + } + restartingSpeechRef.current = false; + speakFromCharIndex( + speechTextRef.current, + speechWordsRef.current, + speechCharIndexRef.current, + currentPageNumberRef.current, + speechRateRef.current, + nextLanguage + ); + }, 80); + }, [isReadingAloud, speakFromCharIndex]); + + return { + isReadingAloud, + speechRate, + speechLanguage, + speechVoice, + supportedLanguageCodes, + handleReadAloud, + handleSpeechRateChange, + handleSpeechLanguageChange, + }; +} diff --git a/frontend/src/core/components/viewer/useViewerRightRailButtons.tsx b/frontend/src/core/components/viewer/useViewerRightRailButtons.tsx index 6da7a33716..bf7b2de833 100644 --- a/frontend/src/core/components/viewer/useViewerRightRailButtons.tsx +++ b/frontend/src/core/components/viewer/useViewerRightRailButtons.tsx @@ -1,6 +1,7 @@ import { useMemo, useState, useEffect, useCallback } from 'react'; -import { ActionIcon, Popover } from '@mantine/core'; +import { ActionIcon, Slider, Popover, Select } from '@mantine/core'; import { useTranslation } from 'react-i18next'; +import { supportedLanguages } from '@app/i18n'; import { useViewer } from '@app/contexts/ViewerContext'; import { useRightRailButtons, RightRailButtonWithAction } from '@app/hooks/useRightRailButtons'; import LocalIcon from '@app/components/shared/LocalIcon'; @@ -15,6 +16,9 @@ import { BASE_PATH, withBasePath } from '@app/constants/app'; import { useRedaction, useRedactionMode } from '@app/contexts/RedactionContext'; import TextFieldsIcon from '@mui/icons-material/TextFields'; import StraightenIcon from '@mui/icons-material/Straighten'; +import VolumeUpIcon from '@mui/icons-material/VolumeUp'; +import StopIcon from '@mui/icons-material/Stop'; +import { useViewerReadAloud } from '@app/components/viewer/useViewerReadAloud'; export function useViewerRightRailButtons( isRulerActive?: boolean, @@ -31,9 +35,10 @@ export function useViewerRightRailButtons( const { requestNavigation } = useNavigationGuard(); const { redactionsApplied, activeType: redactionActiveType } = useRedaction(); const { pendingCount } = useRedactionMode(); + const { isReadingAloud, speechRate, speechLanguage, speechVoice, supportedLanguageCodes, handleReadAloud, handleSpeechRateChange, handleSpeechLanguageChange } = useViewerReadAloud( + i18n.language || 'en-US' + ); - // Subscribe to immediate pan updates so button state stays in sync with actual pan state - // This handles cases where annotation/redaction tools disable pan mode useEffect(() => { return registerImmediatePanUpdate((newIsPanning) => { setIsPanning(newIsPanning); @@ -54,7 +59,6 @@ export function useViewerRightRailButtons( const [isAnnotationsActive, setIsAnnotationsActive] = useState(() => isAnnotationsPath()); - // Update isAnnotationsActive based on selected tool useEffect(() => { if (selectedTool === 'annotate') { setIsAnnotationsActive(true); @@ -73,7 +77,6 @@ export function useViewerRightRailButtons( return () => window.removeEventListener('popstate', handlePopState); }, [isAnnotationsPath]); - // Lift i18n labels out of memo for clarity const searchLabel = t('rightRail.search', 'Search PDF'); const panLabel = t('rightRail.panMode', 'Pan Mode'); const applyRedactionsLabel = t('rightRail.applyRedactionsFirst', 'Apply redactions first'); @@ -85,10 +88,24 @@ export function useViewerRightRailButtons( const printLabel = t('rightRail.print', 'Print PDF'); const annotationsLabel = t('rightRail.annotations', 'Annotations'); const formFillLabel = t('rightRail.formFill', 'Fill Form'); + const rulerLabel = t('rightRail.ruler', 'Ruler / Measure'); + const readAloudLabel = t('rightRail.readAloud', 'Read Aloud'); + const readAloudSpeedLabel = t('rightRail.readAloudSpeed', 'Speed'); const isFormFillActive = (selectedTool as string) === 'formFill'; - const rulerLabel = t('rightRail.ruler', 'Ruler / Measure'); + // Filter languages based on available voices + const filteredLanguages = useMemo(() => + Object.entries(supportedLanguages) + .filter(([code]) => supportedLanguageCodes.size === 0 || supportedLanguageCodes.has(code) || supportedLanguageCodes.has(code.split('-')[0])) + .map(([code, label]) => ({ + value: code, + label: label, + })), + [supportedLanguageCodes] + ); + + const shouldShowLanguageSelector = supportedLanguageCodes.size === 0 || filteredLanguages.length > 1; const viewerButtons = useMemo(() => { const buttons: RightRailButtonWithAction[] = [ @@ -156,7 +173,6 @@ export function useViewerRightRailButtons( onClick: () => { const next = !isRulerActive; setIsRulerActive?.(next); - // Disable pan when activating ruler — they conflict if (next && viewer.getPanState()?.isPanning) { viewer.panActions.togglePan(); setIsPanning(false); @@ -232,6 +248,82 @@ export function useViewerRightRailButtons( viewer.printActions.print(); } }, + { + id: 'viewer-read-aloud', + tooltip: readAloudLabel, + ariaLabel: readAloudLabel, + section: 'top' as const, + order: 57, + active: isReadingAloud, + render: ({ disabled }) => ( + {}} + withinPortal + > + +
+ + + {isReadingAloud ? : } + + +
+ + +
+
+ {readAloudSpeedLabel}: {speechRate.toFixed(1)}x +
+ + {shouldShowLanguageSelector && ( +