From 190cdc471a779ceeb8abf0dd6c967c2ea57a17e8 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sat, 30 Mar 2024 12:44:12 -0600 Subject: [PATCH] Recording fixes (#10728) * Use timerange everywhere and ensure recordings has last 24 hours * Pause recording when selecting timeline * Fix loading layout * Fix updating current time not always working * Simplify dynamic video player * Clean up desktop sizing * Fix current hour * Make padding consistent * Improve spacing for extra cameras * Make back button consistent * Fix preview player not jumping to correct time * Dont use useEffect due to preview changing * Simplify * Fix transition --- web/src/App.tsx | 2 +- web/src/components/card/ReviewCard.tsx | 4 +- web/src/components/player/PreviewPlayer.tsx | 126 +++++++------ .../player/dynamic/DynamicVideoPlayer.tsx | 29 +-- web/src/pages/Events.tsx | 1 + web/src/pages/Logs.tsx | 2 +- web/src/utils/timelineUtil.tsx | 25 ++- web/src/views/events/EventView.tsx | 18 +- web/src/views/events/RecordingView.tsx | 166 +++++++++++------- web/src/views/live/LiveBirdseyeView.tsx | 8 +- web/src/views/live/LiveCameraView.tsx | 6 +- web/src/views/live/LiveDashboardView.tsx | 6 +- 12 files changed, 210 insertions(+), 183 deletions(-) diff --git a/web/src/App.tsx b/web/src/App.tsx index dc82bbc07..c50c53307 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -31,7 +31,7 @@ function App() { {isMobile && }
diff --git a/web/src/components/card/ReviewCard.tsx b/web/src/components/card/ReviewCard.tsx index 5c579e4d4..4c547e4ea 100644 --- a/web/src/components/card/ReviewCard.tsx +++ b/web/src/components/card/ReviewCard.tsx @@ -33,11 +33,11 @@ export default function ReviewCard({ return (
void; onClick?: () => void; }; @@ -143,8 +143,6 @@ function PreviewVideoPlayer({ // initial state - const [loaded, setLoaded] = useState(false); - const [hasCanvas, setHasCanvas] = useState(false); const initialPreview = useMemo(() => { return cameraPreviews.find( (preview) => @@ -164,14 +162,52 @@ function PreviewVideoPlayer({ return; } + setCurrentHourFrame(undefined); + if (isAndroid && isChrome) { // android/chrome glitches when setting currentTime at the same time as onSeeked setTimeout(() => controller.finishedSeeking(), 25); } else { controller.finishedSeeking(); } + // we only want to update on controller change + // eslint-disable-next-line react-hooks/exhaustive-deps }, [controller]); + // canvas to cover preview transition + + const canvasRef = useRef(null); + const [videoSize, setVideoSize] = useState([0, 0]); + + const changeSource = useCallback( + (newPreview: Preview | undefined, video: HTMLVideoElement | null) => { + if (!newPreview || !video) { + setCurrentPreview(newPreview); + return; + } + + if (!canvasRef.current && videoSize[0] > 0) { + const canvas = document.createElement("canvas"); + canvas.width = videoSize[0]; + canvas.height = videoSize[1]; + canvasRef.current = canvas; + } + + const context = canvasRef.current?.getContext("2d"); + + if (context) { + context.drawImage(video, 0, 0, videoSize[0], videoSize[1]); + setCurrentHourFrame(canvasRef.current?.toDataURL("image/webp")); + } + + setCurrentPreview(newPreview); + + // we only want this to change when current preview changes + // eslint-disable-next-line react-hooks/exhaustive-deps + }, + [setCurrentHourFrame, videoSize], + ); + useEffect(() => { if (!controller) { return; @@ -185,8 +221,7 @@ function PreviewVideoPlayer({ ); if (preview != currentPreview) { - setCurrentPreview(preview); - setLoaded(false); + changeSource(preview, previewRef.current); } controller.newPlayback({ @@ -196,63 +231,21 @@ function PreviewVideoPlayer({ // we only want this to change when recordings update // eslint-disable-next-line react-hooks/exhaustive-deps - }, [controller, timeRange]); - - // canvas to cover preview transition - - const canvasRef = useRef(null); - const [videoWidth, videoHeight] = useMemo(() => { - if (!previewRef.current) { - return [0, 0]; - } - - return [previewRef.current.videoWidth, previewRef.current.videoHeight]; - // we know the video size will be known on load - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [loaded]); - // handle switching sources - - useEffect(() => { - if (!currentPreview || !previewRef.current) { - return; - } - - if (canvasRef.current) { - canvasRef.current - .getContext("2d") - ?.drawImage(previewRef.current, 0, 0, videoWidth, videoHeight); - setHasCanvas(true); - } - - if (isSafari) { - setTimeout(() => previewRef.current?.load(), 100); - } else { - previewRef.current.load(); - } - // we only want this to change when current preview changes - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [currentPreview, previewRef]); + }, [controller, timeRange, changeSource]); return (
- {currentHourFrame && ( - - )} - previewRef.current?.load()} /> - {!loaded && !hasCanvas && !currentHourFrame && ( - - )} {cameraPreviews && !currentPreview && (
No Preview Found @@ -362,7 +356,9 @@ class PreviewVideoController extends PreviewController { } override setNewPreviewStartTime(time: number) { - this.timeToSeek = time; + if (this.preview) { + this.timeToSeek = time - this.preview.start; + } } previewReady() { @@ -468,7 +464,7 @@ function PreviewFramesPlayer({ return (
("config"); - // playback behavior - - const grow = useMemo(() => { - if (!config) { - return "aspect-video"; - } - - const aspectRatio = - config.cameras[camera].detect.width / - config.cameras[camera].detect.height; - - if (aspectRatio > 2) { - return ""; - } else if (aspectRatio < 16 / 9) { - return isDesktop ? "" : "aspect-tall"; - } else { - return "aspect-video"; - } - }, [camera, config]); - // controlling playback const playerRef = useRef(null); @@ -169,9 +148,9 @@ export default function DynamicVideoPlayer({ }, [controller, recordings]); return ( -
+ <> -
+ ); } diff --git a/web/src/pages/Events.tsx b/web/src/pages/Events.tsx index f618f4aca..bbd59e49b 100644 --- a/web/src/pages/Events.tsx +++ b/web/src/pages/Events.tsx @@ -338,6 +338,7 @@ export default function Events() { reviewItems={reviews} reviewSummary={reviewSummary} allPreviews={allPreviews} + timeRange={selectedTimeRange} filter={reviewFilter} updateFilter={onUpdateFilter} /> diff --git a/web/src/pages/Logs.tsx b/web/src/pages/Logs.tsx index 77b9f05f0..e8ca9002f 100644 --- a/web/src/pages/Logs.tsx +++ b/web/src/pages/Logs.tsx @@ -53,7 +53,7 @@ function Logs() { ); return ( -
+
endOfThisHour) { break; @@ -146,7 +150,12 @@ export function getChunkedTimeDay(timestamp: number) { start = startDay.getTime() / 1000; } - return { start: startTimestamp, end, ranges: data }; + data.push({ + after: start, + before: Math.floor(timeRange.before), + }); + + return data; } export function getChunkedTimeRange( diff --git a/web/src/views/events/EventView.tsx b/web/src/views/events/EventView.tsx index e2d0c45e8..6d49762e3 100644 --- a/web/src/views/events/EventView.tsx +++ b/web/src/views/events/EventView.tsx @@ -46,7 +46,7 @@ type EventViewProps = { reviews?: ReviewSegment[]; reviewSummary?: ReviewSummary; relevantPreviews?: Preview[]; - timeRange: { before: number; after: number }; + timeRange: TimeRange; filter?: ReviewFilter; severity: ReviewSeverity; startTime?: number; @@ -205,7 +205,7 @@ export default function EventView({ } return ( -
+
{isMobile && ( @@ -492,7 +492,7 @@ function DetectionReview({ <>
{filter?.before == undefined && ( { - controller.setNewPreviewStartTime(currentTime); - }); + setPreviewStart(currentTime); setSelectedRangeIdx(index); } return; @@ -713,7 +713,9 @@ function MotionReview({ Object.values(videoPlayersRef.current).forEach((controller) => { controller.scrubToTimestamp(currentTime); }); - }, [currentTime, currentTimeRange, timeRangeSegments]); + // only refresh when current time or available segments changes + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentTime, timeRangeSegments]); // playback @@ -826,7 +828,7 @@ function MotionReview({ className={`${detectionType ? `outline outline-3 outline-offset-1 outline-severity_${detectionType}` : "outline-0 shadow-none"} rounded-2xl ${grow}`} camera={camera.name} timeRange={currentTimeRange} - startTime={startTime} + startTime={previewStart} cameraPreviews={relevantPreviews || []} isScrubbing={scrubbing} onControllerReady={(controller) => { diff --git a/web/src/views/events/RecordingView.tsx b/web/src/views/events/RecordingView.tsx index 3cea5001a..27f0d060b 100644 --- a/web/src/views/events/RecordingView.tsx +++ b/web/src/views/events/RecordingView.tsx @@ -46,6 +46,7 @@ type RecordingViewProps = { startTime: number; reviewItems?: ReviewSegment[]; reviewSummary?: ReviewSummary; + timeRange: TimeRange; allCameras: string[]; allPreviews?: Preview[]; filter?: ReviewFilter; @@ -56,6 +57,7 @@ export function RecordingView({ startTime, reviewItems, reviewSummary, + timeRange, allCameras, allPreviews, filter, @@ -85,15 +87,18 @@ export function RecordingView({ "timeline", ); - const timeRange = useMemo(() => getChunkedTimeDay(startTime), [startTime]); + const chunkedTimeRange = useMemo( + () => getChunkedTimeDay(timeRange), + [timeRange], + ); const [selectedRangeIdx, setSelectedRangeIdx] = useState( - timeRange.ranges.findIndex((chunk) => { + chunkedTimeRange.findIndex((chunk) => { return chunk.after <= startTime && chunk.before >= startTime; }), ); const currentTimeRange = useMemo( - () => timeRange.ranges[selectedRangeIdx], - [selectedRangeIdx, timeRange], + () => chunkedTimeRange[selectedRangeIdx], + [selectedRangeIdx, chunkedTimeRange], ); // export @@ -108,10 +113,10 @@ export function RecordingView({ return; } - if (selectedRangeIdx < timeRange.ranges.length - 1) { + if (selectedRangeIdx < chunkedTimeRange.length - 1) { setSelectedRangeIdx(selectedRangeIdx + 1); } - }, [selectedRangeIdx, timeRange]); + }, [selectedRangeIdx, chunkedTimeRange]); // scrubbing and timeline state @@ -121,7 +126,7 @@ export function RecordingView({ const updateSelectedSegment = useCallback( (currentTime: number, updateStartTime: boolean) => { - const index = timeRange.ranges.findIndex( + const index = chunkedTimeRange.findIndex( (seg) => seg.after <= currentTime && seg.before >= currentTime, ); @@ -133,7 +138,7 @@ export function RecordingView({ setSelectedRangeIdx(index); } }, - [timeRange], + [chunkedTimeRange], ); useEffect(() => { @@ -189,40 +194,53 @@ export function RecordingView({ // motion timeline data + const getCameraAspect = useCallback( + (cam: string) => { + if (!config) { + return undefined; + } + + const camera = config.cameras[cam]; + + if (!camera) { + return undefined; + } + + return camera.detect.width / camera.detect.height; + }, + [config], + ); + const mainCameraAspect = useMemo(() => { - if (!config) { + const aspectRatio = getCameraAspect(mainCamera); + + if (!aspectRatio) { return "normal"; - } - - const aspectRatio = - config.cameras[mainCamera].detect.width / - config.cameras[mainCamera].detect.height; - - if (aspectRatio > 2) { + } else if (aspectRatio > 2) { return "wide"; } else if (aspectRatio < 16 / 9) { return "tall"; } else { return "normal"; } - }, [config, mainCamera]); + }, [getCameraAspect, mainCamera]); const grow = useMemo(() => { - if (isMobile) { - return ""; - } - if (mainCameraAspect == "wide") { return "w-full aspect-wide"; - } else if (isDesktop && mainCameraAspect == "tall") { - return "h-full aspect-tall flex flex-col justify-center"; + } else if (mainCameraAspect == "tall") { + if (isDesktop) { + return "size-full aspect-tall flex flex-col justify-center"; + } else { + return "size-full"; + } } else { return "w-full aspect-video"; } }, [mainCameraAspect]); return ( -
+
{ + setExportRange(range); + + if (range != undefined) { + mainControllerRef.current?.pause(); + } + }} setMode={setExportMode} /> )} @@ -303,7 +327,7 @@ export function RecordingView({ camera={mainCamera} filter={filter} currentTime={currentTime} - latestTime={timeRange.end} + latestTime={timeRange.before} mode={exportMode} range={exportRange} onUpdateFilter={updateFilter} @@ -314,19 +338,26 @@ export function RecordingView({
-
+
{isDesktop && (
{allCameras.map((cam) => { - if (cam !== mainCamera) { - return ( -
- { - previewRefs.current[cam] = controller; - controller.scrubToTimestamp(startTime); - }} - onClick={() => onSelectCamera(cam)} - /> -
- ); + if (cam == mainCamera) { + return; } - return null; + + return ( +
+ { + previewRefs.current[cam] = controller; + controller.scrubToTimestamp(startTime); + }} + onClick={() => onSelectCamera(cam)} + /> +
+ ); })}
)} @@ -406,7 +442,7 @@ type TimelineProps = { contentRef: MutableRefObject; mainCamera: string; timelineType: TimelineType; - timeRange: { start: number; end: number }; + timeRange: TimeRange; mainCameraReviewItems: ReviewSegment[]; currentTime: number; exportRange?: TimeRange; @@ -429,8 +465,8 @@ function Timeline({ const { data: motionData } = useSWR([ "review/activity/motion", { - before: timeRange.end, - after: timeRange.start, + before: timeRange.before, + after: timeRange.after, scale: SEGMENT_DURATION / 2, cameras: mainCamera, }, @@ -455,7 +491,7 @@ function Timeline({
@@ -465,8 +501,8 @@ function Timeline({ setCurrentTime(review.start_time)} + onClick={() => { + setScrubbing(true); + setCurrentTime(review.start_time); + setScrubbing(false); + }} /> ); })} diff --git a/web/src/views/live/LiveBirdseyeView.tsx b/web/src/views/live/LiveBirdseyeView.tsx index 58655e76c..2fd54735c 100644 --- a/web/src/views/live/LiveBirdseyeView.tsx +++ b/web/src/views/live/LiveBirdseyeView.tsx @@ -116,7 +116,7 @@ export default function LiveBirdseyeView() { className={ fullscreen ? `fixed inset-0 bg-black z-30` - : `size-full flex flex-col ${isMobile ? "landscape:flex-row" : ""}` + : `size-full p-2 flex flex-col ${isMobile ? "landscape:flex-row" : ""}` } >
{!fullscreen ? ( ) : ( diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index 5f01e2698..b406f603b 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -204,7 +204,7 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) { className={ fullscreen ? `fixed inset-0 bg-black z-30` - : `size-full flex flex-col ${isMobile ? "landscape:flex-row" : ""}` + : `size-full p-2 flex flex-col ${isMobile ? "landscape:flex-row" : ""}` } >
navigate(-1)} > @@ -228,7 +228,7 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) { )}
{!isIOS && ( config?.birdseye, [config]); return ( -
+
{isMobile && ( -
+
@@ -164,7 +164,7 @@ export default function LiveDashboardView({ {events && events.length > 0 && ( -
+
{events.map((event) => { return ; })}