Multi cam recording view (#10244)

* Split recording view for mobile and desktop and get desktop working

* Get stuff working well

* Handle onclick for video

* Fix camera grid

* set onclick

web/src/components/player/DynamicVideoPlayer.tsx

@@ -30,6 +30,7 @@ type DynamicVideoPlayerProps = {
   cameraPreviews: Preview[];
   previewOnly?: boolean;
   onControllerReady?: (controller: DynamicVideoController) => void;
+  onClick?: () => void;
 };
 export default function DynamicVideoPlayer({
   className,
@@ -38,6 +39,7 @@ export default function DynamicVideoPlayer({
   cameraPreviews,
   previewOnly = false,
   onControllerReady,
+  onClick,
 }: DynamicVideoPlayerProps) {
   const apiHost = useApiHost();
   const { data: config } = useSWR<FrigateConfig>("config");
@@ -83,6 +85,8 @@ export default function DynamicVideoPlayer({
     );
   }, [camera, config, previewOnly]);
 
+  const [hasRecordingAtTime, setHasRecordingAtTime] = useState(true);
+
   // keyboard control
 
   const onKeyboardShortcut = useCallback(
@@ -145,28 +149,35 @@ export default function DynamicVideoPlayer({
     // we only want to calculate this once
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
-  const initialPreviewSource = useMemo(() => {
-    const preview = cameraPreviews.find(
+  const initialPreview = useMemo(() => {
+    return cameraPreviews.find(
       (preview) =>
         preview.camera == camera &&
         Math.round(preview.start) >= timeRange.start &&
         Math.floor(preview.end) <= timeRange.end,
     );
 
-    if (preview) {
-      return {
-        src: preview.src,
-        type: preview.type,
-      };
-    } else {
-      return undefined;
-    }
-
     // we only want to calculate this once
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
-  const [currentPreview, setCurrentPreview] = useState(initialPreviewSource);
+  const [currentPreview, setCurrentPreview] = useState(initialPreview);
+
+  const onPreviewSeeked = useCallback(() => {
+    if (!controller) {
+      return;
+    }
+
+    controller.finishedSeeking();
+
+    if (currentPreview && previewOnly && previewRef.current && onClick) {
+      setHasRecordingAtTime(
+        controller.hasRecordingAtTime(
+          currentPreview.start + previewRef.current.currentTime,
+        ),
+      );
+    }
+  }, [controller, currentPreview, onClick, previewOnly]);
 
   // state of playback player
 
@@ -177,7 +188,9 @@ export default function DynamicVideoPlayer({
     };
   }, [timeRange]);
   const { data: recordings } = useSWR<Recording[]>(
-    previewOnly ? null : [`${camera}/recordings`, recordingParams],
+    previewOnly && onClick == undefined
+      ? null
+      : [`${camera}/recordings`, recordingParams],
     { revalidateOnFocus: false },
   );
 
@@ -217,7 +230,10 @@ export default function DynamicVideoPlayer({
   }
 
   return (
-    <div className={className}>
+    <div
+      className={`relative ${className ?? ""} ${onClick ? (hasRecordingAtTime ? "cursor-pointer" : "") : ""}`}
+      onClick={onClick}
+    >
       {!previewOnly && (
         <div
           className={`w-full relative ${
@@ -272,7 +288,7 @@ export default function DynamicVideoPlayer({
         autoPlay
         playsInline
         muted
-        onSeeked={() => controller.finishedSeeking()}
+        onSeeked={onPreviewSeeked}
         onLoadedData={() => controller.previewReady()}
         onLoadStart={
           previewOnly && onControllerReady
@@ -286,6 +302,9 @@ export default function DynamicVideoPlayer({
           <source src={currentPreview.src} type={currentPreview.type} />
         )}
       </video>
+      {onClick && !hasRecordingAtTime && (
+        <div className="absolute inset-0 z-10 bg-black bg-opacity-60" />
+      )}
     </div>
   );
 }
@@ -406,7 +425,7 @@ export class DynamicVideoController {
     }
   }
 
-  onPlayerTimeUpdate(listener: (timestamp: number) => void) {
+  onPlayerTimeUpdate(listener: ((timestamp: number) => void) | undefined) {
     this.onPlaybackTimestamp = listener;
   }
 
@@ -508,4 +527,16 @@ export class DynamicVideoController {
     this.previewRef.current?.pause();
     this.readyToScrub = true;
   }
+
+  hasRecordingAtTime(time: number): boolean {
+    if (!this.recordings || this.recordings.length == 0) {
+      return false;
+    }
+
+    return (
+      this.recordings.find(
+        (segment) => segment.start_time <= time && segment.end_time >= time,
+      ) != undefined
+    );
+  }
 }

web/src/pages/Events.tsx

@@ -11,9 +11,13 @@ import {
   ReviewSummary,
 } from "@/types/review";
 import EventView from "@/views/events/EventView";
-import RecordingView from "@/views/events/RecordingView";
+import {
+  DesktopRecordingView,
+  MobileRecordingView,
+} from "@/views/events/RecordingView";
 import axios from "axios";
 import { useCallback, useMemo, useState } from "react";
+import { isMobile } from "react-device-detect";
 import useSWR from "swr";
 import useSWRInfinite from "swr/infinite";
 
@@ -237,6 +241,10 @@ export default function Events() {
   // selected items
 
   const selectedData = useMemo(() => {
+    if (!config) {
+      return undefined;
+    }
+
     if (!selectedReviewId) {
       return undefined;
     }
@@ -245,6 +253,8 @@ export default function Events() {
       return undefined;
     }
 
+    const allCameras = reviewFilter?.cameras ?? Object.keys(config.cameras);
+
     const allReviews = reviewPages.flat();
     const selectedReview = allReviews.find(
       (item) => item.id == selectedReviewId,
@@ -256,11 +266,9 @@ export default function Events() {
 
     return {
       selected: selectedReview,
-      cameraSegments: allReviews.filter(
-        (seg) => seg.camera == selectedReview.camera,
-      ),
-      cameraPreviews: allPreviews?.filter(
-        (seg) => seg.camera == selectedReview.camera,
+      allCameras: allCameras,
+      cameraSegments: allReviews.filter((seg) =>
+        allCameras.includes(seg.camera),
       ),
     };
 
@@ -273,11 +281,24 @@ export default function Events() {
   }
 
   if (selectedData) {
+    if (isMobile) {
       return (
-      <RecordingView
+        <MobileRecordingView
           reviewItems={selectedData.cameraSegments}
           selectedReview={selectedData.selected}
-        relevantPreviews={selectedData.cameraPreviews}
+          relevantPreviews={allPreviews}
+        />
+      );
+    }
+
+    return (
+      <DesktopRecordingView
+        startCamera={selectedData.selected.camera}
+        startTime={selectedData.selected.start_time}
+        allCameras={selectedData.allCameras}
+        severity={selectedData.selected.severity}
+        reviewItems={selectedData.cameraSegments}
+        allPreviews={allPreviews}
       />
     );
   } else {

web/src/views/events/EventView.tsx

@@ -627,8 +627,9 @@ function MotionReview({
             grow = "aspect-video";
           }
           return (
-            <div key={camera.name} className={`${grow}`}>
             <DynamicVideoPlayer
+              key={camera.name}
+              className={`${grow}`}
               camera={camera.name}
               timeRange={timeRangeSegments.ranges[selectedRangeIdx]}
               cameraPreviews={relevantPreviews || []}
@@ -638,7 +639,6 @@ function MotionReview({
                 setPlayerReady(true);
               }}
             />
-            </div>
           );
         })}
       </div>

web/src/views/events/RecordingView.tsx

@@ -4,22 +4,211 @@ import DynamicVideoPlayer, {
 import EventReviewTimeline from "@/components/timeline/EventReviewTimeline";
 import { Button } from "@/components/ui/button";
 import { Preview } from "@/types/preview";
-import { ReviewSegment } from "@/types/review";
+import { ReviewSegment, ReviewSeverity } from "@/types/review";
 import { getChunkedTimeDay } from "@/utils/timelineUtil";
-import { useEffect, useMemo, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { IoMdArrowRoundBack } from "react-icons/io";
 import { useNavigate } from "react-router-dom";
 
-type RecordingViewProps = {
+type DesktopRecordingViewProps = {
+  startCamera: string;
+  startTime: number;
+  severity: ReviewSeverity;
+  reviewItems: ReviewSegment[];
+  allCameras: string[];
+  allPreviews?: Preview[];
+};
+export function DesktopRecordingView({
+  startCamera,
+  startTime,
+  severity,
+  reviewItems,
+  allCameras,
+  allPreviews,
+}: DesktopRecordingViewProps) {
+  const navigate = useNavigate();
+  const contentRef = useRef<HTMLDivElement | null>(null);
+
+  // controller state
+
+  const [playerReady, setPlayerReady] = useState(false);
+  const [mainCamera, setMainCamera] = useState(startCamera);
+  const videoPlayersRef = useRef<{ [camera: string]: DynamicVideoController }>(
+    {},
+  );
+
+  // timeline time
+
+  const timeRange = useMemo(() => getChunkedTimeDay(startTime), [startTime]);
+  const [selectedRangeIdx, setSelectedRangeIdx] = useState(
+    timeRange.ranges.findIndex((chunk) => {
+      return chunk.start <= startTime && chunk.end >= startTime;
+    }),
+  );
+
+  // move to next clip
+  useEffect(() => {
+    if (
+      !videoPlayersRef.current &&
+      Object.values(videoPlayersRef.current).length > 0
+    ) {
+      return;
+    }
+
+    const firstController = Object.values(videoPlayersRef.current)[0];
+
+    if (firstController) {
+      firstController.onClipChangedEvent((dir) => {
+        if (
+          dir == "forward" &&
+          selectedRangeIdx < timeRange.ranges.length - 1
+        ) {
+          setSelectedRangeIdx(selectedRangeIdx + 1);
+        } else if (selectedRangeIdx > 0) {
+          setSelectedRangeIdx(selectedRangeIdx - 1);
+        }
+      });
+    }
+  }, [selectedRangeIdx, timeRange, videoPlayersRef, playerReady]);
+
+  // scrubbing and timeline state
+
+  const [scrubbing, setScrubbing] = useState(false);
+  const [currentTime, setCurrentTime] = useState<number>(startTime);
+
+  useEffect(() => {
+    if (scrubbing) {
+      Object.values(videoPlayersRef.current).forEach((controller) => {
+        controller.scrubToTimestamp(currentTime);
+      });
+    }
+  }, [currentTime, scrubbing]);
+
+  useEffect(() => {
+    if (!scrubbing) {
+      videoPlayersRef.current[mainCamera]?.seekToTimestamp(currentTime, true);
+    }
+
+    // we only want to seek when user stops scrubbing
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [scrubbing]);
+
+  const onSelectCamera = useCallback(
+    (newCam: string) => {
+      videoPlayersRef.current[mainCamera].onPlayerTimeUpdate(undefined);
+      videoPlayersRef.current[mainCamera].scrubToTimestamp(currentTime);
+      videoPlayersRef.current[newCam].seekToTimestamp(currentTime, true);
+      videoPlayersRef.current[newCam].onPlayerTimeUpdate(
+        (timestamp: number) => {
+          setCurrentTime(timestamp);
+
+          allCameras.forEach((cam) => {
+            if (cam != newCam) {
+              videoPlayersRef.current[cam]?.scrubToTimestamp(
+                Math.floor(timestamp),
+              );
+            }
+          });
+        },
+      );
+      setMainCamera(newCam);
+    },
+    [allCameras, currentTime, mainCamera],
+  );
+
+  return (
+    <div ref={contentRef} className="relative size-full">
+      <Button
+        className="absolute top-0 left-0 rounded-lg"
+        onClick={() => navigate(-1)}
+      >
+        <IoMdArrowRoundBack className="size-5 mr-[10px]" />
+        Back
+      </Button>
+
+      <div className="absolute h-32 left-2 right-28 bottom-4 flex justify-center gap-1">
+        {allCameras.map((cam) => {
+          if (cam == mainCamera) {
+            return (
+              <div
+                key={cam}
+                className="fixed left-96 right-96 top-[40%] -translate-y-[50%]"
+              >
+                <DynamicVideoPlayer
+                  camera={cam}
+                  timeRange={timeRange.ranges[selectedRangeIdx]}
+                  cameraPreviews={allPreviews ?? []}
+                  onControllerReady={(controller) => {
+                    videoPlayersRef.current[cam] = controller;
+                    setPlayerReady(true);
+                    controller.onPlayerTimeUpdate((timestamp: number) => {
+                      setCurrentTime(timestamp);
+
+                      allCameras.forEach((otherCam) => {
+                        if (cam != otherCam) {
+                          videoPlayersRef.current[otherCam]?.scrubToTimestamp(
+                            Math.floor(timestamp),
+                          );
+                        }
+                      });
+                    });
+
+                    controller.seekToTimestamp(startTime, true);
+                  }}
+                />
+              </div>
+            );
+          }
+
+          return (
+            <div key={cam} className="aspect-video flex items-center">
+              <DynamicVideoPlayer
+                className="size-full"
+                camera={cam}
+                timeRange={timeRange.ranges[selectedRangeIdx]}
+                cameraPreviews={allPreviews ?? []}
+                previewOnly
+                onControllerReady={(controller) => {
+                  videoPlayersRef.current[cam] = controller;
+                  setPlayerReady(true);
+                  controller.scrubToTimestamp(startTime);
+                }}
+                onClick={() => onSelectCamera(cam)}
+              />
+            </div>
+          );
+        })}
+      </div>
+
+      <div className="absolute overflow-hidden w-56 inset-y-0 right-0">
+        <EventReviewTimeline
+          segmentDuration={30}
+          timestampSpread={15}
+          timelineStart={timeRange.end}
+          timelineEnd={timeRange.start}
+          showHandlebar
+          handlebarTime={currentTime}
+          setHandlebarTime={setCurrentTime}
+          events={reviewItems}
+          severityType={severity}
+          contentRef={contentRef}
+          onHandlebarDraggingChange={(scrubbing) => setScrubbing(scrubbing)}
+        />
+      </div>
+    </div>
+  );
+}
+
+type MobileRecordingViewProps = {
   selectedReview: ReviewSegment;
   reviewItems: ReviewSegment[];
   relevantPreviews?: Preview[];
 };
-export default function RecordingView({
+export function MobileRecordingView({
   selectedReview,
   reviewItems,
   relevantPreviews,
-}: RecordingViewProps) {
+}: MobileRecordingViewProps) {
   const navigate = useNavigate();
   const contentRef = useRef<HTMLDivElement | null>(null);
 
@@ -82,15 +271,12 @@ export default function RecordingView({
 
   return (
     <div ref={contentRef} className="flex flex-col relative w-full h-full">
-      <Button
-        className="md:absolute md:top-0 md:left-0 rounded-lg"
-        onClick={() => navigate(-1)}
-      >
+      <Button className="rounded-lg" onClick={() => navigate(-1)}>
         <IoMdArrowRoundBack className="size-5 mr-[10px]" />
         Back
       </Button>
 
-      <div className="md:absolute md:top-8 md:inset-x-[20%]">
+      <div>
         <DynamicVideoPlayer
           camera={selectedReview.camera}
           timeRange={timeRange.ranges[selectedRangeIdx]}
@@ -110,7 +296,7 @@ export default function RecordingView({
         />
       </div>
 
-      <div className="flex-grow overflow-hidden md:absolute md:w-[100px] md:inset-y-0 md:right-0">
+      <div className="flex-grow overflow-hidden">
         <EventReviewTimeline
           segmentDuration={30}
           timestampSpread={15}