Multi cam recording view (#10244)

* Split recording view for mobile and desktop and get desktop working * Get stuff working well * Handle onclick for video * Fix camera grid * set onclick
2024-11-21 19:07:46 +01:00 · 2024-03-05 05:03:10 -07:00 · 2024-03-05 05:03:10 -07:00 · 30b68e59f2
commit 30b68e59f2
parent bbdb8d36ca
4 changed files with 286 additions and 48 deletions
--- a/web/src/components/player/DynamicVideoPlayer.tsx
+++ b/web/src/components/player/DynamicVideoPlayer.tsx
@ -30,6 +30,7 @@ type DynamicVideoPlayerProps = {
  cameraPreviews: Preview[];
  previewOnly?: boolean;
  onControllerReady?: (controller: DynamicVideoController) => void;
+  onClick?: () => void;
 };
 export default function DynamicVideoPlayer({
  className,
@ -38,6 +39,7 @@ export default function DynamicVideoPlayer({
  cameraPreviews,
  previewOnly = false,
  onControllerReady,
+  onClick,
 }: DynamicVideoPlayerProps) {
  const apiHost = useApiHost();
  const { data: config } = useSWR<FrigateConfig>("config");
@ -83,6 +85,8 @@ export default function DynamicVideoPlayer({
    );
  }, [camera, config, previewOnly]);

+  const [hasRecordingAtTime, setHasRecordingAtTime] = useState(true);
+
  // keyboard control

  const onKeyboardShortcut = useCallback(
@ -145,28 +149,35 @@ export default function DynamicVideoPlayer({
    // we only want to calculate this once
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, []);
-  const initialPreviewSource = useMemo(() => {
-    const preview = cameraPreviews.find(
+  const initialPreview = useMemo(() => {
+    return cameraPreviews.find(
      (preview) =>
        preview.camera == camera &&
        Math.round(preview.start) >= timeRange.start &&
        Math.floor(preview.end) <= timeRange.end,
    );

-    if (preview) {
-      return {
-        src: preview.src,
-        type: preview.type,
-      };
-    } else {
-      return undefined;
-    }
-
    // we only want to calculate this once
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, []);

-  const [currentPreview, setCurrentPreview] = useState(initialPreviewSource);
+  const [currentPreview, setCurrentPreview] = useState(initialPreview);
+
+  const onPreviewSeeked = useCallback(() => {
+    if (!controller) {
+      return;
+    }
+
+    controller.finishedSeeking();
+
+    if (currentPreview && previewOnly && previewRef.current && onClick) {
+      setHasRecordingAtTime(
+        controller.hasRecordingAtTime(
+          currentPreview.start + previewRef.current.currentTime,
+        ),
+      );
+    }
+  }, [controller, currentPreview, onClick, previewOnly]);

  // state of playback player

@ -177,7 +188,9 @@ export default function DynamicVideoPlayer({
    };
  }, [timeRange]);
  const { data: recordings } = useSWR<Recording[]>(
-    previewOnly ? null : [`${camera}/recordings`, recordingParams],
+    previewOnly && onClick == undefined
+      ? null
+      : [`${camera}/recordings`, recordingParams],
    { revalidateOnFocus: false },
  );

@ -217,7 +230,10 @@ export default function DynamicVideoPlayer({
  }

  return (
-    <div className={className}>
+    <div
+      className={`relative ${className ?? ""} ${onClick ? (hasRecordingAtTime ? "cursor-pointer" : "") : ""}`}
+      onClick={onClick}
+    >
      {!previewOnly && (
        <div
          className={`w-full relative ${
@ -272,7 +288,7 @@ export default function DynamicVideoPlayer({
        autoPlay
        playsInline
        muted
-        onSeeked={() => controller.finishedSeeking()}
+        onSeeked={onPreviewSeeked}
        onLoadedData={() => controller.previewReady()}
        onLoadStart={
          previewOnly && onControllerReady
@ -286,6 +302,9 @@ export default function DynamicVideoPlayer({
          <source src={currentPreview.src} type={currentPreview.type} />
        )}
      </video>
+      {onClick && !hasRecordingAtTime && (
+        <div className="absolute inset-0 z-10 bg-black bg-opacity-60" />
+      )}
    </div>
  );
 }
@ -406,7 +425,7 @@ export class DynamicVideoController {
    }
  }

-  onPlayerTimeUpdate(listener: (timestamp: number) => void) {
+  onPlayerTimeUpdate(listener: ((timestamp: number) => void) | undefined) {
    this.onPlaybackTimestamp = listener;
  }

@ -508,4 +527,16 @@ export class DynamicVideoController {
    this.previewRef.current?.pause();
    this.readyToScrub = true;
  }
+
+  hasRecordingAtTime(time: number): boolean {
+    if (!this.recordings || this.recordings.length == 0) {
+      return false;
+    }
+
+    return (
+      this.recordings.find(
+        (segment) => segment.start_time <= time && segment.end_time >= time,
+      ) != undefined
+    );
+  }
 }
--- a/web/src/pages/Events.tsx
+++ b/web/src/pages/Events.tsx
@ -11,9 +11,13 @@ import {
  ReviewSummary,
 } from "@/types/review";
 import EventView from "@/views/events/EventView";
-import RecordingView from "@/views/events/RecordingView";
+import {
+  DesktopRecordingView,
+  MobileRecordingView,
+} from "@/views/events/RecordingView";
 import axios from "axios";
 import { useCallback, useMemo, useState } from "react";
+import { isMobile } from "react-device-detect";
 import useSWR from "swr";
 import useSWRInfinite from "swr/infinite";

@ -237,6 +241,10 @@ export default function Events() {
  // selected items

  const selectedData = useMemo(() => {
+    if (!config) {
+      return undefined;
+    }
+
    if (!selectedReviewId) {
      return undefined;
    }
@ -245,6 +253,8 @@ export default function Events() {
      return undefined;
    }

+    const allCameras = reviewFilter?.cameras ?? Object.keys(config.cameras);
+
    const allReviews = reviewPages.flat();
    const selectedReview = allReviews.find(
      (item) => item.id == selectedReviewId,
@ -256,11 +266,9 @@ export default function Events() {

    return {
      selected: selectedReview,
-      cameraSegments: allReviews.filter(
-        (seg) => seg.camera == selectedReview.camera,
-      ),
-      cameraPreviews: allPreviews?.filter(
-        (seg) => seg.camera == selectedReview.camera,
+      allCameras: allCameras,
+      cameraSegments: allReviews.filter((seg) =>
+        allCameras.includes(seg.camera),
      ),
    };

@ -273,11 +281,24 @@ export default function Events() {
  }

  if (selectedData) {
+    if (isMobile) {
+      return (
+        <MobileRecordingView
+          reviewItems={selectedData.cameraSegments}
+          selectedReview={selectedData.selected}
+          relevantPreviews={allPreviews}
+        />
+      );
+    }
+
    return (
-      <RecordingView
+      <DesktopRecordingView
+        startCamera={selectedData.selected.camera}
+        startTime={selectedData.selected.start_time}
+        allCameras={selectedData.allCameras}
+        severity={selectedData.selected.severity}
        reviewItems={selectedData.cameraSegments}
-        selectedReview={selectedData.selected}
-        relevantPreviews={selectedData.cameraPreviews}
+        allPreviews={allPreviews}
      />
    );
  } else {
--- a/web/src/views/events/EventView.tsx
+++ b/web/src/views/events/EventView.tsx
@ -627,18 +627,18 @@ function MotionReview({
            grow = "aspect-video";
          }
          return (
-            <div key={camera.name} className={`${grow}`}>
-              <DynamicVideoPlayer
-                camera={camera.name}
-                timeRange={timeRangeSegments.ranges[selectedRangeIdx]}
-                cameraPreviews={relevantPreviews || []}
-                previewOnly
-                onControllerReady={(controller) => {
-                  videoPlayersRef.current[camera.name] = controller;
-                  setPlayerReady(true);
-                }}
-              />
-            </div>
+            <DynamicVideoPlayer
+              key={camera.name}
+              className={`${grow}`}
+              camera={camera.name}
+              timeRange={timeRangeSegments.ranges[selectedRangeIdx]}
+              cameraPreviews={relevantPreviews || []}
+              previewOnly
+              onControllerReady={(controller) => {
+                videoPlayersRef.current[camera.name] = controller;
+                setPlayerReady(true);
+              }}
+            />
          );
        })}
      </div>
--- a/web/src/views/events/RecordingView.tsx
+++ b/web/src/views/events/RecordingView.tsx
@ -4,22 +4,211 @@ import DynamicVideoPlayer, {
 import EventReviewTimeline from "@/components/timeline/EventReviewTimeline";
 import { Button } from "@/components/ui/button";
 import { Preview } from "@/types/preview";
-import { ReviewSegment } from "@/types/review";
+import { ReviewSegment, ReviewSeverity } from "@/types/review";
 import { getChunkedTimeDay } from "@/utils/timelineUtil";
-import { useEffect, useMemo, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { IoMdArrowRoundBack } from "react-icons/io";
 import { useNavigate } from "react-router-dom";

-type RecordingViewProps = {
+type DesktopRecordingViewProps = {
+  startCamera: string;
+  startTime: number;
+  severity: ReviewSeverity;
+  reviewItems: ReviewSegment[];
+  allCameras: string[];
+  allPreviews?: Preview[];
+};
+export function DesktopRecordingView({
+  startCamera,
+  startTime,
+  severity,
+  reviewItems,
+  allCameras,
+  allPreviews,
+}: DesktopRecordingViewProps) {
+  const navigate = useNavigate();
+  const contentRef = useRef<HTMLDivElement | null>(null);
+
+  // controller state
+
+  const [playerReady, setPlayerReady] = useState(false);
+  const [mainCamera, setMainCamera] = useState(startCamera);
+  const videoPlayersRef = useRef<{ [camera: string]: DynamicVideoController }>(
+    {},
+  );
+
+  // timeline time
+
+  const timeRange = useMemo(() => getChunkedTimeDay(startTime), [startTime]);
+  const [selectedRangeIdx, setSelectedRangeIdx] = useState(
+    timeRange.ranges.findIndex((chunk) => {
+      return chunk.start <= startTime && chunk.end >= startTime;
+    }),
+  );
+
+  // move to next clip
+  useEffect(() => {
+    if (
+      !videoPlayersRef.current &&
+      Object.values(videoPlayersRef.current).length > 0
+    ) {
+      return;
+    }
+
+    const firstController = Object.values(videoPlayersRef.current)[0];
+
+    if (firstController) {
+      firstController.onClipChangedEvent((dir) => {
+        if (
+          dir == "forward" &&
+          selectedRangeIdx < timeRange.ranges.length - 1
+        ) {
+          setSelectedRangeIdx(selectedRangeIdx + 1);
+        } else if (selectedRangeIdx > 0) {
+          setSelectedRangeIdx(selectedRangeIdx - 1);
+        }
+      });
+    }
+  }, [selectedRangeIdx, timeRange, videoPlayersRef, playerReady]);
+
+  // scrubbing and timeline state
+
+  const [scrubbing, setScrubbing] = useState(false);
+  const [currentTime, setCurrentTime] = useState<number>(startTime);
+
+  useEffect(() => {
+    if (scrubbing) {
+      Object.values(videoPlayersRef.current).forEach((controller) => {
+        controller.scrubToTimestamp(currentTime);
+      });
+    }
+  }, [currentTime, scrubbing]);
+
+  useEffect(() => {
+    if (!scrubbing) {
+      videoPlayersRef.current[mainCamera]?.seekToTimestamp(currentTime, true);
+    }
+
+    // we only want to seek when user stops scrubbing
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [scrubbing]);
+
+  const onSelectCamera = useCallback(
+    (newCam: string) => {
+      videoPlayersRef.current[mainCamera].onPlayerTimeUpdate(undefined);
+      videoPlayersRef.current[mainCamera].scrubToTimestamp(currentTime);
+      videoPlayersRef.current[newCam].seekToTimestamp(currentTime, true);
+      videoPlayersRef.current[newCam].onPlayerTimeUpdate(
+        (timestamp: number) => {
+          setCurrentTime(timestamp);
+
+          allCameras.forEach((cam) => {
+            if (cam != newCam) {
+              videoPlayersRef.current[cam]?.scrubToTimestamp(
+                Math.floor(timestamp),
+              );
+            }
+          });
+        },
+      );
+      setMainCamera(newCam);
+    },
+    [allCameras, currentTime, mainCamera],
+  );
+
+  return (
+    <div ref={contentRef} className="relative size-full">
+      <Button
+        className="absolute top-0 left-0 rounded-lg"
+        onClick={() => navigate(-1)}
+      >
+        <IoMdArrowRoundBack className="size-5 mr-[10px]" />
+        Back
+      </Button>
+
+      <div className="absolute h-32 left-2 right-28 bottom-4 flex justify-center gap-1">
+        {allCameras.map((cam) => {
+          if (cam == mainCamera) {
+            return (
+              <div
+                key={cam}
+                className="fixed left-96 right-96 top-[40%] -translate-y-[50%]"
+              >
+                <DynamicVideoPlayer
+                  camera={cam}
+                  timeRange={timeRange.ranges[selectedRangeIdx]}
+                  cameraPreviews={allPreviews ?? []}
+                  onControllerReady={(controller) => {
+                    videoPlayersRef.current[cam] = controller;
+                    setPlayerReady(true);
+                    controller.onPlayerTimeUpdate((timestamp: number) => {
+                      setCurrentTime(timestamp);
+
+                      allCameras.forEach((otherCam) => {
+                        if (cam != otherCam) {
+                          videoPlayersRef.current[otherCam]?.scrubToTimestamp(
+                            Math.floor(timestamp),
+                          );
+                        }
+                      });
+                    });
+
+                    controller.seekToTimestamp(startTime, true);
+                  }}
+                />
+              </div>
+            );
+          }
+
+          return (
+            <div key={cam} className="aspect-video flex items-center">
+              <DynamicVideoPlayer
+                className="size-full"
+                camera={cam}
+                timeRange={timeRange.ranges[selectedRangeIdx]}
+                cameraPreviews={allPreviews ?? []}
+                previewOnly
+                onControllerReady={(controller) => {
+                  videoPlayersRef.current[cam] = controller;
+                  setPlayerReady(true);
+                  controller.scrubToTimestamp(startTime);
+                }}
+                onClick={() => onSelectCamera(cam)}
+              />
+            </div>
+          );
+        })}
+      </div>
+
+      <div className="absolute overflow-hidden w-56 inset-y-0 right-0">
+        <EventReviewTimeline
+          segmentDuration={30}
+          timestampSpread={15}
+          timelineStart={timeRange.end}
+          timelineEnd={timeRange.start}
+          showHandlebar
+          handlebarTime={currentTime}
+          setHandlebarTime={setCurrentTime}
+          events={reviewItems}
+          severityType={severity}
+          contentRef={contentRef}
+          onHandlebarDraggingChange={(scrubbing) => setScrubbing(scrubbing)}
+        />
+      </div>
+    </div>
+  );
+}
+
+type MobileRecordingViewProps = {
  selectedReview: ReviewSegment;
  reviewItems: ReviewSegment[];
  relevantPreviews?: Preview[];
 };
-export default function RecordingView({
+export function MobileRecordingView({
  selectedReview,
  reviewItems,
  relevantPreviews,
-}: RecordingViewProps) {
+}: MobileRecordingViewProps) {
  const navigate = useNavigate();
  const contentRef = useRef<HTMLDivElement | null>(null);

@ -82,15 +271,12 @@ export default function RecordingView({

  return (
    <div ref={contentRef} className="flex flex-col relative w-full h-full">
-      <Button
-        className="md:absolute md:top-0 md:left-0 rounded-lg"
-        onClick={() => navigate(-1)}
-      >
+      <Button className="rounded-lg" onClick={() => navigate(-1)}>
        <IoMdArrowRoundBack className="size-5 mr-[10px]" />
        Back
      </Button>

-      <div className="md:absolute md:top-8 md:inset-x-[20%]">
+      <div>
        <DynamicVideoPlayer
          camera={selectedReview.camera}
          timeRange={timeRange.ranges[selectedRangeIdx]}
@ -110,7 +296,7 @@ export default function RecordingView({
        />
      </div>

-      <div className="flex-grow overflow-hidden md:absolute md:w-[100px] md:inset-y-0 md:right-0">
+      <div className="flex-grow overflow-hidden">
        <EventReviewTimeline
          segmentDuration={30}
          timestampSpread={15}