From 622dddd2c4dbbb0071f9e8e96213ac76bf160c16 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen <nickmowen213@gmail.com>
Date: Fri, 22 Mar 2024 10:56:53 -0600
Subject: [PATCH] Motion playback (#10609)

* Move controls to separate component and make features configurable

* Allow playback on motion screen

* Simplify layout

* Fix seeking

* Fix playback

* fix preview scrubbing

* Fix player controls visibility

* Use opacity for both dark and light mode
---
 web/src/components/player/HlsVideoPlayer.tsx  | 190 +++---------------
 web/src/components/player/PreviewPlayer.tsx   |   2 +-
 .../player/PreviewThumbnailPlayer.tsx         |  10 +-
 web/src/components/player/VideoControls.tsx   | 170 ++++++++++++++++
 .../player/dynamic/DynamicVideoPlayer.tsx     |  49 +++--
 web/src/views/events/EventView.tsx            |  62 +++++-
 web/src/views/events/RecordingView.tsx        |   2 +-
 7 files changed, 287 insertions(+), 198 deletions(-)
 create mode 100644 web/src/components/player/VideoControls.tsx

diff --git a/web/src/components/player/HlsVideoPlayer.tsx b/web/src/components/player/HlsVideoPlayer.tsx
index d1d7186fe..f8cb3b34c 100644
--- a/web/src/components/player/HlsVideoPlayer.tsx
+++ b/web/src/components/player/HlsVideoPlayer.tsx
@@ -3,31 +3,14 @@ import {
   ReactNode,
   useCallback,
   useEffect,
-  useMemo,
   useRef,
   useState,
 } from "react";
 import Hls from "hls.js";
-import { isDesktop, isMobile, isSafari } from "react-device-detect";
-import { LuPause, LuPlay } from "react-icons/lu";
-import {
-  DropdownMenu,
-  DropdownMenuContent,
-  DropdownMenuRadioGroup,
-  DropdownMenuRadioItem,
-  DropdownMenuTrigger,
-} from "../ui/dropdown-menu";
-import {
-  MdForward10,
-  MdReplay10,
-  MdVolumeDown,
-  MdVolumeMute,
-  MdVolumeOff,
-  MdVolumeUp,
-} from "react-icons/md";
+import { isDesktop, isMobile } from "react-device-detect";
 import useKeyboardListener from "@/hooks/use-keyboard-listener";
-import { Slider } from "../ui/slider-volume";
 import { TransformComponent, TransformWrapper } from "react-zoom-pan-pinch";
+import VideoControls from "./VideoControls";
 
 const HLS_MIME_TYPE = "application/vnd.apple.mpegurl" as const;
 const unsupportedErrorCodes = [
@@ -39,6 +22,7 @@ type HlsVideoPlayerProps = {
   className: string;
   children?: ReactNode;
   videoRef: MutableRefObject<HTMLVideoElement | null>;
+  visible: boolean;
   currentSource: string;
   onClipEnded?: () => void;
   onPlayerLoaded?: () => void;
@@ -49,6 +33,7 @@ export default function HlsVideoPlayer({
   className,
   children,
   videoRef,
+  visible,
   currentSource,
   onClipEnded,
   onPlayerLoaded,
@@ -154,7 +139,7 @@ export default function HlsVideoPlayer({
 
   return (
     <div
-      className={`relative`}
+      className={`relative ${visible ? "visible" : "hidden"}`}
       onMouseOver={
         isDesktop
           ? () => {
@@ -221,157 +206,34 @@ export default function HlsVideoPlayer({
         </TransformComponent>
       </TransformWrapper>
       <VideoControls
+        className="absolute bottom-5 left-1/2 -translate-x-1/2"
         video={videoRef.current}
         isPlaying={isPlaying}
         show={controls}
         controlsOpen={controlsOpen}
         setControlsOpen={setControlsOpen}
+        onPlayPause={(play) => {
+          if (!videoRef.current) {
+            return;
+          }
+
+          if (play) {
+            videoRef.current.play();
+          } else {
+            videoRef.current.pause();
+          }
+        }}
+        onSeek={(diff) => {
+          const currentTime = videoRef.current?.currentTime;
+
+          if (!videoRef.current || !currentTime) {
+            return;
+          }
+
+          videoRef.current.currentTime = Math.max(0, currentTime + diff);
+        }}
       />
       {children}
     </div>
   );
 }
-
-type VideoControlsProps = {
-  video: HTMLVideoElement | null;
-  isPlaying: boolean;
-  show: boolean;
-  controlsOpen: boolean;
-  setControlsOpen: (open: boolean) => void;
-};
-function VideoControls({
-  video,
-  isPlaying,
-  show,
-  controlsOpen,
-  setControlsOpen,
-}: VideoControlsProps) {
-  const playbackRates = useMemo(() => {
-    if (isSafari) {
-      return [0.5, 1, 2];
-    } else {
-      return [0.5, 1, 2, 4, 8, 16];
-    }
-  }, []);
-
-  const onReplay = useCallback(
-    (e: React.MouseEvent<SVGElement>) => {
-      e.stopPropagation();
-
-      const currentTime = video?.currentTime;
-
-      if (!video || !currentTime) {
-        return;
-      }
-
-      video.currentTime = Math.max(0, currentTime - 10);
-    },
-    [video],
-  );
-
-  const onSkip = useCallback(
-    (e: React.MouseEvent<SVGElement>) => {
-      e.stopPropagation();
-
-      const currentTime = video?.currentTime;
-
-      if (!video || !currentTime) {
-        return;
-      }
-
-      video.currentTime = currentTime + 10;
-    },
-    [video],
-  );
-
-  const onTogglePlay = useCallback(
-    (e: React.MouseEvent<HTMLDivElement>) => {
-      e.stopPropagation();
-
-      if (!video) {
-        return;
-      }
-
-      if (isPlaying) {
-        video.pause();
-      } else {
-        video.play();
-      }
-    },
-    [isPlaying, video],
-  );
-
-  // volume control
-
-  const VolumeIcon = useMemo(() => {
-    if (!video || video?.muted) {
-      return MdVolumeOff;
-    } else if (video.volume <= 0.33) {
-      return MdVolumeMute;
-    } else if (video.volume <= 0.67) {
-      return MdVolumeDown;
-    } else {
-      return MdVolumeUp;
-    }
-    // only update when specific fields change
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [video?.volume, video?.muted]);
-
-  if (!video || !show) {
-    return;
-  }
-
-  return (
-    <div
-      className={`absolute bottom-5 left-1/2 -translate-x-1/2 px-4 py-2 flex justify-between items-center gap-8 text-white z-50 bg-black bg-opacity-60 rounded-lg`}
-    >
-      <div className="flex justify-normal items-center gap-2">
-        <VolumeIcon
-          className="size-5"
-          onClick={(e: React.MouseEvent) => {
-            e.stopPropagation();
-            video.muted = !video.muted;
-          }}
-        />
-        {video.muted == false && (
-          <Slider
-            className="w-20"
-            value={[video.volume]}
-            min={0}
-            max={1}
-            step={0.02}
-            onValueChange={(value) => (video.volume = value[0])}
-          />
-        )}
-      </div>
-      <MdReplay10 className="size-5 cursor-pointer" onClick={onReplay} />
-      <div className="cursor-pointer" onClick={onTogglePlay}>
-        {isPlaying ? (
-          <LuPause className="size-5 fill-white" />
-        ) : (
-          <LuPlay className="size-5 fill-white" />
-        )}
-      </div>
-      <MdForward10 className="size-5 cursor-pointer" onClick={onSkip} />
-      <DropdownMenu
-        open={controlsOpen}
-        onOpenChange={(open) => {
-          setControlsOpen(open);
-        }}
-      >
-        <DropdownMenuTrigger>{`${video.playbackRate}x`}</DropdownMenuTrigger>
-        <DropdownMenuContent>
-          <DropdownMenuRadioGroup
-            onValueChange={(rate) => (video.playbackRate = parseInt(rate))}
-          >
-            {playbackRates.map((rate) => (
-              <DropdownMenuRadioItem key={rate} value={rate.toString()}>
-                {rate}x
-              </DropdownMenuRadioItem>
-            ))}
-          </DropdownMenuRadioGroup>
-        </DropdownMenuContent>
-      </DropdownMenu>
-    </div>
-  );
-}
diff --git a/web/src/components/player/PreviewPlayer.tsx b/web/src/components/player/PreviewPlayer.tsx
index 212779f3a..6c4a5a479 100644
--- a/web/src/components/player/PreviewPlayer.tsx
+++ b/web/src/components/player/PreviewPlayer.tsx
@@ -353,7 +353,7 @@ function PreviewFramesPlayer({
 
     return previewFrames.map((frame) =>
       // @ts-expect-error we know this item will exist
-      parseFloat(frame.split("-").slice(undefined, -5)),
+      parseFloat(frame.split("-").at(-1).slice(undefined, -5)),
     );
   }, [previewFrames]);
 
diff --git a/web/src/components/player/PreviewThumbnailPlayer.tsx b/web/src/components/player/PreviewThumbnailPlayer.tsx
index c57e1041c..3c1648e9d 100644
--- a/web/src/components/player/PreviewThumbnailPlayer.tsx
+++ b/web/src/components/player/PreviewThumbnailPlayer.tsx
@@ -171,7 +171,7 @@ export default function PreviewThumbnailPlayer({
       {...swipeHandlers}
     >
       {playingBack && (
-        <div className="absolute inset-0 animate-in fade-in pointer-events-none">
+        <div className="absolute inset-0 animate-in fade-in">
           <PreviewContent
             review={review}
             relevantPreview={relevantPreview}
@@ -486,7 +486,7 @@ function VideoPreview({
   );
 
   return (
-    <div className="relative size-full aspect-video bg-black pointer-events-none">
+    <div className="relative size-full aspect-video bg-black">
       <video
         ref={playerRef}
         className="size-full aspect-video bg-black pointer-events-none"
@@ -500,7 +500,7 @@ function VideoPreview({
       </video>
       <Slider
         ref={sliderRef}
-        className="absolute inset-x-0 bottom-0 z-30 pointer-events-none"
+        className="absolute inset-x-0 bottom-0 z-30"
         value={[progress]}
         onValueChange={onManualSeek}
         onValueCommit={onStopManualSeek}
@@ -654,9 +654,9 @@ function InProgressPreview({
   }
 
   return (
-    <div className="relative size-full flex items-center bg-black pointer-events-none">
+    <div className="relative size-full flex items-center bg-black">
       <img
-        className="size-full object-contain"
+        className="size-full object-contain pointer-events-none"
         src={`${apiHost}api/preview/${previewFrames[key]}/thumbnail.webp`}
         onLoad={handleLoad}
       />
diff --git a/web/src/components/player/VideoControls.tsx b/web/src/components/player/VideoControls.tsx
new file mode 100644
index 000000000..b470ca9bd
--- /dev/null
+++ b/web/src/components/player/VideoControls.tsx
@@ -0,0 +1,170 @@
+import { useCallback, useMemo } from "react";
+import { isSafari } from "react-device-detect";
+import { LuPause, LuPlay } from "react-icons/lu";
+import {
+  DropdownMenu,
+  DropdownMenuContent,
+  DropdownMenuRadioGroup,
+  DropdownMenuRadioItem,
+  DropdownMenuTrigger,
+} from "../ui/dropdown-menu";
+import {
+  MdForward10,
+  MdReplay10,
+  MdVolumeDown,
+  MdVolumeMute,
+  MdVolumeOff,
+  MdVolumeUp,
+} from "react-icons/md";
+import { Slider } from "../ui/slider-volume";
+
+type VideoControls = {
+  volume?: boolean;
+  seek?: boolean;
+  playbackRate?: boolean;
+};
+
+const CONTROLS_DEFAULT: VideoControls = {
+  volume: true,
+  seek: true,
+  playbackRate: true,
+};
+
+type VideoControlsProps = {
+  className?: string;
+  video?: HTMLVideoElement | null;
+  features?: VideoControls;
+  isPlaying: boolean;
+  show: boolean;
+  controlsOpen?: boolean;
+  setControlsOpen?: (open: boolean) => void;
+  onPlayPause: (play: boolean) => void;
+  onSeek: (diff: number) => void;
+};
+export default function VideoControls({
+  className,
+  video,
+  features = CONTROLS_DEFAULT,
+  isPlaying,
+  show,
+  controlsOpen,
+  setControlsOpen,
+  onPlayPause,
+  onSeek,
+}: VideoControlsProps) {
+  const playbackRates = useMemo(() => {
+    if (isSafari) {
+      return [0.5, 1, 2];
+    } else {
+      return [0.5, 1, 2, 4, 8, 16];
+    }
+  }, []);
+
+  const onReplay = useCallback(
+    (e: React.MouseEvent<SVGElement>) => {
+      e.stopPropagation();
+      onSeek(-10);
+    },
+    [onSeek],
+  );
+
+  const onSkip = useCallback(
+    (e: React.MouseEvent<SVGElement>) => {
+      e.stopPropagation();
+      onSeek(10);
+    },
+    [onSeek],
+  );
+
+  const onTogglePlay = useCallback(
+    (e: React.MouseEvent<HTMLDivElement>) => {
+      e.stopPropagation();
+      onPlayPause(!isPlaying);
+    },
+    [isPlaying, onPlayPause],
+  );
+
+  // volume control
+
+  const VolumeIcon = useMemo(() => {
+    if (!video || video?.muted) {
+      return MdVolumeOff;
+    } else if (video.volume <= 0.33) {
+      return MdVolumeMute;
+    } else if (video.volume <= 0.67) {
+      return MdVolumeDown;
+    } else {
+      return MdVolumeUp;
+    }
+    // only update when specific fields change
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [video?.volume, video?.muted]);
+
+  if (!show) {
+    return;
+  }
+
+  return (
+    <div
+      className={`px-4 py-2 flex justify-between items-center gap-8 text-white z-50 bg-secondary-foreground/60 dark:bg-secondary/60 rounded-lg ${className ?? ""}`}
+    >
+      {video && features.volume && (
+        <div className="flex justify-normal items-center gap-2">
+          <VolumeIcon
+            className="size-5"
+            onClick={(e: React.MouseEvent) => {
+              e.stopPropagation();
+              video.muted = !video.muted;
+            }}
+          />
+          {video.muted == false && (
+            <Slider
+              className="w-20"
+              value={[video.volume]}
+              min={0}
+              max={1}
+              step={0.02}
+              onValueChange={(value) => (video.volume = value[0])}
+            />
+          )}
+        </div>
+      )}
+      {features.seek && (
+        <MdReplay10 className="size-5 cursor-pointer" onClick={onReplay} />
+      )}
+      <div className="cursor-pointer" onClick={onTogglePlay}>
+        {isPlaying ? (
+          <LuPause className="size-5 fill-white" />
+        ) : (
+          <LuPlay className="size-5 fill-white" />
+        )}
+      </div>
+      {features.seek && (
+        <MdForward10 className="size-5 cursor-pointer" onClick={onSkip} />
+      )}
+      {video && features.playbackRate && (
+        <DropdownMenu
+          open={controlsOpen == true}
+          onOpenChange={(open) => {
+            if (setControlsOpen) {
+              setControlsOpen(open);
+            }
+          }}
+        >
+          <DropdownMenuTrigger>{`${video.playbackRate}x`}</DropdownMenuTrigger>
+          <DropdownMenuContent>
+            <DropdownMenuRadioGroup
+              onValueChange={(rate) => (video.playbackRate = parseFloat(rate))}
+            >
+              {playbackRates.map((rate) => (
+                <DropdownMenuRadioItem key={rate} value={rate.toString()}>
+                  {rate}x
+                </DropdownMenuRadioItem>
+              ))}
+            </DropdownMenuRadioGroup>
+          </DropdownMenuContent>
+        </DropdownMenu>
+      )}
+    </div>
+  );
+}
diff --git a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx
index c3d2aac9e..62f087cfb 100644
--- a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx
+++ b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx
@@ -165,33 +165,30 @@ export default function DynamicVideoPlayer({
   }, [controller, recordings]);
 
   return (
-    <div className={`relative ${className ?? ""} cursor-pointer`}>
-      <div
-        className={`w-full relative ${isScrubbing || isLoading ? "hidden" : "visible"}`}
-      >
-        <HlsVideoPlayer
-          className={`${grow}`}
-          videoRef={playerRef}
-          currentSource={source}
-          onTimeUpdate={onTimeUpdate}
-          onPlayerLoaded={onPlayerLoaded}
-          onClipEnded={onClipEnded}
-          onPlaying={() => {
-            if (isScrubbing) {
-              playerRef.current?.pause();
-            }
+    <div className={`relative ${className ?? ""}`}>
+      <HlsVideoPlayer
+        className={`w-full ${grow ?? ""}`}
+        videoRef={playerRef}
+        visible={!(isScrubbing || isLoading)}
+        currentSource={source}
+        onTimeUpdate={onTimeUpdate}
+        onPlayerLoaded={onPlayerLoaded}
+        onClipEnded={onClipEnded}
+        onPlaying={() => {
+          if (isScrubbing) {
+            playerRef.current?.pause();
+          }
 
-            setIsLoading(false);
-          }}
-        >
-          {config && focusedItem && (
-            <TimelineEventOverlay
-              timeline={focusedItem}
-              cameraConfig={config.cameras[camera]}
-            />
-          )}
-        </HlsVideoPlayer>
-      </div>
+          setIsLoading(false);
+        }}
+      >
+        {config && focusedItem && (
+          <TimelineEventOverlay
+            timeline={focusedItem}
+            cameraConfig={config.cameras[camera]}
+          />
+        )}
+      </HlsVideoPlayer>
       <PreviewPlayer
         className={`${isScrubbing || isLoading ? "visible" : "hidden"} ${grow}`}
         camera={camera}
diff --git a/web/src/views/events/EventView.tsx b/web/src/views/events/EventView.tsx
index 5c555f402..15c795713 100644
--- a/web/src/views/events/EventView.tsx
+++ b/web/src/views/events/EventView.tsx
@@ -38,6 +38,7 @@ import PreviewPlayer, {
 } from "@/components/player/PreviewPlayer";
 import SummaryTimeline from "@/components/timeline/SummaryTimeline";
 import { RecordingStartingPoint } from "@/types/record";
+import VideoControls from "@/components/player/VideoControls";
 
 type EventViewProps = {
   reviews?: ReviewSegment[];
@@ -678,6 +679,7 @@ function MotionReview({
   );
 
   const [scrubbing, setScrubbing] = useState(false);
+  const [playing, setPlaying] = useState(false);
 
   // move to next clip
 
@@ -704,6 +706,33 @@ function MotionReview({
     });
   }, [currentTime, currentTimeRange, timeRangeSegments]);
 
+  // playback
+
+  useEffect(() => {
+    if (!playing) {
+      return;
+    }
+
+    const startTime = currentTime;
+    let counter = 0;
+    const intervalId = setInterval(() => {
+      counter += 0.5;
+
+      if (startTime + counter >= timeRange.before) {
+        setPlaying(false);
+        return;
+      }
+
+      setCurrentTime(startTime + counter);
+    }, 60);
+
+    return () => {
+      clearInterval(intervalId);
+    };
+    // do not render when current time changes
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [playing]);
+
   if (!relevantPreviews) {
     return <ActivityIndicator />;
   }
@@ -762,9 +791,40 @@ function MotionReview({
           motion_events={motionData ?? []}
           severityType="significant_motion"
           contentRef={contentRef}
-          onHandlebarDraggingChange={(scrubbing) => setScrubbing(scrubbing)}
+          onHandlebarDraggingChange={(scrubbing) => {
+            if (playing && scrubbing) {
+              setPlaying(false);
+            }
+
+            setScrubbing(scrubbing);
+          }}
         />
       </div>
+
+      <VideoControls
+        className="absolute bottom-16 left-1/2 -translate-x-1/2"
+        features={{
+          volume: false,
+          seek: true,
+          playbackRate: false,
+        }}
+        isPlaying={playing}
+        onPlayPause={setPlaying}
+        onSeek={(diff) => {
+          const wasPlaying = playing;
+
+          if (wasPlaying) {
+            setPlaying(false);
+          }
+
+          setCurrentTime(currentTime + diff);
+
+          if (wasPlaying) {
+            setTimeout(() => setPlaying(true), 100);
+          }
+        }}
+        show={currentTime < timeRange.before - 4}
+      />
     </>
   );
 }
diff --git a/web/src/views/events/RecordingView.tsx b/web/src/views/events/RecordingView.tsx
index ad3f3262b..f1a414be0 100644
--- a/web/src/views/events/RecordingView.tsx
+++ b/web/src/views/events/RecordingView.tsx
@@ -147,7 +147,7 @@ export function RecordingView({
         } else {
           updateSelectedSegment(currentTime, true);
         }
-      } else {
+      } else if (playerTime != currentTime) {
         mainControllerRef.current?.play();
       }
     }