Fix motion and recording views (#10318)

* Fix switching between different hours

* Simplify book keeping of different time ranges

* Rewrite player to handle changing of previews outside of video player

* Simplify switching between cameras

* Fix previews causing cameras to not load

* Simplify listeners

* Always clear existing listener

web/src/components/player/DynamicVideoPlayer.tsx

@@ -12,7 +12,6 @@ import TimelineEventOverlay from "../overlay/TimelineDataOverlay";
 import { useApiHost } from "@/api";
 import useSWR from "swr";
 import { FrigateConfig } from "@/types/frigateConfig";
-import ActivityIndicator from "../indicators/activity-indicator";
 import useKeyboardListener from "@/hooks/use-keyboard-listener";
 import { Recording } from "@/types/record";
 import { Preview } from "@/types/preview";
@@ -29,6 +28,7 @@ type DynamicVideoPlayerProps = {
   timeRange: { start: number; end: number };
   cameraPreviews: Preview[];
   previewOnly?: boolean;
+  preloadRecordings: boolean;
   onControllerReady: (controller: DynamicVideoController) => void;
   onClick?: () => void;
 };
@@ -38,6 +38,7 @@ export default function DynamicVideoPlayer({
   timeRange,
   cameraPreviews,
   previewOnly = false,
+  preloadRecordings = true,
   onControllerReady,
   onClick,
 }: DynamicVideoPlayerProps) {
@@ -64,18 +65,19 @@ export default function DynamicVideoPlayer({
 
   // controlling playback
 
-  const playerRef = useRef<Player | undefined>(undefined);
+  const [playerRef, setPlayerRef] = useState<Player | undefined>(undefined);
   const previewRef = useRef<HTMLVideoElement | null>(null);
   const [isScrubbing, setIsScrubbing] = useState(previewOnly);
   const [focusedItem, setFocusedItem] = useState<Timeline | undefined>(
     undefined,
   );
   const controller = useMemo(() => {
-    if (!config) {
+    if (!config || !playerRef || !previewRef.current) {
       return undefined;
     }
 
     return new DynamicVideoController(
+      camera,
       playerRef,
       previewRef,
       (config.cameras[camera]?.detect?.annotation_offset || 0) / 1000,
@@ -83,10 +85,12 @@ export default function DynamicVideoPlayer({
       setIsScrubbing,
       setFocusedItem,
     );
-  }, [camera, config, previewOnly]);
+    // we only want to fire once when players are ready
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [camera, config, playerRef, previewRef]);
 
   useEffect(() => {
-    if (!playerRef.current && !previewRef.current) {
+    if (!controller) {
       return;
     }
 
@@ -96,7 +100,29 @@ export default function DynamicVideoPlayer({
 
     // we only want to fire once when players are ready
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [playerRef, previewRef]);
+  }, [controller]);
+
+  const [initPreviewOnly, setInitPreviewOnly] = useState(previewOnly);
+
+  useEffect(() => {
+    if (!controller || !playerRef) {
+      return;
+    }
+
+    if (previewOnly == initPreviewOnly) {
+      return;
+    }
+
+    if (previewOnly) {
+      playerRef.autoplay(false);
+    } else {
+      controller.seekToTimestamp(playerRef.currentTime() || 0, true);
+    }
+
+    setInitPreviewOnly(previewOnly);
+    // we only want to fire once when players are ready
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [controller, previewOnly]);
 
   const [hasRecordingAtTime, setHasRecordingAtTime] = useState(true);
 
@@ -107,33 +133,33 @@ export default function DynamicVideoPlayer({
       switch (key) {
         case "ArrowLeft":
           if (down) {
-            const currentTime = playerRef.current?.currentTime();
+            const currentTime = playerRef?.currentTime();
 
             if (currentTime) {
-              playerRef.current?.currentTime(Math.max(0, currentTime - 5));
+              playerRef?.currentTime(Math.max(0, currentTime - 5));
             }
           }
           break;
         case "ArrowRight":
           if (down) {
-            const currentTime = playerRef.current?.currentTime();
+            const currentTime = playerRef?.currentTime();
 
             if (currentTime) {
-              playerRef.current?.currentTime(currentTime + 5);
+              playerRef?.currentTime(currentTime + 5);
             }
           }
           break;
         case "m":
-          if (down && !repeat && playerRef.current) {
+          if (down && !repeat && playerRef) {
-            playerRef.current.muted(!playerRef.current.muted());
+            playerRef.muted(!playerRef.muted());
           }
           break;
         case " ":
-          if (down && playerRef.current) {
+          if (down && playerRef) {
-            if (playerRef.current.paused()) {
+            if (playerRef.paused()) {
-              playerRef.current.play();
+              playerRef.play();
             } else {
-              playerRef.current.pause();
+              playerRef.pause();
             }
           }
           break;
@@ -236,31 +262,30 @@ export default function DynamicVideoPlayer({
       recordings: recordings ?? [],
       playbackUri,
       preview,
+      timeRange,
     });
 
     // we only want this to change when recordings update
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [controller, recordings]);
 
-  if (!controller) {
-    return <ActivityIndicator />;
-  }
-
   return (
     <div
       className={`relative ${className ?? ""} ${onClick ? (hasRecordingAtTime ? "cursor-pointer" : "") : ""}`}
       onClick={onClick}
     >
-      {!previewOnly && (
+      {preloadRecordings && (
         <div
           className={`w-full relative ${
-            currentPreview != undefined && isScrubbing ? "hidden" : "visible"
+            previewOnly || (currentPreview != undefined && isScrubbing)
+              ? "hidden"
+              : "visible"
           }`}
         >
           <VideoPlayer
             options={{
               preload: "auto",
-              autoplay: true,
+              autoplay: !previewOnly,
               sources: [initialPlaybackSource],
               aspectRatio: tallVideo ? "16:9" : undefined,
               controlBar: {
@@ -272,17 +297,10 @@ export default function DynamicVideoPlayer({
             }}
             seekOptions={{ forward: 10, backward: 5 }}
             onReady={(player) => {
-              playerRef.current = player;
+              setPlayerRef(player);
-              player.on("playing", () => setFocusedItem(undefined));
-              player.on("timeupdate", () => {
-                controller.updateProgress(player.currentTime() || 0);
-              });
-              player.on("ended", () =>
-                controller.fireClipChangeEvent("forward"),
-              );
             }}
             onDispose={() => {
-              playerRef.current = undefined;
+              setPlayerRef(undefined);
             }}
           >
             {config && focusedItem && (
@@ -303,7 +321,7 @@ export default function DynamicVideoPlayer({
         muted
         disableRemotePlayback
         onSeeked={onPreviewSeeked}
-        onLoadedData={() => controller.previewReady()}
+        onLoadedData={() => controller?.previewReady()}
       >
         {currentPreview != undefined && (
           <source src={currentPreview.src} type={currentPreview.type} />
@@ -315,35 +333,39 @@ export default function DynamicVideoPlayer({
 
 export class DynamicVideoController {
   // main state
-  private playerRef: MutableRefObject<Player | undefined>;
+  public camera = "";
+  private playerRef: Player;
   private previewRef: MutableRefObject<HTMLVideoElement | null>;
   private setScrubbing: (isScrubbing: boolean) => void;
   private setFocusedItem: (timeline: Timeline) => void;
   private playerMode: PlayerMode = "playback";
+  private timeRange: { start: number; end: number } | undefined = undefined;
 
   // playback
   private recordings: Recording[] = [];
-  private onPlaybackTimestamp: ((time: number) => void) | undefined = undefined;
-  private onClipChange: ((dir: "forward" | "backward") => void) | undefined =
-    undefined;
   private annotationOffset: number;
   private timeToStart: number | undefined = undefined;
-  private clipChangeLockout: boolean = true;
 
   // preview
   private preview: Preview | undefined = undefined;
   private timeToSeek: number | undefined = undefined;
   private seeking = false;
-  private readyToScrub = true;
+
+  // listeners
+  private playerProgressListener: (() => void) | null = null;
+  private playerEndedListener: (() => void) | null = null;
+  private canPlayListener: (() => void) | null = null;
 
   constructor(
-    playerRef: MutableRefObject<Player | undefined>,
+    camera: string,
+    playerRef: Player,
     previewRef: MutableRefObject<HTMLVideoElement | null>,
     annotationOffset: number,
     defaultMode: PlayerMode,
     setScrubbing: (isScrubbing: boolean) => void,
     setFocusedItem: (timeline: Timeline) => void,
   ) {
+    this.camera = camera;
     this.playerRef = playerRef;
     this.previewRef = previewRef;
     this.annotationOffset = annotationOffset;
@@ -355,7 +377,7 @@ export class DynamicVideoController {
   newPlayback(newPlayback: DynamicPlayback) {
     this.recordings = newPlayback.recordings;
 
-    this.playerRef.current?.src({
+    this.playerRef.src({
       src: newPlayback.playbackUri,
       type: "application/vnd.apple.mpegurl",
     });
@@ -366,6 +388,13 @@ export class DynamicVideoController {
     }
 
     this.preview = newPlayback.preview;
+
+    if (this.preview) {
+      this.seeking = false;
+      this.timeToSeek = undefined;
+    }
+
+    this.timeRange = newPlayback.timeRange;
   }
 
   seekToTimestamp(time: number, play: boolean = false) {
@@ -396,21 +425,34 @@ export class DynamicVideoController {
         segment.end_time - segment.start_time - (segment.end_time - time);
       return true;
     });
-    this.playerRef.current?.currentTime(seekSeconds);
+    this.playerRef.currentTime(seekSeconds);
 
     if (play) {
-      this.playerRef.current?.play();
+      this.playerRef.play();
+    } else {
+      this.playerRef.pause();
+    }
+  }
+
+  onCanPlay(listener: (() => void) | null) {
+    if (listener) {
+      this.canPlayListener = listener;
+      this.playerRef.on("canplay", this.canPlayListener);
+    } else {
+      if (this.canPlayListener) {
+        this.playerRef.off("canplay", this.canPlayListener);
+        this.canPlayListener = null;
+      }
     }
   }
 
   seekToTimelineItem(timeline: Timeline) {
-    this.playerRef.current?.pause();
+    this.playerRef.pause();
     this.seekToTimestamp(timeline.timestamp + this.annotationOffset);
     this.setFocusedItem(timeline);
   }
 
-  updateProgress(playerTime: number) {
+  getProgress(playerTime: number): number {
-    if (this.onPlaybackTimestamp) {
     // take a player time in seconds and convert to timestamp in timeline
     let timestamp = 0;
     let totalTime = 0;
@@ -425,70 +467,44 @@ export class DynamicVideoController {
       }
     });
 
-      this.onPlaybackTimestamp(timestamp);
+    return timestamp;
-    }
   }
 
   onPlayerTimeUpdate(listener: ((timestamp: number) => void) | undefined) {
-    this.onPlaybackTimestamp = listener;
+    if (this.playerProgressListener) {
+      this.playerRef.off("timeupdate", this.playerProgressListener);
     }
 
-  onClipChangedEvent(listener: (dir: "forward" | "backward") => void) {
+    if (listener) {
-    this.onClipChange = listener;
+      this.playerProgressListener = () =>
+        listener(this.getProgress(this.playerRef.currentTime() || 0));
+      this.playerRef.on("timeupdate", this.playerProgressListener);
+    }
   }
 
-  fireClipChangeEvent(dir: "forward" | "backward") {
+  onClipChangedEvent(listener: ((dir: "forward") => void) | undefined) {
-    if (this.onClipChange) {
+    if (this.playerEndedListener) {
-      this.onClipChange(dir);
+      this.playerRef.off("ended", this.playerEndedListener);
+    }
+
+    if (listener) {
+      this.playerEndedListener = () => listener("forward");
+      this.playerRef.on("ended", this.playerEndedListener);
     }
   }
 
   scrubToTimestamp(time: number) {
-    if (!this.preview) {
+    if (!this.preview || !this.timeRange) {
       return;
     }
 
-    if (!this.readyToScrub) {
+    if (time < this.preview.start || time > this.preview.end) {
-      return;
-    }
-
-    if (time > this.preview.end) {
-      if (this.clipChangeLockout && time - this.preview.end < 30) {
-        return;
-      }
-
-      if (this.playerMode == "scrubbing") {
-        this.playerMode = "playback";
-        this.setScrubbing(false);
-        this.timeToSeek = undefined;
-        this.seeking = false;
-        this.readyToScrub = false;
-        this.clipChangeLockout = true;
-        this.fireClipChangeEvent("forward");
-      }
-      return;
-    }
-
-    if (time < this.preview.start) {
-      if (this.clipChangeLockout && this.preview.start - time < 30) {
-        return;
-      }
-
-      if (this.playerMode == "scrubbing") {
-        this.playerMode = "playback";
-        this.setScrubbing(false);
-        this.timeToSeek = undefined;
-        this.seeking = false;
-        this.readyToScrub = false;
-        this.clipChangeLockout = true;
-        this.fireClipChangeEvent("backward");
-      }
       return;
     }
 
     if (this.playerMode != "scrubbing") {
       this.playerMode = "scrubbing";
-      this.playerRef.current?.pause();
+      this.playerRef.pause();
       this.setScrubbing(true);
     }
 
@@ -514,8 +530,6 @@ export class DynamicVideoController {
       return;
     }
 
-    this.clipChangeLockout = false;
-
     if (
       this.timeToSeek &&
       this.timeToSeek != this.previewRef.current?.currentTime
@@ -529,7 +543,6 @@ export class DynamicVideoController {
 
   previewReady() {
     this.previewRef.current?.pause();
-    this.readyToScrub = true;
   }
 
   hasRecordingAtTime(time: number): boolean {

web/src/types/playback.ts

@@ -5,4 +5,5 @@ export type DynamicPlayback = {
   recordings: Recording[];
   playbackUri: string;
   preview: Preview | undefined;
+  timeRange: { end: number; start: number };
 };

web/src/views/events/EventView.tsx

@@ -579,12 +579,17 @@ function MotionReview({
     () => getChunkedTimeRange(timeRange.after, lastFullHour),
     [lastFullHour, timeRange],
   );
+
   const [selectedRangeIdx, setSelectedRangeIdx] = useState(
     timeRangeSegments.ranges.length - 1,
   );
   const [currentTime, setCurrentTime] = useState<number>(
     timeRangeSegments.ranges[selectedRangeIdx].start,
   );
+  const currentTimeRange = useMemo(
+    () => timeRangeSegments.ranges[selectedRangeIdx],
+    [selectedRangeIdx, timeRangeSegments],
+  );
 
   // move to next clip
 
@@ -600,23 +605,38 @@ function MotionReview({
 
     if (firstController) {
       firstController.onClipChangedEvent((dir) => {
-        if (
+        if (dir == "forward") {
-          dir == "forward" &&
+          if (selectedRangeIdx < timeRangeSegments.ranges.length - 1) {
-          selectedRangeIdx < timeRangeSegments.ranges.length - 1
-        ) {
             setSelectedRangeIdx(selectedRangeIdx + 1);
-        } else if (selectedRangeIdx > 0) {
+          }
-          setSelectedRangeIdx(selectedRangeIdx - 1);
         }
       });
     }
   }, [selectedRangeIdx, timeRangeSegments, videoPlayersRef, playerReady]);
 
   useEffect(() => {
+    if (
+      currentTime > currentTimeRange.end + 60 ||
+      currentTime < currentTimeRange.start - 60
+    ) {
+      const index = timeRangeSegments.ranges.findIndex(
+        (seg) => seg.start <= currentTime && seg.end >= currentTime,
+      );
+
+      if (index != -1) {
+        setSelectedRangeIdx(index);
+      }
+      return;
+    }
+
     Object.values(videoPlayersRef.current).forEach((controller) => {
       controller.scrubToTimestamp(currentTime);
     });
-  }, [currentTime]);
+  }, [currentTime, currentTimeRange, timeRangeSegments]);
+
+  if (!relevantPreviews) {
+    return <ActivityIndicator />;
+  }
 
   return (
     <>
@@ -640,9 +660,10 @@ function MotionReview({
                 key={camera.name}
                 className={`${grow}`}
                 camera={camera.name}
-                timeRange={timeRangeSegments.ranges[selectedRangeIdx]}
+                timeRange={currentTimeRange}
                 cameraPreviews={relevantPreviews || []}
                 previewOnly
+                preloadRecordings={false}
                 onControllerReady={(controller) => {
                   videoPlayersRef.current[camera.name] = controller;
                   setPlayerReady(true);

web/src/views/events/RecordingView.tsx

@@ -49,6 +49,10 @@ export function DesktopRecordingView({
       return chunk.start <= startTime && chunk.end >= startTime;
     }),
   );
+  const currentTimeRange = useMemo(
+    () => timeRange.ranges[selectedRangeIdx],
+    [selectedRangeIdx, timeRange],
+  );
 
   // move to next clip
   useEffect(() => {
@@ -59,21 +63,18 @@ export function DesktopRecordingView({
       return;
     }
 
-    const firstController = Object.values(videoPlayersRef.current)[0];
+    const mainController = videoPlayersRef.current[mainCamera];
 
-    if (firstController) {
+    if (mainController) {
-      firstController.onClipChangedEvent((dir) => {
+      mainController.onClipChangedEvent((dir) => {
-        if (
+        if (dir == "forward") {
-          dir == "forward" &&
+          if (selectedRangeIdx < timeRange.ranges.length - 1) {
-          selectedRangeIdx < timeRange.ranges.length - 1
-        ) {
             setSelectedRangeIdx(selectedRangeIdx + 1);
-        } else if (selectedRangeIdx > 0) {
+          }
-          setSelectedRangeIdx(selectedRangeIdx - 1);
         }
       });
     }
-  }, [selectedRangeIdx, timeRange, videoPlayersRef, playerReady]);
+  }, [selectedRangeIdx, timeRange, videoPlayersRef, playerReady, mainCamera]);
 
   // scrubbing and timeline state
 
@@ -82,11 +83,25 @@ export function DesktopRecordingView({
 
   useEffect(() => {
     if (scrubbing) {
+      if (
+        currentTime > currentTimeRange.end + 60 ||
+        currentTime < currentTimeRange.start - 60
+      ) {
+        const index = timeRange.ranges.findIndex(
+          (seg) => seg.start <= currentTime && seg.end >= currentTime,
+        );
+
+        if (index != -1) {
+          setSelectedRangeIdx(index);
+        }
+        return;
+      }
+
       Object.values(videoPlayersRef.current).forEach((controller) => {
         controller.scrubToTimestamp(currentTime);
       });
     }
-  }, [currentTime, scrubbing]);
+  }, [currentTime, scrubbing, timeRange, currentTimeRange]);
 
   useEffect(() => {
     if (!scrubbing) {
@@ -99,11 +114,16 @@ export function DesktopRecordingView({
 
   const onSelectCamera = useCallback(
     (newCam: string) => {
-      videoPlayersRef.current[mainCamera].onPlayerTimeUpdate(undefined);
+      const lastController = videoPlayersRef.current[mainCamera];
-      videoPlayersRef.current[mainCamera].scrubToTimestamp(currentTime);
+      const newController = videoPlayersRef.current[newCam];
-      videoPlayersRef.current[newCam].seekToTimestamp(currentTime, true);
+      lastController.onPlayerTimeUpdate(undefined);
-      videoPlayersRef.current[newCam].onPlayerTimeUpdate(
+      lastController.onClipChangedEvent(undefined);
-        (timestamp: number) => {
+      lastController.scrubToTimestamp(currentTime);
+      newController.onCanPlay(() => {
+        newController.seekToTimestamp(currentTime, true);
+        newController.onCanPlay(null);
+      });
+      newController.onPlayerTimeUpdate((timestamp: number) => {
         setCurrentTime(timestamp);
 
         allCameras.forEach((cam) => {
@@ -113,8 +133,7 @@ export function DesktopRecordingView({
             );
           }
         });
-        },
+      });
-      );
       setMainCamera(newCam);
     },
     [allCameras, currentTime, mainCamera],
@@ -155,8 +174,9 @@ export function DesktopRecordingView({
               >
                 <DynamicVideoPlayer
                   camera={cam}
-                  timeRange={timeRange.ranges[selectedRangeIdx]}
+                  timeRange={currentTimeRange}
                   cameraPreviews={allPreviews ?? []}
+                  preloadRecordings
                   onControllerReady={(controller) => {
                     videoPlayersRef.current[cam] = controller;
                     setPlayerReady(true);
@@ -172,7 +192,10 @@ export function DesktopRecordingView({
                       });
                     });
 
+                    controller.onCanPlay(() => {
                       controller.seekToTimestamp(startTime, true);
+                      controller.onCanPlay(null);
+                    });
                   }}
                 />
               </div>
@@ -184,9 +207,10 @@ export function DesktopRecordingView({
               <DynamicVideoPlayer
                 className="size-full"
                 camera={cam}
-                timeRange={timeRange.ranges[selectedRangeIdx]}
+                timeRange={currentTimeRange}
                 cameraPreviews={allPreviews ?? []}
                 previewOnly
+                preloadRecordings
                 onControllerReady={(controller) => {
                   videoPlayersRef.current[cam] = controller;
                   setPlayerReady(true);
@@ -265,6 +289,10 @@ export function MobileRecordingView({
       return chunk.start <= startTime && chunk.end >= startTime;
     }),
   );
+  const currentTimeRange = useMemo(
+    () => timeRange.ranges[selectedRangeIdx],
+    [selectedRangeIdx, timeRange],
+  );
 
   // move to next clip
   useEffect(() => {
@@ -273,10 +301,10 @@ export function MobileRecordingView({
     }
 
     controllerRef.current.onClipChangedEvent((dir) => {
-      if (dir == "forward" && selectedRangeIdx < timeRange.ranges.length - 1) {
+      if (dir == "forward") {
+        if (selectedRangeIdx < timeRange.ranges.length - 1) {
           setSelectedRangeIdx(selectedRangeIdx + 1);
-      } else if (selectedRangeIdx > 0) {
+        }
-        setSelectedRangeIdx(selectedRangeIdx - 1);
       }
     });
   }, [playerReady, selectedRangeIdx, timeRange]);
@@ -290,9 +318,23 @@ export function MobileRecordingView({
 
   useEffect(() => {
     if (scrubbing) {
+      if (
+        currentTime > currentTimeRange.end + 60 ||
+        currentTime < currentTimeRange.start - 60
+      ) {
+        const index = timeRange.ranges.findIndex(
+          (seg) => seg.start <= currentTime && seg.end >= currentTime,
+        );
+
+        if (index != -1) {
+          setSelectedRangeIdx(index);
+        }
+        return;
+      }
+
       controllerRef.current?.scrubToTimestamp(currentTime);
     }
-  }, [currentTime, scrubbing]);
+  }, [currentTime, scrubbing, currentTimeRange, timeRange]);
 
   useEffect(() => {
     if (!scrubbing) {
@@ -328,8 +370,9 @@ export function MobileRecordingView({
       <div>
         <DynamicVideoPlayer
           camera={startCamera}
-          timeRange={timeRange.ranges[selectedRangeIdx]}
+          timeRange={currentTimeRange}
           cameraPreviews={relevantPreviews || []}
+          preloadRecordings
           onControllerReady={(controller) => {
             controllerRef.current = controller;
             setPlayerReady(true);