Motion review playback optimizations (#10659)

* handle motion timestamps with ranges

* check for overlaps when checking segment for events

* rename motion color vars to significant_motion for consistency

* safelist significant_motion

* rename vars for clarity and use timeout instead of interval

web/src/components/timeline/EventSegment.tsx

@@ -155,8 +155,8 @@ export function EventSegment({
 
   const severityColors: { [key: number]: string } = {
     1: reviewed
-      ? "from-severity_motion-dimmed/50 to-severity_motion/50"
-      : "from-severity_motion-dimmed to-severity_motion",
+      ? "from-severity_significant_motion-dimmed/50 to-severity_significant_motion/50"
+      : "from-severity_significant_motion-dimmed to-severity_significant_motion",
     2: reviewed
       ? "from-severity_detection-dimmed/50 to-severity_detection/50"
       : "from-severity_detection-dimmed to-severity_detection",

web/src/components/timeline/MotionSegment.tsx

@@ -158,15 +158,15 @@ export function MotionSegment({
       : ""
   }`;
 
-  const animationClassesSecondHalf = `motion-segment ${secondHalfSegmentWidth > 1 ? "hidden" : ""}
+  const animationClassesSecondHalf = `motion-segment ${secondHalfSegmentWidth > 0 ? "hidden" : ""}
     zoom-in-[0.2] ${secondHalfSegmentWidth < 5 ? "duration-200" : "duration-1000"}`;
-  const animationClassesFirstHalf = `motion-segment ${firstHalfSegmentWidth > 1 ? "hidden" : ""}
+  const animationClassesFirstHalf = `motion-segment ${firstHalfSegmentWidth > 0 ? "hidden" : ""}
     zoom-in-[0.2] ${firstHalfSegmentWidth < 5 ? "duration-200" : "duration-1000"}`;
 
   const severityColors: { [key: number]: string } = {
     1: reviewed
-      ? "from-severity_motion-dimmed/50 to-severity_motion/50"
-      : "from-severity_motion-dimmed to-severity_motion",
+      ? "from-severity_significant_motion-dimmed/50 to-severity_significant_motion/50"
+      : "from-severity_significant_motion-dimmed to-severity_significant_motion",
     2: reviewed
       ? "from-severity_detection-dimmed/50 to-severity_detection/50"
       : "from-severity_detection-dimmed to-severity_detection",
@@ -183,14 +183,14 @@ export function MotionSegment({
 
   return (
     <>
-      {(((firstHalfSegmentWidth > 1 || secondHalfSegmentWidth > 1) &&
+      {(((firstHalfSegmentWidth > 0 || secondHalfSegmentWidth > 0) &&
         motionOnly &&
         severity[0] < 2) ||
         !motionOnly) && (
         <div
           key={segmentKey}
           data-segment-id={segmentKey}
-          className={`segment ${firstHalfSegmentWidth > 1 || secondHalfSegmentWidth > 1 ? "has-data" : ""} ${segmentClasses}`}
+          className={`segment ${firstHalfSegmentWidth > 0 || secondHalfSegmentWidth > 0 ? "has-data" : ""} ${segmentClasses}`}
           onClick={segmentClick}
           onTouchEnd={(event) => handleTouchStart(event, segmentClick)}
         >
@@ -228,9 +228,10 @@ export function MotionSegment({
               <div className="flex justify-center">
                 <div
                   key={`${segmentKey}_motion_data_1`}
+                  data-motion-value={secondHalfSegmentWidth}
                   className={`${isDesktop && animationClassesSecondHalf} h-[2px] rounded-full ${severity[0] != 0 ? "bg-motion_review-dimmed" : "bg-motion_review"}`}
                   style={{
-                    width: secondHalfSegmentWidth,
+                    width: secondHalfSegmentWidth || 1,
                   }}
                 ></div>
               </div>
@@ -240,9 +241,10 @@ export function MotionSegment({
               <div className="flex justify-center">
                 <div
                   key={`${segmentKey}_motion_data_2`}
+                  data-motion-value={firstHalfSegmentWidth}
                   className={`${isDesktop && animationClassesFirstHalf} h-[2px] rounded-full ${severity[0] != 0 ? "bg-motion_review-dimmed" : "bg-motion_review"}`}
                   style={{
-                    width: firstHalfSegmentWidth,
+                    width: firstHalfSegmentWidth || 1,
                   }}
                 ></div>
               </div>
@@ -251,7 +253,7 @@ export function MotionSegment({
 
           {!motionOnly &&
             severity.map((severityValue: number, index: number) => {
-              if (severityValue > 1) {
+              if (severityValue > 0) {
                 return (
                   <React.Fragment key={index}>
                     <div className="absolute right-0 h-2 z-10">

web/src/components/timeline/SummarySegment.tsx

@@ -34,7 +34,9 @@ export function SummarySegment({
   const segmentKey = useMemo(() => segmentTime, [segmentTime]);
 
   const severityColors: { [key: number]: string } = {
-    1: reviewed ? "bg-severity_motion/50" : "bg-severity_motion",
+    1: reviewed
+      ? "bg-severity_significant_motion/50"
+      : "bg-severity_significant_motion",
     2: reviewed ? "bg-severity_detection/50" : "bg-severity_detection",
     3: reviewed ? "bg-severity_alert/50" : "bg-severity_alert",
   };

web/src/hooks/use-camera-activity.ts

@@ -4,8 +4,6 @@ import {
   useMotionActivity,
 } from "@/api/ws";
 import { CameraConfig } from "@/types/frigateConfig";
-import { MotionData, ReviewSegment } from "@/types/review";
-import { TimeRange } from "@/types/timeline";
 import { useEffect, useMemo, useState } from "react";
 
 type useCameraActivityReturn = {
@@ -68,57 +66,3 @@ export function useCameraActivity(
       : false,
   };
 }
-
-export function useCameraMotionTimestamps(
-  timeRange: TimeRange,
-  motionOnly: boolean,
-  events: ReviewSegment[],
-  motion: MotionData[],
-) {
-  const timestamps = useMemo(() => {
-    const seekableTimestamps = [];
-    let lastEventIdx = 0;
-    let lastMotionIdx = 0;
-
-    for (let i = timeRange.after; i <= timeRange.before; i += 0.5) {
-      if (!motionOnly) {
-        seekableTimestamps.push(i);
-      } else {
-        const relevantEventIdx = events.findIndex((seg, segIdx) => {
-          if (segIdx < lastEventIdx) {
-            return false;
-          }
-
-          return seg.start_time <= i && seg.end_time >= i;
-        });
-
-        if (relevantEventIdx != -1) {
-          lastEventIdx = relevantEventIdx;
-          continue;
-        }
-
-        const relevantMotionIdx = motion.findIndex((mot, motIdx) => {
-          if (motIdx < lastMotionIdx) {
-            return false;
-          }
-
-          return mot.start_time <= i && mot.start_time + 15 >= i;
-        });
-
-        if (relevantMotionIdx == -1 || motion[relevantMotionIdx].motion == 0) {
-          if (relevantMotionIdx != -1) {
-            lastMotionIdx = relevantMotionIdx;
-          }
-
-          continue;
-        }
-
-        seekableTimestamps.push(i);
-      }
-    }
-
-    return seekableTimestamps;
-  }, [timeRange, motionOnly, events, motion]);
-
-  return timestamps;
-}

web/src/hooks/use-draggable-element.ts

@@ -368,27 +368,10 @@ function useDraggableElement({
 
       const alignedSegmentTime = alignStartDateToTimeline(draggableElementTime);
 
-      let segmentElement = timelineRef.current.querySelector(
+      const segmentElement = timelineRef.current.querySelector(
         `[data-segment-id="${alignedSegmentTime}"]`,
       );
 
-      if (!segmentElement) {
-        // segment not found, maybe we collapsed over a collapsible segment
-        let searchTime = alignedSegmentTime;
-        while (searchTime >= timelineStartAligned - timelineDuration) {
-          // Decrement currentTime by segmentDuration
-          searchTime -= segmentDuration;
-          segmentElement = timelineRef.current.querySelector(
-            `[data-segment-id="${searchTime}"]`,
-          );
-
-          if (segmentElement) {
-            // segmentElement found
-            break;
-          }
-        }
-      }
-
       if (segmentElement) {
         const timelineRect = timelineRef.current.getBoundingClientRect();
         const timelineTopAbsolute = timelineRect.top;
@@ -422,6 +405,37 @@ function useDraggableElement({
     segments,
   ]);
 
+  useEffect(() => {
+    if (timelineRef.current && draggableElementTime && timelineCollapsed) {
+      const alignedSegmentTime = alignStartDateToTimeline(draggableElementTime);
+
+      let segmentElement = timelineRef.current.querySelector(
+        `[data-segment-id="${alignedSegmentTime}"]`,
+      );
+
+      if (!segmentElement) {
+        // segment not found, maybe we collapsed over a collapsible segment
+        let searchTime = alignedSegmentTime;
+        while (searchTime >= timelineStartAligned - timelineDuration) {
+          searchTime -= segmentDuration;
+          segmentElement = timelineRef.current.querySelector(
+            `[data-segment-id="${searchTime}"]`,
+          );
+
+          if (segmentElement) {
+            // found, set time
+            if (setDraggableElementTime) {
+              setDraggableElementTime(searchTime);
+            }
+            break;
+          }
+        }
+      }
+    }
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [timelineCollapsed]);
+
   return { handleMouseDown, handleMouseUp, handleMouseMove };
 }
 

web/src/hooks/use-motion-segment-utils.ts

@@ -33,7 +33,7 @@ export const useMotionSegmentUtils = (
 
   const interpolateMotionAudioData = useCallback(
     (value: number, newMax: number): number => {
-      return Math.ceil((Math.abs(value) / 100.0) * newMax) || 1;
+      return Math.ceil((Math.abs(value) / 100.0) * newMax) || 0;
     },
     [],
   );

web/src/views/events/EventView.tsx

@@ -40,7 +40,6 @@ import SummaryTimeline from "@/components/timeline/SummaryTimeline";
 import { RecordingStartingPoint } from "@/types/record";
 import VideoControls from "@/components/player/VideoControls";
 import { TimeRange } from "@/types/timeline";
-import { useCameraMotionTimestamps } from "@/hooks/use-camera-activity";
 
 type EventViewProps = {
   reviews?: ReviewSegment[];
@@ -247,7 +246,7 @@ export default function EventView({
             value="significant_motion"
             aria-label="Select motion"
           >
-            <MdCircle className="size-2 md:mr-[10px] text-severity_motion" />
+            <MdCircle className="size-2 md:mr-[10px] text-severity_significant_motion" />
             <div className="hidden md:block">Motion</div>
           </ToggleGroupItem>
         </ToggleGroup>
@@ -720,43 +719,111 @@ function MotionReview({
 
   const [playbackRate, setPlaybackRate] = useState(8);
   const [controlsOpen, setControlsOpen] = useState(false);
-  const seekTimestamps = useCameraMotionTimestamps(
-    timeRange,
-    motionOnly,
-    reviewItems?.all ?? [],
-    motionData ?? [],
-  );
+
+  const noMotionRanges = useMemo(() => {
+    if (!motionData || !reviewItems) {
+      return;
+    }
+
+    if (!motionOnly) {
+      return [];
+    }
+
+    const ranges = [];
+    let currentSegmentStart = null;
+    let currentSegmentEnd = null;
+
+    for (let i = 0; i < motionData.length; i = i + segmentDuration / 15) {
+      const motionStart = motionData[i].start_time;
+      const motionEnd = motionStart + segmentDuration;
+
+      const segmentMotion = motionData
+        .slice(i, i + segmentDuration / 15)
+        .some(({ motion }) => motion !== undefined && motion > 0);
+      const overlappingReviewItems = reviewItems.all.some(
+        (item) =>
+          (item.start_time >= motionStart && item.start_time < motionEnd) ||
+          (item.end_time > motionStart && item.end_time <= motionEnd) ||
+          (item.start_time <= motionStart && item.end_time >= motionEnd),
+      );
+
+      if (!segmentMotion || overlappingReviewItems) {
+        if (currentSegmentStart === null) {
+          currentSegmentStart = motionStart;
+        }
+        currentSegmentEnd = motionEnd;
+      } else {
+        if (currentSegmentStart !== null) {
+          ranges.push([currentSegmentStart, currentSegmentEnd]);
+          currentSegmentStart = null;
+          currentSegmentEnd = null;
+        }
+      }
+    }
+
+    if (currentSegmentStart !== null) {
+      ranges.push([currentSegmentStart, currentSegmentEnd]);
+    }
+
+    return ranges;
+  }, [motionData, reviewItems, motionOnly]);
+
+  const nextTimestamp = useMemo(() => {
+    if (!noMotionRanges) {
+      return;
+    }
+    let currentRange = 0;
+    let nextTimestamp = currentTime + 0.5;
+
+    while (currentRange < noMotionRanges.length) {
+      const [start, end] = noMotionRanges[currentRange];
+
+      if (start && end) {
+        // If the current time is before the start of the current range
+        if (currentTime < start) {
+          // The next timestamp is either the start of the current range or currentTime + 0.5, whichever is smaller
+          nextTimestamp = Math.min(start, nextTimestamp);
+          break;
+        }
+        // If the current time is within the current range
+        else if (currentTime >= start && currentTime < end) {
+          // The next timestamp is the end of the current range
+          nextTimestamp = end;
+          currentRange++;
+        }
+        // If the current time is past the end of the current range
+        else {
+          currentRange++;
+        }
+      }
+    }
+
+    return nextTimestamp;
+  }, [currentTime, noMotionRanges]);
+
+  const timeoutIdRef = useRef<NodeJS.Timeout | null>(null);
 
   useEffect(() => {
-    if (!playing) {
-      return;
-    }
-
-    const interval = 500 / playbackRate;
-    const startIdx = seekTimestamps.findIndex((time) => time > currentTime);
-
-    if (!startIdx) {
-      return;
-    }
-
-    let counter = 0;
-    const intervalId = setInterval(() => {
-      counter += 1;
-
-      if (startIdx + counter >= seekTimestamps.length) {
-        setPlaying(false);
+    if (nextTimestamp) {
+      if (!playing && timeoutIdRef.current != null) {
+        clearTimeout(timeoutIdRef.current);
         return;
       }
 
-      setCurrentTime(seekTimestamps[startIdx + counter]);
-    }, interval);
+      const handleTimeout = () => {
+        setCurrentTime(nextTimestamp);
+        timeoutIdRef.current = setTimeout(handleTimeout, 500 / playbackRate);
+      };
 
-    return () => {
-      clearInterval(intervalId);
-    };
-    // do not render when current time changes
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [playing, playbackRate]);
+      timeoutIdRef.current = setTimeout(handleTimeout, 500 / playbackRate);
+
+      return () => {
+        if (timeoutIdRef.current) {
+          clearTimeout(timeoutIdRef.current);
+        }
+      };
+    }
+  }, [playing, playbackRate, nextTimestamp]);
 
   const { alignStartDateToTimeline } = useTimelineUtils({
     segmentDuration,
@@ -767,11 +834,16 @@ function MotionReview({
       if (motionOnly) {
         return null;
       }
-      const segmentTime = alignStartDateToTimeline(currentTime);
+      const segmentStartTime = alignStartDateToTimeline(currentTime);
+      const segmentEndTime = segmentStartTime + segmentDuration;
       const matchingItem = reviewItems?.all.find(
         (item) =>
-          item.start_time >= segmentTime &&
-          item.end_time <= segmentTime + segmentDuration &&
+          ((item.start_time >= segmentStartTime &&
+            item.start_time < segmentEndTime) ||
+            (item.end_time > segmentStartTime &&
+              item.end_time <= segmentEndTime) ||
+            (item.start_time <= segmentStartTime &&
+              item.end_time >= segmentEndTime)) &&
           item.camera === cameraName,
       );
 

web/tailwind.config.js

@@ -9,7 +9,7 @@ module.exports = {
   ],
   safelist: [
     {
-      pattern: /(outline|shadow)-severity_(alert|detection|motion)/,
+      pattern: /(outline|shadow)-severity_(alert|detection|significant_motion)/,
     },
   ],
   theme: {
@@ -87,9 +87,9 @@ module.exports = {
           DEFAULT: "hsl(var(--severity_detection))",
           dimmed: "hsl(var(--severity_detection_dimmed))",
         },
-        severity_motion: {
-          DEFAULT: "hsl(var(--severity_motion))",
-          dimmed: "hsl(var(--severity_motion_dimmed))",
+        severity_significant_motion: {
+          DEFAULT: "hsl(var(--severity_significant_motion))",
+          dimmed: "hsl(var(--severity_significant_motion_dimmed))",
         },
         motion_review: {
           DEFAULT: "hsl(var(--motion_review))",

web/themes/theme-default.css

@@ -71,8 +71,8 @@
     --severity_detection: var(--orange-600);
     --severity_detection_dimmed: var(--orange-400);
 
-    --severity_motion: var(--yellow-400);
-    --severity_motion_dimmed: var(--yellow-200);
+    --severity_significant_motion: var(--yellow-400);
+    --severity_significant_motion_dimmed: var(--yellow-200);
 
     --motion_review: hsl(44, 94%, 50%);
     --motion_review: 44 94% 50%;