From 7b640911281d2fdf0a5094bc9d2c269e7b575e0f Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 24 Mar 2024 21:37:44 -0500 Subject: [PATCH] Motion review playback optimizations (#10659) * handle motion timestamps with ranges * check for overlaps when checking segment for events * rename motion color vars to significant_motion for consistency * safelist significant_motion * rename vars for clarity and use timeout instead of interval --- web/src/components/timeline/EventSegment.tsx | 4 +- web/src/components/timeline/MotionSegment.tsx | 20 +-- .../components/timeline/SummarySegment.tsx | 4 +- web/src/hooks/use-camera-activity.ts | 56 ------- web/src/hooks/use-draggable-element.ts | 50 +++--- web/src/hooks/use-motion-segment-utils.ts | 2 +- web/src/views/events/EventView.tsx | 144 +++++++++++++----- web/tailwind.config.js | 8 +- web/themes/theme-default.css | 4 +- 9 files changed, 163 insertions(+), 129 deletions(-) diff --git a/web/src/components/timeline/EventSegment.tsx b/web/src/components/timeline/EventSegment.tsx index 596c9788e..9fa2d025d 100644 --- a/web/src/components/timeline/EventSegment.tsx +++ b/web/src/components/timeline/EventSegment.tsx @@ -155,8 +155,8 @@ export function EventSegment({ const severityColors: { [key: number]: string } = { 1: reviewed - ? "from-severity_motion-dimmed/50 to-severity_motion/50" - : "from-severity_motion-dimmed to-severity_motion", + ? "from-severity_significant_motion-dimmed/50 to-severity_significant_motion/50" + : "from-severity_significant_motion-dimmed to-severity_significant_motion", 2: reviewed ? "from-severity_detection-dimmed/50 to-severity_detection/50" : "from-severity_detection-dimmed to-severity_detection", diff --git a/web/src/components/timeline/MotionSegment.tsx b/web/src/components/timeline/MotionSegment.tsx index 748024239..279f69a9a 100644 --- a/web/src/components/timeline/MotionSegment.tsx +++ b/web/src/components/timeline/MotionSegment.tsx @@ -158,15 +158,15 @@ export function MotionSegment({ : "" }`; - const animationClassesSecondHalf = `motion-segment ${secondHalfSegmentWidth > 1 ? "hidden" : ""} + const animationClassesSecondHalf = `motion-segment ${secondHalfSegmentWidth > 0 ? "hidden" : ""} zoom-in-[0.2] ${secondHalfSegmentWidth < 5 ? "duration-200" : "duration-1000"}`; - const animationClassesFirstHalf = `motion-segment ${firstHalfSegmentWidth > 1 ? "hidden" : ""} + const animationClassesFirstHalf = `motion-segment ${firstHalfSegmentWidth > 0 ? "hidden" : ""} zoom-in-[0.2] ${firstHalfSegmentWidth < 5 ? "duration-200" : "duration-1000"}`; const severityColors: { [key: number]: string } = { 1: reviewed - ? "from-severity_motion-dimmed/50 to-severity_motion/50" - : "from-severity_motion-dimmed to-severity_motion", + ? "from-severity_significant_motion-dimmed/50 to-severity_significant_motion/50" + : "from-severity_significant_motion-dimmed to-severity_significant_motion", 2: reviewed ? "from-severity_detection-dimmed/50 to-severity_detection/50" : "from-severity_detection-dimmed to-severity_detection", @@ -183,14 +183,14 @@ export function MotionSegment({ return ( <> - {(((firstHalfSegmentWidth > 1 || secondHalfSegmentWidth > 1) && + {(((firstHalfSegmentWidth > 0 || secondHalfSegmentWidth > 0) && motionOnly && severity[0] < 2) || !motionOnly) && (
1 || secondHalfSegmentWidth > 1 ? "has-data" : ""} ${segmentClasses}`} + className={`segment ${firstHalfSegmentWidth > 0 || secondHalfSegmentWidth > 0 ? "has-data" : ""} ${segmentClasses}`} onClick={segmentClick} onTouchEnd={(event) => handleTouchStart(event, segmentClick)} > @@ -228,9 +228,10 @@ export function MotionSegment({
@@ -240,9 +241,10 @@ export function MotionSegment({
@@ -251,7 +253,7 @@ export function MotionSegment({ {!motionOnly && severity.map((severityValue: number, index: number) => { - if (severityValue > 1) { + if (severityValue > 0) { return (
diff --git a/web/src/components/timeline/SummarySegment.tsx b/web/src/components/timeline/SummarySegment.tsx index 9c8e8bca9..34e4b6621 100644 --- a/web/src/components/timeline/SummarySegment.tsx +++ b/web/src/components/timeline/SummarySegment.tsx @@ -34,7 +34,9 @@ export function SummarySegment({ const segmentKey = useMemo(() => segmentTime, [segmentTime]); const severityColors: { [key: number]: string } = { - 1: reviewed ? "bg-severity_motion/50" : "bg-severity_motion", + 1: reviewed + ? "bg-severity_significant_motion/50" + : "bg-severity_significant_motion", 2: reviewed ? "bg-severity_detection/50" : "bg-severity_detection", 3: reviewed ? "bg-severity_alert/50" : "bg-severity_alert", }; diff --git a/web/src/hooks/use-camera-activity.ts b/web/src/hooks/use-camera-activity.ts index 01406d29a..d2fa49671 100644 --- a/web/src/hooks/use-camera-activity.ts +++ b/web/src/hooks/use-camera-activity.ts @@ -4,8 +4,6 @@ import { useMotionActivity, } from "@/api/ws"; import { CameraConfig } from "@/types/frigateConfig"; -import { MotionData, ReviewSegment } from "@/types/review"; -import { TimeRange } from "@/types/timeline"; import { useEffect, useMemo, useState } from "react"; type useCameraActivityReturn = { @@ -68,57 +66,3 @@ export function useCameraActivity( : false, }; } - -export function useCameraMotionTimestamps( - timeRange: TimeRange, - motionOnly: boolean, - events: ReviewSegment[], - motion: MotionData[], -) { - const timestamps = useMemo(() => { - const seekableTimestamps = []; - let lastEventIdx = 0; - let lastMotionIdx = 0; - - for (let i = timeRange.after; i <= timeRange.before; i += 0.5) { - if (!motionOnly) { - seekableTimestamps.push(i); - } else { - const relevantEventIdx = events.findIndex((seg, segIdx) => { - if (segIdx < lastEventIdx) { - return false; - } - - return seg.start_time <= i && seg.end_time >= i; - }); - - if (relevantEventIdx != -1) { - lastEventIdx = relevantEventIdx; - continue; - } - - const relevantMotionIdx = motion.findIndex((mot, motIdx) => { - if (motIdx < lastMotionIdx) { - return false; - } - - return mot.start_time <= i && mot.start_time + 15 >= i; - }); - - if (relevantMotionIdx == -1 || motion[relevantMotionIdx].motion == 0) { - if (relevantMotionIdx != -1) { - lastMotionIdx = relevantMotionIdx; - } - - continue; - } - - seekableTimestamps.push(i); - } - } - - return seekableTimestamps; - }, [timeRange, motionOnly, events, motion]); - - return timestamps; -} diff --git a/web/src/hooks/use-draggable-element.ts b/web/src/hooks/use-draggable-element.ts index d4cd3e713..49499900f 100644 --- a/web/src/hooks/use-draggable-element.ts +++ b/web/src/hooks/use-draggable-element.ts @@ -368,27 +368,10 @@ function useDraggableElement({ const alignedSegmentTime = alignStartDateToTimeline(draggableElementTime); - let segmentElement = timelineRef.current.querySelector( + const segmentElement = timelineRef.current.querySelector( `[data-segment-id="${alignedSegmentTime}"]`, ); - if (!segmentElement) { - // segment not found, maybe we collapsed over a collapsible segment - let searchTime = alignedSegmentTime; - while (searchTime >= timelineStartAligned - timelineDuration) { - // Decrement currentTime by segmentDuration - searchTime -= segmentDuration; - segmentElement = timelineRef.current.querySelector( - `[data-segment-id="${searchTime}"]`, - ); - - if (segmentElement) { - // segmentElement found - break; - } - } - } - if (segmentElement) { const timelineRect = timelineRef.current.getBoundingClientRect(); const timelineTopAbsolute = timelineRect.top; @@ -422,6 +405,37 @@ function useDraggableElement({ segments, ]); + useEffect(() => { + if (timelineRef.current && draggableElementTime && timelineCollapsed) { + const alignedSegmentTime = alignStartDateToTimeline(draggableElementTime); + + let segmentElement = timelineRef.current.querySelector( + `[data-segment-id="${alignedSegmentTime}"]`, + ); + + if (!segmentElement) { + // segment not found, maybe we collapsed over a collapsible segment + let searchTime = alignedSegmentTime; + while (searchTime >= timelineStartAligned - timelineDuration) { + searchTime -= segmentDuration; + segmentElement = timelineRef.current.querySelector( + `[data-segment-id="${searchTime}"]`, + ); + + if (segmentElement) { + // found, set time + if (setDraggableElementTime) { + setDraggableElementTime(searchTime); + } + break; + } + } + } + } + // we know that these deps are correct + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [timelineCollapsed]); + return { handleMouseDown, handleMouseUp, handleMouseMove }; } diff --git a/web/src/hooks/use-motion-segment-utils.ts b/web/src/hooks/use-motion-segment-utils.ts index dfec48358..0482e776e 100644 --- a/web/src/hooks/use-motion-segment-utils.ts +++ b/web/src/hooks/use-motion-segment-utils.ts @@ -33,7 +33,7 @@ export const useMotionSegmentUtils = ( const interpolateMotionAudioData = useCallback( (value: number, newMax: number): number => { - return Math.ceil((Math.abs(value) / 100.0) * newMax) || 1; + return Math.ceil((Math.abs(value) / 100.0) * newMax) || 0; }, [], ); diff --git a/web/src/views/events/EventView.tsx b/web/src/views/events/EventView.tsx index 41101807d..2dfb957d7 100644 --- a/web/src/views/events/EventView.tsx +++ b/web/src/views/events/EventView.tsx @@ -40,7 +40,6 @@ import SummaryTimeline from "@/components/timeline/SummaryTimeline"; import { RecordingStartingPoint } from "@/types/record"; import VideoControls from "@/components/player/VideoControls"; import { TimeRange } from "@/types/timeline"; -import { useCameraMotionTimestamps } from "@/hooks/use-camera-activity"; type EventViewProps = { reviews?: ReviewSegment[]; @@ -247,7 +246,7 @@ export default function EventView({ value="significant_motion" aria-label="Select motion" > - +
Motion
@@ -720,43 +719,111 @@ function MotionReview({ const [playbackRate, setPlaybackRate] = useState(8); const [controlsOpen, setControlsOpen] = useState(false); - const seekTimestamps = useCameraMotionTimestamps( - timeRange, - motionOnly, - reviewItems?.all ?? [], - motionData ?? [], - ); + + const noMotionRanges = useMemo(() => { + if (!motionData || !reviewItems) { + return; + } + + if (!motionOnly) { + return []; + } + + const ranges = []; + let currentSegmentStart = null; + let currentSegmentEnd = null; + + for (let i = 0; i < motionData.length; i = i + segmentDuration / 15) { + const motionStart = motionData[i].start_time; + const motionEnd = motionStart + segmentDuration; + + const segmentMotion = motionData + .slice(i, i + segmentDuration / 15) + .some(({ motion }) => motion !== undefined && motion > 0); + const overlappingReviewItems = reviewItems.all.some( + (item) => + (item.start_time >= motionStart && item.start_time < motionEnd) || + (item.end_time > motionStart && item.end_time <= motionEnd) || + (item.start_time <= motionStart && item.end_time >= motionEnd), + ); + + if (!segmentMotion || overlappingReviewItems) { + if (currentSegmentStart === null) { + currentSegmentStart = motionStart; + } + currentSegmentEnd = motionEnd; + } else { + if (currentSegmentStart !== null) { + ranges.push([currentSegmentStart, currentSegmentEnd]); + currentSegmentStart = null; + currentSegmentEnd = null; + } + } + } + + if (currentSegmentStart !== null) { + ranges.push([currentSegmentStart, currentSegmentEnd]); + } + + return ranges; + }, [motionData, reviewItems, motionOnly]); + + const nextTimestamp = useMemo(() => { + if (!noMotionRanges) { + return; + } + let currentRange = 0; + let nextTimestamp = currentTime + 0.5; + + while (currentRange < noMotionRanges.length) { + const [start, end] = noMotionRanges[currentRange]; + + if (start && end) { + // If the current time is before the start of the current range + if (currentTime < start) { + // The next timestamp is either the start of the current range or currentTime + 0.5, whichever is smaller + nextTimestamp = Math.min(start, nextTimestamp); + break; + } + // If the current time is within the current range + else if (currentTime >= start && currentTime < end) { + // The next timestamp is the end of the current range + nextTimestamp = end; + currentRange++; + } + // If the current time is past the end of the current range + else { + currentRange++; + } + } + } + + return nextTimestamp; + }, [currentTime, noMotionRanges]); + + const timeoutIdRef = useRef(null); useEffect(() => { - if (!playing) { - return; - } - - const interval = 500 / playbackRate; - const startIdx = seekTimestamps.findIndex((time) => time > currentTime); - - if (!startIdx) { - return; - } - - let counter = 0; - const intervalId = setInterval(() => { - counter += 1; - - if (startIdx + counter >= seekTimestamps.length) { - setPlaying(false); + if (nextTimestamp) { + if (!playing && timeoutIdRef.current != null) { + clearTimeout(timeoutIdRef.current); return; } - setCurrentTime(seekTimestamps[startIdx + counter]); - }, interval); + const handleTimeout = () => { + setCurrentTime(nextTimestamp); + timeoutIdRef.current = setTimeout(handleTimeout, 500 / playbackRate); + }; - return () => { - clearInterval(intervalId); - }; - // do not render when current time changes - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [playing, playbackRate]); + timeoutIdRef.current = setTimeout(handleTimeout, 500 / playbackRate); + + return () => { + if (timeoutIdRef.current) { + clearTimeout(timeoutIdRef.current); + } + }; + } + }, [playing, playbackRate, nextTimestamp]); const { alignStartDateToTimeline } = useTimelineUtils({ segmentDuration, @@ -767,11 +834,16 @@ function MotionReview({ if (motionOnly) { return null; } - const segmentTime = alignStartDateToTimeline(currentTime); + const segmentStartTime = alignStartDateToTimeline(currentTime); + const segmentEndTime = segmentStartTime + segmentDuration; const matchingItem = reviewItems?.all.find( (item) => - item.start_time >= segmentTime && - item.end_time <= segmentTime + segmentDuration && + ((item.start_time >= segmentStartTime && + item.start_time < segmentEndTime) || + (item.end_time > segmentStartTime && + item.end_time <= segmentEndTime) || + (item.start_time <= segmentStartTime && + item.end_time >= segmentEndTime)) && item.camera === cameraName, ); diff --git a/web/tailwind.config.js b/web/tailwind.config.js index 58ce1b5fe..916e625d4 100644 --- a/web/tailwind.config.js +++ b/web/tailwind.config.js @@ -9,7 +9,7 @@ module.exports = { ], safelist: [ { - pattern: /(outline|shadow)-severity_(alert|detection|motion)/, + pattern: /(outline|shadow)-severity_(alert|detection|significant_motion)/, }, ], theme: { @@ -87,9 +87,9 @@ module.exports = { DEFAULT: "hsl(var(--severity_detection))", dimmed: "hsl(var(--severity_detection_dimmed))", }, - severity_motion: { - DEFAULT: "hsl(var(--severity_motion))", - dimmed: "hsl(var(--severity_motion_dimmed))", + severity_significant_motion: { + DEFAULT: "hsl(var(--severity_significant_motion))", + dimmed: "hsl(var(--severity_significant_motion_dimmed))", }, motion_review: { DEFAULT: "hsl(var(--motion_review))", diff --git a/web/themes/theme-default.css b/web/themes/theme-default.css index 6dba4474e..b2a0126ba 100644 --- a/web/themes/theme-default.css +++ b/web/themes/theme-default.css @@ -71,8 +71,8 @@ --severity_detection: var(--orange-600); --severity_detection_dimmed: var(--orange-400); - --severity_motion: var(--yellow-400); - --severity_motion_dimmed: var(--yellow-200); + --severity_significant_motion: var(--yellow-400); + --severity_significant_motion_dimmed: var(--yellow-200); --motion_review: hsl(44, 94%, 50%); --motion_review: 44 94% 50%;