From 9e8a42ca0ecb9c043599bbd32df30357dca17b8d Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 5 Mar 2024 12:55:44 -0700 Subject: [PATCH] Motion timeline data (#10245) * Refactor activity api to send motion and audio data * Prepare for using motion data timeline * Get working * reduce to 0 * fix * Formatting * fix typing * add motion data to timelines and allow motion cameas to be selectable * Fix tests * cleanup * Fix not loading preview when changing hours --- frigate/api/app.py | 104 +----------- frigate/api/review.py | 66 +++++++- .../components/player/DynamicVideoPlayer.tsx | 25 +-- .../timeline/MotionReviewTimeline.tsx | 5 +- web/src/components/timeline/MotionSegment.tsx | 5 +- web/src/hooks/use-motion-segment-utils.ts | 8 +- web/src/pages/Events.tsx | 47 ++++-- web/src/pages/UIPlayground.tsx | 28 ++-- web/src/types/review.ts | 6 + web/src/views/events/EventView.tsx | 23 ++- web/src/views/events/RecordingView.tsx | 149 +++++++++++++----- 11 files changed, 269 insertions(+), 197 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index 321bd5758..f4f513e14 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -5,12 +5,9 @@ import json import logging import os import traceback -from collections import defaultdict from datetime import datetime, timedelta from functools import reduce -import numpy as np -import pandas as pd import requests from flask import ( Blueprint, @@ -31,7 +28,7 @@ from frigate.api.review import ReviewBp from frigate.config import FrigateConfig from frigate.const import CONFIG_DIR from frigate.events.external import ExternalEventProcessor -from frigate.models import Event, Recordings, Timeline +from frigate.models import Event, Timeline from frigate.plus import PlusApi from frigate.ptz.onvif import OnvifController from frigate.stats.emitter import StatsEmitter @@ -632,102 +629,3 @@ def hourly_timeline(): "hours": hours, } ) - - -@bp.route("//recording/hourly/activity") -def hourly_timeline_activity(camera_name: str): - """Get hourly summary for timeline.""" - if camera_name not in current_app.frigate_config.cameras: - return make_response( - jsonify({"success": False, "message": "Camera not found"}), - 404, - ) - - before = request.args.get("before", type=float, default=datetime.now()) - after = request.args.get( - "after", type=float, default=datetime.now() - timedelta(hours=1) - ) - tz_name = request.args.get("timezone", default="utc", type=str) - - _, minute_modifier, _ = get_tz_modifiers(tz_name) - minute_offset = int(minute_modifier.split(" ")[0]) - - all_recordings: list[Recordings] = ( - Recordings.select( - Recordings.start_time, - Recordings.duration, - Recordings.objects, - Recordings.motion, - ) - .where(Recordings.camera == camera_name) - .where(Recordings.motion > 0) - .where((Recordings.start_time > after) & (Recordings.end_time < before)) - .order_by(Recordings.start_time.asc()) - .iterator() - ) - - # data format is ex: - # {timestamp: [{ date: 1, count: 1, type: motion }]}] }} - hours: dict[int, list[dict[str, any]]] = defaultdict(list) - - key = datetime.fromtimestamp(after).replace(second=0, microsecond=0) + timedelta( - minutes=minute_offset - ) - check = (key + timedelta(hours=1)).timestamp() - - # set initial start so data is representative of full hour - hours[int(key.timestamp())].append( - [ - key.timestamp(), - 0, - False, - ] - ) - - for recording in all_recordings: - if recording.start_time > check: - hours[int(key.timestamp())].append( - [ - (key + timedelta(minutes=59, seconds=59)).timestamp(), - 0, - False, - ] - ) - key = key + timedelta(hours=1) - check = (key + timedelta(hours=1)).timestamp() - hours[int(key.timestamp())].append( - [ - key.timestamp(), - 0, - False, - ] - ) - - data_type = recording.objects > 0 - count = recording.motion + recording.objects - hours[int(key.timestamp())].append( - [ - recording.start_time + (recording.duration / 2), - 0 if count == 0 else np.log2(count), - data_type, - ] - ) - - # resample data using pandas to get activity on minute to minute basis - for key, data in hours.items(): - df = pd.DataFrame(data, columns=["date", "count", "hasObjects"]) - - # set date as datetime index - df["date"] = pd.to_datetime(df["date"], unit="s") - df.set_index(["date"], inplace=True) - - # normalize data - df = df.resample("T").mean().fillna(0) - - # change types for output - df.index = df.index.astype(int) // (10**9) - df["count"] = df["count"].astype(int) - df["hasObjects"] = df["hasObjects"].astype(bool) - hours[key] = df.reset_index().to_dict("records") - - return jsonify(hours) diff --git a/frigate/api/review.py b/frigate/api/review.py index 0ab524580..2fdc1ef0e 100644 --- a/frigate/api/review.py +++ b/frigate/api/review.py @@ -4,6 +4,7 @@ import logging from datetime import datetime, timedelta from functools import reduce +import pandas as pd from flask import ( Blueprint, jsonify, @@ -12,7 +13,7 @@ from flask import ( ) from peewee import Case, DoesNotExist, fn, operator -from frigate.models import ReviewSegment +from frigate.models import Recordings, ReviewSegment from frigate.util.builtin import get_tz_modifiers logger = logging.getLogger(__name__) @@ -258,3 +259,66 @@ def delete_reviews(ids: str): ReviewSegment.delete().where(ReviewSegment.id << list_of_ids).execute() return make_response(jsonify({"success": True, "message": "Delete reviews"}), 200) + + +@ReviewBp.route("/review/activity") +def review_activity(): + """Get motion and audio activity.""" + before = request.args.get("before", type=float, default=datetime.now().timestamp()) + after = request.args.get( + "after", type=float, default=(datetime.now() - timedelta(hours=1)).timestamp() + ) + + # get scale in seconds + scale = request.args.get("scale", type=int, default=30) + + all_recordings: list[Recordings] = ( + Recordings.select( + Recordings.start_time, + Recordings.duration, + Recordings.objects, + Recordings.motion, + Recordings.dBFS, + ) + .where((Recordings.start_time > after) & (Recordings.end_time < before)) + .order_by(Recordings.start_time.asc()) + .iterator() + ) + + # format is: { timestamp: segment_start_ts, motion: [0-100], audio: [0 - -100] } + # periods where active objects / audio was detected will cause motion / audio to be scaled down + data: list[dict[str, float]] = [] + + for rec in all_recordings: + data.append( + { + "start_time": rec.start_time, + "motion": rec.motion if rec.objects == 0 else 0, + "audio": rec.dBFS if rec.objects == 0 else 0, + } + ) + + # resample data using pandas to get activity on scaled basis + df = pd.DataFrame(data, columns=["start_time", "motion", "audio"]) + + # set date as datetime index + df["start_time"] = pd.to_datetime(df["start_time"], unit="s") + df.set_index(["start_time"], inplace=True) + + # normalize data + df = df.resample(f"{scale}S").mean().fillna(0.0) + df["motion"] = ( + (df["motion"] - df["motion"].min()) + / (df["motion"].max() - df["motion"].min()) + * 100 + ) + df["audio"] = ( + (df["audio"] - df["audio"].max()) + / (df["audio"].min() - df["audio"].max()) + * -100 + ) + + # change types for output + df.index = df.index.astype(int) // (10**9) + normalized = df.reset_index().to_dict("records") + return jsonify(normalized) diff --git a/web/src/components/player/DynamicVideoPlayer.tsx b/web/src/components/player/DynamicVideoPlayer.tsx index 3447d4337..37a4d509a 100644 --- a/web/src/components/player/DynamicVideoPlayer.tsx +++ b/web/src/components/player/DynamicVideoPlayer.tsx @@ -85,6 +85,16 @@ export default function DynamicVideoPlayer({ ); }, [camera, config, previewOnly]); + useEffect(() => { + if (!controller) { + return; + } + + if (onControllerReady) { + onControllerReady(controller); + } + }, [controller, onControllerReady]); + const [hasRecordingAtTime, setHasRecordingAtTime] = useState(true); // keyboard control @@ -215,6 +225,10 @@ export default function DynamicVideoPlayer({ ); setCurrentPreview(preview); + if (preview && previewRef.current) { + previewRef.current.load(); + } + controller.newPlayback({ recordings: recordings ?? [], playbackUri, @@ -283,27 +297,20 @@ export default function DynamicVideoPlayer({ )} {onClick && !hasRecordingAtTime && ( -
+
)}
); diff --git a/web/src/components/timeline/MotionReviewTimeline.tsx b/web/src/components/timeline/MotionReviewTimeline.tsx index 900ceb98d..6cacc0deb 100644 --- a/web/src/components/timeline/MotionReviewTimeline.tsx +++ b/web/src/components/timeline/MotionReviewTimeline.tsx @@ -9,9 +9,8 @@ import { } from "react"; import MotionSegment from "./MotionSegment"; import { useEventUtils } from "@/hooks/use-event-utils"; -import { ReviewSegment, ReviewSeverity } from "@/types/review"; +import { MotionData, ReviewSegment, ReviewSeverity } from "@/types/review"; import ReviewTimeline from "./ReviewTimeline"; -import { MockMotionData } from "@/pages/UIPlayground"; export type MotionReviewTimelineProps = { segmentDuration: number; @@ -25,7 +24,7 @@ export type MotionReviewTimelineProps = { minimapStartTime?: number; minimapEndTime?: number; events: ReviewSegment[]; - motion_events: MockMotionData[]; + motion_events: MotionData[]; severityType: ReviewSeverity; contentRef: RefObject; onHandlebarDraggingChange?: (isDragging: boolean) => void; diff --git a/web/src/components/timeline/MotionSegment.tsx b/web/src/components/timeline/MotionSegment.tsx index fc184e3d1..070b66857 100644 --- a/web/src/components/timeline/MotionSegment.tsx +++ b/web/src/components/timeline/MotionSegment.tsx @@ -1,6 +1,6 @@ import { useEventUtils } from "@/hooks/use-event-utils"; import { useEventSegmentUtils } from "@/hooks/use-event-segment-utils"; -import { ReviewSegment } from "@/types/review"; +import { MotionData, ReviewSegment } from "@/types/review"; import React, { RefObject, useCallback, @@ -10,13 +10,12 @@ import React, { } from "react"; import scrollIntoView from "scroll-into-view-if-needed"; import { MinimapBounds, Tick, Timestamp } from "./segment-metadata"; -import { MockMotionData } from "@/pages/UIPlayground"; import { useMotionSegmentUtils } from "@/hooks/use-motion-segment-utils"; import { isMobile } from "react-device-detect"; type MotionSegmentProps = { events: ReviewSegment[]; - motion_events: MockMotionData[]; + motion_events: MotionData[]; segmentTime: number; segmentDuration: number; timestampSpread: number; diff --git a/web/src/hooks/use-motion-segment-utils.ts b/web/src/hooks/use-motion-segment-utils.ts index 3ec5ee122..7f10069ef 100644 --- a/web/src/hooks/use-motion-segment-utils.ts +++ b/web/src/hooks/use-motion-segment-utils.ts @@ -1,9 +1,9 @@ import { useCallback, useMemo } from "react"; -import { MockMotionData } from "@/pages/UIPlayground"; +import { MotionData } from "@/types/review"; export const useMotionSegmentUtils = ( segmentDuration: number, - motion_events: MockMotionData[], + motion_events: MotionData[], ) => { const halfSegmentDuration = useMemo( () => segmentDuration / 2, @@ -47,7 +47,7 @@ export const useMotionSegmentUtils = ( ); }); - return matchingEvent?.motionValue ?? 0; + return matchingEvent?.motion ?? 0; }, [motion_events, getSegmentStart, getSegmentEnd], ); @@ -61,7 +61,7 @@ export const useMotionSegmentUtils = ( ); }); - return matchingEvent?.audioValue ?? 0; + return matchingEvent?.audio ?? 0; }, [motion_events, getSegmentStart, getSegmentEnd], ); diff --git a/web/src/pages/Events.tsx b/web/src/pages/Events.tsx index 170124262..9ab090fdd 100644 --- a/web/src/pages/Events.tsx +++ b/web/src/pages/Events.tsx @@ -240,22 +240,37 @@ export default function Events() { // selected items - const selectedData = useMemo(() => { + const selectedReviewData = useMemo(() => { if (!config) { return undefined; } - if (!selectedReviewId) { - return undefined; - } - if (!reviewPages) { return undefined; } + if (!selectedReviewId) { + return undefined; + } + const allCameras = reviewFilter?.cameras ?? Object.keys(config.cameras); const allReviews = reviewPages.flat(); + + if (selectedReviewId.startsWith("motion")) { + const motionData = selectedReviewId.split(","); + // format is motion,camera,start_time + return { + camera: motionData[1], + severity: "significant_motion" as ReviewSeverity, + start_time: parseFloat(motionData[2]), + allCameras: allCameras, + cameraSegments: allReviews.filter((seg) => + allCameras.includes(seg.camera), + ), + }; + } + const selectedReview = allReviews.find( (item) => item.id == selectedReviewId, ); @@ -265,7 +280,9 @@ export default function Events() { } return { - selected: selectedReview, + camera: selectedReview.camera, + severity: selectedReview.severity, + start_time: selectedReview.start_time, allCameras: allCameras, cameraSegments: allReviews.filter((seg) => allCameras.includes(seg.camera), @@ -280,12 +297,14 @@ export default function Events() { return ; } - if (selectedData) { + if (selectedReviewData) { if (isMobile) { return ( ); @@ -293,11 +312,11 @@ export default function Events() { return ( ); diff --git a/web/src/pages/UIPlayground.tsx b/web/src/pages/UIPlayground.tsx index 10306e07e..bbfe211b9 100644 --- a/web/src/pages/UIPlayground.tsx +++ b/web/src/pages/UIPlayground.tsx @@ -4,7 +4,12 @@ import useSWR from "swr"; import { FrigateConfig } from "@/types/frigateConfig"; import ActivityIndicator from "@/components/indicators/activity-indicator"; import EventReviewTimeline from "@/components/timeline/EventReviewTimeline"; -import { ReviewData, ReviewSegment, ReviewSeverity } from "@/types/review"; +import { + MotionData, + ReviewData, + ReviewSegment, + ReviewSeverity, +} from "@/types/review"; import { Button } from "@/components/ui/button"; import CameraActivityIndicator from "@/components/indicators/CameraActivityIndicator"; import MotionReviewTimeline from "@/components/timeline/MotionReviewTimeline"; @@ -53,14 +58,7 @@ function ColorSwatch({ name, value }: { name: string; value: string }) { ); } -export type MockMotionData = { - start_time: number; - end_time: number; - motionValue: number; - audioValue: number; -}; - -function generateRandomMotionAudioData(): MockMotionData[] { +function generateRandomMotionAudioData(): MotionData[] { const now = new Date(); const endTime = now.getTime() / 1000; const startTime = endTime - 24 * 60 * 60; // 24 hours ago @@ -72,14 +70,12 @@ function generateRandomMotionAudioData(): MockMotionData[] { startTimestamp < endTime; startTimestamp += interval ) { - const endTimestamp = startTimestamp + interval; - const motionValue = Math.floor(Math.random() * 101); // Random number between 0 and 100 - const audioValue = Math.random() * -100; // Random negative value between -100 and 0 + const motion = Math.floor(Math.random() * 101); // Random number between 0 and 100 + const audio = Math.random() * -100; // Random negative value between -100 and 0 data.push({ start_time: startTimestamp, - end_time: endTimestamp, - motionValue, - audioValue, + motion, + audio, }); } @@ -126,7 +122,7 @@ function UIPlayground() { const { data: config } = useSWR("config"); const contentRef = useRef(null); const [mockEvents, setMockEvents] = useState([]); - const [mockMotionData, setMockMotionData] = useState([]); + const [mockMotionData, setMockMotionData] = useState([]); const [handlebarTime, setHandlebarTime] = useState( Math.floor(Date.now() / 1000) - 15 * 60, ); diff --git a/web/src/types/review.ts b/web/src/types/review.ts index 5030ee003..8e52dc4af 100644 --- a/web/src/types/review.ts +++ b/web/src/types/review.ts @@ -37,3 +37,9 @@ export type ReviewSummary = { total_detection: number; total_motion: number; }; + +export type MotionData = { + start_time: number; + motion: number; + audio: number; +}; diff --git a/web/src/views/events/EventView.tsx b/web/src/views/events/EventView.tsx index 170776281..1b12952ee 100644 --- a/web/src/views/events/EventView.tsx +++ b/web/src/views/events/EventView.tsx @@ -14,6 +14,7 @@ import { useScrollLockout } from "@/hooks/use-mouse-listener"; import { FrigateConfig } from "@/types/frigateConfig"; import { Preview } from "@/types/preview"; import { + MotionData, ReviewFilter, ReviewSegment, ReviewSeverity, @@ -33,6 +34,7 @@ import { isDesktop, isMobile } from "react-device-detect"; import { LuFolderCheck } from "react-icons/lu"; import { MdCircle } from "react-icons/md"; import useSWR from "swr"; +import MotionReviewTimeline from "@/components/timeline/MotionReviewTimeline"; type EventViewProps = { reviewPages?: ReviewSegment[][]; @@ -284,6 +286,7 @@ export default function EventView({ relevantPreviews={relevantPreviews} timeRange={timeRange} filter={filter} + onSelectReview={onSelectReview} /> )}
@@ -526,6 +529,7 @@ type MotionReviewProps = { relevantPreviews?: Preview[]; timeRange: { before: number; after: number }; filter?: ReviewFilter; + onSelectReview: (data: string, ctrl: boolean) => void; }; function MotionReview({ contentRef, @@ -533,6 +537,7 @@ function MotionReview({ relevantPreviews, timeRange, filter, + onSelectReview, }: MotionReviewProps) { const segmentDuration = 30; const { data: config } = useSWR("config"); @@ -561,6 +566,17 @@ function MotionReview({ {}, ); + // motion data + + const { data: motionData } = useSWR([ + "review/activity", + { + before: timeRange.before, + after: timeRange.after, + scale: segmentDuration / 2, + }, + ]); + // timeline time const lastFullHour = useMemo(() => { @@ -580,6 +596,7 @@ function MotionReview({ ); // move to next clip + useEffect(() => { if ( !videoPlayersRef.current && @@ -638,12 +655,15 @@ function MotionReview({ videoPlayersRef.current[camera.name] = controller; setPlayerReady(true); }} + onClick={() => + onSelectReview(`motion,${camera.name},${currentTime}`, false) + } /> ); })}
- diff --git a/web/src/views/events/RecordingView.tsx b/web/src/views/events/RecordingView.tsx index 21bf884ff..30ef6617e 100644 --- a/web/src/views/events/RecordingView.tsx +++ b/web/src/views/events/RecordingView.tsx @@ -2,13 +2,17 @@ import DynamicVideoPlayer, { DynamicVideoController, } from "@/components/player/DynamicVideoPlayer"; import EventReviewTimeline from "@/components/timeline/EventReviewTimeline"; +import MotionReviewTimeline from "@/components/timeline/MotionReviewTimeline"; import { Button } from "@/components/ui/button"; import { Preview } from "@/types/preview"; -import { ReviewSegment, ReviewSeverity } from "@/types/review"; +import { MotionData, ReviewSegment, ReviewSeverity } from "@/types/review"; import { getChunkedTimeDay } from "@/utils/timelineUtil"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { IoMdArrowRoundBack } from "react-icons/io"; import { useNavigate } from "react-router-dom"; +import useSWR from "swr"; + +const SEGMENT_DURATION = 30; type DesktopRecordingViewProps = { startCamera: string; @@ -116,6 +120,21 @@ export function DesktopRecordingView({ [allCameras, currentTime, mainCamera], ); + // motion timeline data + + const { data: motionData } = useSWR( + severity == "significant_motion" + ? [ + "review/activity", + { + before: timeRange.end, + after: timeRange.start, + scale: SEGMENT_DURATION / 2, + }, + ] + : null, + ); + return (
); } type MobileRecordingViewProps = { - selectedReview: ReviewSegment; + startCamera: string; + startTime: number; + severity: ReviewSeverity; reviewItems: ReviewSegment[]; relevantPreviews?: Preview[]; }; export function MobileRecordingView({ - selectedReview, + startCamera, + startTime, + severity, reviewItems, relevantPreviews, }: MobileRecordingViewProps) { @@ -219,16 +259,10 @@ export function MobileRecordingView({ // timeline time - const timeRange = useMemo( - () => getChunkedTimeDay(selectedReview.start_time), - [selectedReview], - ); + const timeRange = useMemo(() => getChunkedTimeDay(startTime), [startTime]); const [selectedRangeIdx, setSelectedRangeIdx] = useState( timeRange.ranges.findIndex((chunk) => { - return ( - chunk.start <= selectedReview.start_time && - chunk.end >= selectedReview.start_time - ); + return chunk.start <= startTime && chunk.end >= startTime; }), ); @@ -251,7 +285,7 @@ export function MobileRecordingView({ const [scrubbing, setScrubbing] = useState(false); const [currentTime, setCurrentTime] = useState( - selectedReview?.start_time || Date.now() / 1000, + startTime || Date.now() / 1000, ); useEffect(() => { @@ -269,6 +303,21 @@ export function MobileRecordingView({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [scrubbing]); + // motion timeline data + + const { data: motionData } = useSWR( + severity == "significant_motion" + ? [ + "review/activity", + { + before: timeRange.end, + after: timeRange.start, + scale: SEGMENT_DURATION / 2, + }, + ] + : null, + ); + return (
);