diff --git a/frigate/api/event.py b/frigate/api/event.py index 247366920..2df32471e 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -336,6 +336,7 @@ def events_explore(limit: int = 10): "sub_label_score", "average_estimated_speed", "velocity_angle", + "path_data", ] }, "event_count": label_counts[event.label], @@ -622,6 +623,7 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends()) "sub_label_score", "average_estimated_speed", "velocity_angle", + "path_data", ] } diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index d49da5a97..fc02dd37a 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -28,6 +28,7 @@ def should_update_db(prev_event: Event, current_event: Event) -> bool: or prev_event["average_estimated_speed"] != current_event["average_estimated_speed"] or prev_event["velocity_angle"] != current_event["velocity_angle"] + or prev_event["path_data"] != current_event["path_data"] ): return True return False @@ -217,6 +218,7 @@ class EventProcessor(threading.Thread): "velocity_angle": event_data["velocity_angle"], "type": "object", "max_severity": event_data.get("max_severity"), + "path_data": event_data.get("path_data"), }, } diff --git a/frigate/track/tracked_object.py b/frigate/track/tracked_object.py index ac57083df..0e7464bc2 100644 --- a/frigate/track/tracked_object.py +++ b/frigate/track/tracked_object.py @@ -2,6 +2,7 @@ import base64 import logging +import math from collections import defaultdict from statistics import median from typing import Optional @@ -66,6 +67,7 @@ class TrackedObject: self.current_estimated_speed = 0 self.average_estimated_speed = 0 self.velocity_angle = 0 + self.path_data = [] self.previous = self.to_dict() @property @@ -148,6 +150,7 @@ class TrackedObject: "attributes": obj_data["attributes"], "current_estimated_speed": self.current_estimated_speed, "velocity_angle": self.velocity_angle, + "path_data": self.path_data, } thumb_update = True @@ -300,6 +303,29 @@ class TrackedObject: if self.obj_data["frame_time"] - self.previous["frame_time"] >= (1 / 3): autotracker_update = True + # update path + width = self.camera_config.detect.width + height = self.camera_config.detect.height + bottom_center = ( + round(obj_data["centroid"][0] / width, 4), + round(obj_data["box"][3] / height, 4), + ) + + # calculate a reasonable movement threshold (e.g., 5% of the frame diagonal) + threshold = 0.05 * math.sqrt(width**2 + height**2) / max(width, height) + + if not self.path_data: + self.path_data.append((bottom_center, obj_data["frame_time"])) + elif ( + math.dist(self.path_data[-1][0], bottom_center) >= threshold + or len(self.path_data) == 1 + ): + # check Euclidean distance before appending + self.path_data.append((bottom_center, obj_data["frame_time"])) + logger.debug( + f"Point tracking: {obj_data['id']}, {bottom_center}, {obj_data['frame_time']}" + ) + self.obj_data.update(obj_data) self.current_zones = current_zones return (thumb_update, significant_change, autotracker_update) @@ -336,6 +362,7 @@ class TrackedObject: "current_estimated_speed": self.current_estimated_speed, "average_estimated_speed": self.average_estimated_speed, "velocity_angle": self.velocity_angle, + "path_data": self.path_data, } if include_thumbnail: diff --git a/web/src/components/overlay/detail/ObjectLifecycle.tsx b/web/src/components/overlay/detail/ObjectLifecycle.tsx index 7481607eb..656ae275c 100644 --- a/web/src/components/overlay/detail/ObjectLifecycle.tsx +++ b/web/src/components/overlay/detail/ObjectLifecycle.tsx @@ -11,7 +11,7 @@ import { CarouselPrevious, } from "@/components/ui/carousel"; import { Button } from "@/components/ui/button"; -import { ObjectLifecycleSequence } from "@/types/timeline"; +import { ClassType, ObjectLifecycleSequence } from "@/types/timeline"; import Heading from "@/components/ui/heading"; import { ReviewDetailPaneType } from "@/types/review"; import { FrigateConfig } from "@/types/frigateConfig"; @@ -53,6 +53,13 @@ import { } from "@/components/ui/context-menu"; import { useNavigate } from "react-router-dom"; +type Position = { + x: number; + y: number; + timestamp: number; + lifecycle_item?: ObjectLifecycleSequence; +}; + type ObjectLifecycleProps = { className?: string; event: Event; @@ -108,6 +115,17 @@ export default function ObjectLifecycle({ [config, event], ); + const getObjectColor = useCallback( + (label: string) => { + const objectColor = config?.model?.colormap[label]; + if (objectColor) { + const reversed = [...objectColor].reverse(); + return reversed; + } + }, + [config], + ); + const getZonePolygon = useCallback( (zoneName: string) => { if (!imgRef.current || !config) { @@ -120,7 +138,7 @@ export default function ObjectLifecycle({ return zonePoints .split(",") - .map(parseFloat) + .map(Number.parseFloat) .reduce((acc, value, index) => { const isXCoordinate = index % 2 === 0; const coordinate = isXCoordinate @@ -158,6 +176,43 @@ export default function ObjectLifecycle({ ); }, [config, event.camera]); + const savedPathPoints = useMemo(() => { + return ( + event.data.path_data?.map(([coords, timestamp]: [number[], number]) => ({ + x: coords[0], + y: coords[1], + timestamp, + lifecycle_item: undefined, + })) || [] + ); + }, [event.data.path_data]); + + const eventSequencePoints = useMemo(() => { + return ( + eventSequence + ?.filter((event) => event.data.box !== undefined) + .map((event) => { + const [left, top, width, height] = event.data.box!; + + return { + x: left + width / 2, // Center x-coordinate + y: top + height, // Bottom y-coordinate + timestamp: event.timestamp, + lifecycle_item: event, + }; + }) || [] + ); + }, [eventSequence]); + + // final object path with timeline points included + const pathPoints = useMemo(() => { + // don't display a path if we don't have any saved path points + if (savedPathPoints.length === 0) return []; + return [...savedPathPoints, ...eventSequencePoints].sort( + (a, b) => a.timestamp - b.timestamp, + ); + }, [savedPathPoints, eventSequencePoints]); + const [timeIndex, setTimeIndex] = useState(0); const handleSetBox = useCallback( @@ -171,12 +226,13 @@ export default function ObjectLifecycle({ top: `${box[1] * imgRect.height}px`, width: `${box[2] * imgRect.width}px`, height: `${box[3] * imgRect.height}px`, + borderColor: `rgb(${getObjectColor(event.label)?.join(",")})`, }; setBoxStyle(style); } }, - [imgRef], + [imgRef, event, getObjectColor], ); // image @@ -254,6 +310,21 @@ export default function ObjectLifecycle({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [mainApi, thumbnailApi]); + const handlePathPointClick = useCallback( + (index: number) => { + if (!mainApi || !thumbnailApi || !eventSequence) return; + const sequenceIndex = eventSequence.findIndex( + (item) => item.timestamp === pathPoints[index].timestamp, + ); + if (sequenceIndex !== -1) { + mainApi.scrollTo(sequenceIndex); + thumbnailApi.scrollTo(sequenceIndex); + setCurrent(sequenceIndex); + } + }, + [mainApi, thumbnailApi, eventSequence, pathPoints], + ); + if (!event.id || !eventSequence || !config || !timeIndex) { return ; } @@ -355,13 +426,33 @@ export default function ObjectLifecycle({ ))} {boxStyle && ( -
+
)} + {pathPoints && pathPoints.length > 0 && ( +
+ + + +
+ )} @@ -699,3 +790,105 @@ function getLifecycleItemDescription(lifecycleItem: ObjectLifecycleSequence) { return `${label} detected`; } } + +type ObjectPathProps = { + positions?: Position[]; + color?: number[]; + width?: number; + pointRadius?: number; + imgRef: React.RefObject; + onPointClick?: (index: number) => void; +}; + +const typeColorMap: Partial> = { + [ClassType.VISIBLE]: [0, 255, 0], // Green + [ClassType.GONE]: [255, 0, 0], // Red + [ClassType.ENTERED_ZONE]: [255, 165, 0], // Orange + [ClassType.ATTRIBUTE]: [128, 0, 128], // Purple + [ClassType.ACTIVE]: [255, 255, 0], // Yellow + [ClassType.STATIONARY]: [128, 128, 128], // Gray + [ClassType.HEARD]: [0, 255, 255], // Cyan + [ClassType.EXTERNAL]: [165, 42, 42], // Brown +}; + +function ObjectPath({ + positions, + color = [0, 0, 255], + width = 2, + pointRadius = 4, + imgRef, + onPointClick, +}: ObjectPathProps) { + const getAbsolutePositions = useCallback(() => { + if (!imgRef.current || !positions) return []; + const imgRect = imgRef.current.getBoundingClientRect(); + return positions.map((pos) => ({ + x: pos.x * imgRect.width, + y: pos.y * imgRect.height, + timestamp: pos.timestamp, + lifecycle_item: pos.lifecycle_item, + })); + }, [positions, imgRef]); + + const generateStraightPath = useCallback((points: Position[]) => { + if (!points || points.length < 2) return ""; + let path = `M ${points[0].x} ${points[0].y}`; + for (let i = 1; i < points.length; i++) { + path += ` L ${points[i].x} ${points[i].y}`; + } + return path; + }, []); + + const getPointColor = (baseColor: number[], type?: ClassType) => { + if (type) { + const typeColor = typeColorMap[type]; + if (typeColor) { + return `rgb(${typeColor.join(",")})`; + } + } + // normal path point + return `rgb(${baseColor.map((c) => Math.max(0, c - 10)).join(",")})`; + }; + + if (!imgRef.current) return null; + const absolutePositions = getAbsolutePositions(); + const lineColor = `rgb(${color.join(",")})`; + + return ( + + + {absolutePositions.map((pos, index) => ( + + + + pos.lifecycle_item && onPointClick && onPointClick(index) + } + style={{ cursor: pos.lifecycle_item ? "pointer" : "default" }} + /> + + + + {pos.lifecycle_item + ? getLifecycleItemDescription(pos.lifecycle_item) + : "Tracked point"} + + + + ))} + + ); +} diff --git a/web/src/types/event.ts b/web/src/types/event.ts index 0e7aa9916..d7c8ca665 100644 --- a/web/src/types/event.ts +++ b/web/src/types/event.ts @@ -22,5 +22,6 @@ export interface Event { area: number; ratio: number; type: "object" | "audio" | "manual"; + path_data: [number[], number][]; }; } diff --git a/web/src/types/timeline.ts b/web/src/types/timeline.ts index 94ef75eba..66366c2f0 100644 --- a/web/src/types/timeline.ts +++ b/web/src/types/timeline.ts @@ -1,3 +1,15 @@ +export enum ClassType { + VISIBLE = "visible", + GONE = "gone", + ENTERED_ZONE = "entered_zone", + ATTRIBUTE = "attribute", + ACTIVE = "active", + STATIONARY = "stationary", + HEARD = "heard", + EXTERNAL = "external", + PATH_POINT = "path_point", +} + export type ObjectLifecycleSequence = { camera: string; timestamp: number; @@ -10,15 +22,7 @@ export type ObjectLifecycleSequence = { attribute: string; zones: string[]; }; - class_type: - | "visible" - | "gone" - | "entered_zone" - | "attribute" - | "active" - | "stationary" - | "heard" - | "external"; + class_type: ClassType; source_id: string; source: string; };