Improve Object Lifecycle pane (#16635)

* add path point tracking to backend

* types

* draw paths on lifecycle pane

* make points clickable

* don't display a path if we don't have any saved path points

* only object lifecycle points should have a click handler

* change to debug log

* better debug log message

frigate/api/event.py

@@ -336,6 +336,7 @@ def events_explore(limit: int = 10):
                         "sub_label_score",
                         "average_estimated_speed",
                         "velocity_angle",
+                        "path_data",
                     ]
                 },
                 "event_count": label_counts[event.label],
@@ -622,6 +623,7 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
                 "sub_label_score",
                 "average_estimated_speed",
                 "velocity_angle",
+                "path_data",
             ]
         }
 

frigate/events/maintainer.py

@@ -28,6 +28,7 @@ def should_update_db(prev_event: Event, current_event: Event) -> bool:
             or prev_event["average_estimated_speed"]
             != current_event["average_estimated_speed"]
             or prev_event["velocity_angle"] != current_event["velocity_angle"]
+            or prev_event["path_data"] != current_event["path_data"]
         ):
             return True
     return False
@@ -217,6 +218,7 @@ class EventProcessor(threading.Thread):
                     "velocity_angle": event_data["velocity_angle"],
                     "type": "object",
                     "max_severity": event_data.get("max_severity"),
+                    "path_data": event_data.get("path_data"),
                 },
             }
 

frigate/track/tracked_object.py

@@ -2,6 +2,7 @@
 
 import base64
 import logging
+import math
 from collections import defaultdict
 from statistics import median
 from typing import Optional
@@ -66,6 +67,7 @@ class TrackedObject:
         self.current_estimated_speed = 0
         self.average_estimated_speed = 0
         self.velocity_angle = 0
+        self.path_data = []
         self.previous = self.to_dict()
 
     @property
@@ -148,6 +150,7 @@ class TrackedObject:
                     "attributes": obj_data["attributes"],
                     "current_estimated_speed": self.current_estimated_speed,
                     "velocity_angle": self.velocity_angle,
+                    "path_data": self.path_data,
                 }
                 thumb_update = True
 
@@ -300,6 +303,29 @@ class TrackedObject:
             if self.obj_data["frame_time"] - self.previous["frame_time"] >= (1 / 3):
                 autotracker_update = True
 
+            # update path
+            width = self.camera_config.detect.width
+            height = self.camera_config.detect.height
+            bottom_center = (
+                round(obj_data["centroid"][0] / width, 4),
+                round(obj_data["box"][3] / height, 4),
+            )
+
+            # calculate a reasonable movement threshold (e.g., 5% of the frame diagonal)
+            threshold = 0.05 * math.sqrt(width**2 + height**2) / max(width, height)
+
+            if not self.path_data:
+                self.path_data.append((bottom_center, obj_data["frame_time"]))
+            elif (
+                math.dist(self.path_data[-1][0], bottom_center) >= threshold
+                or len(self.path_data) == 1
+            ):
+                # check Euclidean distance before appending
+                self.path_data.append((bottom_center, obj_data["frame_time"]))
+                logger.debug(
+                    f"Point tracking: {obj_data['id']}, {bottom_center}, {obj_data['frame_time']}"
+                )
+
         self.obj_data.update(obj_data)
         self.current_zones = current_zones
         return (thumb_update, significant_change, autotracker_update)
@@ -336,6 +362,7 @@ class TrackedObject:
             "current_estimated_speed": self.current_estimated_speed,
             "average_estimated_speed": self.average_estimated_speed,
             "velocity_angle": self.velocity_angle,
+            "path_data": self.path_data,
         }
 
         if include_thumbnail:

web/src/components/overlay/detail/ObjectLifecycle.tsx

@@ -11,7 +11,7 @@ import {
   CarouselPrevious,
 } from "@/components/ui/carousel";
 import { Button } from "@/components/ui/button";
-import { ObjectLifecycleSequence } from "@/types/timeline";
+import { ClassType, ObjectLifecycleSequence } from "@/types/timeline";
 import Heading from "@/components/ui/heading";
 import { ReviewDetailPaneType } from "@/types/review";
 import { FrigateConfig } from "@/types/frigateConfig";
@@ -53,6 +53,13 @@ import {
 } from "@/components/ui/context-menu";
 import { useNavigate } from "react-router-dom";
 
+type Position = {
+  x: number;
+  y: number;
+  timestamp: number;
+  lifecycle_item?: ObjectLifecycleSequence;
+};
+
 type ObjectLifecycleProps = {
   className?: string;
   event: Event;
@@ -108,6 +115,17 @@ export default function ObjectLifecycle({
     [config, event],
   );
 
+  const getObjectColor = useCallback(
+    (label: string) => {
+      const objectColor = config?.model?.colormap[label];
+      if (objectColor) {
+        const reversed = [...objectColor].reverse();
+        return reversed;
+      }
+    },
+    [config],
+  );
+
   const getZonePolygon = useCallback(
     (zoneName: string) => {
       if (!imgRef.current || !config) {
@@ -120,7 +138,7 @@ export default function ObjectLifecycle({
 
       return zonePoints
         .split(",")
-        .map(parseFloat)
+        .map(Number.parseFloat)
         .reduce((acc, value, index) => {
           const isXCoordinate = index % 2 === 0;
           const coordinate = isXCoordinate
@@ -158,6 +176,43 @@ export default function ObjectLifecycle({
     );
   }, [config, event.camera]);
 
+  const savedPathPoints = useMemo(() => {
+    return (
+      event.data.path_data?.map(([coords, timestamp]: [number[], number]) => ({
+        x: coords[0],
+        y: coords[1],
+        timestamp,
+        lifecycle_item: undefined,
+      })) || []
+    );
+  }, [event.data.path_data]);
+
+  const eventSequencePoints = useMemo(() => {
+    return (
+      eventSequence
+        ?.filter((event) => event.data.box !== undefined)
+        .map((event) => {
+          const [left, top, width, height] = event.data.box!;
+
+          return {
+            x: left + width / 2, // Center x-coordinate
+            y: top + height, // Bottom y-coordinate
+            timestamp: event.timestamp,
+            lifecycle_item: event,
+          };
+        }) || []
+    );
+  }, [eventSequence]);
+
+  // final object path with timeline points included
+  const pathPoints = useMemo(() => {
+    // don't display a path if we don't have any saved path points
+    if (savedPathPoints.length === 0) return [];
+    return [...savedPathPoints, ...eventSequencePoints].sort(
+      (a, b) => a.timestamp - b.timestamp,
+    );
+  }, [savedPathPoints, eventSequencePoints]);
+
   const [timeIndex, setTimeIndex] = useState(0);
 
   const handleSetBox = useCallback(
@@ -171,12 +226,13 @@ export default function ObjectLifecycle({
           top: `${box[1] * imgRect.height}px`,
           width: `${box[2] * imgRect.width}px`,
           height: `${box[3] * imgRect.height}px`,
+          borderColor: `rgb(${getObjectColor(event.label)?.join(",")})`,
         };
 
         setBoxStyle(style);
       }
     },
-    [imgRef],
+    [imgRef, event, getObjectColor],
   );
 
   // image
@@ -254,6 +310,21 @@ export default function ObjectLifecycle({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [mainApi, thumbnailApi]);
 
+  const handlePathPointClick = useCallback(
+    (index: number) => {
+      if (!mainApi || !thumbnailApi || !eventSequence) return;
+      const sequenceIndex = eventSequence.findIndex(
+        (item) => item.timestamp === pathPoints[index].timestamp,
+      );
+      if (sequenceIndex !== -1) {
+        mainApi.scrollTo(sequenceIndex);
+        thumbnailApi.scrollTo(sequenceIndex);
+        setCurrent(sequenceIndex);
+      }
+    },
+    [mainApi, thumbnailApi, eventSequence, pathPoints],
+  );
+
   if (!event.id || !eventSequence || !config || !timeIndex) {
     return <ActivityIndicator />;
   }
@@ -355,13 +426,33 @@ export default function ObjectLifecycle({
                 ))}
 
               {boxStyle && (
-                <div
+                <div className="absolute border-2" style={boxStyle}>
-                  className="absolute border-2 border-red-600"
-                  style={boxStyle}
-                >
                   <div className="absolute bottom-[-3px] left-1/2 h-[5px] w-[5px] -translate-x-1/2 transform bg-yellow-500" />
                 </div>
               )}
+              {pathPoints && pathPoints.length > 0 && (
+                <div
+                  className="absolute inset-0 flex items-center justify-center"
+                  style={{
+                    width: imgRef.current?.clientWidth,
+                    height: imgRef.current?.clientHeight,
+                  }}
+                  key="path"
+                >
+                  <svg
+                    viewBox={`0 0 ${imgRef.current?.width} ${imgRef.current?.height}`}
+                    className="absolute inset-0"
+                  >
+                    <ObjectPath
+                      positions={pathPoints} // Use any of the example paths
+                      color={getObjectColor(event.label)}
+                      width={2}
+                      imgRef={imgRef}
+                      onPointClick={handlePathPointClick}
+                    />
+                  </svg>
+                </div>
+              )}
             </ContextMenuTrigger>
             <ContextMenuContent>
               <ContextMenuItem>
@@ -699,3 +790,105 @@ function getLifecycleItemDescription(lifecycleItem: ObjectLifecycleSequence) {
       return `${label} detected`;
   }
 }
+
+type ObjectPathProps = {
+  positions?: Position[];
+  color?: number[];
+  width?: number;
+  pointRadius?: number;
+  imgRef: React.RefObject<HTMLImageElement>;
+  onPointClick?: (index: number) => void;
+};
+
+const typeColorMap: Partial<Record<ClassType, [number, number, number]>> = {
+  [ClassType.VISIBLE]: [0, 255, 0], // Green
+  [ClassType.GONE]: [255, 0, 0], // Red
+  [ClassType.ENTERED_ZONE]: [255, 165, 0], // Orange
+  [ClassType.ATTRIBUTE]: [128, 0, 128], // Purple
+  [ClassType.ACTIVE]: [255, 255, 0], // Yellow
+  [ClassType.STATIONARY]: [128, 128, 128], // Gray
+  [ClassType.HEARD]: [0, 255, 255], // Cyan
+  [ClassType.EXTERNAL]: [165, 42, 42], // Brown
+};
+
+function ObjectPath({
+  positions,
+  color = [0, 0, 255],
+  width = 2,
+  pointRadius = 4,
+  imgRef,
+  onPointClick,
+}: ObjectPathProps) {
+  const getAbsolutePositions = useCallback(() => {
+    if (!imgRef.current || !positions) return [];
+    const imgRect = imgRef.current.getBoundingClientRect();
+    return positions.map((pos) => ({
+      x: pos.x * imgRect.width,
+      y: pos.y * imgRect.height,
+      timestamp: pos.timestamp,
+      lifecycle_item: pos.lifecycle_item,
+    }));
+  }, [positions, imgRef]);
+
+  const generateStraightPath = useCallback((points: Position[]) => {
+    if (!points || points.length < 2) return "";
+    let path = `M ${points[0].x} ${points[0].y}`;
+    for (let i = 1; i < points.length; i++) {
+      path += ` L ${points[i].x} ${points[i].y}`;
+    }
+    return path;
+  }, []);
+
+  const getPointColor = (baseColor: number[], type?: ClassType) => {
+    if (type) {
+      const typeColor = typeColorMap[type];
+      if (typeColor) {
+        return `rgb(${typeColor.join(",")})`;
+      }
+    }
+    // normal path point
+    return `rgb(${baseColor.map((c) => Math.max(0, c - 10)).join(",")})`;
+  };
+
+  if (!imgRef.current) return null;
+  const absolutePositions = getAbsolutePositions();
+  const lineColor = `rgb(${color.join(",")})`;
+
+  return (
+    <g>
+      <path
+        d={generateStraightPath(absolutePositions)}
+        fill="none"
+        stroke={lineColor}
+        strokeWidth={width}
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      {absolutePositions.map((pos, index) => (
+        <Tooltip key={`point-${index}`}>
+          <TooltipTrigger asChild>
+            <circle
+              cx={pos.x}
+              cy={pos.y}
+              r={pointRadius}
+              fill={getPointColor(color, pos.lifecycle_item?.class_type)}
+              stroke="white"
+              strokeWidth={width / 2}
+              onClick={() =>
+                pos.lifecycle_item && onPointClick && onPointClick(index)
+              }
+              style={{ cursor: pos.lifecycle_item ? "pointer" : "default" }}
+            />
+          </TooltipTrigger>
+          <TooltipPortal>
+            <TooltipContent side="top" className="capitalize">
+              {pos.lifecycle_item
+                ? getLifecycleItemDescription(pos.lifecycle_item)
+                : "Tracked point"}
+            </TooltipContent>
+          </TooltipPortal>
+        </Tooltip>
+      ))}
+    </g>
+  );
+}

web/src/types/event.ts

@@ -22,5 +22,6 @@ export interface Event {
     area: number;
     ratio: number;
     type: "object" | "audio" | "manual";
+    path_data: [number[], number][];
   };
 }

web/src/types/timeline.ts

@@ -1,3 +1,15 @@
+export enum ClassType {
+  VISIBLE = "visible",
+  GONE = "gone",
+  ENTERED_ZONE = "entered_zone",
+  ATTRIBUTE = "attribute",
+  ACTIVE = "active",
+  STATIONARY = "stationary",
+  HEARD = "heard",
+  EXTERNAL = "external",
+  PATH_POINT = "path_point",
+}
+
 export type ObjectLifecycleSequence = {
   camera: string;
   timestamp: number;
@@ -10,15 +22,7 @@ export type ObjectLifecycleSequence = {
     attribute: string;
     zones: string[];
   };
-  class_type:
+  class_type: ClassType;
-    | "visible"
-    | "gone"
-    | "entered_zone"
-    | "attribute"
-    | "active"
-    | "stationary"
-    | "heard"
-    | "external";
   source_id: string;
   source: string;
 };