Improve Object Lifecycle pane (#16635)

* add path point tracking to backend * types * draw paths on lifecycle pane * make points clickable * don't display a path if we don't have any saved path points * only object lifecycle points should have a click handler * change to debug log * better debug log message
2025-02-18 00:16:41 +01:00 · 2025-02-17 10:37:17 -06:00 · 2025-02-17 10:37:17 -06:00 · 124d92daa9
commit 124d92daa9
parent 3f07d2d37c
6 changed files with 245 additions and 16 deletions
--- a/frigate/api/event.py
+++ b/frigate/api/event.py
@ -336,6 +336,7 @@ def events_explore(limit: int = 10):
                        "sub_label_score",
                        "average_estimated_speed",
                        "velocity_angle",
+                        "path_data",
                    ]
                },
                "event_count": label_counts[event.label],
@ -622,6 +623,7 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
                "sub_label_score",
                "average_estimated_speed",
                "velocity_angle",
+                "path_data",
            ]
        }

--- a/frigate/events/maintainer.py
+++ b/frigate/events/maintainer.py
@ -28,6 +28,7 @@ def should_update_db(prev_event: Event, current_event: Event) -> bool:
            or prev_event["average_estimated_speed"]
            != current_event["average_estimated_speed"]
            or prev_event["velocity_angle"] != current_event["velocity_angle"]
+            or prev_event["path_data"] != current_event["path_data"]
        ):
            return True
    return False
@ -217,6 +218,7 @@ class EventProcessor(threading.Thread):
                    "velocity_angle": event_data["velocity_angle"],
                    "type": "object",
                    "max_severity": event_data.get("max_severity"),
+                    "path_data": event_data.get("path_data"),
                },
            }

--- a/frigate/track/tracked_object.py
+++ b/frigate/track/tracked_object.py
@ -2,6 +2,7 @@

 import base64
 import logging
+import math
 from collections import defaultdict
 from statistics import median
 from typing import Optional
@ -66,6 +67,7 @@ class TrackedObject:
        self.current_estimated_speed = 0
        self.average_estimated_speed = 0
        self.velocity_angle = 0
+        self.path_data = []
        self.previous = self.to_dict()

    @property
@ -148,6 +150,7 @@ class TrackedObject:
                    "attributes": obj_data["attributes"],
                    "current_estimated_speed": self.current_estimated_speed,
                    "velocity_angle": self.velocity_angle,
+                    "path_data": self.path_data,
                }
                thumb_update = True

@ -300,6 +303,29 @@ class TrackedObject:
            if self.obj_data["frame_time"] - self.previous["frame_time"] >= (1 / 3):
                autotracker_update = True

+            # update path
+            width = self.camera_config.detect.width
+            height = self.camera_config.detect.height
+            bottom_center = (
+                round(obj_data["centroid"][0] / width, 4),
+                round(obj_data["box"][3] / height, 4),
+            )
+
+            # calculate a reasonable movement threshold (e.g., 5% of the frame diagonal)
+            threshold = 0.05 * math.sqrt(width**2 + height**2) / max(width, height)
+
+            if not self.path_data:
+                self.path_data.append((bottom_center, obj_data["frame_time"]))
+            elif (
+                math.dist(self.path_data[-1][0], bottom_center) >= threshold
+                or len(self.path_data) == 1
+            ):
+                # check Euclidean distance before appending
+                self.path_data.append((bottom_center, obj_data["frame_time"]))
+                logger.debug(
+                    f"Point tracking: {obj_data['id']}, {bottom_center}, {obj_data['frame_time']}"
+                )
+
        self.obj_data.update(obj_data)
        self.current_zones = current_zones
        return (thumb_update, significant_change, autotracker_update)
@ -336,6 +362,7 @@ class TrackedObject:
            "current_estimated_speed": self.current_estimated_speed,
            "average_estimated_speed": self.average_estimated_speed,
            "velocity_angle": self.velocity_angle,
+            "path_data": self.path_data,
        }

        if include_thumbnail:
--- a/web/src/components/overlay/detail/ObjectLifecycle.tsx
+++ b/web/src/components/overlay/detail/ObjectLifecycle.tsx
@ -11,7 +11,7 @@ import {
  CarouselPrevious,
 } from "@/components/ui/carousel";
 import { Button } from "@/components/ui/button";
-import { ObjectLifecycleSequence } from "@/types/timeline";
+import { ClassType, ObjectLifecycleSequence } from "@/types/timeline";
 import Heading from "@/components/ui/heading";
 import { ReviewDetailPaneType } from "@/types/review";
 import { FrigateConfig } from "@/types/frigateConfig";
@ -53,6 +53,13 @@ import {
 } from "@/components/ui/context-menu";
 import { useNavigate } from "react-router-dom";

+type Position = {
+  x: number;
+  y: number;
+  timestamp: number;
+  lifecycle_item?: ObjectLifecycleSequence;
+};
+
 type ObjectLifecycleProps = {
  className?: string;
  event: Event;
@ -108,6 +115,17 @@ export default function ObjectLifecycle({
    [config, event],
  );

+  const getObjectColor = useCallback(
+    (label: string) => {
+      const objectColor = config?.model?.colormap[label];
+      if (objectColor) {
+        const reversed = [...objectColor].reverse();
+        return reversed;
+      }
+    },
+    [config],
+  );
+
  const getZonePolygon = useCallback(
    (zoneName: string) => {
      if (!imgRef.current || !config) {
@ -120,7 +138,7 @@ export default function ObjectLifecycle({

      return zonePoints
        .split(",")
-        .map(parseFloat)
+        .map(Number.parseFloat)
        .reduce((acc, value, index) => {
          const isXCoordinate = index % 2 === 0;
          const coordinate = isXCoordinate
@ -158,6 +176,43 @@ export default function ObjectLifecycle({
    );
  }, [config, event.camera]);

+  const savedPathPoints = useMemo(() => {
+    return (
+      event.data.path_data?.map(([coords, timestamp]: [number[], number]) => ({
+        x: coords[0],
+        y: coords[1],
+        timestamp,
+        lifecycle_item: undefined,
+      })) || []
+    );
+  }, [event.data.path_data]);
+
+  const eventSequencePoints = useMemo(() => {
+    return (
+      eventSequence
+        ?.filter((event) => event.data.box !== undefined)
+        .map((event) => {
+          const [left, top, width, height] = event.data.box!;
+
+          return {
+            x: left + width / 2, // Center x-coordinate
+            y: top + height, // Bottom y-coordinate
+            timestamp: event.timestamp,
+            lifecycle_item: event,
+          };
+        }) || []
+    );
+  }, [eventSequence]);
+
+  // final object path with timeline points included
+  const pathPoints = useMemo(() => {
+    // don't display a path if we don't have any saved path points
+    if (savedPathPoints.length === 0) return [];
+    return [...savedPathPoints, ...eventSequencePoints].sort(
+      (a, b) => a.timestamp - b.timestamp,
+    );
+  }, [savedPathPoints, eventSequencePoints]);
+
  const [timeIndex, setTimeIndex] = useState(0);

  const handleSetBox = useCallback(
@ -171,12 +226,13 @@ export default function ObjectLifecycle({
          top: `${box[1] * imgRect.height}px`,
          width: `${box[2] * imgRect.width}px`,
          height: `${box[3] * imgRect.height}px`,
+          borderColor: `rgb(${getObjectColor(event.label)?.join(",")})`,
        };

        setBoxStyle(style);
      }
    },
-    [imgRef],
+    [imgRef, event, getObjectColor],
  );

  // image
@ -254,6 +310,21 @@ export default function ObjectLifecycle({
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [mainApi, thumbnailApi]);

+  const handlePathPointClick = useCallback(
+    (index: number) => {
+      if (!mainApi || !thumbnailApi || !eventSequence) return;
+      const sequenceIndex = eventSequence.findIndex(
+        (item) => item.timestamp === pathPoints[index].timestamp,
+      );
+      if (sequenceIndex !== -1) {
+        mainApi.scrollTo(sequenceIndex);
+        thumbnailApi.scrollTo(sequenceIndex);
+        setCurrent(sequenceIndex);
+      }
+    },
+    [mainApi, thumbnailApi, eventSequence, pathPoints],
+  );
+
  if (!event.id || !eventSequence || !config || !timeIndex) {
    return <ActivityIndicator />;
  }
@ -355,13 +426,33 @@ export default function ObjectLifecycle({
                ))}

              {boxStyle && (
-                <div
-                  className="absolute border-2 border-red-600"
-                  style={boxStyle}
-                >
+                <div className="absolute border-2" style={boxStyle}>
                  <div className="absolute bottom-[-3px] left-1/2 h-[5px] w-[5px] -translate-x-1/2 transform bg-yellow-500" />
                </div>
              )}
+              {pathPoints && pathPoints.length > 0 && (
+                <div
+                  className="absolute inset-0 flex items-center justify-center"
+                  style={{
+                    width: imgRef.current?.clientWidth,
+                    height: imgRef.current?.clientHeight,
+                  }}
+                  key="path"
+                >
+                  <svg
+                    viewBox={`0 0 ${imgRef.current?.width} ${imgRef.current?.height}`}
+                    className="absolute inset-0"
+                  >
+                    <ObjectPath
+                      positions={pathPoints} // Use any of the example paths
+                      color={getObjectColor(event.label)}
+                      width={2}
+                      imgRef={imgRef}
+                      onPointClick={handlePathPointClick}
+                    />
+                  </svg>
+                </div>
+              )}
            </ContextMenuTrigger>
            <ContextMenuContent>
              <ContextMenuItem>
@ -699,3 +790,105 @@ function getLifecycleItemDescription(lifecycleItem: ObjectLifecycleSequence) {
      return `${label} detected`;
  }
 }
+
+type ObjectPathProps = {
+  positions?: Position[];
+  color?: number[];
+  width?: number;
+  pointRadius?: number;
+  imgRef: React.RefObject<HTMLImageElement>;
+  onPointClick?: (index: number) => void;
+};
+
+const typeColorMap: Partial<Record<ClassType, [number, number, number]>> = {
+  [ClassType.VISIBLE]: [0, 255, 0], // Green
+  [ClassType.GONE]: [255, 0, 0], // Red
+  [ClassType.ENTERED_ZONE]: [255, 165, 0], // Orange
+  [ClassType.ATTRIBUTE]: [128, 0, 128], // Purple
+  [ClassType.ACTIVE]: [255, 255, 0], // Yellow
+  [ClassType.STATIONARY]: [128, 128, 128], // Gray
+  [ClassType.HEARD]: [0, 255, 255], // Cyan
+  [ClassType.EXTERNAL]: [165, 42, 42], // Brown
+};
+
+function ObjectPath({
+  positions,
+  color = [0, 0, 255],
+  width = 2,
+  pointRadius = 4,
+  imgRef,
+  onPointClick,
+}: ObjectPathProps) {
+  const getAbsolutePositions = useCallback(() => {
+    if (!imgRef.current || !positions) return [];
+    const imgRect = imgRef.current.getBoundingClientRect();
+    return positions.map((pos) => ({
+      x: pos.x * imgRect.width,
+      y: pos.y * imgRect.height,
+      timestamp: pos.timestamp,
+      lifecycle_item: pos.lifecycle_item,
+    }));
+  }, [positions, imgRef]);
+
+  const generateStraightPath = useCallback((points: Position[]) => {
+    if (!points || points.length < 2) return "";
+    let path = `M ${points[0].x} ${points[0].y}`;
+    for (let i = 1; i < points.length; i++) {
+      path += ` L ${points[i].x} ${points[i].y}`;
+    }
+    return path;
+  }, []);
+
+  const getPointColor = (baseColor: number[], type?: ClassType) => {
+    if (type) {
+      const typeColor = typeColorMap[type];
+      if (typeColor) {
+        return `rgb(${typeColor.join(",")})`;
+      }
+    }
+    // normal path point
+    return `rgb(${baseColor.map((c) => Math.max(0, c - 10)).join(",")})`;
+  };
+
+  if (!imgRef.current) return null;
+  const absolutePositions = getAbsolutePositions();
+  const lineColor = `rgb(${color.join(",")})`;
+
+  return (
+    <g>
+      <path
+        d={generateStraightPath(absolutePositions)}
+        fill="none"
+        stroke={lineColor}
+        strokeWidth={width}
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      {absolutePositions.map((pos, index) => (
+        <Tooltip key={`point-${index}`}>
+          <TooltipTrigger asChild>
+            <circle
+              cx={pos.x}
+              cy={pos.y}
+              r={pointRadius}
+              fill={getPointColor(color, pos.lifecycle_item?.class_type)}
+              stroke="white"
+              strokeWidth={width / 2}
+              onClick={() =>
+                pos.lifecycle_item && onPointClick && onPointClick(index)
+              }
+              style={{ cursor: pos.lifecycle_item ? "pointer" : "default" }}
+            />
+          </TooltipTrigger>
+          <TooltipPortal>
+            <TooltipContent side="top" className="capitalize">
+              {pos.lifecycle_item
+                ? getLifecycleItemDescription(pos.lifecycle_item)
+                : "Tracked point"}
+            </TooltipContent>
+          </TooltipPortal>
+        </Tooltip>
+      ))}
+    </g>
+  );
+}
--- a/web/src/types/event.ts
+++ b/web/src/types/event.ts
@ -22,5 +22,6 @@ export interface Event {
    area: number;
    ratio: number;
    type: "object" | "audio" | "manual";
+    path_data: [number[], number][];
  };
 }
--- a/web/src/types/timeline.ts
+++ b/web/src/types/timeline.ts
@ -1,3 +1,15 @@
+export enum ClassType {
+  VISIBLE = "visible",
+  GONE = "gone",
+  ENTERED_ZONE = "entered_zone",
+  ATTRIBUTE = "attribute",
+  ACTIVE = "active",
+  STATIONARY = "stationary",
+  HEARD = "heard",
+  EXTERNAL = "external",
+  PATH_POINT = "path_point",
+}
+
 export type ObjectLifecycleSequence = {
  camera: string;
  timestamp: number;
@ -10,15 +22,7 @@ export type ObjectLifecycleSequence = {
    attribute: string;
    zones: string[];
  };
-  class_type:
-    | "visible"
-    | "gone"
-    | "entered_zone"
-    | "attribute"
-    | "active"
-    | "stationary"
-    | "heard"
-    | "external";
+  class_type: ClassType;
  source_id: string;
  source: string;
 };