Improve Object Lifecycle pane (#16635)

* add path point tracking to backend

* types

* draw paths on lifecycle pane

* make points clickable

* don't display a path if we don't have any saved path points

* only object lifecycle points should have a click handler

* change to debug log

* better debug log message

web/src/components/overlay/detail/ObjectLifecycle.tsx

@@ -11,7 +11,7 @@ import {
   CarouselPrevious,
 } from "@/components/ui/carousel";
 import { Button } from "@/components/ui/button";
-import { ObjectLifecycleSequence } from "@/types/timeline";
+import { ClassType, ObjectLifecycleSequence } from "@/types/timeline";
 import Heading from "@/components/ui/heading";
 import { ReviewDetailPaneType } from "@/types/review";
 import { FrigateConfig } from "@/types/frigateConfig";
@@ -53,6 +53,13 @@ import {
 } from "@/components/ui/context-menu";
 import { useNavigate } from "react-router-dom";
 
+type Position = {
+  x: number;
+  y: number;
+  timestamp: number;
+  lifecycle_item?: ObjectLifecycleSequence;
+};
+
 type ObjectLifecycleProps = {
   className?: string;
   event: Event;
@@ -108,6 +115,17 @@ export default function ObjectLifecycle({
     [config, event],
   );
 
+  const getObjectColor = useCallback(
+    (label: string) => {
+      const objectColor = config?.model?.colormap[label];
+      if (objectColor) {
+        const reversed = [...objectColor].reverse();
+        return reversed;
+      }
+    },
+    [config],
+  );
+
   const getZonePolygon = useCallback(
     (zoneName: string) => {
       if (!imgRef.current || !config) {
@@ -120,7 +138,7 @@ export default function ObjectLifecycle({
 
       return zonePoints
         .split(",")
-        .map(parseFloat)
+        .map(Number.parseFloat)
         .reduce((acc, value, index) => {
           const isXCoordinate = index % 2 === 0;
           const coordinate = isXCoordinate
@@ -158,6 +176,43 @@ export default function ObjectLifecycle({
     );
   }, [config, event.camera]);
 
+  const savedPathPoints = useMemo(() => {
+    return (
+      event.data.path_data?.map(([coords, timestamp]: [number[], number]) => ({
+        x: coords[0],
+        y: coords[1],
+        timestamp,
+        lifecycle_item: undefined,
+      })) || []
+    );
+  }, [event.data.path_data]);
+
+  const eventSequencePoints = useMemo(() => {
+    return (
+      eventSequence
+        ?.filter((event) => event.data.box !== undefined)
+        .map((event) => {
+          const [left, top, width, height] = event.data.box!;
+
+          return {
+            x: left + width / 2, // Center x-coordinate
+            y: top + height, // Bottom y-coordinate
+            timestamp: event.timestamp,
+            lifecycle_item: event,
+          };
+        }) || []
+    );
+  }, [eventSequence]);
+
+  // final object path with timeline points included
+  const pathPoints = useMemo(() => {
+    // don't display a path if we don't have any saved path points
+    if (savedPathPoints.length === 0) return [];
+    return [...savedPathPoints, ...eventSequencePoints].sort(
+      (a, b) => a.timestamp - b.timestamp,
+    );
+  }, [savedPathPoints, eventSequencePoints]);
+
   const [timeIndex, setTimeIndex] = useState(0);
 
   const handleSetBox = useCallback(
@@ -171,12 +226,13 @@ export default function ObjectLifecycle({
           top: `${box[1] * imgRect.height}px`,
           width: `${box[2] * imgRect.width}px`,
           height: `${box[3] * imgRect.height}px`,
+          borderColor: `rgb(${getObjectColor(event.label)?.join(",")})`,
         };
 
         setBoxStyle(style);
       }
     },
-    [imgRef],
+    [imgRef, event, getObjectColor],
   );
 
   // image
@@ -254,6 +310,21 @@ export default function ObjectLifecycle({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [mainApi, thumbnailApi]);
 
+  const handlePathPointClick = useCallback(
+    (index: number) => {
+      if (!mainApi || !thumbnailApi || !eventSequence) return;
+      const sequenceIndex = eventSequence.findIndex(
+        (item) => item.timestamp === pathPoints[index].timestamp,
+      );
+      if (sequenceIndex !== -1) {
+        mainApi.scrollTo(sequenceIndex);
+        thumbnailApi.scrollTo(sequenceIndex);
+        setCurrent(sequenceIndex);
+      }
+    },
+    [mainApi, thumbnailApi, eventSequence, pathPoints],
+  );
+
   if (!event.id || !eventSequence || !config || !timeIndex) {
     return <ActivityIndicator />;
   }
@@ -355,13 +426,33 @@ export default function ObjectLifecycle({
                 ))}
 
               {boxStyle && (
-                <div
-                  className="absolute border-2 border-red-600"
-                  style={boxStyle}
-                >
+                <div className="absolute border-2" style={boxStyle}>
                   <div className="absolute bottom-[-3px] left-1/2 h-[5px] w-[5px] -translate-x-1/2 transform bg-yellow-500" />
                 </div>
               )}
+              {pathPoints && pathPoints.length > 0 && (
+                <div
+                  className="absolute inset-0 flex items-center justify-center"
+                  style={{
+                    width: imgRef.current?.clientWidth,
+                    height: imgRef.current?.clientHeight,
+                  }}
+                  key="path"
+                >
+                  <svg
+                    viewBox={`0 0 ${imgRef.current?.width} ${imgRef.current?.height}`}
+                    className="absolute inset-0"
+                  >
+                    <ObjectPath
+                      positions={pathPoints} // Use any of the example paths
+                      color={getObjectColor(event.label)}
+                      width={2}
+                      imgRef={imgRef}
+                      onPointClick={handlePathPointClick}
+                    />
+                  </svg>
+                </div>
+              )}
             </ContextMenuTrigger>
             <ContextMenuContent>
               <ContextMenuItem>
@@ -699,3 +790,105 @@ function getLifecycleItemDescription(lifecycleItem: ObjectLifecycleSequence) {
       return `${label} detected`;
   }
 }
+
+type ObjectPathProps = {
+  positions?: Position[];
+  color?: number[];
+  width?: number;
+  pointRadius?: number;
+  imgRef: React.RefObject<HTMLImageElement>;
+  onPointClick?: (index: number) => void;
+};
+
+const typeColorMap: Partial<Record<ClassType, [number, number, number]>> = {
+  [ClassType.VISIBLE]: [0, 255, 0], // Green
+  [ClassType.GONE]: [255, 0, 0], // Red
+  [ClassType.ENTERED_ZONE]: [255, 165, 0], // Orange
+  [ClassType.ATTRIBUTE]: [128, 0, 128], // Purple
+  [ClassType.ACTIVE]: [255, 255, 0], // Yellow
+  [ClassType.STATIONARY]: [128, 128, 128], // Gray
+  [ClassType.HEARD]: [0, 255, 255], // Cyan
+  [ClassType.EXTERNAL]: [165, 42, 42], // Brown
+};
+
+function ObjectPath({
+  positions,
+  color = [0, 0, 255],
+  width = 2,
+  pointRadius = 4,
+  imgRef,
+  onPointClick,
+}: ObjectPathProps) {
+  const getAbsolutePositions = useCallback(() => {
+    if (!imgRef.current || !positions) return [];
+    const imgRect = imgRef.current.getBoundingClientRect();
+    return positions.map((pos) => ({
+      x: pos.x * imgRect.width,
+      y: pos.y * imgRect.height,
+      timestamp: pos.timestamp,
+      lifecycle_item: pos.lifecycle_item,
+    }));
+  }, [positions, imgRef]);
+
+  const generateStraightPath = useCallback((points: Position[]) => {
+    if (!points || points.length < 2) return "";
+    let path = `M ${points[0].x} ${points[0].y}`;
+    for (let i = 1; i < points.length; i++) {
+      path += ` L ${points[i].x} ${points[i].y}`;
+    }
+    return path;
+  }, []);
+
+  const getPointColor = (baseColor: number[], type?: ClassType) => {
+    if (type) {
+      const typeColor = typeColorMap[type];
+      if (typeColor) {
+        return `rgb(${typeColor.join(",")})`;
+      }
+    }
+    // normal path point
+    return `rgb(${baseColor.map((c) => Math.max(0, c - 10)).join(",")})`;
+  };
+
+  if (!imgRef.current) return null;
+  const absolutePositions = getAbsolutePositions();
+  const lineColor = `rgb(${color.join(",")})`;
+
+  return (
+    <g>
+      <path
+        d={generateStraightPath(absolutePositions)}
+        fill="none"
+        stroke={lineColor}
+        strokeWidth={width}
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      {absolutePositions.map((pos, index) => (
+        <Tooltip key={`point-${index}`}>
+          <TooltipTrigger asChild>
+            <circle
+              cx={pos.x}
+              cy={pos.y}
+              r={pointRadius}
+              fill={getPointColor(color, pos.lifecycle_item?.class_type)}
+              stroke="white"
+              strokeWidth={width / 2}
+              onClick={() =>
+                pos.lifecycle_item && onPointClick && onPointClick(index)
+              }
+              style={{ cursor: pos.lifecycle_item ? "pointer" : "default" }}
+            />
+          </TooltipTrigger>
+          <TooltipPortal>
+            <TooltipContent side="top" className="capitalize">
+              {pos.lifecycle_item
+                ? getLifecycleItemDescription(pos.lifecycle_item)
+                : "Tracked point"}
+            </TooltipContent>
+          </TooltipPortal>
+        </Tooltip>
+      ))}
+    </g>
+  );
+}

web/src/types/event.ts

@@ -22,5 +22,6 @@ export interface Event {
     area: number;
     ratio: number;
     type: "object" | "audio" | "manual";
+    path_data: [number[], number][];
   };
 }

web/src/types/timeline.ts

@@ -1,3 +1,15 @@
+export enum ClassType {
+  VISIBLE = "visible",
+  GONE = "gone",
+  ENTERED_ZONE = "entered_zone",
+  ATTRIBUTE = "attribute",
+  ACTIVE = "active",
+  STATIONARY = "stationary",
+  HEARD = "heard",
+  EXTERNAL = "external",
+  PATH_POINT = "path_point",
+}
+
 export type ObjectLifecycleSequence = {
   camera: string;
   timestamp: number;
@@ -10,15 +22,7 @@ export type ObjectLifecycleSequence = {
     attribute: string;
     zones: string[];
   };
-  class_type:
-    | "visible"
-    | "gone"
-    | "entered_zone"
-    | "attribute"
-    | "active"
-    | "stationary"
-    | "heard"
-    | "external";
+  class_type: ClassType;
   source_id: string;
   source: string;
 };