Snap points to edges and create object mask from bounding box (#16488)

web/src/components/overlay/detail/ObjectLifecycle.tsx

@@ -45,6 +45,13 @@ import {
 } from "@/components/ui/tooltip";
 import { AnnotationSettingsPane } from "./AnnotationSettingsPane";
 import { TooltipPortal } from "@radix-ui/react-tooltip";
+import {
+  ContextMenu,
+  ContextMenuContent,
+  ContextMenuItem,
+  ContextMenuTrigger,
+} from "@/components/ui/context-menu";
+import { useNavigate } from "react-router-dom";
 
 type ObjectLifecycleProps = {
   className?: string;
@@ -68,6 +75,7 @@ export default function ObjectLifecycle({
 
   const { data: config } = useSWR<FrigateConfig>("config");
   const apiHost = useApiHost();
+  const navigate = useNavigate();
 
   const [imgLoaded, setImgLoaded] = useState(false);
   const imgRef = useRef<HTMLImageElement>(null);
@@ -293,62 +301,83 @@ export default function ObjectLifecycle({
             imgLoaded ? "visible" : "invisible",
           )}
         >
-          <img
+          <ContextMenu>
-            key={event.id}
+            <ContextMenuTrigger>
-            ref={imgRef}
+              <img
-            className={cn(
+                key={event.id}
-              "max-h-[50dvh] max-w-full select-none rounded-lg object-contain",
+                ref={imgRef}
-            )}
+                className={cn(
-            loading={isSafari ? "eager" : "lazy"}
+                  "max-h-[50dvh] max-w-full select-none rounded-lg object-contain",
-            style={
+                )}
-              isIOS
+                loading={isSafari ? "eager" : "lazy"}
-                ? {
+                style={
-                    WebkitUserSelect: "none",
+                  isIOS
-                    WebkitTouchCallout: "none",
+                    ? {
-                  }
+                        WebkitUserSelect: "none",
-                : undefined
+                        WebkitTouchCallout: "none",
-            }
+                      }
-            draggable={false}
+                    : undefined
-            src={src}
+                }
-            onLoad={() => setImgLoaded(true)}
+                draggable={false}
-            onError={() => setHasError(true)}
+                src={src}
-          />
+                onLoad={() => setImgLoaded(true)}
+                onError={() => setHasError(true)}
+              />
 
-          {showZones &&
+              {showZones &&
-            lifecycleZones?.map((zone) => (
+                lifecycleZones?.map((zone) => (
-              <div
+                  <div
-                className="absolute inset-0 flex items-center justify-center"
+                    className="absolute inset-0 flex items-center justify-center"
-                style={{
-                  width: imgRef.current?.clientWidth,
-                  height: imgRef.current?.clientHeight,
-                }}
-                key={zone}
-              >
-                <svg
-                  viewBox={`0 0 ${imgRef.current?.width} ${imgRef.current?.height}`}
-                  className="absolute inset-0"
-                >
-                  <polygon
-                    points={getZonePolygon(zone)}
-                    className="fill-none stroke-2"
                     style={{
-                      stroke: `rgb(${getZoneColor(zone)?.join(",")})`,
+                      width: imgRef.current?.clientWidth,
-                      fill:
+                      height: imgRef.current?.clientHeight,
-                        selectedZone == zone
-                          ? `rgba(${getZoneColor(zone)?.join(",")}, 0.5)`
-                          : `rgba(${getZoneColor(zone)?.join(",")}, 0.3)`,
-                      strokeWidth: selectedZone == zone ? 4 : 2,
                     }}
-                  />
+                    key={zone}
-                </svg>
+                  >
-              </div>
+                    <svg
-            ))}
+                      viewBox={`0 0 ${imgRef.current?.width} ${imgRef.current?.height}`}
+                      className="absolute inset-0"
+                    >
+                      <polygon
+                        points={getZonePolygon(zone)}
+                        className="fill-none stroke-2"
+                        style={{
+                          stroke: `rgb(${getZoneColor(zone)?.join(",")})`,
+                          fill:
+                            selectedZone == zone
+                              ? `rgba(${getZoneColor(zone)?.join(",")}, 0.5)`
+                              : `rgba(${getZoneColor(zone)?.join(",")}, 0.3)`,
+                          strokeWidth: selectedZone == zone ? 4 : 2,
+                        }}
+                      />
+                    </svg>
+                  </div>
+                ))}
 
-          {boxStyle && (
+              {boxStyle && (
-            <div className="absolute border-2 border-red-600" style={boxStyle}>
+                <div
-              <div className="absolute bottom-[-3px] left-1/2 h-[5px] w-[5px] -translate-x-1/2 transform bg-yellow-500" />
+                  className="absolute border-2 border-red-600"
-            </div>
+                  style={boxStyle}
-          )}
+                >
+                  <div className="absolute bottom-[-3px] left-1/2 h-[5px] w-[5px] -translate-x-1/2 transform bg-yellow-500" />
+                </div>
+              )}
+            </ContextMenuTrigger>
+            <ContextMenuContent>
+              <ContextMenuItem>
+                <div
+                  className="flex w-full cursor-pointer items-center justify-start gap-2 p-2"
+                  onClick={() =>
+                    navigate(
+                      `/settings?page=masks%20/%20zones&camera=${event.camera}&object_mask=${eventSequence?.[current].data.box}`,
+                    )
+                  }
+                >
+                  <div className="text-primary">Create Object Mask</div>
+                </div>
+              </ContextMenuItem>
+            </ContextMenuContent>
+          </ContextMenu>
         </div>
       </div>
 

web/src/components/settings/MotionMaskEditPane.tsx

@@ -33,6 +33,8 @@ type MotionMaskEditPaneProps = {
   setIsLoading: React.Dispatch<React.SetStateAction<boolean>>;
   onSave?: () => void;
   onCancel?: () => void;
+  snapPoints: boolean;
+  setSnapPoints: React.Dispatch<React.SetStateAction<boolean>>;
 };
 
 export default function MotionMaskEditPane({
@@ -45,6 +47,8 @@ export default function MotionMaskEditPane({
   setIsLoading,
   onSave,
   onCancel,
+  snapPoints,
+  setSnapPoints,
 }: MotionMaskEditPaneProps) {
   const { data: config, mutate: updateConfig } =
     useSWR<FrigateConfig>("config");
@@ -252,6 +256,8 @@ export default function MotionMaskEditPane({
             polygons={polygons}
             setPolygons={setPolygons}
             activePolygonIndex={activePolygonIndex}
+            snapPoints={snapPoints}
+            setSnapPoints={setSnapPoints}
           />
         </div>
       )}

web/src/components/settings/ObjectMaskEditPane.tsx

@@ -49,6 +49,8 @@ type ObjectMaskEditPaneProps = {
   setIsLoading: React.Dispatch<React.SetStateAction<boolean>>;
   onSave?: () => void;
   onCancel?: () => void;
+  snapPoints: boolean;
+  setSnapPoints: React.Dispatch<React.SetStateAction<boolean>>;
 };
 
 export default function ObjectMaskEditPane({
@@ -61,6 +63,8 @@ export default function ObjectMaskEditPane({
   setIsLoading,
   onSave,
   onCancel,
+  snapPoints,
+  setSnapPoints,
 }: ObjectMaskEditPaneProps) {
   const { data: config, mutate: updateConfig } =
     useSWR<FrigateConfig>("config");
@@ -272,6 +276,8 @@ export default function ObjectMaskEditPane({
             polygons={polygons}
             setPolygons={setPolygons}
             activePolygonIndex={activePolygonIndex}
+            snapPoints={snapPoints}
+            setSnapPoints={setSnapPoints}
           />
         </div>
       )}

web/src/components/settings/PolygonCanvas.tsx

@@ -6,6 +6,7 @@ import type { KonvaEventObject } from "konva/lib/Node";
 import { Polygon, PolygonType } from "@/types/canvas";
 import { useApiHost } from "@/api";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
+import { snapPointToLines } from "@/utils/canvasUtil";
 
 type PolygonCanvasProps = {
   containerRef: RefObject<HTMLDivElement>;
@@ -18,6 +19,7 @@ type PolygonCanvasProps = {
   hoveredPolygonIndex: number | null;
   selectedZoneMask: PolygonType[] | undefined;
   activeLine?: number;
+  snapPoints: boolean;
 };
 
 export function PolygonCanvas({
@@ -31,6 +33,7 @@ export function PolygonCanvas({
   hoveredPolygonIndex,
   selectedZoneMask,
   activeLine,
+  snapPoints,
 }: PolygonCanvasProps) {
   const [isLoaded, setIsLoaded] = useState(false);
   const [image, setImage] = useState<HTMLImageElement | undefined>();
@@ -156,9 +159,23 @@ export function PolygonCanvas({
           intersection?.getClassName() !== "Circle") ||
         (activePolygon.isFinished && intersection?.name() == "unfilled-line")
       ) {
+        let newPoint = [mousePos.x, mousePos.y];
+
+        if (snapPoints) {
+          // Snap to other polygons' edges
+          const otherPolygons = polygons.filter(
+            (_, i) => i !== activePolygonIndex,
+          );
+          const snappedPos = snapPointToLines(newPoint, otherPolygons, 10);
+
+          if (snappedPos) {
+            newPoint = snappedPos;
+          }
+        }
+
         const { updatedPoints, updatedPointsOrder } = addPointToPolygon(
           activePolygon,
-          [mousePos.x, mousePos.y],
+          newPoint,
         );
 
         updatedPolygons[activePolygonIndex] = {
@@ -184,11 +201,24 @@ export function PolygonCanvas({
     if (stage) {
       // we add an unfilled line for adding points when finished
       const index = e.target.index - (activePolygon.isFinished ? 2 : 1);
-      const pos = [e.target._lastPos!.x, e.target._lastPos!.y];
+      let pos = [e.target._lastPos!.x, e.target._lastPos!.y];
-      if (pos[0] < 0) pos[0] = 0;
+
-      if (pos[1] < 0) pos[1] = 0;
+      if (snapPoints) {
-      if (pos[0] > stage.width()) pos[0] = stage.width();
+        // Snap to other polygons' edges
-      if (pos[1] > stage.height()) pos[1] = stage.height();
+        const otherPolygons = polygons.filter(
+          (_, i) => i !== activePolygonIndex,
+        );
+        const snappedPos = snapPointToLines(pos, otherPolygons, 10); // 10 is the snap threshold
+
+        if (snappedPos) {
+          pos = snappedPos;
+        }
+      }
+
+      // Constrain to stage boundaries
+      pos[0] = Math.max(0, Math.min(pos[0], stage.width()));
+      pos[1] = Math.max(0, Math.min(pos[1], stage.height()));
+
       updatedPolygons[activePolygonIndex] = {
         ...activePolygon,
         points: [
@@ -291,6 +321,16 @@ export function PolygonCanvas({
                 handlePointDragMove={handlePointDragMove}
                 handleGroupDragEnd={handleGroupDragEnd}
                 activeLine={activeLine}
+                snapPoints={snapPoints}
+                snapToLines={(point) =>
+                  snapPoints
+                    ? snapPointToLines(
+                        point,
+                        polygons.filter((_, i) => i !== index),
+                        10,
+                      )
+                    : null
+                }
               />
             ),
         )}
@@ -310,6 +350,16 @@ export function PolygonCanvas({
               handlePointDragMove={handlePointDragMove}
               handleGroupDragEnd={handleGroupDragEnd}
               activeLine={activeLine}
+              snapPoints={snapPoints}
+              snapToLines={(point) =>
+                snapPoints
+                  ? snapPointToLines(
+                      point,
+                      polygons.filter((_, i) => i !== activePolygonIndex),
+                      10,
+                    )
+                  : null
+              }
             />
           )}
       </Layer>

web/src/components/settings/PolygonDrawer.tsx

@@ -28,6 +28,8 @@ type PolygonDrawerProps = {
   handlePointDragMove: (e: KonvaEventObject<MouseEvent | TouchEvent>) => void;
   handleGroupDragEnd: (e: KonvaEventObject<MouseEvent | TouchEvent>) => void;
   activeLine?: number;
+  snapToLines: (point: number[]) => number[] | null;
+  snapPoints: boolean;
 };
 
 export default function PolygonDrawer({
@@ -41,6 +43,8 @@ export default function PolygonDrawer({
   handlePointDragMove,
   handleGroupDragEnd,
   activeLine,
+  snapToLines,
+  snapPoints,
 }: PolygonDrawerProps) {
   const vertexRadius = 6;
   const flattenedPoints = useMemo(() => flattenPoints(points), [points]);
@@ -218,15 +222,32 @@ export default function PolygonDrawer({
             onMouseOver={handleMouseOverPoint}
             onMouseOut={handleMouseOutPoint}
             draggable={isActive}
-            onDragMove={isActive ? handlePointDragMove : undefined}
+            onDragMove={(e) => {
+              if (isActive) {
+                if (snapPoints) {
+                  const snappedPos = snapToLines([e.target.x(), e.target.y()]);
+                  if (snappedPos) {
+                    e.target.position({ x: snappedPos[0], y: snappedPos[1] });
+                  }
+                }
+                handlePointDragMove(e);
+              }
+            }}
             dragBoundFunc={(pos) => {
               if (stageRef.current) {
-                return dragBoundFunc(
+                const boundPos = dragBoundFunc(
                   stageRef.current.width(),
                   stageRef.current.height(),
                   vertexRadius,
                   pos,
                 );
+                if (snapPoints) {
+                  const snappedPos = snapToLines([boundPos.x, boundPos.y]);
+                  return snappedPos
+                    ? { x: snappedPos[0], y: snappedPos[1] }
+                    : boundPos;
+                }
+                return boundPos;
               } else {
                 return pos;
               }

web/src/components/settings/PolygonEditControls.tsx

@@ -2,17 +2,23 @@ import { Polygon } from "@/types/canvas";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
 import { MdOutlineRestartAlt, MdUndo } from "react-icons/md";
 import { Button } from "../ui/button";
+import { TbPolygon, TbPolygonOff } from "react-icons/tb";
+import { cn } from "@/lib/utils";
 
 type PolygonEditControlsProps = {
   polygons: Polygon[];
   setPolygons: React.Dispatch<React.SetStateAction<Polygon[]>>;
   activePolygonIndex: number | undefined;
+  snapPoints: boolean;
+  setSnapPoints: React.Dispatch<React.SetStateAction<boolean>>;
 };
 
 export default function PolygonEditControls({
   polygons,
   setPolygons,
   activePolygonIndex,
+  snapPoints,
+  setSnapPoints,
 }: PolygonEditControlsProps) {
   const undo = () => {
     if (activePolygonIndex === undefined || !polygons) {
@@ -97,6 +103,25 @@ export default function PolygonEditControls({
         </TooltipTrigger>
         <TooltipContent>Reset</TooltipContent>
       </Tooltip>
+      <Tooltip>
+        <TooltipTrigger asChild>
+          <Button
+            variant={snapPoints ? "select" : "default"}
+            className={cn("size-6 rounded-md p-1")}
+            aria-label="Snap points"
+            onClick={() => setSnapPoints((prev) => !prev)}
+          >
+            {snapPoints ? (
+              <TbPolygon className="text-primary" />
+            ) : (
+              <TbPolygonOff className="text-secondary-foreground" />
+            )}
+          </Button>
+        </TooltipTrigger>
+        <TooltipContent>
+          {snapPoints ? "Don't snap points" : "Snap points"}
+        </TooltipContent>
+      </Tooltip>
     </div>
   );
 }

web/src/components/settings/ZoneEditPane.tsx

@@ -41,6 +41,8 @@ type ZoneEditPaneProps = {
   onSave?: () => void;
   onCancel?: () => void;
   setActiveLine: React.Dispatch<React.SetStateAction<number | undefined>>;
+  snapPoints: boolean;
+  setSnapPoints: React.Dispatch<React.SetStateAction<boolean>>;
 };
 
 export default function ZoneEditPane({
@@ -54,6 +56,8 @@ export default function ZoneEditPane({
   onSave,
   onCancel,
   setActiveLine,
+  snapPoints,
+  setSnapPoints,
 }: ZoneEditPaneProps) {
   const { data: config, mutate: updateConfig } =
     useSWR<FrigateConfig>("config");
@@ -483,6 +487,8 @@ export default function ZoneEditPane({
             polygons={polygons}
             setPolygons={setPolygons}
             activePolygonIndex={activePolygonIndex}
+            snapPoints={snapPoints}
+            setSnapPoints={setSnapPoints}
           />
         </div>
       )}

web/src/pages/Settings.tsx

@@ -124,7 +124,7 @@ export default function Settings() {
     if (allSettingsViews.includes(page as SettingsType)) {
       setPage(page as SettingsType);
     }
-    return true;
+    return false;
   });
 
   useSearchEffect("camera", (camera: string) => {
@@ -132,7 +132,7 @@ export default function Settings() {
     if (cameraNames.includes(camera)) {
       setSelectedCamera(camera);
     }
-    return true;
+    return false;
   });
 
   useEffect(() => {

web/src/utils/canvasUtil.ts

@@ -1,4 +1,5 @@
 import { Vector2d } from "konva/lib/types";
+import { Polygon } from "@/types/canvas";
 
 export const getAveragePoint = (points: number[]): Vector2d => {
   let totalX = 0;
@@ -100,3 +101,72 @@ export const masksAreIdentical = (arr1: string[], arr2: string[]): boolean => {
   }
   return true;
 };
+
+export function snapPointToLines(
+  point: number[],
+  polygons: Polygon[],
+  threshold: number,
+): number[] | null {
+  for (const polygon of polygons) {
+    if (!polygon.isFinished) continue;
+
+    for (let i = 0; i < polygon.points.length; i++) {
+      const start = polygon.points[i];
+      const end = polygon.points[(i + 1) % polygon.points.length];
+
+      const snappedPoint = snapPointToLine(point, start, end, threshold);
+      if (snappedPoint) {
+        return snappedPoint;
+      }
+    }
+  }
+
+  return null;
+}
+
+function snapPointToLine(
+  point: number[],
+  lineStart: number[],
+  lineEnd: number[],
+  threshold: number,
+): number[] | null {
+  const [x, y] = point;
+  const [x1, y1] = lineStart;
+  const [x2, y2] = lineEnd;
+
+  const A = x - x1;
+  const B = y - y1;
+  const C = x2 - x1;
+  const D = y2 - y1;
+
+  const dot = A * C + B * D;
+  const lenSq = C * C + D * D;
+  let param = -1;
+
+  if (lenSq !== 0) {
+    param = dot / lenSq;
+  }
+
+  let xx, yy;
+
+  if (param < 0) {
+    xx = x1;
+    yy = y1;
+  } else if (param > 1) {
+    xx = x2;
+    yy = y2;
+  } else {
+    xx = x1 + param * C;
+    yy = y1 + param * D;
+  }
+
+  const dx = x - xx;
+  const dy = y - yy;
+  const distance = Math.sqrt(dx * dx + dy * dy);
+
+  if (distance <= threshold) {
+    return [xx, yy];
+  }
+
+  return null;
+}

web/src/views/settings/MasksAndZonesView.tsx

@@ -37,6 +37,7 @@ import PolygonItem from "@/components/settings/PolygonItem";
 import { Link } from "react-router-dom";
 import { isDesktop } from "react-device-detect";
 import { StatusBarMessagesContext } from "@/context/statusbar-provider";
+import { useSearchEffect } from "@/hooks/use-overlay-state";
 
 type MasksAndZoneViewProps = {
   selectedCamera: string;
@@ -62,6 +63,7 @@ export default function MasksAndZonesView({
   const containerRef = useRef<HTMLDivElement | null>(null);
   const [editPane, setEditPane] = useState<PolygonType | undefined>(undefined);
   const [activeLine, setActiveLine] = useState<number | undefined>();
+  const [snapPoints, setSnapPoints] = useState(false);
 
   const { addMessage } = useContext(StatusBarMessagesContext)!;
 
@@ -142,7 +144,7 @@ export default function MasksAndZonesView({
     }
   }, [scaledHeight, aspectRatio]);
 
-  const handleNewPolygon = (type: PolygonType) => {
+  const handleNewPolygon = (type: PolygonType, coordinates?: number[][]) => {
     if (!cameraConfig) {
       return;
     }
@@ -161,9 +163,9 @@ export default function MasksAndZonesView({
     setEditingPolygons([
       ...(allPolygons || []),
       {
-        points: [],
+        points: coordinates ?? [],
         distances: [],
-        isFinished: false,
+        isFinished: coordinates ? true : false,
         type,
         typeIndex: 9999,
         name: "",
@@ -373,6 +375,48 @@ export default function MasksAndZonesView({
     }
   }, [selectedCamera]);
 
+  useSearchEffect("object_mask", (coordinates: string) => {
+    if (!scaledWidth || !scaledHeight || isLoading) {
+      return false;
+    }
+    // convert box points string to points array
+    const points = coordinates.split(",").map((p) => parseFloat(p));
+
+    const [x1, y1, w, h] = points;
+
+    // bottom center
+    const centerX = x1 + w / 2;
+    const bottomY = y1 + h;
+
+    const centerXAbs = centerX * scaledWidth;
+    const bottomYAbs = bottomY * scaledHeight;
+
+    // padding and clamp
+    const minPadding = 0.1 * w * scaledWidth;
+    const maxPadding = 0.3 * w * scaledWidth;
+    const padding = Math.min(
+      Math.max(minPadding, 0.15 * w * scaledWidth),
+      maxPadding,
+    );
+
+    const top = Math.max(0, bottomYAbs - padding);
+    const bottom = Math.min(scaledHeight, bottomYAbs + padding);
+    const left = Math.max(0, centerXAbs - padding);
+    const right = Math.min(scaledWidth, centerXAbs + padding);
+
+    const paddedBox = [
+      [left, top],
+      [right, top],
+      [right, bottom],
+      [left, bottom],
+    ];
+
+    setEditPane("object_mask");
+    setActivePolygonIndex(undefined);
+    handleNewPolygon("object_mask", paddedBox);
+    return true;
+  });
+
   useEffect(() => {
     document.title = "Mask and Zone Editor - Frigate";
   }, []);
@@ -399,6 +443,8 @@ export default function MasksAndZonesView({
                 onCancel={handleCancel}
                 onSave={handleSave}
                 setActiveLine={setActiveLine}
+                snapPoints={snapPoints}
+                setSnapPoints={setSnapPoints}
               />
             )}
             {editPane == "motion_mask" && (
@@ -412,6 +458,8 @@ export default function MasksAndZonesView({
                 setIsLoading={setIsLoading}
                 onCancel={handleCancel}
                 onSave={handleSave}
+                snapPoints={snapPoints}
+                setSnapPoints={setSnapPoints}
               />
             )}
             {editPane == "object_mask" && (
@@ -425,6 +473,8 @@ export default function MasksAndZonesView({
                 setIsLoading={setIsLoading}
                 onCancel={handleCancel}
                 onSave={handleSave}
+                snapPoints={snapPoints}
+                setSnapPoints={setSnapPoints}
               />
             )}
             {editPane === undefined && (
@@ -662,6 +712,7 @@ export default function MasksAndZonesView({
                   hoveredPolygonIndex={hoveredPolygonIndex}
                   selectedZoneMask={selectedZoneMask}
                   activeLine={activeLine}
+                  snapPoints={true}
                 />
               ) : (
                 <Skeleton className="size-full" />