From c8cec63cb9f411e778badb0f2b342700e7f1dd52 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 9 Feb 2025 15:48:23 -0600 Subject: [PATCH] Object area debugging and improvements (#16432) * add ability to specify min and max area as percentages * debug draw area and ratio * docs * update for best percentage --- docs/docs/configuration/object_filters.md | 2 +- docs/docs/configuration/reference.md | 8 +- frigate/config/camera/objects.py | 10 +- frigate/config/config.py | 8 + frigate/util/config.py | 30 +++ .../components/overlay/DebugDrawingLayer.tsx | 177 ++++++++++++++++++ .../overlay/detail/ObjectLifecycle.tsx | 27 ++- web/src/views/settings/ObjectSettingsView.tsx | 111 ++++++++++- 8 files changed, 352 insertions(+), 21 deletions(-) create mode 100644 web/src/components/overlay/DebugDrawingLayer.tsx diff --git a/docs/docs/configuration/object_filters.md b/docs/docs/configuration/object_filters.md index ca7260094..3f36086c0 100644 --- a/docs/docs/configuration/object_filters.md +++ b/docs/docs/configuration/object_filters.md @@ -34,7 +34,7 @@ False positives can also be reduced by filtering a detection based on its shape. ### Object Area -`min_area` and `max_area` filter on the area of an objects bounding box in pixels and can be used to reduce false positives that are outside the range of expected sizes. For example when a leaf is detected as a dog or when a large tree is detected as a person, these can be reduced by adding a `min_area` / `max_area` filter. +`min_area` and `max_area` filter on the area of an objects bounding box and can be used to reduce false positives that are outside the range of expected sizes. For example when a leaf is detected as a dog or when a large tree is detected as a person, these can be reduced by adding a `min_area` / `max_area` filter. These values can either be in pixels or as a percentage of the frame (for example, 0.12 represents 12% of the frame). ### Object Proportions diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index 7b682e3de..6c95fa27d 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -312,9 +312,11 @@ objects: # Optional: filters to reduce false positives for specific object types filters: person: - # Optional: minimum width*height of the bounding box for the detected object (default: 0) + # Optional: minimum size of the bounding box for the detected object (default: 0). + # Can be specified as an integer for width*height in pixels or as a decimal representing the percentage of the frame (0.000001 to 0.99). min_area: 5000 - # Optional: maximum width*height of the bounding box for the detected object (default: 24000000) + # Optional: maximum size of the bounding box for the detected object (default: 24000000). + # Can be specified as an integer for width*height in pixels or as a decimal representing the percentage of the frame (0.000001 to 0.99). max_area: 100000 # Optional: minimum width/height of the bounding box for the detected object (default: 0) min_ratio: 0.5 @@ -559,7 +561,7 @@ genai: # Optional: Restream configuration # Uses https://github.com/AlexxIT/go2rtc (v1.9.2) # NOTE: The default go2rtc API port (1984) must be used, -# changing this port for the integrated go2rtc instance is not supported. +# changing this port for the integrated go2rtc instance is not supported. go2rtc: # Optional: Live stream configuration for WebUI. diff --git a/frigate/config/camera/objects.py b/frigate/config/camera/objects.py index 578f8e677..0d559b6ce 100644 --- a/frigate/config/camera/objects.py +++ b/frigate/config/camera/objects.py @@ -11,11 +11,13 @@ DEFAULT_TRACKED_OBJECTS = ["person"] class FilterConfig(FrigateBaseModel): - min_area: int = Field( - default=0, title="Minimum area of bounding box for object to be counted." + min_area: Union[int, float] = Field( + default=0, + title="Minimum area of bounding box for object to be counted. Can be pixels (int) or percentage (float between 0.000001 and 0.99).", ) - max_area: int = Field( - default=24000000, title="Maximum area of bounding box for object to be counted." + max_area: Union[int, float] = Field( + default=24000000, + title="Maximum area of bounding box for object to be counted. Can be pixels (int) or percentage (float between 0.000001 and 0.99).", ) min_ratio: float = Field( default=0, diff --git a/frigate/config/config.py b/frigate/config/config.py index f3b17c5fa..c4c502d26 100644 --- a/frigate/config/config.py +++ b/frigate/config/config.py @@ -29,6 +29,7 @@ from frigate.util.builtin import ( ) from frigate.util.config import ( StreamInfoRetriever, + convert_area_to_pixels, find_config_file, get_relative_coordinates, migrate_frigate_config, @@ -148,6 +149,13 @@ class RuntimeFilterConfig(FilterConfig): if mask is not None: config["mask"] = create_mask(frame_shape, mask) + # Convert min_area and max_area to pixels if they're percentages + if "min_area" in config: + config["min_area"] = convert_area_to_pixels(config["min_area"], frame_shape) + + if "max_area" in config: + config["max_area"] = convert_area_to_pixels(config["max_area"], frame_shape) + super().__init__(**config) def dict(self, **kwargs): diff --git a/frigate/util/config.py b/frigate/util/config.py index d456c7557..a8664ea4e 100644 --- a/frigate/util/config.py +++ b/frigate/util/config.py @@ -347,6 +347,36 @@ def get_relative_coordinates( return mask +def convert_area_to_pixels( + area_value: Union[int, float], frame_shape: tuple[int, int] +) -> int: + """ + Convert area specification to pixels. + + Args: + area_value: Area value (pixels or percentage) + frame_shape: Tuple of (height, width) for the frame + + Returns: + Area in pixels + """ + # If already an integer, assume it's in pixels + if isinstance(area_value, int): + return area_value + + # Check if it's a percentage + if isinstance(area_value, float): + if 0.000001 <= area_value <= 0.99: + frame_area = frame_shape[0] * frame_shape[1] + return max(1, int(frame_area * area_value)) + else: + raise ValueError( + f"Percentage must be between 0.000001 and 0.99, got {area_value}" + ) + + raise TypeError(f"Unexpected type for area: {type(area_value)}") + + class StreamInfoRetriever: def __init__(self) -> None: self.stream_cache: dict[str, tuple[int, int]] = {} diff --git a/web/src/components/overlay/DebugDrawingLayer.tsx b/web/src/components/overlay/DebugDrawingLayer.tsx new file mode 100644 index 000000000..b45ef3f81 --- /dev/null +++ b/web/src/components/overlay/DebugDrawingLayer.tsx @@ -0,0 +1,177 @@ +import React, { useState, useRef, useCallback, useMemo } from "react"; +import { Stage, Layer, Rect } from "react-konva"; +import { KonvaEventObject } from "konva/lib/Node"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import Konva from "konva"; +import { useResizeObserver } from "@/hooks/resize-observer"; + +type DebugDrawingLayerProps = { + containerRef: React.RefObject; + cameraWidth: number; + cameraHeight: number; +}; + +function DebugDrawingLayer({ + containerRef, + cameraWidth, + cameraHeight, +}: DebugDrawingLayerProps) { + const [rectangle, setRectangle] = useState<{ + x: number; + y: number; + width: number; + height: number; + } | null>(null); + const [isDrawing, setIsDrawing] = useState(false); + const [showPopover, setShowPopover] = useState(false); + const stageRef = useRef(null); + + const [{ width: containerWidth }] = useResizeObserver(containerRef); + + const imageSize = useMemo(() => { + const aspectRatio = cameraWidth / cameraHeight; + const imageWidth = containerWidth; + const imageHeight = imageWidth / aspectRatio; + return { width: imageWidth, height: imageHeight }; + }, [containerWidth, cameraWidth, cameraHeight]); + + const handleMouseDown = (e: KonvaEventObject) => { + const pos = e.target.getStage()?.getPointerPosition(); + if (pos) { + setIsDrawing(true); + setRectangle({ x: pos.x, y: pos.y, width: 0, height: 0 }); + } + }; + + const handleMouseMove = (e: KonvaEventObject) => { + if (!isDrawing) return; + + const pos = e.target.getStage()?.getPointerPosition(); + if (pos && rectangle) { + setRectangle({ + ...rectangle, + width: pos.x - rectangle.x, + height: pos.y - rectangle.y, + }); + } + }; + + const handleMouseUp = () => { + setIsDrawing(false); + if (rectangle) { + setShowPopover(true); + } + }; + + const convertToRealCoordinates = useCallback( + (x: number, y: number, width: number, height: number) => { + const scaleX = cameraWidth / imageSize.width; + const scaleY = cameraHeight / imageSize.height; + return { + x: x * scaleX, + y: y * scaleY, + width: width * scaleX, + height: height * scaleY, + }; + }, + [cameraWidth, cameraHeight, imageSize.width, imageSize.height], + ); + + const calculateArea = useCallback(() => { + if (!rectangle) return 0; + const { width, height } = convertToRealCoordinates( + 0, + 0, + Math.abs(rectangle.width), + Math.abs(rectangle.height), + ); + return width * height; + }, [rectangle, convertToRealCoordinates]); + + const calculateAreaPercentage = useCallback(() => { + if (!rectangle) return 0; + const { width, height } = convertToRealCoordinates( + 0, + 0, + Math.abs(rectangle.width), + Math.abs(rectangle.height), + ); + return (width * height) / (cameraWidth * cameraHeight); + }, [rectangle, convertToRealCoordinates, cameraWidth, cameraHeight]); + + const calculateRatio = useCallback(() => { + if (!rectangle) return 0; + const { width, height } = convertToRealCoordinates( + 0, + 0, + Math.abs(rectangle.width), + Math.abs(rectangle.height), + ); + return width / height; + }, [rectangle, convertToRealCoordinates]); + + return ( +
+ + + {rectangle && ( + + )} + + + {showPopover && rectangle && ( + + +
+ + +
+
+ Area:{" "} + + px: {calculateArea().toFixed(0)} + + + %: {calculateAreaPercentage().toFixed(4)} + +
+
+ Ratio:{" "} + + {" "} + {calculateRatio().toFixed(2)} + +
+
+ + + )} +
+ ); +} + +export default DebugDrawingLayer; diff --git a/web/src/components/overlay/detail/ObjectLifecycle.tsx b/web/src/components/overlay/detail/ObjectLifecycle.tsx index 9512e4a7e..d61b5fa56 100644 --- a/web/src/components/overlay/detail/ObjectLifecycle.tsx +++ b/web/src/components/overlay/detail/ObjectLifecycle.tsx @@ -490,12 +490,27 @@ export default function ObjectLifecycle({ Area

{Array.isArray(item.data.box) && - item.data.box.length >= 4 - ? Math.round( - detectArea * - (item.data.box[2] * item.data.box[3]), - ) - : "N/A"} + item.data.box.length >= 4 ? ( + <> +
+ px:{" "} + {Math.round( + detectArea * + (item.data.box[2] * item.data.box[3]), + )} +
+
+ %:{" "} + {( + (detectArea * + (item.data.box[2] * item.data.box[3])) / + detectArea + ).toFixed(4)} +
+ + ) : ( + "N/A" + )}
diff --git a/web/src/views/settings/ObjectSettingsView.tsx b/web/src/views/settings/ObjectSettingsView.tsx index 7b8a08d2e..ea1083ec1 100644 --- a/web/src/views/settings/ObjectSettingsView.tsx +++ b/web/src/views/settings/ObjectSettingsView.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import ActivityIndicator from "@/components/indicators/activity-indicator"; import AutoUpdatingCameraImage from "@/components/camera/AutoUpdatingCameraImage"; import { CameraConfig, FrigateConfig } from "@/types/frigateConfig"; @@ -23,6 +23,9 @@ import { getIconForLabel } from "@/utils/iconUtil"; import { capitalizeFirstLetter } from "@/utils/stringUtil"; import { LuExternalLink, LuInfo } from "react-icons/lu"; import { Link } from "react-router-dom"; +import DebugDrawingLayer from "@/components/overlay/DebugDrawingLayer"; +import { Separator } from "@/components/ui/separator"; +import { isDesktop } from "react-device-detect"; type ObjectSettingsViewProps = { selectedCamera?: string; @@ -37,6 +40,8 @@ export default function ObjectSettingsView({ }: ObjectSettingsViewProps) { const { data: config } = useSWR("config"); + const containerRef = useRef(null); + const DEBUG_OPTIONS = [ { param: "bbox", @@ -130,6 +135,12 @@ export default function ObjectSettingsView({ [options, setOptions], ); + const [debugDraw, setDebugDraw] = useState(false); + + useEffect(() => { + setDebugDraw(false); + }, [selectedCamera]); + const cameraConfig = useMemo(() => { if (config && selectedCamera) { return config.cameras[selectedCamera]; @@ -234,7 +245,7 @@ export default function ObjectSettingsView({ Info - + {info} @@ -256,18 +267,74 @@ export default function ObjectSettingsView({ ))} + {isDesktop && ( + <> + +
+
+
+ + + + +
+ + Info +
+ + + Enable this option to draw a rectangle on the + camera image to show its area and ratio. These + values can then be used to set object shape filter + parameters in your config. +
+ + Read the documentation{" "} + + +
+ + +
+
+ Draw a rectangle on the image to view area and ratio + details +
+
+ { + setDebugDraw(isChecked); + }} + /> +
+ + )} - {ObjectList(memoizedObjects)} + {cameraConfig ? (
-
+
+ {debugDraw && ( + + )}
) : ( @@ -284,7 +358,12 @@ export default function ObjectSettingsView({ ); } -function ObjectList(objects?: ObjectType[]) { +type ObjectListProps = { + cameraConfig: CameraConfig; + objects?: ObjectType[]; +}; + +function ObjectList({ cameraConfig, objects }: ObjectListProps) { const { data: config } = useSWR("config"); const colormap = useMemo(() => { @@ -326,7 +405,7 @@ function ObjectList(objects?: ObjectType[]) { {capitalizeFirstLetter(obj.label.replaceAll("_", " "))}
-
+

@@ -351,7 +430,25 @@ function ObjectList(objects?: ObjectType[]) {

Area

- {obj.area ? obj.area.toString() : "-"} + {obj.area ? ( + <> +
+ px: {obj.area.toString()} +
+
+ %:{" "} + {( + obj.area / + (cameraConfig.detect.width * + cameraConfig.detect.height) + ) + .toFixed(4) + .toString()} +
+ + ) : ( + "-" + )}