From 6d2457ebad91e7fdece141659b1bf9c8097aec81 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Wed, 1 May 2024 09:07:56 -0500 Subject: [PATCH] Revamp object debug view (#11186) * revamp object debug view * fix vite * remove console log * don't display empty fields * clarify masks as motion masks * add descriptions * color and spacing * add sub_label to camera activity * add sub_label to type * rename to debug --- frigate/api/app.py | 1 + frigate/object_processing.py | 12 +- .../components/settings/ObjectSettings.tsx | 258 +++++++++++++++++- web/src/hooks/use-camera-activity.ts | 45 +-- web/src/pages/Settings.tsx | 8 +- web/src/types/frigateConfig.ts | 1 + web/src/types/ws.ts | 4 + 7 files changed, 301 insertions(+), 28 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index 5d0bce78b..67ad072a7 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -164,6 +164,7 @@ def config(): camera_dict["zones"][zone_name]["color"] = zone.color config["plus"] = {"enabled": current_app.plus_api.is_active()} + config["model"]["colormap"] = config_obj.model.colormap for detector_config in config["detectors"].values(): detector_config["model"]["labelmap"] = ( diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 6dd70781e..676c175ec 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -739,15 +739,25 @@ class CameraState: if not obj.false_positive: label = object_type + sub_label = None if obj.obj_data.get("sub_label"): if obj.obj_data.get("sub_label")[0] in ALL_ATTRIBUTE_LABELS: label = obj.obj_data["sub_label"][0] else: label = f"{object_type}-verified" + sub_label = obj.obj_data["sub_label"][0] camera_activity["objects"].append( - {"id": obj.obj_data["id"], "label": label, "stationary": not active} + { + "id": obj.obj_data["id"], + "label": label, + "stationary": not active, + "area": obj.obj_data["area"], + "ratio": obj.obj_data["ratio"], + "score": obj.obj_data["score"], + "sub_label": sub_label, + } ) # if the object's thumbnail is not from the current frame diff --git a/web/src/components/settings/ObjectSettings.tsx b/web/src/components/settings/ObjectSettings.tsx index 92dac2d7d..15d51aa54 100644 --- a/web/src/components/settings/ObjectSettings.tsx +++ b/web/src/components/settings/ObjectSettings.tsx @@ -1,24 +1,104 @@ -import { useEffect, useMemo } from "react"; -import DebugCameraImage from "../camera/DebugCameraImage"; -import { FrigateConfig } from "@/types/frigateConfig"; +import { useCallback, useEffect, useMemo } from "react"; +import ActivityIndicator from "@/components/indicators/activity-indicator"; +import AutoUpdatingCameraImage from "@/components/camera/AutoUpdatingCameraImage"; +import { CameraConfig, FrigateConfig } from "@/types/frigateConfig"; +import { Toaster } from "@/components/ui/sonner"; +import { Label } from "@/components/ui/label"; import useSWR from "swr"; -import ActivityIndicator from "../indicators/activity-indicator"; +import Heading from "../ui/heading"; +import { Switch } from "../ui/switch"; +import { usePersistence } from "@/hooks/use-persistence"; +import { Skeleton } from "../ui/skeleton"; +import { useCameraActivity } from "@/hooks/use-camera-activity"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "../ui/tabs"; +import { ObjectType } from "@/types/ws"; +import useDeepMemo from "@/hooks/use-deep-memo"; +import { Card } from "../ui/card"; +import { getIconForLabel } from "@/utils/iconUtil"; +import { capitalizeFirstLetter } from "@/utils/stringUtil"; type ObjectSettingsProps = { selectedCamera?: string; }; +type Options = { [key: string]: boolean }; + +const emptyObject = Object.freeze({}); + export default function ObjectSettings({ selectedCamera, }: ObjectSettingsProps) { const { data: config } = useSWR("config"); + const DEBUG_OPTIONS = [ + { + param: "bbox", + title: "Bounding boxes", + description: "Show bounding boxes around detected objects", + }, + { + param: "timestamp", + title: "Timestamp", + description: "Overlay a timestamp on the image", + }, + { + param: "zones", + title: "Zones", + description: "Show an outline of any defined zones", + }, + { + param: "mask", + title: "Motion masks", + description: "Show motion mask polygons", + }, + { + param: "motion", + title: "Motion boxes", + description: "Show boxes around areas where motion is detected", + }, + { + param: "regions", + title: "Regions", + description: + "Show a box of the region of interest sent to the object detector", + }, + ]; + + const [options, setOptions] = usePersistence( + `${selectedCamera}-feed`, + emptyObject, + ); + + const handleSetOption = useCallback( + (id: string, value: boolean) => { + const newOptions = { ...options, [id]: value }; + setOptions(newOptions); + }, + [options, setOptions], + ); + const cameraConfig = useMemo(() => { if (config && selectedCamera) { return config.cameras[selectedCamera]; } }, [config, selectedCamera]); + const { objects } = useCameraActivity(cameraConfig ?? ({} as CameraConfig)); + + const memoizedObjects = useDeepMemo(objects); + + const searchParams = useMemo( + () => + new URLSearchParams( + Object.keys(options || {}).reduce((memo, key) => { + //@ts-expect-error we know this is correct + memo.push([key, options[key] === true ? "1" : "0"]); + return memo; + }, []), + ), + [options], + ); + useEffect(() => { document.title = "Object Settings - Frigate"; }, []); @@ -28,8 +108,174 @@ export default function ObjectSettings({ } return ( -
- +
+ +
+ + Debug + +
+

+ Frigate uses your detectors{" "} + {config + ? "(" + + Object.keys(config?.detectors) + .map((detector) => capitalizeFirstLetter(detector)) + .join(",") + + ")" + : ""}{" "} + to detect objects in your camera's video stream. +

+

+ Debugging view shows a real-time view of detected objects and their + statistics. The object list shows a time-delayed summary of detected + objects. +

+
+ + + + Debugging + Object List + + +
+
+
+ {DEBUG_OPTIONS.map(({ param, title, description }) => ( +
+
+ +
+ {description} +
+
+ { + handleSetOption(param, isChecked); + }} + /> +
+ ))} +
+
+
+ + + {ObjectList(memoizedObjects)} + + +
+ + {cameraConfig ? ( +
+
+ +
+
+ ) : ( + + )} +
+ ); +} + +function ObjectList(objects?: ObjectType[]) { + const { data: config } = useSWR("config"); + + const colormap = useMemo(() => { + if (!config) { + return; + } + + return config.model?.colormap; + }, [config]); + + const getColorForObjectName = useCallback( + (objectName: string) => { + return colormap && colormap[objectName] + ? `rgb(${colormap[objectName][2]}, ${colormap[objectName][1]}, ${colormap[objectName][0]})` + : "rgb(128, 128, 128)"; + }, + [colormap], + ); + + return ( +
+ {objects && objects.length > 0 ? ( + objects.map((obj) => { + return ( + +
+
+
+ {getIconForLabel(obj.label, "size-5 text-white")} +
+
+ {capitalizeFirstLetter(obj.label)} +
+
+
+
+
+

+ Score +

+ {obj.score + ? (obj.score * 100).toFixed(1).toString() + : "-"} + % +
+
+
+
+

+ Ratio +

+ {obj.ratio ? obj.ratio.toFixed(2).toString() : "-"} +
+
+
+
+

+ Area +

+ {obj.area ? obj.area.toString() : "-"} +
+
+
+
+ + ); + }) + ) : ( +
No objects
+ )}
); } diff --git a/web/src/hooks/use-camera-activity.ts b/web/src/hooks/use-camera-activity.ts index 4e3b8b4be..074c2be52 100644 --- a/web/src/hooks/use-camera-activity.ts +++ b/web/src/hooks/use-camera-activity.ts @@ -5,10 +5,11 @@ import { } from "@/api/ws"; import { ATTRIBUTE_LABELS, CameraConfig } from "@/types/frigateConfig"; import { MotionData, ReviewSegment } from "@/types/review"; -import { useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { useTimelineUtils } from "./use-timeline-utils"; import { ObjectType } from "@/types/ws"; import useDeepMemo from "./use-deep-memo"; +import { isEqual } from "lodash"; type useCameraActivityReturn = { activeTracking: boolean; @@ -29,10 +30,9 @@ export function useCameraActivity( useEffect(() => { if (updatedCameraState) { - console.log(`the initial objects are ${JSON.stringify(updatedCameraState.objects)}`) setObjects(updatedCameraState.objects); } - }, [updatedCameraState]); + }, [updatedCameraState, camera]); // handle camera activity @@ -45,12 +45,21 @@ export function useCameraActivity( const { payload: event } = useFrigateEvents(); const updatedEvent = useDeepMemo(event); + const handleSetObjects = useCallback( + (newObjects: ObjectType[]) => { + if (!isEqual(objects, newObjects)) { + setObjects(newObjects); + } + }, + [objects], + ); + useEffect(() => { if (!updatedEvent) { return; } - if (updatedEvent.after.camera != camera.name) { + if (updatedEvent.after.camera !== camera.name) { return; } @@ -58,23 +67,26 @@ export function useCameraActivity( (obj) => obj.id === updatedEvent.after.id, ); - if (updatedEvent.type == "end") { - if (updatedEventIndex != -1) { - const newActiveObjects = [...objects]; - newActiveObjects.splice(updatedEventIndex, 1); - setObjects(newActiveObjects); + let newObjects: ObjectType[] = [...objects]; + + if (updatedEvent.type === "end") { + if (updatedEventIndex !== -1) { + newObjects.splice(updatedEventIndex, 1); } } else { - if (updatedEventIndex == -1) { + if (updatedEventIndex === -1) { // add unknown updatedEvent to list if not stationary if (!updatedEvent.after.stationary) { const newActiveObject: ObjectType = { id: updatedEvent.after.id, label: updatedEvent.after.label, stationary: updatedEvent.after.stationary, + area: updatedEvent.after.area, + ratio: updatedEvent.after.ratio, + score: updatedEvent.after.score, + sub_label: updatedEvent.after.sub_label?.[0] ?? "", }; - const newActiveObjects = [...objects, newActiveObject]; - setObjects(newActiveObjects); + newObjects = [...objects, newActiveObject]; } } else { const newObjects = [...objects]; @@ -94,16 +106,17 @@ export function useCameraActivity( newObjects[updatedEventIndex].label = label; newObjects[updatedEventIndex].stationary = updatedEvent.after.stationary; - setObjects(newObjects); } } - }, [camera, updatedEvent, objects]); + + handleSetObjects(newObjects); + }, [camera, updatedEvent, objects, handleSetObjects]); return { activeTracking: hasActiveObjects, activeMotion: detectingMotion - ? detectingMotion == "ON" - : initialCameraState?.motion == true, + ? detectingMotion === "ON" + : initialCameraState?.motion === true, objects, }; } diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx index d0bd80102..96f48f6d9 100644 --- a/web/src/pages/Settings.tsx +++ b/web/src/pages/Settings.tsx @@ -37,9 +37,9 @@ import scrollIntoView from "scroll-into-view-if-needed"; export default function Settings() { const settingsViews = [ "general", - "objects", "masks / zones", "motion tuner", + "debug", ] as const; type SettingsType = (typeof settingsViews)[number]; @@ -135,7 +135,7 @@ export default function Settings() {
- {(page == "objects" || + {(page == "debug" || page == "masks / zones" || page == "motion tuner") && (
@@ -155,9 +155,7 @@ export default function Settings() {
{page == "general" && } - {page == "objects" && ( - - )} + {page == "debug" && } {page == "masks / zones" && ( | null; diff --git a/web/src/types/ws.ts b/web/src/types/ws.ts index fb7a963c4..0fae44b07 100644 --- a/web/src/types/ws.ts +++ b/web/src/types/ws.ts @@ -45,6 +45,10 @@ export type ObjectType = { id: string; label: string; stationary: boolean; + area: number; + ratio: number; + score: number; + sub_label: string; }; export interface FrigateCameraState {