Revamp object debug view (#11186)

* revamp object debug view

* fix vite

* remove console log

* don't display empty fields

* clarify masks as motion masks

* add descriptions

* color and spacing

* add sub_label to camera activity

* add sub_label to type

* rename to debug

frigate/api/app.py

@@ -164,6 +164,7 @@ def config():
             camera_dict["zones"][zone_name]["color"] = zone.color
 
     config["plus"] = {"enabled": current_app.plus_api.is_active()}
+    config["model"]["colormap"] = config_obj.model.colormap
 
     for detector_config in config["detectors"].values():
         detector_config["model"]["labelmap"] = (

frigate/object_processing.py

@@ -739,15 +739,25 @@ class CameraState:
 
             if not obj.false_positive:
                 label = object_type
+                sub_label = None
 
                 if obj.obj_data.get("sub_label"):
                     if obj.obj_data.get("sub_label")[0] in ALL_ATTRIBUTE_LABELS:
                         label = obj.obj_data["sub_label"][0]
                     else:
                         label = f"{object_type}-verified"
+                        sub_label = obj.obj_data["sub_label"][0]
 
                 camera_activity["objects"].append(
-                    {"id": obj.obj_data["id"], "label": label, "stationary": not active}
+                    {
+                        "id": obj.obj_data["id"],
+                        "label": label,
+                        "stationary": not active,
+                        "area": obj.obj_data["area"],
+                        "ratio": obj.obj_data["ratio"],
+                        "score": obj.obj_data["score"],
+                        "sub_label": sub_label,
+                    }
                 )
 
             # if the object's thumbnail is not from the current frame

web/src/components/settings/ObjectSettings.tsx

@@ -1,24 +1,104 @@
-import { useEffect, useMemo } from "react";
+import { useCallback, useEffect, useMemo } from "react";
-import DebugCameraImage from "../camera/DebugCameraImage";
+import ActivityIndicator from "@/components/indicators/activity-indicator";
-import { FrigateConfig } from "@/types/frigateConfig";
+import AutoUpdatingCameraImage from "@/components/camera/AutoUpdatingCameraImage";
+import { CameraConfig, FrigateConfig } from "@/types/frigateConfig";
+import { Toaster } from "@/components/ui/sonner";
+import { Label } from "@/components/ui/label";
 import useSWR from "swr";
-import ActivityIndicator from "../indicators/activity-indicator";
+import Heading from "../ui/heading";
+import { Switch } from "../ui/switch";
+import { usePersistence } from "@/hooks/use-persistence";
+import { Skeleton } from "../ui/skeleton";
+import { useCameraActivity } from "@/hooks/use-camera-activity";
+import { Tabs, TabsContent, TabsList, TabsTrigger } from "../ui/tabs";
+import { ObjectType } from "@/types/ws";
+import useDeepMemo from "@/hooks/use-deep-memo";
+import { Card } from "../ui/card";
+import { getIconForLabel } from "@/utils/iconUtil";
+import { capitalizeFirstLetter } from "@/utils/stringUtil";
 
 type ObjectSettingsProps = {
   selectedCamera?: string;
 };
 
+type Options = { [key: string]: boolean };
+
+const emptyObject = Object.freeze({});
+
 export default function ObjectSettings({
   selectedCamera,
 }: ObjectSettingsProps) {
   const { data: config } = useSWR<FrigateConfig>("config");
 
+  const DEBUG_OPTIONS = [
+    {
+      param: "bbox",
+      title: "Bounding boxes",
+      description: "Show bounding boxes around detected objects",
+    },
+    {
+      param: "timestamp",
+      title: "Timestamp",
+      description: "Overlay a timestamp on the image",
+    },
+    {
+      param: "zones",
+      title: "Zones",
+      description: "Show an outline of any defined zones",
+    },
+    {
+      param: "mask",
+      title: "Motion masks",
+      description: "Show motion mask polygons",
+    },
+    {
+      param: "motion",
+      title: "Motion boxes",
+      description: "Show boxes around areas where motion is detected",
+    },
+    {
+      param: "regions",
+      title: "Regions",
+      description:
+        "Show a box of the region of interest sent to the object detector",
+    },
+  ];
+
+  const [options, setOptions] = usePersistence<Options>(
+    `${selectedCamera}-feed`,
+    emptyObject,
+  );
+
+  const handleSetOption = useCallback(
+    (id: string, value: boolean) => {
+      const newOptions = { ...options, [id]: value };
+      setOptions(newOptions);
+    },
+    [options, setOptions],
+  );
+
   const cameraConfig = useMemo(() => {
     if (config && selectedCamera) {
       return config.cameras[selectedCamera];
     }
   }, [config, selectedCamera]);
 
+  const { objects } = useCameraActivity(cameraConfig ?? ({} as CameraConfig));
+
+  const memoizedObjects = useDeepMemo(objects);
+
+  const searchParams = useMemo(
+    () =>
+      new URLSearchParams(
+        Object.keys(options || {}).reduce((memo, key) => {
+          //@ts-expect-error we know this is correct
+          memo.push([key, options[key] === true ? "1" : "0"]);
+          return memo;
+        }, []),
+      ),
+    [options],
+  );
+
   useEffect(() => {
     document.title = "Object Settings - Frigate";
   }, []);
@@ -28,8 +108,174 @@ export default function ObjectSettings({
   }
 
   return (
-    <div className="flex flex-col h-50">
+    <div className="flex flex-col md:flex-row size-full">
-      <DebugCameraImage cameraConfig={cameraConfig} className="size-full" />
+      <Toaster position="top-center" />
+      <div className="flex flex-col h-full w-full overflow-y-auto mt-2 md:mt-0 mb-10 md:mb-0 md:w-3/12 order-last md:order-none md:mr-2 rounded-lg border-secondary-foreground border-[1px] p-2 bg-background_alt">
+        <Heading as="h3" className="my-2">
+          Debug
+        </Heading>
+        <div className="text-sm text-muted-foreground mb-5 space-y-3">
+          <p>
+            Frigate uses your detectors{" "}
+            {config
+              ? "(" +
+                Object.keys(config?.detectors)
+                  .map((detector) => capitalizeFirstLetter(detector))
+                  .join(",") +
+                ")"
+              : ""}{" "}
+            to detect objects in your camera's video stream.
+          </p>
+          <p>
+            Debugging view shows a real-time view of detected objects and their
+            statistics. The object list shows a time-delayed summary of detected
+            objects.
+          </p>
+        </div>
+
+        <Tabs defaultValue="debug" className="w-full">
+          <TabsList className="grid w-full grid-cols-2">
+            <TabsTrigger value="debug">Debugging</TabsTrigger>
+            <TabsTrigger value="objectlist">Object List</TabsTrigger>
+          </TabsList>
+          <TabsContent value="debug">
+            <div className="flex flex-col w-full space-y-6">
+              <div className="mt-2 space-y-6">
+                <div className="my-2.5 flex flex-col gap-2.5">
+                  {DEBUG_OPTIONS.map(({ param, title, description }) => (
+                    <div
+                      key={param}
+                      className="flex flex-row w-full justify-between items-center"
+                    >
+                      <div className="flex flex-col mb-2">
+                        <Label
+                          className="w-full text-primary capitalize cursor-pointer mb-2"
+                          htmlFor={param}
+                        >
+                          {title}
+                        </Label>
+                        <div className="text-xs text-muted-foreground">
+                          {description}
+                        </div>
+                      </div>
+                      <Switch
+                        key={param}
+                        className="ml-1"
+                        id={param}
+                        checked={options && options[param]}
+                        onCheckedChange={(isChecked) => {
+                          handleSetOption(param, isChecked);
+                        }}
+                      />
+                    </div>
+                  ))}
+                </div>
+              </div>
+            </div>
+          </TabsContent>
+          <TabsContent value="objectlist">
+            {ObjectList(memoizedObjects)}
+          </TabsContent>
+        </Tabs>
+      </div>
+
+      {cameraConfig ? (
+        <div className="flex md:w-7/12 md:grow md:h-dvh md:max-h-full">
+          <div className="size-full min-h-10">
+            <AutoUpdatingCameraImage
+              camera={cameraConfig.name}
+              searchParams={searchParams}
+              showFps={false}
+              className="size-full"
+              cameraClasses="relative w-full h-full flex flex-col justify-start"
+            />
+          </div>
+        </div>
+      ) : (
+        <Skeleton className="size-full rounded-lg md:rounded-2xl" />
+      )}
+    </div>
+  );
+}
+
+function ObjectList(objects?: ObjectType[]) {
+  const { data: config } = useSWR<FrigateConfig>("config");
+
+  const colormap = useMemo(() => {
+    if (!config) {
+      return;
+    }
+
+    return config.model?.colormap;
+  }, [config]);
+
+  const getColorForObjectName = useCallback(
+    (objectName: string) => {
+      return colormap && colormap[objectName]
+        ? `rgb(${colormap[objectName][2]}, ${colormap[objectName][1]}, ${colormap[objectName][0]})`
+        : "rgb(128, 128, 128)";
+    },
+    [colormap],
+  );
+
+  return (
+    <div className="flex flex-col w-full overflow-y-auto">
+      {objects && objects.length > 0 ? (
+        objects.map((obj) => {
+          return (
+            <Card className="text-sm p-2 mb-1" key={obj.id}>
+              <div className="flex flex-row items-center gap-3 pb-1">
+                <div className="flex flex-row flex-1 items-center justify-start p-3 pl-1">
+                  <div
+                    className="p-2 rounded-lg"
+                    style={{
+                      backgroundColor: obj.stationary
+                        ? "rgb(110,110,110)"
+                        : getColorForObjectName(obj.label),
+                    }}
+                  >
+                    {getIconForLabel(obj.label, "size-5 text-white")}
+                  </div>
+                  <div className="ml-3 text-lg">
+                    {capitalizeFirstLetter(obj.label)}
+                  </div>
+                </div>
+                <div className="flex flex-row w-8/12 items-end justify-end">
+                  <div className="mr-2 text-md w-1/3">
+                    <div className="flex flex-col items-end justify-end">
+                      <p className="text-sm mb-1.5 text-primary-variant">
+                        Score
+                      </p>
+                      {obj.score
+                        ? (obj.score * 100).toFixed(1).toString()
+                        : "-"}
+                      %
+                    </div>
+                  </div>
+                  <div className="mr-2 text-md w-1/3">
+                    <div className="flex flex-col items-end justify-end">
+                      <p className="text-sm mb-1.5 text-primary-variant">
+                        Ratio
+                      </p>
+                      {obj.ratio ? obj.ratio.toFixed(2).toString() : "-"}
+                    </div>
+                  </div>
+                  <div className="mr-2 text-md w-1/3">
+                    <div className="flex flex-col items-end justify-end">
+                      <p className="text-sm mb-1.5 text-primary-variant">
+                        Area
+                      </p>
+                      {obj.area ? obj.area.toString() : "-"}
+                    </div>
+                  </div>
+                </div>
+              </div>
+            </Card>
+          );
+        })
+      ) : (
+        <div className="p-3 text-center">No objects</div>
+      )}
     </div>
   );
 }

web/src/hooks/use-camera-activity.ts

@@ -5,10 +5,11 @@ import {
 } from "@/api/ws";
 import { ATTRIBUTE_LABELS, CameraConfig } from "@/types/frigateConfig";
 import { MotionData, ReviewSegment } from "@/types/review";
-import { useEffect, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
 import { useTimelineUtils } from "./use-timeline-utils";
 import { ObjectType } from "@/types/ws";
 import useDeepMemo from "./use-deep-memo";
+import { isEqual } from "lodash";
 
 type useCameraActivityReturn = {
   activeTracking: boolean;
@@ -29,10 +30,9 @@ export function useCameraActivity(
 
   useEffect(() => {
     if (updatedCameraState) {
-      console.log(`the initial objects are ${JSON.stringify(updatedCameraState.objects)}`)
       setObjects(updatedCameraState.objects);
     }
-  }, [updatedCameraState]);
+  }, [updatedCameraState, camera]);
 
   // handle camera activity
 
@@ -45,12 +45,21 @@ export function useCameraActivity(
   const { payload: event } = useFrigateEvents();
   const updatedEvent = useDeepMemo(event);
 
+  const handleSetObjects = useCallback(
+    (newObjects: ObjectType[]) => {
+      if (!isEqual(objects, newObjects)) {
+        setObjects(newObjects);
+      }
+    },
+    [objects],
+  );
+
   useEffect(() => {
     if (!updatedEvent) {
       return;
     }
 
-    if (updatedEvent.after.camera != camera.name) {
+    if (updatedEvent.after.camera !== camera.name) {
       return;
     }
 
@@ -58,23 +67,26 @@ export function useCameraActivity(
       (obj) => obj.id === updatedEvent.after.id,
     );
 
-    if (updatedEvent.type == "end") {
+    let newObjects: ObjectType[] = [...objects];
-      if (updatedEventIndex != -1) {
+
-        const newActiveObjects = [...objects];
+    if (updatedEvent.type === "end") {
-        newActiveObjects.splice(updatedEventIndex, 1);
+      if (updatedEventIndex !== -1) {
-        setObjects(newActiveObjects);
+        newObjects.splice(updatedEventIndex, 1);
       }
     } else {
-      if (updatedEventIndex == -1) {
+      if (updatedEventIndex === -1) {
         // add unknown updatedEvent to list if not stationary
         if (!updatedEvent.after.stationary) {
           const newActiveObject: ObjectType = {
             id: updatedEvent.after.id,
             label: updatedEvent.after.label,
             stationary: updatedEvent.after.stationary,
+            area: updatedEvent.after.area,
+            ratio: updatedEvent.after.ratio,
+            score: updatedEvent.after.score,
+            sub_label: updatedEvent.after.sub_label?.[0] ?? "",
           };
-          const newActiveObjects = [...objects, newActiveObject];
+          newObjects = [...objects, newActiveObject];
-          setObjects(newActiveObjects);
         }
       } else {
         const newObjects = [...objects];
@@ -94,16 +106,17 @@ export function useCameraActivity(
         newObjects[updatedEventIndex].label = label;
         newObjects[updatedEventIndex].stationary =
           updatedEvent.after.stationary;
-        setObjects(newObjects);
       }
     }
-  }, [camera, updatedEvent, objects]);
+
+    handleSetObjects(newObjects);
+  }, [camera, updatedEvent, objects, handleSetObjects]);
 
   return {
     activeTracking: hasActiveObjects,
     activeMotion: detectingMotion
-      ? detectingMotion == "ON"
+      ? detectingMotion === "ON"
-      : initialCameraState?.motion == true,
+      : initialCameraState?.motion === true,
     objects,
   };
 }

web/src/pages/Settings.tsx

@@ -37,9 +37,9 @@ import scrollIntoView from "scroll-into-view-if-needed";
 export default function Settings() {
   const settingsViews = [
     "general",
-    "objects",
     "masks / zones",
     "motion tuner",
+    "debug",
   ] as const;
 
   type SettingsType = (typeof settingsViews)[number];
@@ -135,7 +135,7 @@ export default function Settings() {
             <ScrollBar orientation="horizontal" className="h-0" />
           </div>
         </ScrollArea>
-        {(page == "objects" ||
+        {(page == "debug" ||
           page == "masks / zones" ||
           page == "motion tuner") && (
           <div className="flex items-center gap-2 ml-2 flex-shrink-0">
@@ -155,9 +155,7 @@ export default function Settings() {
       </div>
       <div className="mt-2 flex flex-col items-start w-full h-full md:h-dvh md:pb-24">
         {page == "general" && <General />}
-        {page == "objects" && (
+        {page == "debug" && <ObjectSettings selectedCamera={selectedCamera} />}
-          <ObjectSettings selectedCamera={selectedCamera} />
-        )}
         {page == "masks / zones" && (
           <MasksAndZones
             selectedCamera={selectedCamera}

web/src/types/frigateConfig.ts

@@ -324,6 +324,7 @@ export interface FrigateConfig {
     model_type: string;
     path: string | null;
     width: number;
+    colormap: { [key: string]: [number, number, number] };
   };
 
   motion: Record<string, unknown> | null;

web/src/types/ws.ts

@@ -45,6 +45,10 @@ export type ObjectType = {
   id: string;
   label: string;
   stationary: boolean;
+  area: number;
+  ratio: number;
+  score: number;
+  sub_label: string;
 };
 
 export interface FrigateCameraState {