Review genai updates (#19448)

* Include extra level for normal activity

* Add dynamic toggling

* Update docs

* Add different threshold for genai

* Adjust webUI for object and review description feature

* Adjust config

* Send on startup

* Cleanup config setting

* Set config

* Fix config name

docs/docs/configuration/genai.md

@@ -39,7 +39,7 @@ By default, descriptions will be generated for all tracked objects and all zones
 
 Optionally, you can generate the description using a snapshot (if enabled) by setting `use_snapshot` to `True`. By default, this is set to `False`, which sends the uncompressed images from the `detect` stream collected over the object's lifetime to the model. Once the object lifecycle ends, only a single compressed and cropped thumbnail is saved with the tracked object. Using a snapshot might be useful when you want to _regenerate_ a tracked object's description as it will provide the AI with a higher-quality image (typically downscaled by the AI itself) than the cropped/compressed thumbnail. Using a snapshot otherwise has a trade-off in that only a single image is sent to your provider, which will limit the model's ability to determine object movement or direction.
 
-Generative AI can also be toggled dynamically for a camera via MQTT with the topic `frigate/<camera_name>/genai/set`. See the [MQTT documentation](/integrations/mqtt/#frigatecamera_namegenaiset).
+Generative AI can also be toggled dynamically for a camera via MQTT with the topic `frigate/<camera_name>/object_descriptions/set`. See the [MQTT documentation](/integrations/mqtt/#frigatecamera_nameobjectdescriptionsset).
 
 ## Ollama
 

docs/docs/integrations/mqtt.md

@@ -411,13 +411,21 @@ Topic to turn review detections for a camera on or off. Expected values are `ON`
 
 Topic with current state of review detections for a camera. Published values are `ON` and `OFF`.
 
-### `frigate/<camera_name>/genai/set`
+### `frigate/<camera_name>/object_descriptions/set`
 
-Topic to turn generative AI for a camera on or off. Expected values are `ON` and `OFF`.
+Topic to turn generative AI object descriptions for a camera on or off. Expected values are `ON` and `OFF`.
 
-### `frigate/<camera_name>/genai/state`
+### `frigate/<camera_name>/object_descriptions/state`
 
-Topic with current state of generative AI for a camera. Published values are `ON` and `OFF`.
+Topic with current state of generative AI object descriptions for a camera. Published values are `ON` and `OFF`.
+
+### `frigate/<camera_name>/review_descriptions/set`
+
+Topic to turn generative AI review descriptions for a camera on or off. Expected values are `ON` and `OFF`.
+
+### `frigate/<camera_name>/review_descriptions/state`
+
+Topic with current state of generative AI review descriptions for a camera. Published values are `ON` and `OFF`.
 
 ### `frigate/<camera_name>/birdseye/set`
 

frigate/comms/dispatcher.py

@@ -75,7 +75,8 @@ class Dispatcher:
             "birdseye_mode": self._on_birdseye_mode_command,
             "review_alerts": self._on_alerts_command,
             "review_detections": self._on_detections_command,
-            "genai": self._on_genai_command,
+            "object_descriptions": self._on_object_description_command,
+            "review_descriptions": self._on_review_description_command,
         }
         self._global_settings_handlers: dict[str, Callable] = {
             "notifications": self._on_global_notification_command,
@@ -218,7 +219,12 @@ class Dispatcher:
                     ].onvif.autotracking.enabled,
                     "alerts": self.config.cameras[camera].review.alerts.enabled,
                     "detections": self.config.cameras[camera].review.detections.enabled,
-                    "genai": self.config.cameras[camera].objects.genai.enabled,
+                    "object_descriptions": self.config.cameras[
+                        camera
+                    ].objects.genai.enabled,
+                    "review_descriptions": self.config.cameras[
+                        camera
+                    ].review.genai.enabled,
                 }
 
             self.publish("camera_activity", json.dumps(camera_status))
@@ -752,8 +758,8 @@ class Dispatcher:
         )
         self.publish(f"{camera_name}/review_detections/state", payload, retain=True)
 
-    def _on_genai_command(self, camera_name: str, payload: str) -> None:
+    def _on_object_description_command(self, camera_name: str, payload: str) -> None:
-        """Callback for GenAI topic."""
+        """Callback for object description topic."""
         genai_settings = self.config.cameras[camera_name].objects.genai
 
         if payload == "ON":
@@ -764,15 +770,40 @@ class Dispatcher:
                 return
 
             if not genai_settings.enabled:
-                logger.info(f"Turning on GenAI for {camera_name}")
+                logger.info(f"Turning on object descriptions for {camera_name}")
                 genai_settings.enabled = True
         elif payload == "OFF":
             if genai_settings.enabled:
-                logger.info(f"Turning off GenAI for {camera_name}")
+                logger.info(f"Turning off object descriptions for {camera_name}")
                 genai_settings.enabled = False
 
         self.config_updater.publish_update(
-            CameraConfigUpdateTopic(CameraConfigUpdateEnum.genai, camera_name),
+            CameraConfigUpdateTopic(CameraConfigUpdateEnum.object_genai, camera_name),
             genai_settings,
         )
-        self.publish(f"{camera_name}/genai/state", payload, retain=True)
+        self.publish(f"{camera_name}/object_descriptions/state", payload, retain=True)
+
+    def _on_review_description_command(self, camera_name: str, payload: str) -> None:
+        """Callback for review description topic."""
+        genai_settings = self.config.cameras[camera_name].review.genai
+
+        if payload == "ON":
+            if not self.config.cameras[camera_name].review.genai.enabled_in_config:
+                logger.error(
+                    "GenAI Alerts or Detections must be enabled in the config to be turned on via MQTT."
+                )
+                return
+
+            if not genai_settings.enabled:
+                logger.info(f"Turning on review descriptions for {camera_name}")
+                genai_settings.enabled = True
+        elif payload == "OFF":
+            if genai_settings.enabled:
+                logger.info(f"Turning off review descriptions for {camera_name}")
+                genai_settings.enabled = False
+
+        self.config_updater.publish_update(
+            CameraConfigUpdateTopic(CameraConfigUpdateEnum.review_genai, camera_name),
+            genai_settings,
+        )
+        self.publish(f"{camera_name}/review_descriptions/state", payload, retain=True)

frigate/comms/mqtt.py

@@ -123,10 +123,15 @@ class MqttClient(Communicator):
                 retain=True,
             )
             self.publish(
-                f"{camera_name}/genai/state",
+                f"{camera_name}/object_descriptions/state",
                 "ON" if camera.objects.genai.enabled_in_config else "OFF",
                 retain=True,
             )
+            self.publish(
+                f"{camera_name}/review_descriptions/state",
+                "ON" if camera.review.genai.enabled_in_config else "OFF",
+                retain=True,
+            )
 
         if self.config.notifications.enabled_in_config:
             self.publish(

frigate/config/camera/review.py

@@ -63,7 +63,11 @@ class DetectionsConfig(FrigateBaseModel):
 
 
 class GenAIReviewConfig(FrigateBaseModel):
-    alerts: bool = Field(default=False, title="Enable GenAI for alerts.")
+    enabled: bool = Field(
+        default=False,
+        title="Enable GenAI descriptions for review items.",
+    )
+    alerts: bool = Field(default=True, title="Enable GenAI for alerts.")
     detections: bool = Field(default=False, title="Enable GenAI for detections.")
     debug_save_thumbnails: bool = Field(
         default=False,

frigate/config/camera/updater.py

@@ -17,13 +17,14 @@ class CameraConfigUpdateEnum(str, Enum):
     birdseye = "birdseye"
     detect = "detect"
     enabled = "enabled"
-    genai = "genai"
     motion = "motion"  # includes motion and motion masks
     notifications = "notifications"
     objects = "objects"
+    object_genai = "object_genai"
     record = "record"
     remove = "remove"  # for removing a camera
     review = "review"
+    review_genai = "review_genai"
     semantic_search = "semantic_search"  # for semantic search triggers
     snapshots = "snapshots"
     zones = "zones"
@@ -98,7 +99,7 @@ class CameraConfigUpdateSubscriber:
             config.detect = updated_config
         elif update_type == CameraConfigUpdateEnum.enabled:
             config.enabled = updated_config
-        elif update_type == CameraConfigUpdateEnum.genai:
+        elif update_type == CameraConfigUpdateEnum.object_genai:
             config.objects.genai = updated_config
         elif update_type == CameraConfigUpdateEnum.motion:
             config.motion = updated_config
@@ -110,6 +111,8 @@ class CameraConfigUpdateSubscriber:
             config.record = updated_config
         elif update_type == CameraConfigUpdateEnum.review:
             config.review = updated_config
+        elif update_type == CameraConfigUpdateEnum.review_genai:
+            config.review.genai = updated_config
         elif update_type == CameraConfigUpdateEnum.semantic_search:
             config.semantic_search = updated_config
         elif update_type == CameraConfigUpdateEnum.snapshots:

frigate/config/config.py

@@ -611,8 +611,7 @@ class FrigateConfig(FrigateBaseModel):
                 camera_config.objects.genai.enabled
             )
             camera_config.review.genai.enabled_in_config = (
-                camera_config.review.genai.alerts
+                camera_config.review.genai.enabled
-                or camera_config.review.genai.detections
             )
 
             # Add default filters

frigate/data_processing/post/review_descriptions.py

@@ -46,6 +46,11 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         if data_type != PostProcessDataEnum.review:
             return
 
+        camera = data["after"]["camera"]
+
+        if not self.config.cameras[camera].review.genai.enabled:
+            return
+
         id = data["after"]["id"]
 
         if data["type"] == "new" or data["type"] == "update":
@@ -91,7 +96,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                 return
 
             final_data = data["after"]
-            camera = final_data["camera"]
 
             if (
                 final_data["severity"] == "alert"

frigate/data_processing/post/types.py

@@ -8,9 +8,8 @@ class ReviewMetadata(BaseModel):
     confidence: float = Field(
         description="A float between 0 and 1 representing your overall confidence in this analysis."
     )
-    potential_threat_level: int | None = Field(
+    potential_threat_level: int = Field(
-        default=None,
+        ge=0,
-        ge=1,
         le=3,
         description="An integer representing the potential threat level (1-3). 1: Minor anomaly. 2: Moderate concern. 3: High threat. Only include this field if a clear security concern is observable; otherwise, omit it.",
     )

frigate/embeddings/maintainer.py

@@ -102,7 +102,8 @@ class EmbeddingMaintainer(threading.Thread):
             [
                 CameraConfigUpdateEnum.add,
                 CameraConfigUpdateEnum.remove,
-                CameraConfigUpdateEnum.genai,
+                CameraConfigUpdateEnum.object_genai,
+                CameraConfigUpdateEnum.review_genai,
                 CameraConfigUpdateEnum.semantic_search,
             ],
         )

frigate/genai/__init__.py

@@ -62,10 +62,10 @@ class GenAIClient:
         - `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences.
         - `confidence` (float): A number 0–1 for overall confidence in the analysis.
         - `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern:
+            - 0 = Normal activity is occurring
             - 1 = Unusual but not overtly threatening
             - 2 = Suspicious or potentially harmful
             - 3 = Clear and immediate threat
-            Omit this field if no concern is evident.
 
         **IMPORTANT:**
         - Values must be plain strings, floats, or integers — no nested objects, no extra commentary.

web/public/locales/en/views/settings.json

@@ -150,9 +150,13 @@
       "title": "Streams",
       "desc": "Temporarily disable a camera until Frigate restarts. Disabling a camera completely stops Frigate's processing of this camera's streams. Detection, recording, and debugging will be unavailable.<br /> <em>Note: This does not disable go2rtc restreams.</em>"
     },
-    "genai": {
+    "object_descriptions": {
-      "title": "Generative AI",
+      "title": "Generative AI Object Descriptions",
-      "desc": "Temporarily enable/disable Generative AI for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera."
+      "desc": "Temporarily enable/disable Generative AI object descriptions for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera."
+    },
+    "review_descriptions": {
+      "title": "Generative AI Review Descriptions",
+      "desc": "Temporarily enable/disable Generative AI review descriptions for this camera. When disabled, AI generated descriptions will not be requested for review items on this camera."
     },
     "review": {
       "title": "Review",

web/src/api/ws.tsx

@@ -68,7 +68,8 @@ function useValue(): useValueReturn {
         autotracking,
         alerts,
         detections,
-        genai,
+        object_descriptions,
+        review_descriptions,
       } = state["config"];
       cameraStates[`${name}/recordings/state`] = record ? "ON" : "OFF";
       cameraStates[`${name}/enabled/state`] = enabled ? "ON" : "OFF";
@@ -90,7 +91,12 @@ function useValue(): useValueReturn {
       cameraStates[`${name}/review_detections/state`] = detections
         ? "ON"
         : "OFF";
-      cameraStates[`${name}/genai/state`] = genai ? "ON" : "OFF";
+      cameraStates[`${name}/object_descriptions/state`] = object_descriptions
+        ? "ON"
+        : "OFF";
+      cameraStates[`${name}/review_descriptions/state`] = review_descriptions
+        ? "ON"
+        : "OFF";
     });
 
     setWsState((prevState) => ({
@@ -278,14 +284,31 @@ export function useDetectionsState(camera: string): {
   return { payload: payload as ToggleableSetting, send };
 }
 
-export function useGenAIState(camera: string): {
+export function useObjectDescriptionState(camera: string): {
   payload: ToggleableSetting;
   send: (payload: ToggleableSetting, retain?: boolean) => void;
 } {
   const {
     value: { payload },
     send,
-  } = useWs(`${camera}/genai/state`, `${camera}/genai/set`);
+  } = useWs(
+    `${camera}/object_descriptions/state`,
+    `${camera}/object_descriptions/set`,
+  );
+  return { payload: payload as ToggleableSetting, send };
+}
+
+export function useReviewDescriptionState(camera: string): {
+  payload: ToggleableSetting;
+  send: (payload: ToggleableSetting, retain?: boolean) => void;
+} {
+  const {
+    value: { payload },
+    send,
+  } = useWs(
+    `${camera}/review_descriptions/state`,
+    `${camera}/review_descriptions/set`,
+  );
   return { payload: payload as ToggleableSetting, send };
 }
 

web/src/types/frigateConfig.ts

@@ -218,6 +218,12 @@ export interface CameraConfig {
         mode: string;
       };
     };
+    genai?: {
+      enabled: boolean;
+      enabled_in_config: boolean;
+      alerts: boolean;
+      detections: boolean;
+    };
   };
   rtmp: {
     enabled: boolean;

web/src/types/graph.ts

@@ -23,6 +23,11 @@ export const EmbeddingThreshold = {
   error: 1000,
 } as Threshold;
 
+export const GenAIThreshold = {
+  warning: 30000,
+  error: 60000,
+} as Threshold;
+
 export const DetectorTempThreshold = {
   warning: 72,
   error: 80,

web/src/types/ws.ts

@@ -64,7 +64,8 @@ export interface FrigateCameraState {
     autotracking: boolean;
     alerts: boolean;
     detections: boolean;
-    genai: boolean;
+    object_descriptions: boolean;
+    review_descriptions: boolean;
   };
   motion: boolean;
   objects: ObjectType[];

web/src/views/settings/CameraSettingsView.tsx

@@ -35,7 +35,8 @@ import {
   useAlertsState,
   useDetectionsState,
   useEnabledState,
-  useGenAIState,
+  useObjectDescriptionState,
+  useReviewDescriptionState,
 } from "@/api/ws";
 import CameraEditForm from "@/components/settings/CameraEditForm";
 import { LuPlus } from "react-icons/lu";
@@ -150,8 +151,10 @@ export default function CameraSettingsView({
   const { payload: detectionsState, send: sendDetections } =
     useDetectionsState(selectedCamera);
 
-  const { payload: genAIState, send: sendGenAI } =
+  const { payload: objDescState, send: sendObjDesc } =
-    useGenAIState(selectedCamera);
+    useObjectDescriptionState(selectedCamera);
+  const { payload: revDescState, send: sendRevDesc } =
+    useReviewDescriptionState(selectedCamera);
 
   const handleCheckedChange = useCallback(
     (isChecked: boolean) => {
@@ -418,7 +421,9 @@ export default function CameraSettingsView({
                   <Separator className="my-2 flex bg-secondary" />
 
                   <Heading as="h4" className="my-2">
-                    <Trans ns="views/settings">camera.genai.title</Trans>
+                    <Trans ns="views/settings">
+                      camera.object_descriptions.title
+                    </Trans>
                   </Heading>
 
                   <div className="mb-5 mt-2 flex max-w-5xl flex-col gap-2 space-y-3 text-sm text-primary-variant">
@@ -426,9 +431,9 @@ export default function CameraSettingsView({
                       <Switch
                         id="alerts-enabled"
                         className="mr-3"
-                        checked={genAIState == "ON"}
+                        checked={objDescState == "ON"}
                         onCheckedChange={(isChecked) => {
-                          sendGenAI(isChecked ? "ON" : "OFF");
+                          sendObjDesc(isChecked ? "ON" : "OFF");
                         }}
                       />
                       <div className="space-y-0.5">
@@ -438,7 +443,44 @@ export default function CameraSettingsView({
                       </div>
                     </div>
                     <div className="mt-3 text-sm text-muted-foreground">
-                      <Trans ns="views/settings">camera.genai.desc</Trans>
+                      <Trans ns="views/settings">
+                        camera.object_descriptions.desc
+                      </Trans>
+                    </div>
+                  </div>
+                </>
+              )}
+
+              {cameraConfig?.review?.genai?.enabled_in_config && (
+                <>
+                  <Separator className="my-2 flex bg-secondary" />
+
+                  <Heading as="h4" className="my-2">
+                    <Trans ns="views/settings">
+                      camera.review_descriptions.title
+                    </Trans>
+                  </Heading>
+
+                  <div className="mb-5 mt-2 flex max-w-5xl flex-col gap-2 space-y-3 text-sm text-primary-variant">
+                    <div className="flex flex-row items-center">
+                      <Switch
+                        id="alerts-enabled"
+                        className="mr-3"
+                        checked={revDescState == "ON"}
+                        onCheckedChange={(isChecked) => {
+                          sendRevDesc(isChecked ? "ON" : "OFF");
+                        }}
+                      />
+                      <div className="space-y-0.5">
+                        <Label htmlFor="genai-enabled">
+                          <Trans>button.enabled</Trans>
+                        </Label>
+                      </div>
+                    </div>
+                    <div className="mt-3 text-sm text-muted-foreground">
+                      <Trans ns="views/settings">
+                        camera.review_descriptions.desc
+                      </Trans>
                     </div>
                   </div>
                 </>

web/src/views/system/EnrichmentMetrics.tsx

@@ -1,8 +1,8 @@
 import useSWR from "swr";
 import { FrigateStats } from "@/types/stats";
-import { useEffect, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
 import { useFrigateStats } from "@/api/ws";
-import { EmbeddingThreshold } from "@/types/graph";
+import { EmbeddingThreshold, GenAIThreshold, Threshold } from "@/types/graph";
 import { Skeleton } from "@/components/ui/skeleton";
 import { ThresholdBarGraph } from "@/components/graph/SystemGraph";
 import { cn } from "@/lib/utils";
@@ -50,6 +50,14 @@ export default function EnrichmentMetrics({
     }
   }, [initialStats, updatedStats, statsHistory, lastUpdated, setLastUpdated]);
 
+  const getThreshold = useCallback((key: string) => {
+    if (key.includes("description")) {
+      return GenAIThreshold;
+    }
+
+    return EmbeddingThreshold;
+  }, []);
+
   // timestamps
 
   const updateTimes = useMemo(
@@ -65,7 +73,11 @@ export default function EnrichmentMetrics({
     }
 
     const series: {
-      [key: string]: { name: string; data: { x: number; y: number }[] };
+      [key: string]: {
+        name: string;
+        metrics: Threshold;
+        data: { x: number; y: number }[];
+      };
     } = {};
 
     statsHistory.forEach((stats, statsIdx) => {
@@ -79,6 +91,7 @@ export default function EnrichmentMetrics({
         if (!(key in series)) {
           series[key] = {
             name: t("enrichments.embeddings." + rawKey),
+            metrics: getThreshold(rawKey),
             data: [],
           };
         }
@@ -87,7 +100,7 @@ export default function EnrichmentMetrics({
       });
     });
     return Object.values(series);
-  }, [statsHistory, t]);
+  }, [statsHistory, t, getThreshold]);
 
   return (
     <>
@@ -112,7 +125,7 @@ export default function EnrichmentMetrics({
                       graphId={`${series.name}-inference`}
                       name={series.name}
                       unit="ms"
-                      threshold={EmbeddingThreshold}
+                      threshold={series.metrics}
                       updateTimes={updateTimes}
                       data={[series]}
                     />