Dynamically enable/disable GenAI (#19139)

* config

* dispatcher and mqtt

* docs

* use config updater

* add switch to frontend

docs/docs/configuration/genai.md

@@ -21,7 +21,7 @@ genai:
   model: gemini-1.5-flash
 
 cameras:
-  front_camera: 
+  front_camera:
     genai:
       enabled: True # <- enable GenAI for your front camera
       use_snapshot: True
@@ -30,7 +30,7 @@ cameras:
       required_zones:
         - steps
   indoor_camera:
-    genai: 
+    genai:
       enabled: False # <- disable GenAI for your indoor camera
 ```
 
@@ -38,6 +38,8 @@ By default, descriptions will be generated for all tracked objects and all zones
 
 Optionally, you can generate the description using a snapshot (if enabled) by setting `use_snapshot` to `True`. By default, this is set to `False`, which sends the uncompressed images from the `detect` stream collected over the object's lifetime to the model. Once the object lifecycle ends, only a single compressed and cropped thumbnail is saved with the tracked object. Using a snapshot might be useful when you want to _regenerate_ a tracked object's description as it will provide the AI with a higher-quality image (typically downscaled by the AI itself) than the cropped/compressed thumbnail. Using a snapshot otherwise has a trade-off in that only a single image is sent to your provider, which will limit the model's ability to determine object movement or direction.
 
+Generative AI can also be toggled dynamically for a camera via MQTT with the topic `frigate/<camera_name>/genai/set`. See the [MQTT documentation](/integrations/mqtt/#frigatecamera_namegenaiset).
+
 ## Ollama
 
 :::warning
@@ -202,7 +204,7 @@ genai:
     car: "Observe the primary vehicle in these images. Focus on its movement, direction, or purpose (e.g., parking, approaching, circling). If it's a delivery vehicle, mention the company."
 ```
 
-Prompts can also be overriden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire. 
+Prompts can also be overriden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire.
 
 ```yaml
 cameras:

docs/docs/integrations/mqtt.md

@@ -411,6 +411,14 @@ Topic to turn review detections for a camera on or off. Expected values are `ON`
 
 Topic with current state of review detections for a camera. Published values are `ON` and `OFF`.
 
+### `frigate/<camera_name>/genai/set`
+
+Topic to turn generative AI for a camera on or off. Expected values are `ON` and `OFF`.
+
+### `frigate/<camera_name>/genai/state`
+
+Topic with current state of generative AI for a camera. Published values are `ON` and `OFF`.
+
 ### `frigate/<camera_name>/birdseye/set`
 
 Topic to turn Birdseye for a camera on and off. Expected values are `ON` and `OFF`. Birdseye mode

frigate/comms/dispatcher.py

@@ -75,6 +75,7 @@ class Dispatcher:
             "birdseye_mode": self._on_birdseye_mode_command,
             "review_alerts": self._on_alerts_command,
             "review_detections": self._on_detections_command,
+            "genai": self._on_genai_command,
         }
         self._global_settings_handlers: dict[str, Callable] = {
             "notifications": self._on_global_notification_command,
@@ -207,6 +208,7 @@ class Dispatcher:
                     ].onvif.autotracking.enabled,
                     "alerts": self.config.cameras[camera].review.alerts.enabled,
                     "detections": self.config.cameras[camera].review.detections.enabled,
+                    "genai": self.config.cameras[camera].genai.enabled,
                 }
 
             self.publish("camera_activity", json.dumps(camera_status))
@@ -737,3 +739,28 @@ class Dispatcher:
             review_settings,
         )
         self.publish(f"{camera_name}/review_detections/state", payload, retain=True)
+
+    def _on_genai_command(self, camera_name: str, payload: str) -> None:
+        """Callback for GenAI topic."""
+        genai_settings = self.config.cameras[camera_name].genai
+
+        if payload == "ON":
+            if not self.config.cameras[camera_name].genai.enabled_in_config:
+                logger.error(
+                    "GenAI must be enabled in the config to be turned on via MQTT."
+                )
+                return
+
+            if not genai_settings.enabled:
+                logger.info(f"Turning on GenAI for {camera_name}")
+                genai_settings.enabled = True
+        elif payload == "OFF":
+            if genai_settings.enabled:
+                logger.info(f"Turning off GenAI for {camera_name}")
+                genai_settings.enabled = False
+
+        self.config_updater.publish_update(
+            CameraConfigUpdateTopic(CameraConfigUpdateEnum.genai, camera_name),
+            genai_settings,
+        )
+        self.publish(f"{camera_name}/genai/state", payload, retain=True)

frigate/comms/mqtt.py

@@ -122,6 +122,11 @@ class MqttClient(Communicator):  # type: ignore[misc]
                 "ON" if camera.review.detections.enabled_in_config else "OFF",
                 retain=True,
             )
+            self.publish(
+                f"{camera_name}/genai/state",
+                "ON" if camera.genai.enabled_in_config else "OFF",
+                retain=True,
+            )
 
         if self.config.notifications.enabled_in_config:
             self.publish(
@@ -215,6 +220,7 @@ class MqttClient(Communicator):  # type: ignore[misc]
             "birdseye_mode",
             "review_alerts",
             "review_detections",
+            "genai",
         ]
 
         for name in self.config.cameras.keys():

frigate/config/camera/genai.py

@@ -58,6 +58,10 @@ class GenAICameraConfig(BaseModel):
         title="What triggers to use to send frames to generative AI for a tracked object.",
     )
 
+    enabled_in_config: Optional[bool] = Field(
+        default=None, title="Keep track of original state of generative AI."
+    )
+
     @field_validator("required_zones", mode="before")
     @classmethod
     def validate_required_zones(cls, v):

frigate/config/camera/updater.py

@@ -17,6 +17,7 @@ class CameraConfigUpdateEnum(str, Enum):
     birdseye = "birdseye"
     detect = "detect"
     enabled = "enabled"
+    genai = "genai"
     motion = "motion"  # includes motion and motion masks
     notifications = "notifications"
     objects = "objects"
@@ -97,6 +98,8 @@ class CameraConfigUpdateSubscriber:
             config.detect = updated_config
         elif update_type == CameraConfigUpdateEnum.enabled:
             config.enabled = updated_config
+        elif update_type == CameraConfigUpdateEnum.genai:
+            config.genai = updated_config
         elif update_type == CameraConfigUpdateEnum.motion:
             config.motion = updated_config
         elif update_type == CameraConfigUpdateEnum.notifications:

frigate/config/config.py

@@ -606,6 +606,7 @@ class FrigateConfig(FrigateBaseModel):
             camera_config.review.detections.enabled_in_config = (
                 camera_config.review.detections.enabled
             )
+            camera_config.genai.enabled_in_config = camera_config.genai.enabled
 
             # Add default filters
             object_keys = camera_config.objects.track

frigate/embeddings/maintainer.py

@@ -100,6 +100,7 @@ class EmbeddingMaintainer(threading.Thread):
             [
                 CameraConfigUpdateEnum.add,
                 CameraConfigUpdateEnum.remove,
+                CameraConfigUpdateEnum.genai,
                 CameraConfigUpdateEnum.semantic_search,
             ],
         )

web/public/locales/en/views/settings.json

@@ -150,6 +150,10 @@
       "title": "Streams",
       "desc": "Temporarily disable a camera until Frigate restarts. Disabling a camera completely stops Frigate's processing of this camera's streams. Detection, recording, and debugging will be unavailable.<br /> <em>Note: This does not disable go2rtc restreams.</em>"
     },
+    "genai": {
+      "title": "Generative AI",
+      "desc": "Temporarily enable/disable Generative AI for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera."
+    },
     "review": {
       "title": "Review",
       "desc": "Temporarily enable/disable alerts and detections for this camera until Frigate restarts. When disabled, no new review items will be generated. ",

web/src/api/ws.tsx

@@ -68,6 +68,7 @@ function useValue(): useValueReturn {
         autotracking,
         alerts,
         detections,
+        genai,
       } = state["config"];
       cameraStates[`${name}/recordings/state`] = record ? "ON" : "OFF";
       cameraStates[`${name}/enabled/state`] = enabled ? "ON" : "OFF";
@@ -89,6 +90,7 @@ function useValue(): useValueReturn {
       cameraStates[`${name}/review_detections/state`] = detections
         ? "ON"
         : "OFF";
+      cameraStates[`${name}/genai/state`] = genai ? "ON" : "OFF";
     });
 
     setWsState((prevState) => ({
@@ -276,6 +278,17 @@ export function useDetectionsState(camera: string): {
   return { payload: payload as ToggleableSetting, send };
 }
 
+export function useGenAIState(camera: string): {
+  payload: ToggleableSetting;
+  send: (payload: ToggleableSetting, retain?: boolean) => void;
+} {
+  const {
+    value: { payload },
+    send,
+  } = useWs(`${camera}/genai/state`, `${camera}/genai/set`);
+  return { payload: payload as ToggleableSetting, send };
+}
+
 export function usePtzCommand(camera: string): {
   payload: string;
   send: (payload: string, retain?: boolean) => void;

web/src/types/ws.ts

@@ -64,6 +64,7 @@ export interface FrigateCameraState {
     autotracking: boolean;
     alerts: boolean;
     detections: boolean;
+    genai: boolean;
   };
   motion: boolean;
   objects: ObjectType[];

web/src/views/settings/CameraSettingsView.tsx

@@ -29,9 +29,14 @@ import { cn } from "@/lib/utils";
 import { Trans, useTranslation } from "react-i18next";
 import { Switch } from "@/components/ui/switch";
 import { Label } from "@/components/ui/label";
-import { useAlertsState, useDetectionsState, useEnabledState } from "@/api/ws";
 import { useDocDomain } from "@/hooks/use-doc-domain";
 import { getTranslatedLabel } from "@/utils/i18n";
+import {
+  useAlertsState,
+  useDetectionsState,
+  useEnabledState,
+  useGenAIState,
+} from "@/api/ws";
 import CameraEditForm from "@/components/settings/CameraEditForm";
 import { LuPlus } from "react-icons/lu";
 import {
@@ -145,6 +150,9 @@ export default function CameraSettingsView({
   const { payload: detectionsState, send: sendDetections } =
     useDetectionsState(selectedCamera);
 
+  const { payload: genAIState, send: sendGenAI } =
+    useGenAIState(selectedCamera);
+
   const handleCheckedChange = useCallback(
     (isChecked: boolean) => {
       if (!isChecked) {
@@ -405,6 +413,36 @@ export default function CameraSettingsView({
                   </div>
                 </div>
               </div>
+              {config?.genai?.enabled && (
+                <>
+                  <Separator className="my-2 flex bg-secondary" />
+
+                  <Heading as="h4" className="my-2">
+                    <Trans ns="views/settings">camera.genai.title</Trans>
+                  </Heading>
+
+                  <div className="mb-5 mt-2 flex max-w-5xl flex-col gap-2 space-y-3 text-sm text-primary-variant">
+                    <div className="flex flex-row items-center">
+                      <Switch
+                        id="alerts-enabled"
+                        className="mr-3"
+                        checked={genAIState == "ON"}
+                        onCheckedChange={(isChecked) => {
+                          sendGenAI(isChecked ? "ON" : "OFF");
+                        }}
+                      />
+                      <div className="space-y-0.5">
+                        <Label htmlFor="genai-enabled">
+                          <Trans>button.enabled</Trans>
+                        </Label>
+                      </div>
+                    </div>
+                    <div className="mt-3 text-sm text-muted-foreground">
+                      <Trans ns="views/settings">camera.genai.desc</Trans>
+                    </div>
+                  </div>
+                </>
+              )}
 
               <Separator className="my-2 flex bg-secondary" />