From 99885b4bdc4148b86d315663a2f09a3b9443712b Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 14 Jul 2025 07:58:43 -0500 Subject: [PATCH] Dynamically enable/disable GenAI (#19139) * config * dispatcher and mqtt * docs * use config updater * add switch to frontend --- docs/docs/configuration/genai.md | 2 + docs/docs/integrations/mqtt.md | 8 ++++ frigate/comms/dispatcher.py | 27 +++++++++++++ frigate/comms/mqtt.py | 6 +++ frigate/config/camera/genai.py | 4 ++ frigate/config/camera/updater.py | 3 ++ frigate/config/config.py | 1 + frigate/embeddings/maintainer.py | 1 + web/public/locales/en/views/settings.json | 4 ++ web/src/api/ws.tsx | 13 ++++++ web/src/types/ws.ts | 1 + web/src/views/settings/CameraSettingsView.tsx | 40 ++++++++++++++++++- 12 files changed, 109 insertions(+), 1 deletion(-) diff --git a/docs/docs/configuration/genai.md b/docs/docs/configuration/genai.md index 6e2389933..c6e8a34f6 100644 --- a/docs/docs/configuration/genai.md +++ b/docs/docs/configuration/genai.md @@ -27,6 +27,8 @@ cameras: enabled: False ``` +Generative AI can also be toggled dynamically for a camera via MQTT with the topic `frigate//genai/set`. See the [MQTT documentation](/integrations/mqtt/#frigatecamera_namegenaiset). + ## Ollama :::warning diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md index 78206762f..b41b604b9 100644 --- a/docs/docs/integrations/mqtt.md +++ b/docs/docs/integrations/mqtt.md @@ -397,6 +397,14 @@ Topic to turn review detections for a camera on or off. Expected values are `ON` Topic with current state of review detections for a camera. Published values are `ON` and `OFF`. +### `frigate//genai/set` + +Topic to turn generative AI for a camera on or off. Expected values are `ON` and `OFF`. + +### `frigate//genai/state` + +Topic with current state of generative AI for a camera. Published values are `ON` and `OFF`. + ### `frigate//birdseye/set` Topic to turn Birdseye for a camera on and off. Expected values are `ON` and `OFF`. Birdseye mode diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index 0a9c439f4..6782ec529 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -75,6 +75,7 @@ class Dispatcher: "birdseye_mode": self._on_birdseye_mode_command, "review_alerts": self._on_alerts_command, "review_detections": self._on_detections_command, + "genai": self._on_genai_command, } self._global_settings_handlers: dict[str, Callable] = { "notifications": self._on_global_notification_command, @@ -207,6 +208,7 @@ class Dispatcher: ].onvif.autotracking.enabled, "alerts": self.config.cameras[camera].review.alerts.enabled, "detections": self.config.cameras[camera].review.detections.enabled, + "genai": self.config.cameras[camera].genai.enabled, } self.publish("camera_activity", json.dumps(camera_status)) @@ -737,3 +739,28 @@ class Dispatcher: review_settings, ) self.publish(f"{camera_name}/review_detections/state", payload, retain=True) + + def _on_genai_command(self, camera_name: str, payload: str) -> None: + """Callback for GenAI topic.""" + genai_settings = self.config.cameras[camera_name].genai + + if payload == "ON": + if not self.config.cameras[camera_name].genai.enabled_in_config: + logger.error( + "GenAI must be enabled in the config to be turned on via MQTT." + ) + return + + if not genai_settings.enabled: + logger.info(f"Turning on GenAI for {camera_name}") + genai_settings.enabled = True + elif payload == "OFF": + if genai_settings.enabled: + logger.info(f"Turning off GenAI for {camera_name}") + genai_settings.enabled = False + + self.config_updater.publish_update( + CameraConfigUpdateTopic(CameraConfigUpdateEnum.genai, camera_name), + genai_settings, + ) + self.publish(f"{camera_name}/genai/state", payload, retain=True) diff --git a/frigate/comms/mqtt.py b/frigate/comms/mqtt.py index e487b30ee..b0f85387e 100644 --- a/frigate/comms/mqtt.py +++ b/frigate/comms/mqtt.py @@ -122,6 +122,11 @@ class MqttClient(Communicator): # type: ignore[misc] "ON" if camera.review.detections.enabled_in_config else "OFF", retain=True, ) + self.publish( + f"{camera_name}/genai/state", + "ON" if camera.genai.enabled_in_config else "OFF", + retain=True, + ) if self.config.notifications.enabled_in_config: self.publish( @@ -215,6 +220,7 @@ class MqttClient(Communicator): # type: ignore[misc] "birdseye_mode", "review_alerts", "review_detections", + "genai", ] for name in self.config.cameras.keys(): diff --git a/frigate/config/camera/genai.py b/frigate/config/camera/genai.py index 6ef93682b..efc3b0711 100644 --- a/frigate/config/camera/genai.py +++ b/frigate/config/camera/genai.py @@ -58,6 +58,10 @@ class GenAICameraConfig(BaseModel): title="What triggers to use to send frames to generative AI for a tracked object.", ) + enabled_in_config: Optional[bool] = Field( + default=None, title="Keep track of original state of generative AI." + ) + @field_validator("required_zones", mode="before") @classmethod def validate_required_zones(cls, v): diff --git a/frigate/config/camera/updater.py b/frigate/config/camera/updater.py index 756e370db..3e8b7acb3 100644 --- a/frigate/config/camera/updater.py +++ b/frigate/config/camera/updater.py @@ -17,6 +17,7 @@ class CameraConfigUpdateEnum(str, Enum): birdseye = "birdseye" detect = "detect" enabled = "enabled" + genai = "genai" motion = "motion" # includes motion and motion masks notifications = "notifications" objects = "objects" @@ -97,6 +98,8 @@ class CameraConfigUpdateSubscriber: config.detect = updated_config elif update_type == CameraConfigUpdateEnum.enabled: config.enabled = updated_config + elif update_type == CameraConfigUpdateEnum.genai: + config.genai = updated_config elif update_type == CameraConfigUpdateEnum.motion: config.motion = updated_config elif update_type == CameraConfigUpdateEnum.notifications: diff --git a/frigate/config/config.py b/frigate/config/config.py index 62c931c96..83bf59ec0 100644 --- a/frigate/config/config.py +++ b/frigate/config/config.py @@ -606,6 +606,7 @@ class FrigateConfig(FrigateBaseModel): camera_config.review.detections.enabled_in_config = ( camera_config.review.detections.enabled ) + camera_config.genai.enabled_in_config = camera_config.genai.enabled # Add default filters object_keys = camera_config.objects.track diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index ec8e20a48..1f6558221 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -100,6 +100,7 @@ class EmbeddingMaintainer(threading.Thread): [ CameraConfigUpdateEnum.add, CameraConfigUpdateEnum.remove, + CameraConfigUpdateEnum.genai, CameraConfigUpdateEnum.semantic_search, ], ) diff --git a/web/public/locales/en/views/settings.json b/web/public/locales/en/views/settings.json index ffdfe6d1a..8fa2451a4 100644 --- a/web/public/locales/en/views/settings.json +++ b/web/public/locales/en/views/settings.json @@ -150,6 +150,10 @@ "title": "Streams", "desc": "Disabling a camera completely stops Frigate's processing of this camera's streams. Detection, recording, and debugging will be unavailable.
Note: This does not disable go2rtc restreams." }, + "genai": { + "title": "Generative AI", + "desc": "Temporarily enable/disable Generative AI for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera." + }, "review": { "title": "Review", "desc": "Enable/disable alerts and detections for this camera. When disabled, no new review items will be generated.", diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx index cc3ea05bf..af25e6a51 100644 --- a/web/src/api/ws.tsx +++ b/web/src/api/ws.tsx @@ -68,6 +68,7 @@ function useValue(): useValueReturn { autotracking, alerts, detections, + genai, } = state["config"]; cameraStates[`${name}/recordings/state`] = record ? "ON" : "OFF"; cameraStates[`${name}/enabled/state`] = enabled ? "ON" : "OFF"; @@ -89,6 +90,7 @@ function useValue(): useValueReturn { cameraStates[`${name}/review_detections/state`] = detections ? "ON" : "OFF"; + cameraStates[`${name}/genai/state`] = genai ? "ON" : "OFF"; }); setWsState((prevState) => ({ @@ -276,6 +278,17 @@ export function useDetectionsState(camera: string): { return { payload: payload as ToggleableSetting, send }; } +export function useGenAIState(camera: string): { + payload: ToggleableSetting; + send: (payload: ToggleableSetting, retain?: boolean) => void; +} { + const { + value: { payload }, + send, + } = useWs(`${camera}/genai/state`, `${camera}/genai/set`); + return { payload: payload as ToggleableSetting, send }; +} + export function usePtzCommand(camera: string): { payload: string; send: (payload: string, retain?: boolean) => void; diff --git a/web/src/types/ws.ts b/web/src/types/ws.ts index 7fad6e953..9ecb9cc6f 100644 --- a/web/src/types/ws.ts +++ b/web/src/types/ws.ts @@ -64,6 +64,7 @@ export interface FrigateCameraState { autotracking: boolean; alerts: boolean; detections: boolean; + genai: boolean; }; motion: boolean; objects: ObjectType[]; diff --git a/web/src/views/settings/CameraSettingsView.tsx b/web/src/views/settings/CameraSettingsView.tsx index db2490f53..ea0e2854f 100644 --- a/web/src/views/settings/CameraSettingsView.tsx +++ b/web/src/views/settings/CameraSettingsView.tsx @@ -29,7 +29,12 @@ import { cn } from "@/lib/utils"; import { Trans, useTranslation } from "react-i18next"; import { Switch } from "@/components/ui/switch"; import { Label } from "@/components/ui/label"; -import { useAlertsState, useDetectionsState, useEnabledState } from "@/api/ws"; +import { + useAlertsState, + useDetectionsState, + useEnabledState, + useGenAIState, +} from "@/api/ws"; import CameraEditForm from "@/components/settings/CameraEditForm"; import { LuPlus } from "react-icons/lu"; import { @@ -142,6 +147,9 @@ export default function CameraSettingsView({ const { payload: detectionsState, send: sendDetections } = useDetectionsState(selectedCamera); + const { payload: genAIState, send: sendGenAI } = + useGenAIState(selectedCamera); + const handleCheckedChange = useCallback( (isChecked: boolean) => { if (!isChecked) { @@ -402,6 +410,36 @@ export default function CameraSettingsView({ + {config?.genai?.enabled && ( + <> + + + + camera.genai.title + + +
+
+ { + sendGenAI(isChecked ? "ON" : "OFF"); + }} + /> +
+ +
+
+
+ camera.genai.desc +
+
+ + )}