diff --git a/docs/docs/configuration/genai.md b/docs/docs/configuration/genai.md index 0fd86d3e4..75b3eae1a 100644 --- a/docs/docs/configuration/genai.md +++ b/docs/docs/configuration/genai.md @@ -39,7 +39,7 @@ By default, descriptions will be generated for all tracked objects and all zones Optionally, you can generate the description using a snapshot (if enabled) by setting `use_snapshot` to `True`. By default, this is set to `False`, which sends the uncompressed images from the `detect` stream collected over the object's lifetime to the model. Once the object lifecycle ends, only a single compressed and cropped thumbnail is saved with the tracked object. Using a snapshot might be useful when you want to _regenerate_ a tracked object's description as it will provide the AI with a higher-quality image (typically downscaled by the AI itself) than the cropped/compressed thumbnail. Using a snapshot otherwise has a trade-off in that only a single image is sent to your provider, which will limit the model's ability to determine object movement or direction. -Generative AI can also be toggled dynamically for a camera via MQTT with the topic `frigate//genai/set`. See the [MQTT documentation](/integrations/mqtt/#frigatecamera_namegenaiset). +Generative AI can also be toggled dynamically for a camera via MQTT with the topic `frigate//object_descriptions/set`. See the [MQTT documentation](/integrations/mqtt/#frigatecamera_nameobjectdescriptionsset). ## Ollama diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md index f4e540adc..ba1e1302f 100644 --- a/docs/docs/integrations/mqtt.md +++ b/docs/docs/integrations/mqtt.md @@ -411,13 +411,21 @@ Topic to turn review detections for a camera on or off. Expected values are `ON` Topic with current state of review detections for a camera. Published values are `ON` and `OFF`. -### `frigate//genai/set` +### `frigate//object_descriptions/set` -Topic to turn generative AI for a camera on or off. Expected values are `ON` and `OFF`. +Topic to turn generative AI object descriptions for a camera on or off. Expected values are `ON` and `OFF`. -### `frigate//genai/state` +### `frigate//object_descriptions/state` -Topic with current state of generative AI for a camera. Published values are `ON` and `OFF`. +Topic with current state of generative AI object descriptions for a camera. Published values are `ON` and `OFF`. + +### `frigate//review_descriptions/set` + +Topic to turn generative AI review descriptions for a camera on or off. Expected values are `ON` and `OFF`. + +### `frigate//review_descriptions/state` + +Topic with current state of generative AI review descriptions for a camera. Published values are `ON` and `OFF`. ### `frigate//birdseye/set` diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index 43c47fc51..93956068c 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -75,7 +75,8 @@ class Dispatcher: "birdseye_mode": self._on_birdseye_mode_command, "review_alerts": self._on_alerts_command, "review_detections": self._on_detections_command, - "genai": self._on_genai_command, + "object_descriptions": self._on_object_description_command, + "review_descriptions": self._on_review_description_command, } self._global_settings_handlers: dict[str, Callable] = { "notifications": self._on_global_notification_command, @@ -218,7 +219,12 @@ class Dispatcher: ].onvif.autotracking.enabled, "alerts": self.config.cameras[camera].review.alerts.enabled, "detections": self.config.cameras[camera].review.detections.enabled, - "genai": self.config.cameras[camera].objects.genai.enabled, + "object_descriptions": self.config.cameras[ + camera + ].objects.genai.enabled, + "review_descriptions": self.config.cameras[ + camera + ].review.genai.enabled, } self.publish("camera_activity", json.dumps(camera_status)) @@ -752,8 +758,8 @@ class Dispatcher: ) self.publish(f"{camera_name}/review_detections/state", payload, retain=True) - def _on_genai_command(self, camera_name: str, payload: str) -> None: - """Callback for GenAI topic.""" + def _on_object_description_command(self, camera_name: str, payload: str) -> None: + """Callback for object description topic.""" genai_settings = self.config.cameras[camera_name].objects.genai if payload == "ON": @@ -764,15 +770,40 @@ class Dispatcher: return if not genai_settings.enabled: - logger.info(f"Turning on GenAI for {camera_name}") + logger.info(f"Turning on object descriptions for {camera_name}") genai_settings.enabled = True elif payload == "OFF": if genai_settings.enabled: - logger.info(f"Turning off GenAI for {camera_name}") + logger.info(f"Turning off object descriptions for {camera_name}") genai_settings.enabled = False self.config_updater.publish_update( - CameraConfigUpdateTopic(CameraConfigUpdateEnum.genai, camera_name), + CameraConfigUpdateTopic(CameraConfigUpdateEnum.object_genai, camera_name), genai_settings, ) - self.publish(f"{camera_name}/genai/state", payload, retain=True) + self.publish(f"{camera_name}/object_descriptions/state", payload, retain=True) + + def _on_review_description_command(self, camera_name: str, payload: str) -> None: + """Callback for review description topic.""" + genai_settings = self.config.cameras[camera_name].review.genai + + if payload == "ON": + if not self.config.cameras[camera_name].review.genai.enabled_in_config: + logger.error( + "GenAI Alerts or Detections must be enabled in the config to be turned on via MQTT." + ) + return + + if not genai_settings.enabled: + logger.info(f"Turning on review descriptions for {camera_name}") + genai_settings.enabled = True + elif payload == "OFF": + if genai_settings.enabled: + logger.info(f"Turning off review descriptions for {camera_name}") + genai_settings.enabled = False + + self.config_updater.publish_update( + CameraConfigUpdateTopic(CameraConfigUpdateEnum.review_genai, camera_name), + genai_settings, + ) + self.publish(f"{camera_name}/review_descriptions/state", payload, retain=True) diff --git a/frigate/comms/mqtt.py b/frigate/comms/mqtt.py index 6e8c79486..0af56e259 100644 --- a/frigate/comms/mqtt.py +++ b/frigate/comms/mqtt.py @@ -123,10 +123,15 @@ class MqttClient(Communicator): retain=True, ) self.publish( - f"{camera_name}/genai/state", + f"{camera_name}/object_descriptions/state", "ON" if camera.objects.genai.enabled_in_config else "OFF", retain=True, ) + self.publish( + f"{camera_name}/review_descriptions/state", + "ON" if camera.review.genai.enabled_in_config else "OFF", + retain=True, + ) if self.config.notifications.enabled_in_config: self.publish( diff --git a/frigate/config/camera/review.py b/frigate/config/camera/review.py index 8e3c0b01c..cb01e5107 100644 --- a/frigate/config/camera/review.py +++ b/frigate/config/camera/review.py @@ -63,7 +63,11 @@ class DetectionsConfig(FrigateBaseModel): class GenAIReviewConfig(FrigateBaseModel): - alerts: bool = Field(default=False, title="Enable GenAI for alerts.") + enabled: bool = Field( + default=False, + title="Enable GenAI descriptions for review items.", + ) + alerts: bool = Field(default=True, title="Enable GenAI for alerts.") detections: bool = Field(default=False, title="Enable GenAI for detections.") debug_save_thumbnails: bool = Field( default=False, diff --git a/frigate/config/camera/updater.py b/frigate/config/camera/updater.py index c2439040c..125094f10 100644 --- a/frigate/config/camera/updater.py +++ b/frigate/config/camera/updater.py @@ -17,13 +17,14 @@ class CameraConfigUpdateEnum(str, Enum): birdseye = "birdseye" detect = "detect" enabled = "enabled" - genai = "genai" motion = "motion" # includes motion and motion masks notifications = "notifications" objects = "objects" + object_genai = "object_genai" record = "record" remove = "remove" # for removing a camera review = "review" + review_genai = "review_genai" semantic_search = "semantic_search" # for semantic search triggers snapshots = "snapshots" zones = "zones" @@ -98,7 +99,7 @@ class CameraConfigUpdateSubscriber: config.detect = updated_config elif update_type == CameraConfigUpdateEnum.enabled: config.enabled = updated_config - elif update_type == CameraConfigUpdateEnum.genai: + elif update_type == CameraConfigUpdateEnum.object_genai: config.objects.genai = updated_config elif update_type == CameraConfigUpdateEnum.motion: config.motion = updated_config @@ -110,6 +111,8 @@ class CameraConfigUpdateSubscriber: config.record = updated_config elif update_type == CameraConfigUpdateEnum.review: config.review = updated_config + elif update_type == CameraConfigUpdateEnum.review_genai: + config.review.genai = updated_config elif update_type == CameraConfigUpdateEnum.semantic_search: config.semantic_search = updated_config elif update_type == CameraConfigUpdateEnum.snapshots: diff --git a/frigate/config/config.py b/frigate/config/config.py index 36e319410..de41c1d24 100644 --- a/frigate/config/config.py +++ b/frigate/config/config.py @@ -611,8 +611,7 @@ class FrigateConfig(FrigateBaseModel): camera_config.objects.genai.enabled ) camera_config.review.genai.enabled_in_config = ( - camera_config.review.genai.alerts - or camera_config.review.genai.detections + camera_config.review.genai.enabled ) # Add default filters diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 72ea0f9cb..3b7d1d6d9 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -46,6 +46,11 @@ class ReviewDescriptionProcessor(PostProcessorApi): if data_type != PostProcessDataEnum.review: return + camera = data["after"]["camera"] + + if not self.config.cameras[camera].review.genai.enabled: + return + id = data["after"]["id"] if data["type"] == "new" or data["type"] == "update": @@ -91,7 +96,6 @@ class ReviewDescriptionProcessor(PostProcessorApi): return final_data = data["after"] - camera = final_data["camera"] if ( final_data["severity"] == "alert" diff --git a/frigate/data_processing/post/types.py b/frigate/data_processing/post/types.py index cae50d80c..d79a063b5 100644 --- a/frigate/data_processing/post/types.py +++ b/frigate/data_processing/post/types.py @@ -8,9 +8,8 @@ class ReviewMetadata(BaseModel): confidence: float = Field( description="A float between 0 and 1 representing your overall confidence in this analysis." ) - potential_threat_level: int | None = Field( - default=None, - ge=1, + potential_threat_level: int = Field( + ge=0, le=3, description="An integer representing the potential threat level (1-3). 1: Minor anomaly. 2: Moderate concern. 3: High threat. Only include this field if a clear security concern is observable; otherwise, omit it.", ) diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index f7b1db647..ae613b623 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -102,7 +102,8 @@ class EmbeddingMaintainer(threading.Thread): [ CameraConfigUpdateEnum.add, CameraConfigUpdateEnum.remove, - CameraConfigUpdateEnum.genai, + CameraConfigUpdateEnum.object_genai, + CameraConfigUpdateEnum.review_genai, CameraConfigUpdateEnum.semantic_search, ], ) diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index a79cf71ee..bcb95e756 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -62,10 +62,10 @@ class GenAIClient: - `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences. - `confidence` (float): A number 0–1 for overall confidence in the analysis. - `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern: + - 0 = Normal activity is occurring - 1 = Unusual but not overtly threatening - 2 = Suspicious or potentially harmful - 3 = Clear and immediate threat - Omit this field if no concern is evident. **IMPORTANT:** - Values must be plain strings, floats, or integers — no nested objects, no extra commentary. diff --git a/web/public/locales/en/views/settings.json b/web/public/locales/en/views/settings.json index 83da403ef..516ddf9f2 100644 --- a/web/public/locales/en/views/settings.json +++ b/web/public/locales/en/views/settings.json @@ -150,9 +150,13 @@ "title": "Streams", "desc": "Temporarily disable a camera until Frigate restarts. Disabling a camera completely stops Frigate's processing of this camera's streams. Detection, recording, and debugging will be unavailable.
Note: This does not disable go2rtc restreams." }, - "genai": { - "title": "Generative AI", - "desc": "Temporarily enable/disable Generative AI for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera." + "object_descriptions": { + "title": "Generative AI Object Descriptions", + "desc": "Temporarily enable/disable Generative AI object descriptions for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera." + }, + "review_descriptions": { + "title": "Generative AI Review Descriptions", + "desc": "Temporarily enable/disable Generative AI review descriptions for this camera. When disabled, AI generated descriptions will not be requested for review items on this camera." }, "review": { "title": "Review", diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx index af25e6a51..0cef235a0 100644 --- a/web/src/api/ws.tsx +++ b/web/src/api/ws.tsx @@ -68,7 +68,8 @@ function useValue(): useValueReturn { autotracking, alerts, detections, - genai, + object_descriptions, + review_descriptions, } = state["config"]; cameraStates[`${name}/recordings/state`] = record ? "ON" : "OFF"; cameraStates[`${name}/enabled/state`] = enabled ? "ON" : "OFF"; @@ -90,7 +91,12 @@ function useValue(): useValueReturn { cameraStates[`${name}/review_detections/state`] = detections ? "ON" : "OFF"; - cameraStates[`${name}/genai/state`] = genai ? "ON" : "OFF"; + cameraStates[`${name}/object_descriptions/state`] = object_descriptions + ? "ON" + : "OFF"; + cameraStates[`${name}/review_descriptions/state`] = review_descriptions + ? "ON" + : "OFF"; }); setWsState((prevState) => ({ @@ -278,14 +284,31 @@ export function useDetectionsState(camera: string): { return { payload: payload as ToggleableSetting, send }; } -export function useGenAIState(camera: string): { +export function useObjectDescriptionState(camera: string): { payload: ToggleableSetting; send: (payload: ToggleableSetting, retain?: boolean) => void; } { const { value: { payload }, send, - } = useWs(`${camera}/genai/state`, `${camera}/genai/set`); + } = useWs( + `${camera}/object_descriptions/state`, + `${camera}/object_descriptions/set`, + ); + return { payload: payload as ToggleableSetting, send }; +} + +export function useReviewDescriptionState(camera: string): { + payload: ToggleableSetting; + send: (payload: ToggleableSetting, retain?: boolean) => void; +} { + const { + value: { payload }, + send, + } = useWs( + `${camera}/review_descriptions/state`, + `${camera}/review_descriptions/set`, + ); return { payload: payload as ToggleableSetting, send }; } diff --git a/web/src/types/frigateConfig.ts b/web/src/types/frigateConfig.ts index 576b4d750..22a35bdd9 100644 --- a/web/src/types/frigateConfig.ts +++ b/web/src/types/frigateConfig.ts @@ -218,6 +218,12 @@ export interface CameraConfig { mode: string; }; }; + genai?: { + enabled: boolean; + enabled_in_config: boolean; + alerts: boolean; + detections: boolean; + }; }; rtmp: { enabled: boolean; diff --git a/web/src/types/graph.ts b/web/src/types/graph.ts index bfa9dd331..894a2feec 100644 --- a/web/src/types/graph.ts +++ b/web/src/types/graph.ts @@ -23,6 +23,11 @@ export const EmbeddingThreshold = { error: 1000, } as Threshold; +export const GenAIThreshold = { + warning: 30000, + error: 60000, +} as Threshold; + export const DetectorTempThreshold = { warning: 72, error: 80, diff --git a/web/src/types/ws.ts b/web/src/types/ws.ts index 9ecb9cc6f..f2e45bda4 100644 --- a/web/src/types/ws.ts +++ b/web/src/types/ws.ts @@ -64,7 +64,8 @@ export interface FrigateCameraState { autotracking: boolean; alerts: boolean; detections: boolean; - genai: boolean; + object_descriptions: boolean; + review_descriptions: boolean; }; motion: boolean; objects: ObjectType[]; diff --git a/web/src/views/settings/CameraSettingsView.tsx b/web/src/views/settings/CameraSettingsView.tsx index a58e1c711..61f327c6e 100644 --- a/web/src/views/settings/CameraSettingsView.tsx +++ b/web/src/views/settings/CameraSettingsView.tsx @@ -35,7 +35,8 @@ import { useAlertsState, useDetectionsState, useEnabledState, - useGenAIState, + useObjectDescriptionState, + useReviewDescriptionState, } from "@/api/ws"; import CameraEditForm from "@/components/settings/CameraEditForm"; import { LuPlus } from "react-icons/lu"; @@ -150,8 +151,10 @@ export default function CameraSettingsView({ const { payload: detectionsState, send: sendDetections } = useDetectionsState(selectedCamera); - const { payload: genAIState, send: sendGenAI } = - useGenAIState(selectedCamera); + const { payload: objDescState, send: sendObjDesc } = + useObjectDescriptionState(selectedCamera); + const { payload: revDescState, send: sendRevDesc } = + useReviewDescriptionState(selectedCamera); const handleCheckedChange = useCallback( (isChecked: boolean) => { @@ -418,7 +421,9 @@ export default function CameraSettingsView({ - camera.genai.title + + camera.object_descriptions.title +
@@ -426,9 +431,9 @@ export default function CameraSettingsView({ { - sendGenAI(isChecked ? "ON" : "OFF"); + sendObjDesc(isChecked ? "ON" : "OFF"); }} />
@@ -438,7 +443,44 @@ export default function CameraSettingsView({
- camera.genai.desc + + camera.object_descriptions.desc + +
+ + + )} + + {cameraConfig?.review?.genai?.enabled_in_config && ( + <> + + + + + camera.review_descriptions.title + + + +
+
+ { + sendRevDesc(isChecked ? "ON" : "OFF"); + }} + /> +
+ +
+
+
+ + camera.review_descriptions.desc +
diff --git a/web/src/views/system/EnrichmentMetrics.tsx b/web/src/views/system/EnrichmentMetrics.tsx index 0e9c38f4d..b762f6a7f 100644 --- a/web/src/views/system/EnrichmentMetrics.tsx +++ b/web/src/views/system/EnrichmentMetrics.tsx @@ -1,8 +1,8 @@ import useSWR from "swr"; import { FrigateStats } from "@/types/stats"; -import { useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { useFrigateStats } from "@/api/ws"; -import { EmbeddingThreshold } from "@/types/graph"; +import { EmbeddingThreshold, GenAIThreshold, Threshold } from "@/types/graph"; import { Skeleton } from "@/components/ui/skeleton"; import { ThresholdBarGraph } from "@/components/graph/SystemGraph"; import { cn } from "@/lib/utils"; @@ -50,6 +50,14 @@ export default function EnrichmentMetrics({ } }, [initialStats, updatedStats, statsHistory, lastUpdated, setLastUpdated]); + const getThreshold = useCallback((key: string) => { + if (key.includes("description")) { + return GenAIThreshold; + } + + return EmbeddingThreshold; + }, []); + // timestamps const updateTimes = useMemo( @@ -65,7 +73,11 @@ export default function EnrichmentMetrics({ } const series: { - [key: string]: { name: string; data: { x: number; y: number }[] }; + [key: string]: { + name: string; + metrics: Threshold; + data: { x: number; y: number }[]; + }; } = {}; statsHistory.forEach((stats, statsIdx) => { @@ -79,6 +91,7 @@ export default function EnrichmentMetrics({ if (!(key in series)) { series[key] = { name: t("enrichments.embeddings." + rawKey), + metrics: getThreshold(rawKey), data: [], }; } @@ -87,7 +100,7 @@ export default function EnrichmentMetrics({ }); }); return Object.values(series); - }, [statsHistory, t]); + }, [statsHistory, t, getThreshold]); return ( <> @@ -112,7 +125,7 @@ export default function EnrichmentMetrics({ graphId={`${series.name}-inference`} name={series.name} unit="ms" - threshold={EmbeddingThreshold} + threshold={series.metrics} updateTimes={updateTimes} data={[series]} />