Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2025-08-13 13:47:36 +02:00
Code Issues Projects Releases Wiki Activity

Review genai updates (#19448)

Browse Source 
* Include extra level for normal activity

* Add dynamic toggling

* Update docs

* Add different threshold for genai

* Adjust webUI for object and review description feature

* Adjust config

* Send on startup

* Cleanup config setting

* Set config

* Fix config name
This commit is contained in:
Nicolas Mowen 2025-08-10 07:38:04 -06:00 committed by GitHub
parent f8ca91643e
commit 6c50e69172
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 193 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/docs/configuration/genai.md
View File

@ -39,7 +39,7 @@ By default, descriptions will be generated for all tracked objects and all zones
Optionally, you can generate the description using a snapshot (if enabled) by setting `use_snapshot` to `True`. By default, this is set to `False`, which sends the uncompressed images from the `detect` stream collected over the object's lifetime to the model. Once the object lifecycle ends, only a single compressed and cropped thumbnail is saved with the tracked object. Using a snapshot might be useful when you want to _regenerate_ a tracked object's description as it will provide the AI with a higher-quality image (typically downscaled by the AI itself) than the cropped/compressed thumbnail. Using a snapshot otherwise has a trade-off in that only a single image is sent to your provider, which will limit the model's ability to determine object movement or direction.
Generative AI can also be toggled dynamically for a camera via MQTT with the topic `frigate/<camera_name>/genai/set`. See the [MQTT documentation](/integrations/mqtt/#frigatecamera_namegenaiset).
Generative AI can also be toggled dynamically for a camera via MQTT with the topic `frigate/<camera_name>/object_descriptions/set`. See the [MQTT documentation](/integrations/mqtt/#frigatecamera_nameobjectdescriptionsset).
## Ollama

16
docs/docs/integrations/mqtt.md
View File

@ -411,13 +411,21 @@ Topic to turn review detections for a camera on or off. Expected values are `ON`
Topic with current state of review detections for a camera. Published values are `ON` and `OFF`.
### `frigate/<camera_name>/genai/set`
### `frigate/<camera_name>/object_descriptions/set`
Topic to turn generative AI for a camera on or off. Expected values are `ON` and `OFF`.
Topic to turn generative AI object descriptions for a camera on or off. Expected values are `ON` and `OFF`.
### `frigate/<camera_name>/genai/state`
### `frigate/<camera_name>/object_descriptions/state`
Topic with current state of generative AI for a camera. Published values are `ON` and `OFF`.
Topic with current state of generative AI object descriptions for a camera. Published values are `ON` and `OFF`.
### `frigate/<camera_name>/review_descriptions/set`
Topic to turn generative AI review descriptions for a camera on or off. Expected values are `ON` and `OFF`.
### `frigate/<camera_name>/review_descriptions/state`
Topic with current state of generative AI review descriptions for a camera. Published values are `ON` and `OFF`.
### `frigate/<camera_name>/birdseye/set`

47
frigate/comms/dispatcher.py
View File

@ -75,7 +75,8 @@ class Dispatcher:
"birdseye_mode": self._on_birdseye_mode_command,
"review_alerts": self._on_alerts_command,
"review_detections": self._on_detections_command,
"genai": self._on_genai_command,
"object_descriptions": self._on_object_description_command,
"review_descriptions": self._on_review_description_command,
}
self._global_settings_handlers: dict[str, Callable] = {
"notifications": self._on_global_notification_command,
@ -218,7 +219,12 @@ class Dispatcher:
].onvif.autotracking.enabled,
"alerts": self.config.cameras[camera].review.alerts.enabled,
"detections": self.config.cameras[camera].review.detections.enabled,
"genai": self.config.cameras[camera].objects.genai.enabled,
"object_descriptions": self.config.cameras[
camera
].objects.genai.enabled,
"review_descriptions": self.config.cameras[
camera
].review.genai.enabled,
}
self.publish("camera_activity", json.dumps(camera_status))
@ -752,8 +758,8 @@ class Dispatcher:
)
self.publish(f"{camera_name}/review_detections/state", payload, retain=True)
def _on_genai_command(self, camera_name: str, payload: str) -> None:
"""Callback for GenAI topic."""
def _on_object_description_command(self, camera_name: str, payload: str) -> None:
"""Callback for object description topic."""
genai_settings = self.config.cameras[camera_name].objects.genai
if payload == "ON":
@ -764,15 +770,40 @@ class Dispatcher:
return
if not genai_settings.enabled:
logger.info(f"Turning on GenAI for {camera_name}")
logger.info(f"Turning on object descriptions for {camera_name}")
genai_settings.enabled = True
elif payload == "OFF":
if genai_settings.enabled:
logger.info(f"Turning off GenAI for {camera_name}")
logger.info(f"Turning off object descriptions for {camera_name}")
genai_settings.enabled = False
self.config_updater.publish_update(
CameraConfigUpdateTopic(CameraConfigUpdateEnum.genai, camera_name),
CameraConfigUpdateTopic(CameraConfigUpdateEnum.object_genai, camera_name),
genai_settings,
)
self.publish(f"{camera_name}/genai/state", payload, retain=True)
self.publish(f"{camera_name}/object_descriptions/state", payload, retain=True)
def _on_review_description_command(self, camera_name: str, payload: str) -> None:
"""Callback for review description topic."""
genai_settings = self.config.cameras[camera_name].review.genai
if payload == "ON":
if not self.config.cameras[camera_name].review.genai.enabled_in_config:
logger.error(
"GenAI Alerts or Detections must be enabled in the config to be turned on via MQTT."
)
return
if not genai_settings.enabled:
logger.info(f"Turning on review descriptions for {camera_name}")
genai_settings.enabled = True
elif payload == "OFF":
if genai_settings.enabled:
logger.info(f"Turning off review descriptions for {camera_name}")
genai_settings.enabled = False
self.config_updater.publish_update(
CameraConfigUpdateTopic(CameraConfigUpdateEnum.review_genai, camera_name),
genai_settings,
)
self.publish(f"{camera_name}/review_descriptions/state", payload, retain=True)

7
frigate/comms/mqtt.py
View File

@ -123,10 +123,15 @@ class MqttClient(Communicator):
retain=True,
)
self.publish(
f"{camera_name}/genai/state",
f"{camera_name}/object_descriptions/state",
"ON" if camera.objects.genai.enabled_in_config else "OFF",
retain=True,
)
self.publish(
f"{camera_name}/review_descriptions/state",
"ON" if camera.review.genai.enabled_in_config else "OFF",
retain=True,
)
if self.config.notifications.enabled_in_config:
self.publish(

6
frigate/config/camera/review.py
View File

@ -63,7 +63,11 @@ class DetectionsConfig(FrigateBaseModel):
class GenAIReviewConfig(FrigateBaseModel):
alerts: bool = Field(default=False, title="Enable GenAI for alerts.")
enabled: bool = Field(
default=False,
title="Enable GenAI descriptions for review items.",
)
alerts: bool = Field(default=True, title="Enable GenAI for alerts.")
detections: bool = Field(default=False, title="Enable GenAI for detections.")
debug_save_thumbnails: bool = Field(
default=False,

7
frigate/config/camera/updater.py
View File

@ -17,13 +17,14 @@ class CameraConfigUpdateEnum(str, Enum):
birdseye = "birdseye"
detect = "detect"
enabled = "enabled"
genai = "genai"
motion = "motion" # includes motion and motion masks
notifications = "notifications"
objects = "objects"
object_genai = "object_genai"
record = "record"
remove = "remove" # for removing a camera
review = "review"
review_genai = "review_genai"
semantic_search = "semantic_search" # for semantic search triggers
snapshots = "snapshots"
zones = "zones"
@ -98,7 +99,7 @@ class CameraConfigUpdateSubscriber:
config.detect = updated_config
elif update_type == CameraConfigUpdateEnum.enabled:
config.enabled = updated_config
elif update_type == CameraConfigUpdateEnum.genai:
elif update_type == CameraConfigUpdateEnum.object_genai:
config.objects.genai = updated_config
elif update_type == CameraConfigUpdateEnum.motion:
config.motion = updated_config
@ -110,6 +111,8 @@ class CameraConfigUpdateSubscriber:
config.record = updated_config
elif update_type == CameraConfigUpdateEnum.review:
config.review = updated_config
elif update_type == CameraConfigUpdateEnum.review_genai:
config.review.genai = updated_config
elif update_type == CameraConfigUpdateEnum.semantic_search:
config.semantic_search = updated_config
elif update_type == CameraConfigUpdateEnum.snapshots:

3
frigate/config/config.py
View File

@ -611,8 +611,7 @@ class FrigateConfig(FrigateBaseModel):
camera_config.objects.genai.enabled
)
camera_config.review.genai.enabled_in_config = (
camera_config.review.genai.alerts
or camera_config.review.genai.detections
camera_config.review.genai.enabled
)
# Add default filters

6
frigate/data_processing/post/review_descriptions.py
View File

@ -46,6 +46,11 @@ class ReviewDescriptionProcessor(PostProcessorApi):
if data_type != PostProcessDataEnum.review:
return
camera = data["after"]["camera"]
if not self.config.cameras[camera].review.genai.enabled:
return
id = data["after"]["id"]
if data["type"] == "new" or data["type"] == "update":
@ -91,7 +96,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
return
final_data = data["after"]
camera = final_data["camera"]
if (
final_data["severity"] == "alert"

5
frigate/data_processing/post/types.py
View File

@ -8,9 +8,8 @@ class ReviewMetadata(BaseModel):
confidence: float = Field(
description="A float between 0 and 1 representing your overall confidence in this analysis."
)
potential_threat_level: int | None = Field(
default=None,
ge=1,
potential_threat_level: int = Field(
ge=0,
le=3,
description="An integer representing the potential threat level (1-3). 1: Minor anomaly. 2: Moderate concern. 3: High threat. Only include this field if a clear security concern is observable; otherwise, omit it.",
)

3
frigate/embeddings/maintainer.py
View File

@ -102,7 +102,8 @@ class EmbeddingMaintainer(threading.Thread):
[
CameraConfigUpdateEnum.add,
CameraConfigUpdateEnum.remove,
CameraConfigUpdateEnum.genai,
CameraConfigUpdateEnum.object_genai,
CameraConfigUpdateEnum.review_genai,
CameraConfigUpdateEnum.semantic_search,
],
)

2
frigate/genai/__init__.py
View File

@ -62,10 +62,10 @@ class GenAIClient:
- `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences.
- `confidence` (float): A number 0–1 for overall confidence in the analysis.
- `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern:
- 0 = Normal activity is occurring
- 1 = Unusual but not overtly threatening
- 2 = Suspicious or potentially harmful
- 3 = Clear and immediate threat
Omit this field if no concern is evident.
**IMPORTANT:**
- Values must be plain strings, floats, or integers — no nested objects, no extra commentary.

10
web/public/locales/en/views/settings.json
View File

@ -150,9 +150,13 @@
"title": "Streams",
"desc": "Temporarily disable a camera until Frigate restarts. Disabling a camera completely stops Frigate's processing of this camera's streams. Detection, recording, and debugging will be unavailable.<br /> <em>Note: This does not disable go2rtc restreams.</em>"
},
"genai": {
"title": "Generative AI",
"desc": "Temporarily enable/disable Generative AI for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera."
"object_descriptions": {
"title": "Generative AI Object Descriptions",
"desc": "Temporarily enable/disable Generative AI object descriptions for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera."
},
"review_descriptions": {
"title": "Generative AI Review Descriptions",
"desc": "Temporarily enable/disable Generative AI review descriptions for this camera. When disabled, AI generated descriptions will not be requested for review items on this camera."
},
"review": {
"title": "Review",

31
web/src/api/ws.tsx
View File

@ -68,7 +68,8 @@ function useValue(): useValueReturn {
autotracking,
alerts,
detections,
genai,
object_descriptions,
review_descriptions,
} = state["config"];
cameraStates[`${name}/recordings/state`] = record ? "ON" : "OFF";
cameraStates[`${name}/enabled/state`] = enabled ? "ON" : "OFF";
@ -90,7 +91,12 @@ function useValue(): useValueReturn {
cameraStates[`${name}/review_detections/state`] = detections
? "ON"
: "OFF";
cameraStates[`${name}/genai/state`] = genai ? "ON" : "OFF";
cameraStates[`${name}/object_descriptions/state`] = object_descriptions
? "ON"
: "OFF";
cameraStates[`${name}/review_descriptions/state`] = review_descriptions
? "ON"
: "OFF";
});
setWsState((prevState) => ({
@ -278,14 +284,31 @@ export function useDetectionsState(camera: string): {
return { payload: payload as ToggleableSetting, send };
}
export function useGenAIState(camera: string): {
export function useObjectDescriptionState(camera: string): {
payload: ToggleableSetting;
send: (payload: ToggleableSetting, retain?: boolean) => void;
} {
const {
value: { payload },
send,
} = useWs(`${camera}/genai/state`, `${camera}/genai/set`);
} = useWs(
`${camera}/object_descriptions/state`,
`${camera}/object_descriptions/set`,
);
return { payload: payload as ToggleableSetting, send };
}
export function useReviewDescriptionState(camera: string): {
payload: ToggleableSetting;
send: (payload: ToggleableSetting, retain?: boolean) => void;
} {
const {
value: { payload },
send,
} = useWs(
`${camera}/review_descriptions/state`,
`${camera}/review_descriptions/set`,
);
return { payload: payload as ToggleableSetting, send };
}

6
web/src/types/frigateConfig.ts
View File

@ -218,6 +218,12 @@ export interface CameraConfig {
mode: string;
};
};
genai?: {
enabled: boolean;
enabled_in_config: boolean;
alerts: boolean;
detections: boolean;
};
};
rtmp: {
enabled: boolean;

5
web/src/types/graph.ts
View File

@ -23,6 +23,11 @@ export const EmbeddingThreshold = {
error: 1000,
} as Threshold;
export const GenAIThreshold = {
warning: 30000,
error: 60000,
} as Threshold;
export const DetectorTempThreshold = {
warning: 72,
error: 80,

3
web/src/types/ws.ts
View File

@ -64,7 +64,8 @@ export interface FrigateCameraState {
autotracking: boolean;
alerts: boolean;
detections: boolean;
genai: boolean;
object_descriptions: boolean;
review_descriptions: boolean;
};
motion: boolean;
objects: ObjectType[];

56
web/src/views/settings/CameraSettingsView.tsx
View File

@ -35,7 +35,8 @@ import {
useAlertsState,
useDetectionsState,
useEnabledState,
useGenAIState,
useObjectDescriptionState,
useReviewDescriptionState,
} from "@/api/ws";
import CameraEditForm from "@/components/settings/CameraEditForm";
import { LuPlus } from "react-icons/lu";
@ -150,8 +151,10 @@ export default function CameraSettingsView({
const { payload: detectionsState, send: sendDetections } =
useDetectionsState(selectedCamera);
const { payload: genAIState, send: sendGenAI } =
useGenAIState(selectedCamera);
const { payload: objDescState, send: sendObjDesc } =
useObjectDescriptionState(selectedCamera);
const { payload: revDescState, send: sendRevDesc } =
useReviewDescriptionState(selectedCamera);
const handleCheckedChange = useCallback(
(isChecked: boolean) => {
@ -418,7 +421,9 @@ export default function CameraSettingsView({
<Separator className="my-2 flex bg-secondary" />
<Heading as="h4" className="my-2">
<Trans ns="views/settings">camera.genai.title</Trans>
<Trans ns="views/settings">
camera.object_descriptions.title
</Trans>
</Heading>
<div className="mb-5 mt-2 flex max-w-5xl flex-col gap-2 space-y-3 text-sm text-primary-variant">
@ -426,9 +431,9 @@ export default function CameraSettingsView({
<Switch
id="alerts-enabled"
className="mr-3"
checked={genAIState == "ON"}
checked={objDescState == "ON"}
onCheckedChange={(isChecked) => {
sendGenAI(isChecked ? "ON" : "OFF");
sendObjDesc(isChecked ? "ON" : "OFF");
}}
/>
<div className="space-y-0.5">
@ -438,7 +443,44 @@ export default function CameraSettingsView({
</div>
</div>
<div className="mt-3 text-sm text-muted-foreground">
<Trans ns="views/settings">camera.genai.desc</Trans>
<Trans ns="views/settings">
camera.object_descriptions.desc
</Trans>
</div>
</div>
</>
)}
{cameraConfig?.review?.genai?.enabled_in_config && (
<>
<Separator className="my-2 flex bg-secondary" />
<Heading as="h4" className="my-2">
<Trans ns="views/settings">
camera.review_descriptions.title
</Trans>
</Heading>
<div className="mb-5 mt-2 flex max-w-5xl flex-col gap-2 space-y-3 text-sm text-primary-variant">
<div className="flex flex-row items-center">
<Switch
id="alerts-enabled"
className="mr-3"
checked={revDescState == "ON"}
onCheckedChange={(isChecked) => {
sendRevDesc(isChecked ? "ON" : "OFF");
}}
/>
<div className="space-y-0.5">
<Label htmlFor="genai-enabled">
<Trans>button.enabled</Trans>
</Label>
</div>
</div>
<div className="mt-3 text-sm text-muted-foreground">
<Trans ns="views/settings">
camera.review_descriptions.desc
</Trans>
</div>
</div>
</>

23
web/src/views/system/EnrichmentMetrics.tsx
View File

@ -1,8 +1,8 @@
import useSWR from "swr";
import { FrigateStats } from "@/types/stats";
import { useEffect, useMemo, useState } from "react";
import { useCallback, useEffect, useMemo, useState } from "react";
import { useFrigateStats } from "@/api/ws";
import { EmbeddingThreshold } from "@/types/graph";
import { EmbeddingThreshold, GenAIThreshold, Threshold } from "@/types/graph";
import { Skeleton } from "@/components/ui/skeleton";
import { ThresholdBarGraph } from "@/components/graph/SystemGraph";
import { cn } from "@/lib/utils";
@ -50,6 +50,14 @@ export default function EnrichmentMetrics({
}
}, [initialStats, updatedStats, statsHistory, lastUpdated, setLastUpdated]);
const getThreshold = useCallback((key: string) => {
if (key.includes("description")) {
return GenAIThreshold;
}
return EmbeddingThreshold;
}, []);
// timestamps
const updateTimes = useMemo(
@ -65,7 +73,11 @@ export default function EnrichmentMetrics({
}
const series: {
[key: string]: { name: string; data: { x: number; y: number }[] };
[key: string]: {
name: string;
metrics: Threshold;
data: { x: number; y: number }[];
};
} = {};
statsHistory.forEach((stats, statsIdx) => {
@ -79,6 +91,7 @@ export default function EnrichmentMetrics({
if (!(key in series)) {
series[key] = {
name: t("enrichments.embeddings." + rawKey),
metrics: getThreshold(rawKey),
data: [],
};
}
@ -87,7 +100,7 @@ export default function EnrichmentMetrics({
});
});
return Object.values(series);
}, [statsHistory, t]);
}, [statsHistory, t, getThreshold]);
return (
<>
@ -112,7 +125,7 @@ export default function EnrichmentMetrics({
graphId={`${series.name}-inference`}
name={series.name}
unit="ms"
threshold={EmbeddingThreshold}
threshold={series.metrics}
updateTimes={updateTimes}
data={[series]}
/>