Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2025-08-13 13:47:36 +02:00
Code Issues Projects Releases Wiki Activity

Migrate object genai configuration (#19437)

Browse Source 
* Move genAI object to objects section

* Adjust config propogation behavior

* Refactor genai config usage

* Automatic migration

* Always start the embeddings process

* Always init embeddings

* Config fixes

* Adjust reference config

* Adjust docs

* Formatting

* Fix
This commit is contained in:
Nicolas Mowen 2025-08-08 16:33:11 -06:00 committed by GitHub
parent 6d078e565a
commit 52295fcac4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 251 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/docs/configuration/genai.md
View File

@ -15,13 +15,13 @@ To use Generative AI, you must define a single provider at the global level of y
```yaml ```yaml
genai: genai:
enabled: True
provider: gemini provider: gemini
api_key: "{FRIGATE_GEMINI_API_KEY}" api_key: "{FRIGATE_GEMINI_API_KEY}"
model: gemini-1.5-flash model: gemini-1.5-flash
cameras: cameras:
front_camera: front_camera:
objects:
genai: genai:
enabled: True # <- enable GenAI for your front camera enabled: True # <- enable GenAI for your front camera
use_snapshot: True use_snapshot: True
@ -30,6 +30,7 @@ cameras:
required_zones: required_zones:
- steps - steps
indoor_camera: indoor_camera:
objects:
genai: genai:
enabled: False # <- disable GenAI for your indoor camera enabled: False # <- disable GenAI for your indoor camera
``` ```
@ -68,7 +69,6 @@ You should have at least 8 GB of RAM available (or VRAM if running on GPU) to ru
```yaml ```yaml
genai: genai:
enabled: True
provider: ollama provider: ollama
base_url: http://localhost:11434 base_url: http://localhost:11434
model: llava:7b model: llava:7b
@ -95,7 +95,6 @@ To start using Gemini, you must first get an API key from [Google AI Studio](htt
```yaml ```yaml
genai: genai:
enabled: True
provider: gemini provider: gemini
api_key: "{FRIGATE_GEMINI_API_KEY}" api_key: "{FRIGATE_GEMINI_API_KEY}"
model: gemini-1.5-flash model: gemini-1.5-flash
@ -117,7 +116,6 @@ To start using OpenAI, you must first [create an API key](https://platform.opena
```yaml ```yaml
genai: genai:
enabled: True
provider: openai provider: openai
api_key: "{FRIGATE_OPENAI_API_KEY}" api_key: "{FRIGATE_OPENAI_API_KEY}"
model: gpt-4o model: gpt-4o
@ -145,7 +143,6 @@ To start using Azure OpenAI, you must first [create a resource](https://learn.mi
```yaml ```yaml
genai: genai:
enabled: True
provider: azure_openai provider: azure_openai
base_url: https://example-endpoint.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2023-03-15-preview base_url: https://example-endpoint.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2023-03-15-preview
api_key: "{FRIGATE_OPENAI_API_KEY}" api_key: "{FRIGATE_OPENAI_API_KEY}"
@ -188,22 +185,25 @@ You are also able to define custom prompts in your configuration.
```yaml ```yaml
genai: genai:
enabled: True
provider: ollama provider: ollama
base_url: http://localhost:11434 base_url: http://localhost:11434
model: llava model: llava
objects:
prompt: "Analyze the {label} in these images from the {camera} security camera. Focus on the actions, behavior, and potential intent of the {label}, rather than just describing its appearance." prompt: "Analyze the {label} in these images from the {camera} security camera. Focus on the actions, behavior, and potential intent of the {label}, rather than just describing its appearance."
object_prompts: object_prompts:
person: "Examine the main person in these images. What are they doing and what might their actions suggest about their intent (e.g., approaching a door, leaving an area, standing still)? Do not describe the surroundings or static details." person: "Examine the main person in these images. What are they doing and what might their actions suggest about their intent (e.g., approaching a door, leaving an area, standing still)? Do not describe the surroundings or static details."
car: "Observe the primary vehicle in these images. Focus on its movement, direction, or purpose (e.g., parking, approaching, circling). If it's a delivery vehicle, mention the company." car: "Observe the primary vehicle in these images. Focus on its movement, direction, or purpose (e.g., parking, approaching, circling). If it's a delivery vehicle, mention the company."
``` ```
Prompts can also be overriden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire. Prompts can also be overridden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire.
```yaml ```yaml
cameras: cameras:
front_door: front_door:
objects:
genai: genai:
enabled: True
use_snapshot: True use_snapshot: True
prompt: "Analyze the {label} in these images from the {camera} security camera at the front door. Focus on the actions and potential intent of the {label}." prompt: "Analyze the {label} in these images from the {camera} security camera at the front door. Focus on the actions and potential intent of the {label}."
object_prompts: object_prompts:

62
docs/docs/configuration/reference.md
View File

@ -339,6 +339,33 @@ objects:
# Optional: mask to prevent this object type from being detected in certain areas (default: no mask) # Optional: mask to prevent this object type from being detected in certain areas (default: no mask)
# Checks based on the bottom center of the bounding box of the object # Checks based on the bottom center of the bounding box of the object
mask: 0.000,0.000,0.781,0.000,0.781,0.278,0.000,0.278 mask: 0.000,0.000,0.781,0.000,0.781,0.278,0.000,0.278
# Optional: Configuration for AI generated tracked object descriptions
genai:
# Optional: Enable AI object description generation (default: shown below)
enabled: False
# Optional: Use the object snapshot instead of thumbnails for description generation (default: shown below)
use_snapshot: False
# Optional: The default prompt for generating descriptions. Can use replacement
# variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
# Optional: Object specific prompts to customize description results
# Format: {label}: {prompt}
object_prompts:
person: "My special person prompt."
# Optional: objects to generate descriptions for (default: all objects that are tracked)
objects:
- person
- cat
# Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify)
required_zones: []
# Optional: What triggers to use to send frames for a tracked object to generative AI (default: shown below)
send_triggers:
# Once the object is no longer tracked
tracked_object_end: True
# Optional: After X many significant updates are received (default: shown below)
after_significant_updates: None
# Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below)
debug_save_thumbnails: False
# Optional: Review configuration # Optional: Review configuration
# NOTE: Can be overridden at the camera level # NOTE: Can be overridden at the camera level
@ -612,13 +639,6 @@ genai:
base_url: http://localhost::11434 base_url: http://localhost::11434
# Required if gemini or openai # Required if gemini or openai
api_key: "{FRIGATE_GENAI_API_KEY}" api_key: "{FRIGATE_GENAI_API_KEY}"
# Optional: The default prompt for generating descriptions. Can use replacement
# variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
# Optional: Object specific prompts to customize description results
# Format: {label}: {prompt}
object_prompts:
person: "My special person prompt."
# Optional: Configuration for audio transcription # Optional: Configuration for audio transcription
# NOTE: only the enabled option can be overridden at the camera level # NOTE: only the enabled option can be overridden at the camera level
@ -857,34 +877,6 @@ cameras:
actions: actions:
- notification - notification
# Optional: Configuration for AI generated tracked object descriptions
genai:
# Optional: Enable AI description generation (default: shown below)
enabled: False
# Optional: Use the object snapshot instead of thumbnails for description generation (default: shown below)
use_snapshot: False
# Optional: The default prompt for generating descriptions. Can use replacement
# variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
# Optional: Object specific prompts to customize description results
# Format: {label}: {prompt}
object_prompts:
person: "My special person prompt."
# Optional: objects to generate descriptions for (default: all objects that are tracked)
objects:
- person
- cat
# Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify)
required_zones: []
# Optional: What triggers to use to send frames for a tracked object to generative AI (default: shown below)
send_triggers:
# Once the object is no longer tracked
tracked_object_end: True
# Optional: After X many significant updates are received (default: shown below)
after_significant_updates: None
# Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below)
debug_save_thumbnails: False
# Optional # Optional
ui: ui:
# Optional: Set a timezone to use in the UI (default: use browser local time) # Optional: Set a timezone to use in the UI (default: use browser local time)

2
frigate/api/event.py
View File

@ -1230,7 +1230,7 @@ def regenerate_description(
camera_config = request.app.frigate_config.cameras[event.camera] camera_config = request.app.frigate_config.cameras[event.camera]
if camera_config.genai.enabled or params.force: if camera_config.objects.genai.enabled or params.force:
request.app.event_metadata_updater.publish( request.app.event_metadata_updater.publish(
(event.id, params.source, params.force), (event.id, params.source, params.force),
EventMetadataTypeEnum.regenerate_description.value, EventMetadataTypeEnum.regenerate_description.value,

25
frigate/app.py
View File

@ -246,18 +246,7 @@ class FrigateApp:
logger.info(f"Review process started: {review_segment_process.pid}") logger.info(f"Review process started: {review_segment_process.pid}")
def init_embeddings_manager(self) -> None: def init_embeddings_manager(self) -> None:
genai_cameras = [ # always start the embeddings process
c for c in self.config.cameras.values() if c.enabled and c.genai.enabled
]
if (
not self.config.semantic_search.enabled
and not genai_cameras
and not self.config.lpr.enabled
and not self.config.face_recognition.enabled
):
return
embedding_process = EmbeddingProcess( embedding_process = EmbeddingProcess(
self.config, self.embeddings_metrics, self.stop_event self.config, self.embeddings_metrics, self.stop_event
) )
@ -309,18 +298,6 @@ class FrigateApp:
migrate_exports(self.config.ffmpeg, list(self.config.cameras.keys())) migrate_exports(self.config.ffmpeg, list(self.config.cameras.keys()))
def init_embeddings_client(self) -> None: def init_embeddings_client(self) -> None:
genai_cameras = [
c
for c in self.config.cameras.values()
if c.enabled_in_config and c.genai.enabled
]
if (
self.config.semantic_search.enabled
or self.config.lpr.enabled
or genai_cameras
or self.config.face_recognition.enabled
):
# Create a client for other processes to use # Create a client for other processes to use
self.embeddings = EmbeddingsContext(self.db) self.embeddings = EmbeddingsContext(self.db)

6
frigate/comms/dispatcher.py
View File

@ -209,7 +209,7 @@ class Dispatcher:
].onvif.autotracking.enabled, ].onvif.autotracking.enabled,
"alerts": self.config.cameras[camera].review.alerts.enabled, "alerts": self.config.cameras[camera].review.alerts.enabled,
"detections": self.config.cameras[camera].review.detections.enabled, "detections": self.config.cameras[camera].review.detections.enabled,
"genai": self.config.cameras[camera].genai.enabled, "genai": self.config.cameras[camera].objects.genai.enabled,
} }
self.publish("camera_activity", json.dumps(camera_status)) self.publish("camera_activity", json.dumps(camera_status))
@ -744,10 +744,10 @@ class Dispatcher:
def _on_genai_command(self, camera_name: str, payload: str) -> None: def _on_genai_command(self, camera_name: str, payload: str) -> None:
"""Callback for GenAI topic.""" """Callback for GenAI topic."""
genai_settings = self.config.cameras[camera_name].genai genai_settings = self.config.cameras[camera_name].objects.genai
if payload == "ON": if payload == "ON":
if not self.config.cameras[camera_name].genai.enabled_in_config: if not self.config.cameras[camera_name].objects.genai.enabled_in_config:
logger.error( logger.error(
"GenAI must be enabled in the config to be turned on via MQTT." "GenAI must be enabled in the config to be turned on via MQTT."
) )

2
frigate/comms/mqtt.py
View File

@ -124,7 +124,7 @@ class MqttClient(Communicator):
) )
self.publish( self.publish(
f"{camera_name}/genai/state", f"{camera_name}/genai/state",
"ON" if camera.genai.enabled_in_config else "OFF", "ON" if camera.objects.genai.enabled_in_config else "OFF",
retain=True, retain=True,
) )

4
frigate/config/camera/camera.py
View File

@ -28,7 +28,6 @@ from .audio import AudioConfig
from .birdseye import BirdseyeCameraConfig from .birdseye import BirdseyeCameraConfig
from .detect import DetectConfig from .detect import DetectConfig
from .ffmpeg import CameraFfmpegConfig, CameraInput from .ffmpeg import CameraFfmpegConfig, CameraInput
from .genai import GenAICameraConfig
from .live import CameraLiveConfig from .live import CameraLiveConfig
from .motion import MotionConfig from .motion import MotionConfig
from .mqtt import CameraMqttConfig from .mqtt import CameraMqttConfig
@ -71,9 +70,6 @@ class CameraConfig(FrigateBaseModel):
default_factory=CameraFaceRecognitionConfig, title="Face recognition config." default_factory=CameraFaceRecognitionConfig, title="Face recognition config."
) )
ffmpeg: CameraFfmpegConfig = Field(title="FFmpeg configuration for the camera.") ffmpeg: CameraFfmpegConfig = Field(title="FFmpeg configuration for the camera.")
genai: GenAICameraConfig = Field(
default_factory=GenAICameraConfig, title="Generative AI configuration."
)
live: CameraLiveConfig = Field( live: CameraLiveConfig = Field(
default_factory=CameraLiveConfig, title="Live playback settings." default_factory=CameraLiveConfig, title="Live playback settings."
) )

70
frigate/config/camera/genai.py
View File

@ -1,12 +1,12 @@
from enum import Enum from enum import Enum
from typing import Optional, Union from typing import Optional
from pydantic import BaseModel, Field, field_validator from pydantic import Field
from ..base import FrigateBaseModel from ..base import FrigateBaseModel
from ..env import EnvString from ..env import EnvString
__all__ = ["GenAIConfig", "GenAICameraConfig", "GenAIProviderEnum"] __all__ = ["GenAIConfig", "GenAIProviderEnum"]
class GenAIProviderEnum(str, Enum): class GenAIProviderEnum(str, Enum):
@ -16,70 +16,8 @@ class GenAIProviderEnum(str, Enum):
ollama = "ollama" ollama = "ollama"
class GenAISendTriggersConfig(BaseModel):
tracked_object_end: bool = Field(
default=True, title="Send once the object is no longer tracked."
)
after_significant_updates: Optional[int] = Field(
default=None,
title="Send an early request to generative AI when X frames accumulated.",
ge=1,
)
# uses BaseModel because some global attributes are not available at the camera level
class GenAICameraConfig(BaseModel):
enabled: bool = Field(default=False, title="Enable GenAI for camera.")
use_snapshot: bool = Field(
default=False, title="Use snapshots for generating descriptions."
)
prompt: str = Field(
default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
title="Default caption prompt.",
)
object_prompts: dict[str, str] = Field(
default_factory=dict, title="Object specific prompts."
)
objects: Union[str, list[str]] = Field(
default_factory=list,
title="List of objects to run generative AI for.",
)
required_zones: Union[str, list[str]] = Field(
default_factory=list,
title="List of required zones to be entered in order to run generative AI.",
)
debug_save_thumbnails: bool = Field(
default=False,
title="Save thumbnails sent to generative AI for debugging purposes.",
)
send_triggers: GenAISendTriggersConfig = Field(
default_factory=GenAISendTriggersConfig,
title="What triggers to use to send frames to generative AI for a tracked object.",
)
enabled_in_config: Optional[bool] = Field(
default=None, title="Keep track of original state of generative AI."
)
@field_validator("required_zones", mode="before")
@classmethod
def validate_required_zones(cls, v):
if isinstance(v, str) and "," not in v:
return [v]
return v
class GenAIConfig(FrigateBaseModel): class GenAIConfig(FrigateBaseModel):
enabled: bool = Field(default=False, title="Enable GenAI.") """Primary GenAI Config to define GenAI Provider."""
prompt: str = Field(
default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
title="Default caption prompt.",
)
object_prompts: dict[str, str] = Field(
default_factory=dict, title="Object specific prompts."
)
api_key: Optional[EnvString] = Field(default=None, title="Provider API key.") api_key: Optional[EnvString] = Field(default=None, title="Provider API key.")
base_url: Optional[str] = Field(default=None, title="Provider base url.") base_url: Optional[str] = Field(default=None, title="Provider base url.")

61
frigate/config/camera/objects.py
View File

@ -1,10 +1,10 @@
from typing import Any, Optional, Union from typing import Any, Optional, Union
from pydantic import Field, PrivateAttr, field_serializer from pydantic import Field, PrivateAttr, field_serializer, field_validator
from ..base import FrigateBaseModel from ..base import FrigateBaseModel
__all__ = ["ObjectConfig", "FilterConfig"] __all__ = ["ObjectConfig", "GenAIObjectConfig", "FilterConfig"]
DEFAULT_TRACKED_OBJECTS = ["person"] DEFAULT_TRACKED_OBJECTS = ["person"]
@ -49,12 +49,69 @@ class FilterConfig(FrigateBaseModel):
return None return None
class GenAIObjectTriggerConfig(FrigateBaseModel):
tracked_object_end: bool = Field(
default=True, title="Send once the object is no longer tracked."
)
after_significant_updates: Optional[int] = Field(
default=None,
title="Send an early request to generative AI when X frames accumulated.",
ge=1,
)
class GenAIObjectConfig(FrigateBaseModel):
enabled: bool = Field(default=False, title="Enable GenAI for camera.")
use_snapshot: bool = Field(
default=False, title="Use snapshots for generating descriptions."
)
prompt: str = Field(
default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
title="Default caption prompt.",
)
object_prompts: dict[str, str] = Field(
default_factory=dict, title="Object specific prompts."
)
objects: Union[str, list[str]] = Field(
default_factory=list,
title="List of objects to run generative AI for.",
)
required_zones: Union[str, list[str]] = Field(
default_factory=list,
title="List of required zones to be entered in order to run generative AI.",
)
debug_save_thumbnails: bool = Field(
default=False,
title="Save thumbnails sent to generative AI for debugging purposes.",
)
send_triggers: GenAIObjectTriggerConfig = Field(
default_factory=GenAIObjectTriggerConfig,
title="What triggers to use to send frames to generative AI for a tracked object.",
)
enabled_in_config: Optional[bool] = Field(
default=None, title="Keep track of original state of generative AI."
)
@field_validator("required_zones", mode="before")
@classmethod
def validate_required_zones(cls, v):
if isinstance(v, str) and "," not in v:
return [v]
return v
class ObjectConfig(FrigateBaseModel): class ObjectConfig(FrigateBaseModel):
track: list[str] = Field(default=DEFAULT_TRACKED_OBJECTS, title="Objects to track.") track: list[str] = Field(default=DEFAULT_TRACKED_OBJECTS, title="Objects to track.")
filters: dict[str, FilterConfig] = Field( filters: dict[str, FilterConfig] = Field(
default_factory=dict, title="Object filters." default_factory=dict, title="Object filters."
) )
mask: Union[str, list[str]] = Field(default="", title="Object mask.") mask: Union[str, list[str]] = Field(default="", title="Object mask.")
genai: GenAIObjectConfig = Field(
default_factory=GenAIObjectConfig,
title="Config for using genai to analyze objects.",
)
_all_objects: list[str] = PrivateAttr() _all_objects: list[str] = PrivateAttr()
@property @property

2
frigate/config/camera/updater.py
View File

@ -99,7 +99,7 @@ class CameraConfigUpdateSubscriber:
elif update_type == CameraConfigUpdateEnum.enabled: elif update_type == CameraConfigUpdateEnum.enabled:
config.enabled = updated_config config.enabled = updated_config
elif update_type == CameraConfigUpdateEnum.genai: elif update_type == CameraConfigUpdateEnum.genai:
config.genai = updated_config config.objects.genai = updated_config
elif update_type == CameraConfigUpdateEnum.motion: elif update_type == CameraConfigUpdateEnum.motion:
config.motion = updated_config config.motion = updated_config
elif update_type == CameraConfigUpdateEnum.notifications: elif update_type == CameraConfigUpdateEnum.notifications:

13
frigate/config/config.py
View File

@ -352,6 +352,11 @@ class FrigateConfig(FrigateBaseModel):
default_factory=ModelConfig, title="Detection model configuration." default_factory=ModelConfig, title="Detection model configuration."
) )
# GenAI config
genai: GenAIConfig = Field(
default_factory=GenAIConfig, title="Generative AI configuration."
)
# Camera config # Camera config
cameras: Dict[str, CameraConfig] = Field(title="Camera configuration.") cameras: Dict[str, CameraConfig] = Field(title="Camera configuration.")
audio: AudioConfig = Field( audio: AudioConfig = Field(
@ -366,9 +371,6 @@ class FrigateConfig(FrigateBaseModel):
ffmpeg: FfmpegConfig = Field( ffmpeg: FfmpegConfig = Field(
default_factory=FfmpegConfig, title="Global FFmpeg configuration." default_factory=FfmpegConfig, title="Global FFmpeg configuration."
) )
genai: GenAIConfig = Field(
default_factory=GenAIConfig, title="Generative AI configuration."
)
live: CameraLiveConfig = Field( live: CameraLiveConfig = Field(
default_factory=CameraLiveConfig, title="Live playback settings." default_factory=CameraLiveConfig, title="Live playback settings."
) )
@ -458,7 +460,6 @@ class FrigateConfig(FrigateBaseModel):
"live": ..., "live": ...,
"objects": ..., "objects": ...,
"review": ..., "review": ...,
"genai": ...,
"motion": ..., "motion": ...,
"notifications": ..., "notifications": ...,
"detect": ..., "detect": ...,
@ -606,7 +607,9 @@ class FrigateConfig(FrigateBaseModel):
camera_config.review.detections.enabled_in_config = ( camera_config.review.detections.enabled_in_config = (
camera_config.review.detections.enabled camera_config.review.detections.enabled
) )
camera_config.genai.enabled_in_config = camera_config.genai.enabled camera_config.objects.genai.enabled_in_config = (
camera_config.objects.genai.enabled
)
# Add default filters # Add default filters
object_keys = camera_config.objects.track object_keys = camera_config.objects.track

44
frigate/embeddings/maintainer.py
View File

@ -30,7 +30,7 @@ from frigate.comms.recordings_updater import (
RecordingsDataTypeEnum, RecordingsDataTypeEnum,
) )
from frigate.comms.review_updater import ReviewDataSubscriber from frigate.comms.review_updater import ReviewDataSubscriber
from frigate.config import FrigateConfig from frigate.config import CameraConfig, FrigateConfig
from frigate.config.camera.camera import CameraTypeEnum from frigate.config.camera.camera import CameraTypeEnum
from frigate.config.camera.updater import ( from frigate.config.camera.updater import (
CameraConfigUpdateEnum, CameraConfigUpdateEnum,
@ -329,7 +329,10 @@ class EmbeddingMaintainer(threading.Thread):
camera_config = self.config.cameras[camera] camera_config = self.config.cameras[camera]
# no need to process updated objects if face recognition, lpr, genai are disabled # no need to process updated objects if face recognition, lpr, genai are disabled
if not camera_config.genai.enabled and len(self.realtime_processors) == 0: if (
not camera_config.objects.genai.enabled
and len(self.realtime_processors) == 0
):
return return
# Create our own thumbnail based on the bounding box and the frame time # Create our own thumbnail based on the bounding box and the frame time
@ -367,23 +370,23 @@ class EmbeddingMaintainer(threading.Thread):
# check if we're configured to send an early request after a minimum number of updates received # check if we're configured to send an early request after a minimum number of updates received
if ( if (
self.genai_client is not None self.genai_client is not None
and camera_config.genai.send_triggers.after_significant_updates and camera_config.objects.genai.send_triggers.after_significant_updates
): ):
if ( if (
len(self.tracked_events.get(data["id"], [])) len(self.tracked_events.get(data["id"], []))
>= camera_config.genai.send_triggers.after_significant_updates >= camera_config.objects.genai.send_triggers.after_significant_updates
and data["id"] not in self.early_request_sent and data["id"] not in self.early_request_sent
): ):
if data["has_clip"] and data["has_snapshot"]: if data["has_clip"] and data["has_snapshot"]:
event: Event = Event.get(Event.id == data["id"]) event: Event = Event.get(Event.id == data["id"])
if ( if (
not camera_config.genai.objects not camera_config.objects.genai.objects
or event.label in camera_config.genai.objects or event.label in camera_config.objects.genai.objects
) and ( ) and (
not camera_config.genai.required_zones not camera_config.objects.genai.required_zones
or set(data["entered_zones"]) or set(data["entered_zones"])
& set(camera_config.genai.required_zones) & set(camera_config.objects.genai.required_zones)
): ):
logger.debug(f"{camera} sending early request to GenAI") logger.debug(f"{camera} sending early request to GenAI")
@ -436,16 +439,17 @@ class EmbeddingMaintainer(threading.Thread):
# Run GenAI # Run GenAI
if ( if (
camera_config.genai.enabled camera_config.objects.genai.enabled
and camera_config.genai.send_triggers.tracked_object_end and camera_config.objects.genai.send_triggers.tracked_object_end
and self.genai_client is not None and self.genai_client is not None
and ( and (
not camera_config.genai.objects not camera_config.objects.genai.objects
or event.label in camera_config.genai.objects or event.label in camera_config.objects.genai.objects
) )
and ( and (
not camera_config.genai.required_zones not camera_config.objects.genai.required_zones
or set(event.zones) & set(camera_config.genai.required_zones) or set(event.zones)
& set(camera_config.objects.genai.required_zones)
) )
): ):
self._process_genai_description(event, camera_config, thumbnail) self._process_genai_description(event, camera_config, thumbnail)
@ -624,8 +628,10 @@ class EmbeddingMaintainer(threading.Thread):
self.embeddings.embed_thumbnail(event_id, thumbnail) self.embeddings.embed_thumbnail(event_id, thumbnail)
def _process_genai_description(self, event, camera_config, thumbnail) -> None: def _process_genai_description(
if event.has_snapshot and camera_config.genai.use_snapshot: self, event: Event, camera_config: CameraConfig, thumbnail
) -> None:
if event.has_snapshot and camera_config.objects.genai.use_snapshot:
snapshot_image = self._read_and_crop_snapshot(event, camera_config) snapshot_image = self._read_and_crop_snapshot(event, camera_config)
if not snapshot_image: if not snapshot_image:
return return
@ -637,7 +643,7 @@ class EmbeddingMaintainer(threading.Thread):
embed_image = ( embed_image = (
[snapshot_image] [snapshot_image]
if event.has_snapshot and camera_config.genai.use_snapshot if event.has_snapshot and camera_config.objects.genai.use_snapshot
else ( else (
[data["thumbnail"] for data in self.tracked_events[event.id]] [data["thumbnail"] for data in self.tracked_events[event.id]]
if num_thumbnails > 0 if num_thumbnails > 0
@ -645,7 +651,7 @@ class EmbeddingMaintainer(threading.Thread):
) )
) )
if camera_config.genai.debug_save_thumbnails and num_thumbnails > 0: if camera_config.objects.genai.debug_save_thumbnails and num_thumbnails > 0:
logger.debug(f"Saving {num_thumbnails} thumbnails for event {event.id}") logger.debug(f"Saving {num_thumbnails} thumbnails for event {event.id}")
Path(os.path.join(CLIPS_DIR, f"genai-requests/{event.id}")).mkdir( Path(os.path.join(CLIPS_DIR, f"genai-requests/{event.id}")).mkdir(
@ -775,7 +781,7 @@ class EmbeddingMaintainer(threading.Thread):
return return
camera_config = self.config.cameras[event.camera] camera_config = self.config.cameras[event.camera]
if not camera_config.genai.enabled and not force: if not camera_config.objects.genai.enabled and not force:
logger.error(f"GenAI not enabled for camera {event.camera}") logger.error(f"GenAI not enabled for camera {event.camera}")
return return

14
frigate/genai/__init__.py
View File

@ -40,9 +40,9 @@ class GenAIClient:
event: Event, event: Event,
) -> Optional[str]: ) -> Optional[str]:
"""Generate a description for the frame.""" """Generate a description for the frame."""
prompt = camera_config.genai.object_prompts.get( prompt = camera_config.objects.genai.object_prompts.get(
event.label, event.label,
camera_config.genai.prompt, camera_config.objects.genai.prompt,
).format(**model_to_dict(event)) ).format(**model_to_dict(event))
logger.debug(f"Sending images to genai provider with prompt: {prompt}") logger.debug(f"Sending images to genai provider with prompt: {prompt}")
return self._send(prompt, thumbnails) return self._send(prompt, thumbnails)
@ -58,16 +58,10 @@ class GenAIClient:
def get_genai_client(config: FrigateConfig) -> Optional[GenAIClient]: def get_genai_client(config: FrigateConfig) -> Optional[GenAIClient]:
"""Get the GenAI client.""" """Get the GenAI client."""
genai_config = config.genai
genai_cameras = [
c for c in config.cameras.values() if c.enabled and c.genai.enabled
]
if genai_cameras or genai_config.enabled:
load_providers() load_providers()
provider = PROVIDERS.get(genai_config.provider) provider = PROVIDERS.get(config.genai.provider)
if provider: if provider:
return provider(genai_config) return provider(config.genai)
return None return None

23
frigate/util/config.py
View File

@ -371,6 +371,22 @@ def migrate_017_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
del new_config["record"]["retain"] del new_config["record"]["retain"]
# migrate global genai to new objects config
global_genai = config.get("genai", {})
if global_genai:
new_genai_config = {}
new_object_config = config.get("objects", {})
new_object_config["genai"] = {}
for key in global_genai.keys():
if key not in ["provider", "base_url", "api_key"]:
new_object_config["genai"][key] = global_genai[key]
else:
new_genai_config[key] = global_genai[key]
config["genai"] = new_genai_config
for name, camera in config.get("cameras", {}).items(): for name, camera in config.get("cameras", {}).items():
camera_config: dict[str, dict[str, Any]] = camera.copy() camera_config: dict[str, dict[str, Any]] = camera.copy()
camera_record_retain = camera_config.get("record", {}).get("retain") camera_record_retain = camera_config.get("record", {}).get("retain")
@ -392,6 +408,13 @@ def migrate_017_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
del camera_config["record"]["retain"] del camera_config["record"]["retain"]
camera_genai = camera_config.get("genai", {})
if camera_genai:
new_object_config = config.get("objects", {})
new_object_config["genai"] = camera_genai
del camera_config["genai"]
new_config["cameras"][name] = camera_config new_config["cameras"][name] = camera_config
new_config["version"] = "0.17-0" new_config["version"] = "0.17-0"

21
web/src/components/overlay/detail/SearchDetailDialog.tsx
View File

@ -936,14 +936,17 @@ function ObjectDetailsTab({
</div> </div>
</div> </div>
<div className="flex flex-col gap-1.5"> <div className="flex flex-col gap-1.5">
{config?.cameras[search.camera].genai.enabled && {config?.cameras[search.camera].objects.genai.enabled &&
!search.end_time && !search.end_time &&
(config.cameras[search.camera].genai.required_zones.length === 0 || (config.cameras[search.camera].objects.genai.required_zones.length ===
0 ||
search.zones.some((zone) => search.zones.some((zone) =>
config.cameras[search.camera].genai.required_zones.includes(zone), config.cameras[search.camera].objects.genai.required_zones.includes(
zone,
),
)) && )) &&
(config.cameras[search.camera].genai.objects.length === 0 || (config.cameras[search.camera].objects.genai.objects.length === 0 ||
config.cameras[search.camera].genai.objects.includes( config.cameras[search.camera].objects.genai.objects.includes(
search.label, search.label,
)) ? ( )) ? (
<> <>
@ -972,7 +975,8 @@ function ObjectDetailsTab({
)} )}
<div className="flex w-full flex-row justify-end gap-2"> <div className="flex w-full flex-row justify-end gap-2">
{config?.cameras[search.camera].genai.enabled && search.end_time && ( {config?.cameras[search.camera].objects.genai.enabled &&
search.end_time && (
<div className="flex items-start"> <div className="flex items-start">
<Button <Button
className="rounded-r-none border-r-0" className="rounded-r-none border-r-0"
@ -1011,8 +1015,9 @@ function ObjectDetailsTab({
)} )}
</div> </div>
)} )}
{((config?.cameras[search.camera].genai.enabled && search.end_time) || {((config?.cameras[search.camera].objects.genai.enabled &&
!config?.cameras[search.camera].genai.enabled) && ( search.end_time) ||
!config?.cameras[search.camera].objects.genai.enabled) && (
<Button <Button
variant="select" variant="select"
aria-label={t("button.save", { ns: "common" })} aria-label={t("button.save", { ns: "common" })}

20
web/src/types/frigateConfig.ts
View File

@ -94,13 +94,6 @@ export interface CameraConfig {
cmd: string; cmd: string;
roles: string[]; roles: string[];
}[]; }[];
genai: {
enabled: string;
prompt: string;
object_prompts: { [key: string]: string };
required_zones: string[];
objects: string[];
};
live: { live: {
height: number; height: number;
quality: number; quality: number;
@ -146,6 +139,14 @@ export interface CameraConfig {
}; };
mask: string; mask: string;
track: string[]; track: string[];
genai: {
enabled: boolean;
enabled_in_config: boolean;
prompt: string;
object_prompts: { [key: string]: string };
required_zones: string[];
objects: string[];
};
}; };
onvif: { onvif: {
autotracking: { autotracking: {
@ -406,15 +407,10 @@ export interface FrigateConfig {
}; };
genai: { genai: {
enabled: boolean;
provider: string; provider: string;
base_url?: string; base_url?: string;
api_key?: string; api_key?: string;
model: string; model: string;
prompt: string;
object_prompts: { [key: string]: string };
required_zones: string[];
objects: string[];
}; };
go2rtc: { go2rtc: {

2
web/src/views/settings/CameraSettingsView.tsx
View File

@ -413,7 +413,7 @@ export default function CameraSettingsView({
</div> </div>
</div> </div>
</div> </div>
{config?.genai?.enabled && ( {cameraConfig?.objects?.genai?.enabled_in_config && (
<> <>
<Separator className="my-2 flex bg-secondary" /> <Separator className="my-2 flex bg-secondary" />