Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2025-08-13 13:47:36 +02:00
Code Issues Projects Releases Wiki Activity

Migrate object genai configuration (#19437)

Browse Source 
* Move genAI object to objects section

* Adjust config propogation behavior

* Refactor genai config usage

* Automatic migration

* Always start the embeddings process

* Always init embeddings

* Config fixes

* Adjust reference config

* Adjust docs

* Formatting

* Fix
This commit is contained in:
Nicolas Mowen 2025-08-08 16:33:11 -06:00 committed by GitHub
parent 6d078e565a
commit 52295fcac4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 251 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
docs/docs/configuration/genai.md
View File

@ -15,23 +15,24 @@ To use Generative AI, you must define a single provider at the global level of y
```yaml
genai:
enabled: True
provider: gemini
api_key: "{FRIGATE_GEMINI_API_KEY}"
model: gemini-1.5-flash
cameras:
front_camera:
objects:
genai:
enabled: True # <- enable GenAI for your front camera
use_snapshot: True
objects:
- person
required_zones:
- steps
enabled: True # <- enable GenAI for your front camera
use_snapshot: True
objects:
- person
required_zones:
- steps
indoor_camera:
genai:
enabled: False # <- disable GenAI for your indoor camera
objects:
genai:
enabled: False # <- disable GenAI for your indoor camera
```
By default, descriptions will be generated for all tracked objects and all zones. But you can also optionally specify `objects` and `required_zones` to only generate descriptions for certain tracked objects or zones.
@ -68,7 +69,6 @@ You should have at least 8 GB of RAM available (or VRAM if running on GPU) to ru
```yaml
genai:
enabled: True
provider: ollama
base_url: http://localhost:11434
model: llava:7b
@ -95,7 +95,6 @@ To start using Gemini, you must first get an API key from [Google AI Studio](htt
```yaml
genai:
enabled: True
provider: gemini
api_key: "{FRIGATE_GEMINI_API_KEY}"
model: gemini-1.5-flash
@ -117,7 +116,6 @@ To start using OpenAI, you must first [create an API key](https://platform.opena
```yaml
genai:
enabled: True
provider: openai
api_key: "{FRIGATE_OPENAI_API_KEY}"
model: gpt-4o
@ -145,7 +143,6 @@ To start using Azure OpenAI, you must first [create a resource](https://learn.mi
```yaml
genai:
enabled: True
provider: azure_openai
base_url: https://example-endpoint.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2023-03-15-preview
api_key: "{FRIGATE_OPENAI_API_KEY}"
@ -188,32 +185,35 @@ You are also able to define custom prompts in your configuration.
```yaml
genai:
enabled: True
provider: ollama
base_url: http://localhost:11434
model: llava
prompt: "Analyze the {label} in these images from the {camera} security camera. Focus on the actions, behavior, and potential intent of the {label}, rather than just describing its appearance."
object_prompts:
person: "Examine the main person in these images. What are they doing and what might their actions suggest about their intent (e.g., approaching a door, leaving an area, standing still)? Do not describe the surroundings or static details."
car: "Observe the primary vehicle in these images. Focus on its movement, direction, or purpose (e.g., parking, approaching, circling). If it's a delivery vehicle, mention the company."
objects:
prompt: "Analyze the {label} in these images from the {camera} security camera. Focus on the actions, behavior, and potential intent of the {label}, rather than just describing its appearance."
object_prompts:
person: "Examine the main person in these images. What are they doing and what might their actions suggest about their intent (e.g., approaching a door, leaving an area, standing still)? Do not describe the surroundings or static details."
car: "Observe the primary vehicle in these images. Focus on its movement, direction, or purpose (e.g., parking, approaching, circling). If it's a delivery vehicle, mention the company."
```
Prompts can also be overriden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire.
Prompts can also be overridden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire.
```yaml
cameras:
front_door:
genai:
use_snapshot: True
prompt: "Analyze the {label} in these images from the {camera} security camera at the front door. Focus on the actions and potential intent of the {label}."
object_prompts:
person: "Examine the person in these images. What are they doing, and how might their actions suggest their purpose (e.g., delivering something, approaching, leaving)? If they are carrying or interacting with a package, include details about its source or destination."
cat: "Observe the cat in these images. Focus on its movement and intent (e.g., wandering, hunting, interacting with objects). If the cat is near the flower pots or engaging in any specific actions, mention it."
objects:
- person
- cat
required_zones:
- steps
objects:
genai:
enabled: True
use_snapshot: True
prompt: "Analyze the {label} in these images from the {camera} security camera at the front door. Focus on the actions and potential intent of the {label}."
object_prompts:
person: "Examine the person in these images. What are they doing, and how might their actions suggest their purpose (e.g., delivering something, approaching, leaving)? If they are carrying or interacting with a package, include details about its source or destination."
cat: "Observe the cat in these images. Focus on its movement and intent (e.g., wandering, hunting, interacting with objects). If the cat is near the flower pots or engaging in any specific actions, mention it."
objects:
- person
- cat
required_zones:
- steps
```
### Experiment with prompts

62
docs/docs/configuration/reference.md
View File

@ -339,6 +339,33 @@ objects:
# Optional: mask to prevent this object type from being detected in certain areas (default: no mask)
# Checks based on the bottom center of the bounding box of the object
mask: 0.000,0.000,0.781,0.000,0.781,0.278,0.000,0.278
# Optional: Configuration for AI generated tracked object descriptions
genai:
# Optional: Enable AI object description generation (default: shown below)
enabled: False
# Optional: Use the object snapshot instead of thumbnails for description generation (default: shown below)
use_snapshot: False
# Optional: The default prompt for generating descriptions. Can use replacement
# variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
# Optional: Object specific prompts to customize description results
# Format: {label}: {prompt}
object_prompts:
person: "My special person prompt."
# Optional: objects to generate descriptions for (default: all objects that are tracked)
objects:
- person
- cat
# Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify)
required_zones: []
# Optional: What triggers to use to send frames for a tracked object to generative AI (default: shown below)
send_triggers:
# Once the object is no longer tracked
tracked_object_end: True
# Optional: After X many significant updates are received (default: shown below)
after_significant_updates: None
# Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below)
debug_save_thumbnails: False
# Optional: Review configuration
# NOTE: Can be overridden at the camera level
@ -612,13 +639,6 @@ genai:
base_url: http://localhost::11434
# Required if gemini or openai
api_key: "{FRIGATE_GENAI_API_KEY}"
# Optional: The default prompt for generating descriptions. Can use replacement
# variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
# Optional: Object specific prompts to customize description results
# Format: {label}: {prompt}
object_prompts:
person: "My special person prompt."
# Optional: Configuration for audio transcription
# NOTE: only the enabled option can be overridden at the camera level
@ -857,34 +877,6 @@ cameras:
actions:
- notification
# Optional: Configuration for AI generated tracked object descriptions
genai:
# Optional: Enable AI description generation (default: shown below)
enabled: False
# Optional: Use the object snapshot instead of thumbnails for description generation (default: shown below)
use_snapshot: False
# Optional: The default prompt for generating descriptions. Can use replacement
# variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
# Optional: Object specific prompts to customize description results
# Format: {label}: {prompt}
object_prompts:
person: "My special person prompt."
# Optional: objects to generate descriptions for (default: all objects that are tracked)
objects:
- person
- cat
# Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify)
required_zones: []
# Optional: What triggers to use to send frames for a tracked object to generative AI (default: shown below)
send_triggers:
# Once the object is no longer tracked
tracked_object_end: True
# Optional: After X many significant updates are received (default: shown below)
after_significant_updates: None
# Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below)
debug_save_thumbnails: False
# Optional
ui:
# Optional: Set a timezone to use in the UI (default: use browser local time)

2
frigate/api/event.py
View File

@ -1230,7 +1230,7 @@ def regenerate_description(
camera_config = request.app.frigate_config.cameras[event.camera]
if camera_config.genai.enabled or params.force:
if camera_config.objects.genai.enabled or params.force:
request.app.event_metadata_updater.publish(
(event.id, params.source, params.force),
EventMetadataTypeEnum.regenerate_description.value,

29
frigate/app.py
View File

@ -246,18 +246,7 @@ class FrigateApp:
logger.info(f"Review process started: {review_segment_process.pid}")
def init_embeddings_manager(self) -> None:
genai_cameras = [
c for c in self.config.cameras.values() if c.enabled and c.genai.enabled
]
if (
not self.config.semantic_search.enabled
and not genai_cameras
and not self.config.lpr.enabled
and not self.config.face_recognition.enabled
):
return
# always start the embeddings process
embedding_process = EmbeddingProcess(
self.config, self.embeddings_metrics, self.stop_event
)
@ -309,20 +298,8 @@ class FrigateApp:
migrate_exports(self.config.ffmpeg, list(self.config.cameras.keys()))
def init_embeddings_client(self) -> None:
genai_cameras = [
c
for c in self.config.cameras.values()
if c.enabled_in_config and c.genai.enabled
]
if (
self.config.semantic_search.enabled
or self.config.lpr.enabled
or genai_cameras
or self.config.face_recognition.enabled
):
# Create a client for other processes to use
self.embeddings = EmbeddingsContext(self.db)
# Create a client for other processes to use
self.embeddings = EmbeddingsContext(self.db)
def init_inter_process_communicator(self) -> None:
self.inter_process_communicator = InterProcessCommunicator()

6
frigate/comms/dispatcher.py
View File

@ -209,7 +209,7 @@ class Dispatcher:
].onvif.autotracking.enabled,
"alerts": self.config.cameras[camera].review.alerts.enabled,
"detections": self.config.cameras[camera].review.detections.enabled,
"genai": self.config.cameras[camera].genai.enabled,
"genai": self.config.cameras[camera].objects.genai.enabled,
}
self.publish("camera_activity", json.dumps(camera_status))
@ -744,10 +744,10 @@ class Dispatcher:
def _on_genai_command(self, camera_name: str, payload: str) -> None:
"""Callback for GenAI topic."""
genai_settings = self.config.cameras[camera_name].genai
genai_settings = self.config.cameras[camera_name].objects.genai
if payload == "ON":
if not self.config.cameras[camera_name].genai.enabled_in_config:
if not self.config.cameras[camera_name].objects.genai.enabled_in_config:
logger.error(
"GenAI must be enabled in the config to be turned on via MQTT."
)

2
frigate/comms/mqtt.py
View File

@ -124,7 +124,7 @@ class MqttClient(Communicator):
)
self.publish(
f"{camera_name}/genai/state",
"ON" if camera.genai.enabled_in_config else "OFF",
"ON" if camera.objects.genai.enabled_in_config else "OFF",
retain=True,
)

4
frigate/config/camera/camera.py
View File

@ -28,7 +28,6 @@ from .audio import AudioConfig
from .birdseye import BirdseyeCameraConfig
from .detect import DetectConfig
from .ffmpeg import CameraFfmpegConfig, CameraInput
from .genai import GenAICameraConfig
from .live import CameraLiveConfig
from .motion import MotionConfig
from .mqtt import CameraMqttConfig
@ -71,9 +70,6 @@ class CameraConfig(FrigateBaseModel):
default_factory=CameraFaceRecognitionConfig, title="Face recognition config."
)
ffmpeg: CameraFfmpegConfig = Field(title="FFmpeg configuration for the camera.")
genai: GenAICameraConfig = Field(
default_factory=GenAICameraConfig, title="Generative AI configuration."
)
live: CameraLiveConfig = Field(
default_factory=CameraLiveConfig, title="Live playback settings."
)

70
frigate/config/camera/genai.py
View File

@ -1,12 +1,12 @@
from enum import Enum
from typing import Optional, Union
from typing import Optional
from pydantic import BaseModel, Field, field_validator
from pydantic import Field
from ..base import FrigateBaseModel
from ..env import EnvString
__all__ = ["GenAIConfig", "GenAICameraConfig", "GenAIProviderEnum"]
__all__ = ["GenAIConfig", "GenAIProviderEnum"]
class GenAIProviderEnum(str, Enum):
@ -16,70 +16,8 @@ class GenAIProviderEnum(str, Enum):
ollama = "ollama"
class GenAISendTriggersConfig(BaseModel):
tracked_object_end: bool = Field(
default=True, title="Send once the object is no longer tracked."
)
after_significant_updates: Optional[int] = Field(
default=None,
title="Send an early request to generative AI when X frames accumulated.",
ge=1,
)
# uses BaseModel because some global attributes are not available at the camera level
class GenAICameraConfig(BaseModel):
enabled: bool = Field(default=False, title="Enable GenAI for camera.")
use_snapshot: bool = Field(
default=False, title="Use snapshots for generating descriptions."
)
prompt: str = Field(
default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
title="Default caption prompt.",
)
object_prompts: dict[str, str] = Field(
default_factory=dict, title="Object specific prompts."
)
objects: Union[str, list[str]] = Field(
default_factory=list,
title="List of objects to run generative AI for.",
)
required_zones: Union[str, list[str]] = Field(
default_factory=list,
title="List of required zones to be entered in order to run generative AI.",
)
debug_save_thumbnails: bool = Field(
default=False,
title="Save thumbnails sent to generative AI for debugging purposes.",
)
send_triggers: GenAISendTriggersConfig = Field(
default_factory=GenAISendTriggersConfig,
title="What triggers to use to send frames to generative AI for a tracked object.",
)
enabled_in_config: Optional[bool] = Field(
default=None, title="Keep track of original state of generative AI."
)
@field_validator("required_zones", mode="before")
@classmethod
def validate_required_zones(cls, v):
if isinstance(v, str) and "," not in v:
return [v]
return v
class GenAIConfig(FrigateBaseModel):
enabled: bool = Field(default=False, title="Enable GenAI.")
prompt: str = Field(
default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
title="Default caption prompt.",
)
object_prompts: dict[str, str] = Field(
default_factory=dict, title="Object specific prompts."
)
"""Primary GenAI Config to define GenAI Provider."""
api_key: Optional[EnvString] = Field(default=None, title="Provider API key.")
base_url: Optional[str] = Field(default=None, title="Provider base url.")

61
frigate/config/camera/objects.py
View File

@ -1,10 +1,10 @@
from typing import Any, Optional, Union
from pydantic import Field, PrivateAttr, field_serializer
from pydantic import Field, PrivateAttr, field_serializer, field_validator
from ..base import FrigateBaseModel
__all__ = ["ObjectConfig", "FilterConfig"]
__all__ = ["ObjectConfig", "GenAIObjectConfig", "FilterConfig"]
DEFAULT_TRACKED_OBJECTS = ["person"]
@ -49,12 +49,69 @@ class FilterConfig(FrigateBaseModel):
return None
class GenAIObjectTriggerConfig(FrigateBaseModel):
tracked_object_end: bool = Field(
default=True, title="Send once the object is no longer tracked."
)
after_significant_updates: Optional[int] = Field(
default=None,
title="Send an early request to generative AI when X frames accumulated.",
ge=1,
)
class GenAIObjectConfig(FrigateBaseModel):
enabled: bool = Field(default=False, title="Enable GenAI for camera.")
use_snapshot: bool = Field(
default=False, title="Use snapshots for generating descriptions."
)
prompt: str = Field(
default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
title="Default caption prompt.",
)
object_prompts: dict[str, str] = Field(
default_factory=dict, title="Object specific prompts."
)
objects: Union[str, list[str]] = Field(
default_factory=list,
title="List of objects to run generative AI for.",
)
required_zones: Union[str, list[str]] = Field(
default_factory=list,
title="List of required zones to be entered in order to run generative AI.",
)
debug_save_thumbnails: bool = Field(
default=False,
title="Save thumbnails sent to generative AI for debugging purposes.",
)
send_triggers: GenAIObjectTriggerConfig = Field(
default_factory=GenAIObjectTriggerConfig,
title="What triggers to use to send frames to generative AI for a tracked object.",
)
enabled_in_config: Optional[bool] = Field(
default=None, title="Keep track of original state of generative AI."
)
@field_validator("required_zones", mode="before")
@classmethod
def validate_required_zones(cls, v):
if isinstance(v, str) and "," not in v:
return [v]
return v
class ObjectConfig(FrigateBaseModel):
track: list[str] = Field(default=DEFAULT_TRACKED_OBJECTS, title="Objects to track.")
filters: dict[str, FilterConfig] = Field(
default_factory=dict, title="Object filters."
)
mask: Union[str, list[str]] = Field(default="", title="Object mask.")
genai: GenAIObjectConfig = Field(
default_factory=GenAIObjectConfig,
title="Config for using genai to analyze objects.",
)
_all_objects: list[str] = PrivateAttr()
@property

2
frigate/config/camera/updater.py
View File

@ -99,7 +99,7 @@ class CameraConfigUpdateSubscriber:
elif update_type == CameraConfigUpdateEnum.enabled:
config.enabled = updated_config
elif update_type == CameraConfigUpdateEnum.genai:
config.genai = updated_config
config.objects.genai = updated_config
elif update_type == CameraConfigUpdateEnum.motion:
config.motion = updated_config
elif update_type == CameraConfigUpdateEnum.notifications:

13
frigate/config/config.py
View File

@ -352,6 +352,11 @@ class FrigateConfig(FrigateBaseModel):
default_factory=ModelConfig, title="Detection model configuration."
)
# GenAI config
genai: GenAIConfig = Field(
default_factory=GenAIConfig, title="Generative AI configuration."
)
# Camera config
cameras: Dict[str, CameraConfig] = Field(title="Camera configuration.")
audio: AudioConfig = Field(
@ -366,9 +371,6 @@ class FrigateConfig(FrigateBaseModel):
ffmpeg: FfmpegConfig = Field(
default_factory=FfmpegConfig, title="Global FFmpeg configuration."
)
genai: GenAIConfig = Field(
default_factory=GenAIConfig, title="Generative AI configuration."
)
live: CameraLiveConfig = Field(
default_factory=CameraLiveConfig, title="Live playback settings."
)
@ -458,7 +460,6 @@ class FrigateConfig(FrigateBaseModel):
"live": ...,
"objects": ...,
"review": ...,
"genai": ...,
"motion": ...,
"notifications": ...,
"detect": ...,
@ -606,7 +607,9 @@ class FrigateConfig(FrigateBaseModel):
camera_config.review.detections.enabled_in_config = (
camera_config.review.detections.enabled
)
camera_config.genai.enabled_in_config = camera_config.genai.enabled
camera_config.objects.genai.enabled_in_config = (
camera_config.objects.genai.enabled
)
# Add default filters
object_keys = camera_config.objects.track

44
frigate/embeddings/maintainer.py
View File

@ -30,7 +30,7 @@ from frigate.comms.recordings_updater import (
RecordingsDataTypeEnum,
)
from frigate.comms.review_updater import ReviewDataSubscriber
from frigate.config import FrigateConfig
from frigate.config import CameraConfig, FrigateConfig
from frigate.config.camera.camera import CameraTypeEnum
from frigate.config.camera.updater import (
CameraConfigUpdateEnum,
@ -329,7 +329,10 @@ class EmbeddingMaintainer(threading.Thread):
camera_config = self.config.cameras[camera]
# no need to process updated objects if face recognition, lpr, genai are disabled
if not camera_config.genai.enabled and len(self.realtime_processors) == 0:
if (
not camera_config.objects.genai.enabled
and len(self.realtime_processors) == 0
):
return
# Create our own thumbnail based on the bounding box and the frame time
@ -367,23 +370,23 @@ class EmbeddingMaintainer(threading.Thread):
# check if we're configured to send an early request after a minimum number of updates received
if (
self.genai_client is not None
and camera_config.genai.send_triggers.after_significant_updates
and camera_config.objects.genai.send_triggers.after_significant_updates
):
if (
len(self.tracked_events.get(data["id"], []))
>= camera_config.genai.send_triggers.after_significant_updates
>= camera_config.objects.genai.send_triggers.after_significant_updates
and data["id"] not in self.early_request_sent
):
if data["has_clip"] and data["has_snapshot"]:
event: Event = Event.get(Event.id == data["id"])
if (
not camera_config.genai.objects
or event.label in camera_config.genai.objects
not camera_config.objects.genai.objects
or event.label in camera_config.objects.genai.objects
) and (
not camera_config.genai.required_zones
not camera_config.objects.genai.required_zones
or set(data["entered_zones"])
& set(camera_config.genai.required_zones)
& set(camera_config.objects.genai.required_zones)
):
logger.debug(f"{camera} sending early request to GenAI")
@ -436,16 +439,17 @@ class EmbeddingMaintainer(threading.Thread):
# Run GenAI
if (
camera_config.genai.enabled
and camera_config.genai.send_triggers.tracked_object_end
camera_config.objects.genai.enabled
and camera_config.objects.genai.send_triggers.tracked_object_end
and self.genai_client is not None
and (
not camera_config.genai.objects
or event.label in camera_config.genai.objects
not camera_config.objects.genai.objects
or event.label in camera_config.objects.genai.objects
)
and (
not camera_config.genai.required_zones
or set(event.zones) & set(camera_config.genai.required_zones)
not camera_config.objects.genai.required_zones
or set(event.zones)
& set(camera_config.objects.genai.required_zones)
)
):
self._process_genai_description(event, camera_config, thumbnail)
@ -624,8 +628,10 @@ class EmbeddingMaintainer(threading.Thread):
self.embeddings.embed_thumbnail(event_id, thumbnail)
def _process_genai_description(self, event, camera_config, thumbnail) -> None:
if event.has_snapshot and camera_config.genai.use_snapshot:
def _process_genai_description(
self, event: Event, camera_config: CameraConfig, thumbnail
) -> None:
if event.has_snapshot and camera_config.objects.genai.use_snapshot:
snapshot_image = self._read_and_crop_snapshot(event, camera_config)
if not snapshot_image:
return
@ -637,7 +643,7 @@ class EmbeddingMaintainer(threading.Thread):
embed_image = (
[snapshot_image]
if event.has_snapshot and camera_config.genai.use_snapshot
if event.has_snapshot and camera_config.objects.genai.use_snapshot
else (
[data["thumbnail"] for data in self.tracked_events[event.id]]
if num_thumbnails > 0
@ -645,7 +651,7 @@ class EmbeddingMaintainer(threading.Thread):
)
)
if camera_config.genai.debug_save_thumbnails and num_thumbnails > 0:
if camera_config.objects.genai.debug_save_thumbnails and num_thumbnails > 0:
logger.debug(f"Saving {num_thumbnails} thumbnails for event {event.id}")
Path(os.path.join(CLIPS_DIR, f"genai-requests/{event.id}")).mkdir(
@ -775,7 +781,7 @@ class EmbeddingMaintainer(threading.Thread):
return
camera_config = self.config.cameras[event.camera]
if not camera_config.genai.enabled and not force:
if not camera_config.objects.genai.enabled and not force:
logger.error(f"GenAI not enabled for camera {event.camera}")
return

18
frigate/genai/__init__.py
View File

@ -40,9 +40,9 @@ class GenAIClient:
event: Event,
) -> Optional[str]:
"""Generate a description for the frame."""
prompt = camera_config.genai.object_prompts.get(
prompt = camera_config.objects.genai.object_prompts.get(
event.label,
camera_config.genai.prompt,
camera_config.objects.genai.prompt,
).format(**model_to_dict(event))
logger.debug(f"Sending images to genai provider with prompt: {prompt}")
return self._send(prompt, thumbnails)
@ -58,16 +58,10 @@ class GenAIClient:
def get_genai_client(config: FrigateConfig) -> Optional[GenAIClient]:
"""Get the GenAI client."""
genai_config = config.genai
genai_cameras = [
c for c in config.cameras.values() if c.enabled and c.genai.enabled
]
if genai_cameras or genai_config.enabled:
load_providers()
provider = PROVIDERS.get(genai_config.provider)
if provider:
return provider(genai_config)
load_providers()
provider = PROVIDERS.get(config.genai.provider)
if provider:
return provider(config.genai)
return None

23
frigate/util/config.py
View File

@ -371,6 +371,22 @@ def migrate_017_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
del new_config["record"]["retain"]
# migrate global genai to new objects config
global_genai = config.get("genai", {})
if global_genai:
new_genai_config = {}
new_object_config = config.get("objects", {})
new_object_config["genai"] = {}
for key in global_genai.keys():
if key not in ["provider", "base_url", "api_key"]:
new_object_config["genai"][key] = global_genai[key]
else:
new_genai_config[key] = global_genai[key]
config["genai"] = new_genai_config
for name, camera in config.get("cameras", {}).items():
camera_config: dict[str, dict[str, Any]] = camera.copy()
camera_record_retain = camera_config.get("record", {}).get("retain")
@ -392,6 +408,13 @@ def migrate_017_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
del camera_config["record"]["retain"]
camera_genai = camera_config.get("genai", {})
if camera_genai:
new_object_config = config.get("objects", {})
new_object_config["genai"] = camera_genai
del camera_config["genai"]
new_config["cameras"][name] = camera_config
new_config["version"] = "0.17-0"

97
web/src/components/overlay/detail/SearchDetailDialog.tsx
View File

@ -936,14 +936,17 @@ function ObjectDetailsTab({
</div>
</div>
<div className="flex flex-col gap-1.5">
{config?.cameras[search.camera].genai.enabled &&
{config?.cameras[search.camera].objects.genai.enabled &&
!search.end_time &&
(config.cameras[search.camera].genai.required_zones.length === 0 ||
(config.cameras[search.camera].objects.genai.required_zones.length ===
0 ||
search.zones.some((zone) =>
config.cameras[search.camera].genai.required_zones.includes(zone),
config.cameras[search.camera].objects.genai.required_zones.includes(
zone,
),
)) &&
(config.cameras[search.camera].genai.objects.length === 0 ||
config.cameras[search.camera].genai.objects.includes(
(config.cameras[search.camera].objects.genai.objects.length === 0 ||
config.cameras[search.camera].objects.genai.objects.includes(
search.label,
)) ? (
<>
@ -972,47 +975,49 @@ function ObjectDetailsTab({
)}
<div className="flex w-full flex-row justify-end gap-2">
{config?.cameras[search.camera].genai.enabled && search.end_time && (
<div className="flex items-start">
<Button
className="rounded-r-none border-r-0"
aria-label={t("details.button.regenerate.label")}
onClick={() => regenerateDescription("thumbnails")}
>
{t("details.button.regenerate.title")}
</Button>
{search.has_snapshot && (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button
className="rounded-l-none border-l-0 px-2"
aria-label={t("details.expandRegenerationMenu")}
>
<FaChevronDown className="size-3" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent>
<DropdownMenuItem
className="cursor-pointer"
aria-label={t("details.regenerateFromSnapshot")}
onClick={() => regenerateDescription("snapshot")}
>
{t("details.regenerateFromSnapshot")}
</DropdownMenuItem>
<DropdownMenuItem
className="cursor-pointer"
aria-label={t("details.regenerateFromThumbnails")}
onClick={() => regenerateDescription("thumbnails")}
>
{t("details.regenerateFromThumbnails")}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
)}
</div>
)}
{((config?.cameras[search.camera].genai.enabled && search.end_time) ||
!config?.cameras[search.camera].genai.enabled) && (
{config?.cameras[search.camera].objects.genai.enabled &&
search.end_time && (
<div className="flex items-start">
<Button
className="rounded-r-none border-r-0"
aria-label={t("details.button.regenerate.label")}
onClick={() => regenerateDescription("thumbnails")}
>
{t("details.button.regenerate.title")}
</Button>
{search.has_snapshot && (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button
className="rounded-l-none border-l-0 px-2"
aria-label={t("details.expandRegenerationMenu")}
>
<FaChevronDown className="size-3" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent>
<DropdownMenuItem
className="cursor-pointer"
aria-label={t("details.regenerateFromSnapshot")}
onClick={() => regenerateDescription("snapshot")}
>
{t("details.regenerateFromSnapshot")}
</DropdownMenuItem>
<DropdownMenuItem
className="cursor-pointer"
aria-label={t("details.regenerateFromThumbnails")}
onClick={() => regenerateDescription("thumbnails")}
>
{t("details.regenerateFromThumbnails")}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
)}
</div>
)}
{((config?.cameras[search.camera].objects.genai.enabled &&
search.end_time) ||
!config?.cameras[search.camera].objects.genai.enabled) && (
<Button
variant="select"
aria-label={t("button.save", { ns: "common" })}

20
web/src/types/frigateConfig.ts
View File

@ -94,13 +94,6 @@ export interface CameraConfig {
cmd: string;
roles: string[];
}[];
genai: {
enabled: string;
prompt: string;
object_prompts: { [key: string]: string };
required_zones: string[];
objects: string[];
};
live: {
height: number;
quality: number;
@ -146,6 +139,14 @@ export interface CameraConfig {
};
mask: string;
track: string[];
genai: {
enabled: boolean;
enabled_in_config: boolean;
prompt: string;
object_prompts: { [key: string]: string };
required_zones: string[];
objects: string[];
};
};
onvif: {
autotracking: {
@ -406,15 +407,10 @@ export interface FrigateConfig {
};
genai: {
enabled: boolean;
provider: string;
base_url?: string;
api_key?: string;
model: string;
prompt: string;
object_prompts: { [key: string]: string };
required_zones: string[];
objects: string[];
};
go2rtc: {

2
web/src/views/settings/CameraSettingsView.tsx
View File

@ -413,7 +413,7 @@ export default function CameraSettingsView({
</div>
</div>
</div>
{config?.genai?.enabled && (
{cameraConfig?.objects?.genai?.enabled_in_config && (
<>
<Separator className="my-2 flex bg-secondary" />