mirror of
https://github.com/blakeblackshear/frigate.git
synced 2025-07-30 13:48:07 +02:00
Embeddings tweaks (#16864)
* make semantic search optional * config * frontend metrics * docs * tweak * fixes * also check genai cameras for embeddings context
This commit is contained in:
parent
db4152c4ca
commit
8d2f461350
@ -9,7 +9,7 @@ Frigate has support for CV2 Local Binary Pattern Face Recognizer to recognize fa
|
||||
|
||||
## Configuration
|
||||
|
||||
Face recognition is disabled by default and requires semantic search to be enabled, face recognition must be enabled in your config file before it can be used. Semantic Search and face recognition are global configuration settings.
|
||||
Face recognition is disabled by default, face recognition must be enabled in your config file before it can be used. Face recognition is a global configuration setting.
|
||||
|
||||
```yaml
|
||||
face_recognition:
|
||||
@ -36,6 +36,7 @@ The accuracy of face recognition is heavily dependent on the quality of data giv
|
||||
:::tip
|
||||
|
||||
When choosing images to include in the face training set it is recommended to always follow these recommendations:
|
||||
|
||||
- If it is difficult to make out details in a persons face it will not be helpful in training.
|
||||
- Avoid images with under/over-exposure.
|
||||
- Avoid blurry / pixelated images.
|
||||
@ -52,4 +53,4 @@ Then it is recommended to use the `Face Library` tab in Frigate to select and tr
|
||||
|
||||
### Step 2 - Expanding The Dataset
|
||||
|
||||
Once straight-on images are performing well, start choosing slightly off-angle images to include for training. It is important to still choose images where enough face detail is visible to recognize someone.
|
||||
Once straight-on images are performing well, start choosing slightly off-angle images to include for training. It is important to still choose images where enough face detail is visible to recognize someone.
|
||||
|
@ -7,12 +7,6 @@ Generative AI can be used to automatically generate descriptive text based on th
|
||||
|
||||
Requests for a description are sent off automatically to your AI provider at the end of the tracked object's lifecycle. Descriptions can also be regenerated manually via the Frigate UI.
|
||||
|
||||
:::info
|
||||
|
||||
Semantic Search must be enabled to use Generative AI.
|
||||
|
||||
:::
|
||||
|
||||
## Configuration
|
||||
|
||||
Generative AI can be enabled for all cameras or only for specific cameras. There are currently 3 native providers available to integrate with Frigate. Other providers that support the OpenAI standard API can also be used. See the OpenAI section below.
|
||||
|
@ -570,7 +570,6 @@ lpr:
|
||||
known_plates: {}
|
||||
|
||||
# Optional: Configuration for AI generated tracked object descriptions
|
||||
# NOTE: Semantic Search must be enabled for this to do anything.
|
||||
# WARNING: Depending on the provider, this will send thumbnails over the internet
|
||||
# to Google or OpenAI's LLMs to generate descriptions. It can be overridden at
|
||||
# the camera level (enabled: False) to enhance privacy for indoor cameras.
|
||||
|
@ -1083,10 +1083,7 @@ def regenerate_description(
|
||||
|
||||
camera_config = request.app.frigate_config.cameras[event.camera]
|
||||
|
||||
if (
|
||||
request.app.frigate_config.semantic_search.enabled
|
||||
and camera_config.genai.enabled
|
||||
):
|
||||
if camera_config.genai.enabled:
|
||||
request.app.event_metadata_updater.publish((event.id, params.source))
|
||||
|
||||
return JSONResponse(
|
||||
|
@ -93,7 +93,13 @@ class FrigateApp:
|
||||
self.log_queue: Queue = mp.Queue()
|
||||
self.camera_metrics: dict[str, CameraMetrics] = {}
|
||||
self.embeddings_metrics: DataProcessorMetrics | None = (
|
||||
DataProcessorMetrics() if config.semantic_search.enabled else None
|
||||
DataProcessorMetrics()
|
||||
if (
|
||||
config.semantic_search.enabled
|
||||
or config.lpr.enabled
|
||||
or config.face_recognition.enabled
|
||||
)
|
||||
else None
|
||||
)
|
||||
self.ptz_metrics: dict[str, PTZMetrics] = {}
|
||||
self.processes: dict[str, int] = {}
|
||||
@ -236,7 +242,16 @@ class FrigateApp:
|
||||
logger.info(f"Review process started: {review_segment_process.pid}")
|
||||
|
||||
def init_embeddings_manager(self) -> None:
|
||||
if not self.config.semantic_search.enabled:
|
||||
genai_cameras = [
|
||||
c for c in self.config.cameras.values() if c.enabled and c.genai.enabled
|
||||
]
|
||||
|
||||
if (
|
||||
not self.config.semantic_search.enabled
|
||||
and not genai_cameras
|
||||
and not self.config.lpr.enabled
|
||||
and not self.config.face_recognition.enabled
|
||||
):
|
||||
return
|
||||
|
||||
embedding_process = util.Process(
|
||||
@ -293,7 +308,16 @@ class FrigateApp:
|
||||
migrate_exports(self.config.ffmpeg, list(self.config.cameras.keys()))
|
||||
|
||||
def init_embeddings_client(self) -> None:
|
||||
if self.config.semantic_search.enabled:
|
||||
genai_cameras = [
|
||||
c for c in self.config.cameras.values() if c.enabled and c.genai.enabled
|
||||
]
|
||||
|
||||
if (
|
||||
self.config.semantic_search.enabled
|
||||
or self.config.lpr.enabled
|
||||
or genai_cameras
|
||||
or self.config.face_recognition.enabled
|
||||
):
|
||||
# Create a client for other processes to use
|
||||
self.embeddings = EmbeddingsContext(self.db)
|
||||
|
||||
|
@ -172,16 +172,6 @@ class RestreamConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
|
||||
def verify_semantic_search_dependent_configs(config: FrigateConfig) -> None:
|
||||
"""Verify that semantic search is enabled if required features are enabled."""
|
||||
if not config.semantic_search.enabled:
|
||||
if config.genai.enabled:
|
||||
raise ValueError("Genai requires semantic search to be enabled.")
|
||||
|
||||
if config.face_recognition.enabled:
|
||||
raise ValueError("Face recognition requires semantic to be enabled.")
|
||||
|
||||
|
||||
def verify_config_roles(camera_config: CameraConfig) -> None:
|
||||
"""Verify that roles are setup in the config correctly."""
|
||||
assigned_roles = list(
|
||||
@ -647,7 +637,6 @@ class FrigateConfig(FrigateBaseModel):
|
||||
detector_config.model = model
|
||||
self.detectors[key] = detector_config
|
||||
|
||||
verify_semantic_search_dependent_configs(self)
|
||||
return self
|
||||
|
||||
@field_validator("cameras")
|
||||
|
@ -28,10 +28,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def manage_embeddings(config: FrigateConfig, metrics: DataProcessorMetrics) -> None:
|
||||
# Only initialize embeddings if semantic search is enabled
|
||||
if not config.semantic_search.enabled:
|
||||
return
|
||||
|
||||
stop_event = mp.Event()
|
||||
|
||||
def receiveSignal(signalNumber: int, frame: Optional[FrameType]) -> None:
|
||||
|
@ -71,11 +71,14 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
super().__init__(name="embeddings_maintainer")
|
||||
self.config = config
|
||||
self.metrics = metrics
|
||||
self.embeddings = Embeddings(config, db, metrics)
|
||||
self.embeddings = None
|
||||
|
||||
# Check if we need to re-index events
|
||||
if config.semantic_search.reindex:
|
||||
self.embeddings.reindex()
|
||||
if config.semantic_search.enabled:
|
||||
self.embeddings = Embeddings(config, db, metrics)
|
||||
|
||||
# Check if we need to re-index events
|
||||
if config.semantic_search.reindex:
|
||||
self.embeddings.reindex()
|
||||
|
||||
# create communication for updating event descriptions
|
||||
self.requestor = InterProcessRequestor()
|
||||
@ -152,30 +155,30 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
|
||||
def _handle_request(topic: str, data: dict[str, any]) -> str:
|
||||
try:
|
||||
if topic == EmbeddingsRequestEnum.embed_description.value:
|
||||
return serialize(
|
||||
self.embeddings.embed_description(
|
||||
data["id"], data["description"]
|
||||
),
|
||||
pack=False,
|
||||
)
|
||||
elif topic == EmbeddingsRequestEnum.embed_thumbnail.value:
|
||||
thumbnail = base64.b64decode(data["thumbnail"])
|
||||
return serialize(
|
||||
self.embeddings.embed_thumbnail(data["id"], thumbnail),
|
||||
pack=False,
|
||||
)
|
||||
elif topic == EmbeddingsRequestEnum.generate_search.value:
|
||||
return serialize(
|
||||
self.embeddings.embed_description("", data, upsert=False),
|
||||
pack=False,
|
||||
)
|
||||
else:
|
||||
processors = [self.realtime_processors, self.post_processors]
|
||||
for processor_list in processors:
|
||||
for processor in processor_list:
|
||||
resp = processor.handle_request(topic, data)
|
||||
|
||||
# First handle the embedding-specific topics when semantic search is enabled
|
||||
if self.config.semantic_search.enabled:
|
||||
if topic == EmbeddingsRequestEnum.embed_description.value:
|
||||
return serialize(
|
||||
self.embeddings.embed_description(
|
||||
data["id"], data["description"]
|
||||
),
|
||||
pack=False,
|
||||
)
|
||||
elif topic == EmbeddingsRequestEnum.embed_thumbnail.value:
|
||||
thumbnail = base64.b64decode(data["thumbnail"])
|
||||
return serialize(
|
||||
self.embeddings.embed_thumbnail(data["id"], thumbnail),
|
||||
pack=False,
|
||||
)
|
||||
elif topic == EmbeddingsRequestEnum.generate_search.value:
|
||||
return serialize(
|
||||
self.embeddings.embed_description("", data, upsert=False),
|
||||
pack=False,
|
||||
)
|
||||
processors = [self.realtime_processors, self.post_processors]
|
||||
for processor_list in processors:
|
||||
for processor in processor_list:
|
||||
resp = processor.handle_request(topic, data)
|
||||
if resp is not None:
|
||||
return resp
|
||||
except Exception as e:
|
||||
@ -432,6 +435,9 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
|
||||
def _embed_thumbnail(self, event_id: str, thumbnail: bytes) -> None:
|
||||
"""Embed the thumbnail for an event."""
|
||||
if not self.config.semantic_search.enabled:
|
||||
return
|
||||
|
||||
self.embeddings.embed_thumbnail(event_id, thumbnail)
|
||||
|
||||
def _embed_description(self, event: Event, thumbnails: list[bytes]) -> None:
|
||||
@ -457,7 +463,8 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
)
|
||||
|
||||
# Embed the description
|
||||
self.embeddings.embed_description(event.id, description)
|
||||
if self.config.semantic_search.enabled:
|
||||
self.embeddings.embed_description(event.id, description)
|
||||
|
||||
logger.debug(
|
||||
"Generated description for %s (%d images): %s",
|
||||
|
@ -282,16 +282,24 @@ def stats_snapshot(
|
||||
}
|
||||
stats["detection_fps"] = round(total_detection_fps, 2)
|
||||
|
||||
if config.semantic_search.enabled:
|
||||
embeddings_metrics = stats_tracking["embeddings_metrics"]
|
||||
stats["embeddings"] = {
|
||||
"image_embedding_speed": round(
|
||||
embeddings_metrics.image_embeddings_fps.value * 1000, 2
|
||||
),
|
||||
"text_embedding_speed": round(
|
||||
embeddings_metrics.text_embeddings_sps.value * 1000, 2
|
||||
),
|
||||
}
|
||||
stats["embeddings"] = {}
|
||||
|
||||
# Get metrics if available
|
||||
embeddings_metrics = stats_tracking.get("embeddings_metrics")
|
||||
|
||||
if embeddings_metrics:
|
||||
# Add metrics based on what's enabled
|
||||
if config.semantic_search.enabled:
|
||||
stats["embeddings"].update(
|
||||
{
|
||||
"image_embedding_speed": round(
|
||||
embeddings_metrics.image_embeddings_fps.value * 1000, 2
|
||||
),
|
||||
"text_embedding_speed": round(
|
||||
embeddings_metrics.text_embeddings_sps.value * 1000, 2
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
if config.face_recognition.enabled:
|
||||
stats["embeddings"]["face_recognition_speed"] = round(
|
||||
@ -302,6 +310,7 @@ def stats_snapshot(
|
||||
stats["embeddings"]["plate_recognition_speed"] = round(
|
||||
embeddings_metrics.alpr_pps.value * 1000, 2
|
||||
)
|
||||
|
||||
if "license_plate" not in config.objects.all_objects:
|
||||
stats["embeddings"]["yolov9_plate_detection_speed"] = round(
|
||||
embeddings_metrics.yolov9_lpr_fps.value * 1000, 2
|
||||
|
@ -28,7 +28,11 @@ function System() {
|
||||
const metrics = useMemo(() => {
|
||||
const metrics = [...allMetrics];
|
||||
|
||||
if (!config?.semantic_search.enabled) {
|
||||
if (
|
||||
!config?.semantic_search.enabled &&
|
||||
!config?.lpr.enabled &&
|
||||
!config?.face_recognition.enabled
|
||||
) {
|
||||
const index = metrics.indexOf("features");
|
||||
metrics.splice(index, 1);
|
||||
}
|
||||
|
@ -363,6 +363,10 @@ export interface FrigateConfig {
|
||||
|
||||
camera_groups: { [groupName: string]: CameraGroupConfig };
|
||||
|
||||
lpr: {
|
||||
enabled: boolean;
|
||||
};
|
||||
|
||||
logger: {
|
||||
default: string;
|
||||
logs: Record<string, string>;
|
||||
|
Loading…
Reference in New Issue
Block a user