Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2025-08-22 13:47:29 +02:00
Code Issues Projects Releases Wiki Activity

Embeddings tweaks (#16864)

Browse Source 
* make semantic search optional

* config

* frontend metrics

* docs

* tweak

* fixes

* also check genai cameras for embeddings context
This commit is contained in:
Josh Hawkins 2025-02-28 12:43:08 -06:00 committed by GitHub
parent db4152c4ca
commit 8d2f461350
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 95 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/docs/configuration/face_recognition.md
View File

@ -9,7 +9,7 @@ Frigate has support for CV2 Local Binary Pattern Face Recognizer to recognize fa
## Configuration
Face recognition is disabled by default and requires semantic search to be enabled, face recognition must be enabled in your config file before it can be used. Semantic Search and face recognition are global configuration settings.
Face recognition is disabled by default, face recognition must be enabled in your config file before it can be used. Face recognition is a global configuration setting.
```yaml
face_recognition:
@ -36,6 +36,7 @@ The accuracy of face recognition is heavily dependent on the quality of data giv
:::tip
When choosing images to include in the face training set it is recommended to always follow these recommendations:
- If it is difficult to make out details in a persons face it will not be helpful in training.
- Avoid images with under/over-exposure.
- Avoid blurry / pixelated images.
@ -52,4 +53,4 @@ Then it is recommended to use the `Face Library` tab in Frigate to select and tr
### Step 2 - Expanding The Dataset
Once straight-on images are performing well, start choosing slightly off-angle images to include for training. It is important to still choose images where enough face detail is visible to recognize someone.
Once straight-on images are performing well, start choosing slightly off-angle images to include for training. It is important to still choose images where enough face detail is visible to recognize someone.

6
docs/docs/configuration/genai.md
View File

@ -7,12 +7,6 @@ Generative AI can be used to automatically generate descriptive text based on th
Requests for a description are sent off automatically to your AI provider at the end of the tracked object's lifecycle. Descriptions can also be regenerated manually via the Frigate UI.
:::info
Semantic Search must be enabled to use Generative AI.
:::
## Configuration
Generative AI can be enabled for all cameras or only for specific cameras. There are currently 3 native providers available to integrate with Frigate. Other providers that support the OpenAI standard API can also be used. See the OpenAI section below.

1
docs/docs/configuration/reference.md
View File

@ -570,7 +570,6 @@ lpr:
known_plates: {}
# Optional: Configuration for AI generated tracked object descriptions
# NOTE: Semantic Search must be enabled for this to do anything.
# WARNING: Depending on the provider, this will send thumbnails over the internet
# to Google or OpenAI's LLMs to generate descriptions. It can be overridden at
# the camera level (enabled: False) to enhance privacy for indoor cameras.

5
frigate/api/event.py
View File

@ -1083,10 +1083,7 @@ def regenerate_description(
camera_config = request.app.frigate_config.cameras[event.camera]
if (
request.app.frigate_config.semantic_search.enabled
and camera_config.genai.enabled
):
if camera_config.genai.enabled:
request.app.event_metadata_updater.publish((event.id, params.source))
return JSONResponse(

30
frigate/app.py
View File

@ -93,7 +93,13 @@ class FrigateApp:
self.log_queue: Queue = mp.Queue()
self.camera_metrics: dict[str, CameraMetrics] = {}
self.embeddings_metrics: DataProcessorMetrics | None = (
DataProcessorMetrics() if config.semantic_search.enabled else None
DataProcessorMetrics()
if (
config.semantic_search.enabled
or config.lpr.enabled
or config.face_recognition.enabled
)
else None
)
self.ptz_metrics: dict[str, PTZMetrics] = {}
self.processes: dict[str, int] = {}
@ -236,7 +242,16 @@ class FrigateApp:
logger.info(f"Review process started: {review_segment_process.pid}")
def init_embeddings_manager(self) -> None:
if not self.config.semantic_search.enabled:
genai_cameras = [
c for c in self.config.cameras.values() if c.enabled and c.genai.enabled
]
if (
not self.config.semantic_search.enabled
and not genai_cameras
and not self.config.lpr.enabled
and not self.config.face_recognition.enabled
):
return
embedding_process = util.Process(
@ -293,7 +308,16 @@ class FrigateApp:
migrate_exports(self.config.ffmpeg, list(self.config.cameras.keys()))
def init_embeddings_client(self) -> None:
if self.config.semantic_search.enabled:
genai_cameras = [
c for c in self.config.cameras.values() if c.enabled and c.genai.enabled
]
if (
self.config.semantic_search.enabled
or self.config.lpr.enabled
or genai_cameras
or self.config.face_recognition.enabled
):
# Create a client for other processes to use
self.embeddings = EmbeddingsContext(self.db)

11
frigate/config/config.py
View File

@ -172,16 +172,6 @@ class RestreamConfig(BaseModel):
model_config = ConfigDict(extra="allow")
def verify_semantic_search_dependent_configs(config: FrigateConfig) -> None:
"""Verify that semantic search is enabled if required features are enabled."""
if not config.semantic_search.enabled:
if config.genai.enabled:
raise ValueError("Genai requires semantic search to be enabled.")
if config.face_recognition.enabled:
raise ValueError("Face recognition requires semantic to be enabled.")
def verify_config_roles(camera_config: CameraConfig) -> None:
"""Verify that roles are setup in the config correctly."""
assigned_roles = list(
@ -647,7 +637,6 @@ class FrigateConfig(FrigateBaseModel):
detector_config.model = model
self.detectors[key] = detector_config
verify_semantic_search_dependent_configs(self)
return self
@field_validator("cameras")

4
frigate/embeddings/__init__.py
View File

@ -28,10 +28,6 @@ logger = logging.getLogger(__name__)
def manage_embeddings(config: FrigateConfig, metrics: DataProcessorMetrics) -> None:
# Only initialize embeddings if semantic search is enabled
if not config.semantic_search.enabled:
return
stop_event = mp.Event()
def receiveSignal(signalNumber: int, frame: Optional[FrameType]) -> None:

65
frigate/embeddings/maintainer.py
View File

@ -71,11 +71,14 @@ class EmbeddingMaintainer(threading.Thread):
super().__init__(name="embeddings_maintainer")
self.config = config
self.metrics = metrics
self.embeddings = Embeddings(config, db, metrics)
self.embeddings = None
# Check if we need to re-index events
if config.semantic_search.reindex:
self.embeddings.reindex()
if config.semantic_search.enabled:
self.embeddings = Embeddings(config, db, metrics)
# Check if we need to re-index events
if config.semantic_search.reindex:
self.embeddings.reindex()
# create communication for updating event descriptions
self.requestor = InterProcessRequestor()
@ -152,30 +155,30 @@ class EmbeddingMaintainer(threading.Thread):
def _handle_request(topic: str, data: dict[str, any]) -> str:
try:
if topic == EmbeddingsRequestEnum.embed_description.value:
return serialize(
self.embeddings.embed_description(
data["id"], data["description"]
),
pack=False,
)
elif topic == EmbeddingsRequestEnum.embed_thumbnail.value:
thumbnail = base64.b64decode(data["thumbnail"])
return serialize(
self.embeddings.embed_thumbnail(data["id"], thumbnail),
pack=False,
)
elif topic == EmbeddingsRequestEnum.generate_search.value:
return serialize(
self.embeddings.embed_description("", data, upsert=False),
pack=False,
)
else:
processors = [self.realtime_processors, self.post_processors]
for processor_list in processors:
for processor in processor_list:
resp = processor.handle_request(topic, data)
# First handle the embedding-specific topics when semantic search is enabled
if self.config.semantic_search.enabled:
if topic == EmbeddingsRequestEnum.embed_description.value:
return serialize(
self.embeddings.embed_description(
data["id"], data["description"]
),
pack=False,
)
elif topic == EmbeddingsRequestEnum.embed_thumbnail.value:
thumbnail = base64.b64decode(data["thumbnail"])
return serialize(
self.embeddings.embed_thumbnail(data["id"], thumbnail),
pack=False,
)
elif topic == EmbeddingsRequestEnum.generate_search.value:
return serialize(
self.embeddings.embed_description("", data, upsert=False),
pack=False,
)
processors = [self.realtime_processors, self.post_processors]
for processor_list in processors:
for processor in processor_list:
resp = processor.handle_request(topic, data)
if resp is not None:
return resp
except Exception as e:
@ -432,6 +435,9 @@ class EmbeddingMaintainer(threading.Thread):
def _embed_thumbnail(self, event_id: str, thumbnail: bytes) -> None:
"""Embed the thumbnail for an event."""
if not self.config.semantic_search.enabled:
return
self.embeddings.embed_thumbnail(event_id, thumbnail)
def _embed_description(self, event: Event, thumbnails: list[bytes]) -> None:
@ -457,7 +463,8 @@ class EmbeddingMaintainer(threading.Thread):
)
# Embed the description
self.embeddings.embed_description(event.id, description)
if self.config.semantic_search.enabled:
self.embeddings.embed_description(event.id, description)
logger.debug(
"Generated description for %s (%d images): %s",

29
frigate/stats/util.py
View File

@ -282,16 +282,24 @@ def stats_snapshot(
}
stats["detection_fps"] = round(total_detection_fps, 2)
if config.semantic_search.enabled:
embeddings_metrics = stats_tracking["embeddings_metrics"]
stats["embeddings"] = {
"image_embedding_speed": round(
embeddings_metrics.image_embeddings_fps.value * 1000, 2
),
"text_embedding_speed": round(
embeddings_metrics.text_embeddings_sps.value * 1000, 2
),
}
stats["embeddings"] = {}
# Get metrics if available
embeddings_metrics = stats_tracking.get("embeddings_metrics")
if embeddings_metrics:
# Add metrics based on what's enabled
if config.semantic_search.enabled:
stats["embeddings"].update(
{
"image_embedding_speed": round(
embeddings_metrics.image_embeddings_fps.value * 1000, 2
),
"text_embedding_speed": round(
embeddings_metrics.text_embeddings_sps.value * 1000, 2
),
}
)
if config.face_recognition.enabled:
stats["embeddings"]["face_recognition_speed"] = round(
@ -302,6 +310,7 @@ def stats_snapshot(
stats["embeddings"]["plate_recognition_speed"] = round(
embeddings_metrics.alpr_pps.value * 1000, 2
)
if "license_plate" not in config.objects.all_objects:
stats["embeddings"]["yolov9_plate_detection_speed"] = round(
embeddings_metrics.yolov9_lpr_fps.value * 1000, 2

6
web/src/pages/System.tsx
View File

@ -28,7 +28,11 @@ function System() {
const metrics = useMemo(() => {
const metrics = [...allMetrics];
if (!config?.semantic_search.enabled) {
if (
!config?.semantic_search.enabled &&
!config?.lpr.enabled &&
!config?.face_recognition.enabled
) {
const index = metrics.indexOf("features");
metrics.splice(index, 1);
}

4
web/src/types/frigateConfig.ts
View File

@ -363,6 +363,10 @@ export interface FrigateConfig {
camera_groups: { [groupName: string]: CameraGroupConfig };
lpr: {
enabled: boolean;
};
logger: {
default: string;
logs: Record<string, string>;