From 8d2f461350ed6f1881eee15d7089014e24a9b0ed Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 28 Feb 2025 12:43:08 -0600 Subject: [PATCH] Embeddings tweaks (#16864) * make semantic search optional * config * frontend metrics * docs * tweak * fixes * also check genai cameras for embeddings context --- docs/docs/configuration/face_recognition.md | 5 +- docs/docs/configuration/genai.md | 6 -- docs/docs/configuration/reference.md | 1 - frigate/api/event.py | 5 +- frigate/app.py | 30 +++++++++- frigate/config/config.py | 11 ---- frigate/embeddings/__init__.py | 4 -- frigate/embeddings/maintainer.py | 65 ++++++++++++--------- frigate/stats/util.py | 29 +++++---- web/src/pages/System.tsx | 6 +- web/src/types/frigateConfig.ts | 4 ++ 11 files changed, 95 insertions(+), 71 deletions(-) diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md index aaab92e6d..4d934afce 100644 --- a/docs/docs/configuration/face_recognition.md +++ b/docs/docs/configuration/face_recognition.md @@ -9,7 +9,7 @@ Frigate has support for CV2 Local Binary Pattern Face Recognizer to recognize fa ## Configuration -Face recognition is disabled by default and requires semantic search to be enabled, face recognition must be enabled in your config file before it can be used. Semantic Search and face recognition are global configuration settings. +Face recognition is disabled by default, face recognition must be enabled in your config file before it can be used. Face recognition is a global configuration setting. ```yaml face_recognition: @@ -36,6 +36,7 @@ The accuracy of face recognition is heavily dependent on the quality of data giv :::tip When choosing images to include in the face training set it is recommended to always follow these recommendations: + - If it is difficult to make out details in a persons face it will not be helpful in training. - Avoid images with under/over-exposure. - Avoid blurry / pixelated images. @@ -52,4 +53,4 @@ Then it is recommended to use the `Face Library` tab in Frigate to select and tr ### Step 2 - Expanding The Dataset -Once straight-on images are performing well, start choosing slightly off-angle images to include for training. It is important to still choose images where enough face detail is visible to recognize someone. \ No newline at end of file +Once straight-on images are performing well, start choosing slightly off-angle images to include for training. It is important to still choose images where enough face detail is visible to recognize someone. diff --git a/docs/docs/configuration/genai.md b/docs/docs/configuration/genai.md index 23f1c06be..e46107a82 100644 --- a/docs/docs/configuration/genai.md +++ b/docs/docs/configuration/genai.md @@ -7,12 +7,6 @@ Generative AI can be used to automatically generate descriptive text based on th Requests for a description are sent off automatically to your AI provider at the end of the tracked object's lifecycle. Descriptions can also be regenerated manually via the Frigate UI. -:::info - -Semantic Search must be enabled to use Generative AI. - -::: - ## Configuration Generative AI can be enabled for all cameras or only for specific cameras. There are currently 3 native providers available to integrate with Frigate. Other providers that support the OpenAI standard API can also be used. See the OpenAI section below. diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index c64272214..b53d9268f 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -570,7 +570,6 @@ lpr: known_plates: {} # Optional: Configuration for AI generated tracked object descriptions -# NOTE: Semantic Search must be enabled for this to do anything. # WARNING: Depending on the provider, this will send thumbnails over the internet # to Google or OpenAI's LLMs to generate descriptions. It can be overridden at # the camera level (enabled: False) to enhance privacy for indoor cameras. diff --git a/frigate/api/event.py b/frigate/api/event.py index bb1bf7395..9a5578bae 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -1083,10 +1083,7 @@ def regenerate_description( camera_config = request.app.frigate_config.cameras[event.camera] - if ( - request.app.frigate_config.semantic_search.enabled - and camera_config.genai.enabled - ): + if camera_config.genai.enabled: request.app.event_metadata_updater.publish((event.id, params.source)) return JSONResponse( diff --git a/frigate/app.py b/frigate/app.py index 400d4bca0..8b63ab0a0 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -93,7 +93,13 @@ class FrigateApp: self.log_queue: Queue = mp.Queue() self.camera_metrics: dict[str, CameraMetrics] = {} self.embeddings_metrics: DataProcessorMetrics | None = ( - DataProcessorMetrics() if config.semantic_search.enabled else None + DataProcessorMetrics() + if ( + config.semantic_search.enabled + or config.lpr.enabled + or config.face_recognition.enabled + ) + else None ) self.ptz_metrics: dict[str, PTZMetrics] = {} self.processes: dict[str, int] = {} @@ -236,7 +242,16 @@ class FrigateApp: logger.info(f"Review process started: {review_segment_process.pid}") def init_embeddings_manager(self) -> None: - if not self.config.semantic_search.enabled: + genai_cameras = [ + c for c in self.config.cameras.values() if c.enabled and c.genai.enabled + ] + + if ( + not self.config.semantic_search.enabled + and not genai_cameras + and not self.config.lpr.enabled + and not self.config.face_recognition.enabled + ): return embedding_process = util.Process( @@ -293,7 +308,16 @@ class FrigateApp: migrate_exports(self.config.ffmpeg, list(self.config.cameras.keys())) def init_embeddings_client(self) -> None: - if self.config.semantic_search.enabled: + genai_cameras = [ + c for c in self.config.cameras.values() if c.enabled and c.genai.enabled + ] + + if ( + self.config.semantic_search.enabled + or self.config.lpr.enabled + or genai_cameras + or self.config.face_recognition.enabled + ): # Create a client for other processes to use self.embeddings = EmbeddingsContext(self.db) diff --git a/frigate/config/config.py b/frigate/config/config.py index 39ee31411..d2ca9a6f5 100644 --- a/frigate/config/config.py +++ b/frigate/config/config.py @@ -172,16 +172,6 @@ class RestreamConfig(BaseModel): model_config = ConfigDict(extra="allow") -def verify_semantic_search_dependent_configs(config: FrigateConfig) -> None: - """Verify that semantic search is enabled if required features are enabled.""" - if not config.semantic_search.enabled: - if config.genai.enabled: - raise ValueError("Genai requires semantic search to be enabled.") - - if config.face_recognition.enabled: - raise ValueError("Face recognition requires semantic to be enabled.") - - def verify_config_roles(camera_config: CameraConfig) -> None: """Verify that roles are setup in the config correctly.""" assigned_roles = list( @@ -647,7 +637,6 @@ class FrigateConfig(FrigateBaseModel): detector_config.model = model self.detectors[key] = detector_config - verify_semantic_search_dependent_configs(self) return self @field_validator("cameras") diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py index 18673c4e9..56bd097d6 100644 --- a/frigate/embeddings/__init__.py +++ b/frigate/embeddings/__init__.py @@ -28,10 +28,6 @@ logger = logging.getLogger(__name__) def manage_embeddings(config: FrigateConfig, metrics: DataProcessorMetrics) -> None: - # Only initialize embeddings if semantic search is enabled - if not config.semantic_search.enabled: - return - stop_event = mp.Event() def receiveSignal(signalNumber: int, frame: Optional[FrameType]) -> None: diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index a18ca7a7f..c9b6062c9 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -71,11 +71,14 @@ class EmbeddingMaintainer(threading.Thread): super().__init__(name="embeddings_maintainer") self.config = config self.metrics = metrics - self.embeddings = Embeddings(config, db, metrics) + self.embeddings = None - # Check if we need to re-index events - if config.semantic_search.reindex: - self.embeddings.reindex() + if config.semantic_search.enabled: + self.embeddings = Embeddings(config, db, metrics) + + # Check if we need to re-index events + if config.semantic_search.reindex: + self.embeddings.reindex() # create communication for updating event descriptions self.requestor = InterProcessRequestor() @@ -152,30 +155,30 @@ class EmbeddingMaintainer(threading.Thread): def _handle_request(topic: str, data: dict[str, any]) -> str: try: - if topic == EmbeddingsRequestEnum.embed_description.value: - return serialize( - self.embeddings.embed_description( - data["id"], data["description"] - ), - pack=False, - ) - elif topic == EmbeddingsRequestEnum.embed_thumbnail.value: - thumbnail = base64.b64decode(data["thumbnail"]) - return serialize( - self.embeddings.embed_thumbnail(data["id"], thumbnail), - pack=False, - ) - elif topic == EmbeddingsRequestEnum.generate_search.value: - return serialize( - self.embeddings.embed_description("", data, upsert=False), - pack=False, - ) - else: - processors = [self.realtime_processors, self.post_processors] - for processor_list in processors: - for processor in processor_list: - resp = processor.handle_request(topic, data) - + # First handle the embedding-specific topics when semantic search is enabled + if self.config.semantic_search.enabled: + if topic == EmbeddingsRequestEnum.embed_description.value: + return serialize( + self.embeddings.embed_description( + data["id"], data["description"] + ), + pack=False, + ) + elif topic == EmbeddingsRequestEnum.embed_thumbnail.value: + thumbnail = base64.b64decode(data["thumbnail"]) + return serialize( + self.embeddings.embed_thumbnail(data["id"], thumbnail), + pack=False, + ) + elif topic == EmbeddingsRequestEnum.generate_search.value: + return serialize( + self.embeddings.embed_description("", data, upsert=False), + pack=False, + ) + processors = [self.realtime_processors, self.post_processors] + for processor_list in processors: + for processor in processor_list: + resp = processor.handle_request(topic, data) if resp is not None: return resp except Exception as e: @@ -432,6 +435,9 @@ class EmbeddingMaintainer(threading.Thread): def _embed_thumbnail(self, event_id: str, thumbnail: bytes) -> None: """Embed the thumbnail for an event.""" + if not self.config.semantic_search.enabled: + return + self.embeddings.embed_thumbnail(event_id, thumbnail) def _embed_description(self, event: Event, thumbnails: list[bytes]) -> None: @@ -457,7 +463,8 @@ class EmbeddingMaintainer(threading.Thread): ) # Embed the description - self.embeddings.embed_description(event.id, description) + if self.config.semantic_search.enabled: + self.embeddings.embed_description(event.id, description) logger.debug( "Generated description for %s (%d images): %s", diff --git a/frigate/stats/util.py b/frigate/stats/util.py index 3d836868e..287c384cd 100644 --- a/frigate/stats/util.py +++ b/frigate/stats/util.py @@ -282,16 +282,24 @@ def stats_snapshot( } stats["detection_fps"] = round(total_detection_fps, 2) - if config.semantic_search.enabled: - embeddings_metrics = stats_tracking["embeddings_metrics"] - stats["embeddings"] = { - "image_embedding_speed": round( - embeddings_metrics.image_embeddings_fps.value * 1000, 2 - ), - "text_embedding_speed": round( - embeddings_metrics.text_embeddings_sps.value * 1000, 2 - ), - } + stats["embeddings"] = {} + + # Get metrics if available + embeddings_metrics = stats_tracking.get("embeddings_metrics") + + if embeddings_metrics: + # Add metrics based on what's enabled + if config.semantic_search.enabled: + stats["embeddings"].update( + { + "image_embedding_speed": round( + embeddings_metrics.image_embeddings_fps.value * 1000, 2 + ), + "text_embedding_speed": round( + embeddings_metrics.text_embeddings_sps.value * 1000, 2 + ), + } + ) if config.face_recognition.enabled: stats["embeddings"]["face_recognition_speed"] = round( @@ -302,6 +310,7 @@ def stats_snapshot( stats["embeddings"]["plate_recognition_speed"] = round( embeddings_metrics.alpr_pps.value * 1000, 2 ) + if "license_plate" not in config.objects.all_objects: stats["embeddings"]["yolov9_plate_detection_speed"] = round( embeddings_metrics.yolov9_lpr_fps.value * 1000, 2 diff --git a/web/src/pages/System.tsx b/web/src/pages/System.tsx index 491149be2..05eed5b3e 100644 --- a/web/src/pages/System.tsx +++ b/web/src/pages/System.tsx @@ -28,7 +28,11 @@ function System() { const metrics = useMemo(() => { const metrics = [...allMetrics]; - if (!config?.semantic_search.enabled) { + if ( + !config?.semantic_search.enabled && + !config?.lpr.enabled && + !config?.face_recognition.enabled + ) { const index = metrics.indexOf("features"); metrics.splice(index, 1); } diff --git a/web/src/types/frigateConfig.ts b/web/src/types/frigateConfig.ts index d021fde0f..4ec4de853 100644 --- a/web/src/types/frigateConfig.ts +++ b/web/src/types/frigateConfig.ts @@ -363,6 +363,10 @@ export interface FrigateConfig { camera_groups: { [groupName: string]: CameraGroupConfig }; + lpr: { + enabled: boolean; + }; + logger: { default: string; logs: Record;