From 4b4053d174d580325e95f494a13d2d575a1a8577 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 28 Apr 2025 16:43:03 -0600 Subject: [PATCH] Add face and lpr to tracked object update topic (#17940) * Send tracked object updates for face and license_plate objects * Update docs * Add to type enum * Add camera to object description update * Formatting * Consolidate yue-Hant * Add missing --- docs/docs/integrations/mqtt.md | 31 ++++++++++++++++++- frigate/comms/dispatcher.py | 1 + .../common/license_plate/mixin.py | 16 ++++++++++ frigate/data_processing/post/license_plate.py | 3 ++ frigate/data_processing/real_time/face.py | 24 ++++++++++++-- .../real_time/license_plate.py | 3 ++ frigate/embeddings/maintainer.py | 3 ++ frigate/types.py | 2 ++ web/public/locales/en/views/faceLibrary.json | 1 + .../locales/{yue_Hant => yue-Hant}/audio.json | 0 .../components/auth.json | 0 .../components/icons.json | 0 .../components/input.json | 0 .../{yue_Hant => yue-Hant}/objects.json | 0 .../views/configEditor.json | 0 .../{yue_Hant => yue-Hant}/views/events.json | 0 .../views/recording.json | 0 17 files changed, 81 insertions(+), 3 deletions(-) rename web/public/locales/{yue_Hant => yue-Hant}/audio.json (100%) rename web/public/locales/{yue_Hant => yue-Hant}/components/auth.json (100%) rename web/public/locales/{yue_Hant => yue-Hant}/components/icons.json (100%) rename web/public/locales/{yue_Hant => yue-Hant}/components/input.json (100%) rename web/public/locales/{yue_Hant => yue-Hant}/objects.json (100%) rename web/public/locales/{yue_Hant => yue-Hant}/views/configEditor.json (100%) rename web/public/locales/{yue_Hant => yue-Hant}/views/events.json (100%) rename web/public/locales/{yue_Hant => yue-Hant}/views/recording.json (100%) diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md index 4139efaf3..d6dcaa3fb 100644 --- a/docs/docs/integrations/mqtt.md +++ b/docs/docs/integrations/mqtt.md @@ -104,7 +104,9 @@ Message published for each changed tracked object. The first message is publishe ### `frigate/tracked_object_update` -Message published for updates to tracked object metadata, for example when GenAI runs and returns a tracked object description. +Message published for updates to tracked object metadata, for example: + +#### Generative AI Description Update ```json { @@ -114,6 +116,33 @@ Message published for updates to tracked object metadata, for example when GenAI } ``` +#### Face Recognition Update + +```json +{ + "type": "face", + "id": "1607123955.475377-mxklsc", + "name": "John", + "score": 0.95, + "camera": "front_door_cam", + "timestamp": 1607123958.748393, +} +``` + +#### License Plate Recognition Update + +```json +{ + "type": "lpr", + "id": "1607123955.475377-mxklsc", + "name": "John's Car", + "plate": "123ABC", + "score": 0.95, + "camera": "driveway_cam", + "timestamp": 1607123958.748393, +} +``` + ### `frigate/reviews` Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. When additional objects are detected or when a zone change occurs, it will publish a, `update` message with the same id. When the review activity has ended a final `end` message is published. diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index 4c0b0a8ff..87891ec88 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -135,6 +135,7 @@ class Dispatcher: "type": TrackedObjectUpdateTypesEnum.description, "id": event.id, "description": event.data["description"], + "camera": event.camera, } ), ) diff --git a/frigate/data_processing/common/license_plate/mixin.py b/frigate/data_processing/common/license_plate/mixin.py index e471aa304..8dea639d6 100644 --- a/frigate/data_processing/common/license_plate/mixin.py +++ b/frigate/data_processing/common/license_plate/mixin.py @@ -2,6 +2,7 @@ import base64 import datetime +import json import logging import math import os @@ -23,6 +24,7 @@ from frigate.comms.event_metadata_updater import ( ) from frigate.const import CLIPS_DIR from frigate.embeddings.onnx.lpr_embedding import LPR_EMBEDDING_SIZE +from frigate.types import TrackedObjectUpdateTypesEnum from frigate.util.builtin import EventsPerSecond from frigate.util.image import area @@ -1510,6 +1512,20 @@ class LicensePlateProcessingMixin: ) # always publish to recognized_license_plate field + self.requestor.send_data( + "tracked_object_update", + json.dumps( + { + "type": TrackedObjectUpdateTypesEnum.lpr, + "name": sub_label, + "plate": top_plate, + "score": avg_confidence, + "id": id, + "camera": camera, + "timestamp": start, + } + ), + ) self.sub_label_publisher.publish( EventMetadataTypeEnum.recognized_license_plate, (id, top_plate, avg_confidence), diff --git a/frigate/data_processing/post/license_plate.py b/frigate/data_processing/post/license_plate.py index e439fb763..e7d4ca6e5 100644 --- a/frigate/data_processing/post/license_plate.py +++ b/frigate/data_processing/post/license_plate.py @@ -9,6 +9,7 @@ from peewee import DoesNotExist from frigate.comms.embeddings_updater import EmbeddingsRequestEnum from frigate.comms.event_metadata_updater import EventMetadataPublisher +from frigate.comms.inter_process import InterProcessRequestor from frigate.config import FrigateConfig from frigate.data_processing.common.license_plate.mixin import ( WRITE_DEBUG_IMAGES, @@ -31,11 +32,13 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi): def __init__( self, config: FrigateConfig, + requestor: InterProcessRequestor, sub_label_publisher: EventMetadataPublisher, metrics: DataProcessorMetrics, model_runner: LicensePlateModelRunner, detected_license_plates: dict[str, dict[str, any]], ): + self.requestor = self.requestor self.detected_license_plates = detected_license_plates self.model_runner = model_runner self.lpr_config = config.lpr diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py index fb55bb7d2..d91ab9b80 100644 --- a/frigate/data_processing/real_time/face.py +++ b/frigate/data_processing/real_time/face.py @@ -2,6 +2,7 @@ import base64 import datetime +import json import logging import os import random @@ -17,6 +18,7 @@ from frigate.comms.event_metadata_updater import ( EventMetadataPublisher, EventMetadataTypeEnum, ) +from frigate.comms.inter_process import InterProcessRequestor from frigate.config import FrigateConfig from frigate.const import FACE_DIR, MODEL_CACHE_DIR from frigate.data_processing.common.face.model import ( @@ -24,6 +26,7 @@ from frigate.data_processing.common.face.model import ( FaceNetRecognizer, FaceRecognizer, ) +from frigate.types import TrackedObjectUpdateTypesEnum from frigate.util.builtin import EventsPerSecond from frigate.util.image import area @@ -42,11 +45,13 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): def __init__( self, config: FrigateConfig, + requestor: InterProcessRequestor, sub_label_publisher: EventMetadataPublisher, metrics: DataProcessorMetrics, ): super().__init__(config, metrics) self.face_config = config.face_recognition + self.requestor = requestor self.sub_label_publisher = sub_label_publisher self.face_detector: cv2.FaceDetectorYN = None self.requires_face_detection = "face" not in self.config.objects.all_objects @@ -157,8 +162,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): def process_frame(self, obj_data: dict[str, any], frame: np.ndarray): """Look for faces in image.""" self.metrics.face_rec_fps.value = self.faces_per_second.eps() + camera = obj_data["camera"] - if not self.config.cameras[obj_data["camera"]].face_recognition.enabled: + if not self.config.cameras[camera].face_recognition.enabled: return start = datetime.datetime.now().timestamp() @@ -245,7 +251,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): if ( not face_box or area(face_box) - < self.config.cameras[obj_data["camera"]].face_recognition.min_area + < self.config.cameras[camera].face_recognition.min_area ): logger.debug(f"Invalid face box {face}") return @@ -286,6 +292,20 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): self.person_face_history[id] ) + self.requestor.send_data( + "tracked_object_update", + json.dumps( + { + "type": TrackedObjectUpdateTypesEnum.face, + "name": weighted_sub_label, + "score": weighted_score, + "id": id, + "camera": camera, + "timestamp": start, + } + ), + ) + if weighted_score >= self.face_config.recognition_threshold: self.sub_label_publisher.publish( EventMetadataTypeEnum.sub_label, diff --git a/frigate/data_processing/real_time/license_plate.py b/frigate/data_processing/real_time/license_plate.py index 95d53a343..864968073 100644 --- a/frigate/data_processing/real_time/license_plate.py +++ b/frigate/data_processing/real_time/license_plate.py @@ -5,6 +5,7 @@ import logging import numpy as np from frigate.comms.event_metadata_updater import EventMetadataPublisher +from frigate.comms.inter_process import InterProcessRequestor from frigate.config import FrigateConfig from frigate.data_processing.common.license_plate.mixin import ( LicensePlateProcessingMixin, @@ -23,11 +24,13 @@ class LicensePlateRealTimeProcessor(LicensePlateProcessingMixin, RealTimeProcess def __init__( self, config: FrigateConfig, + requestor: InterProcessRequestor, sub_label_publisher: EventMetadataPublisher, metrics: DataProcessorMetrics, model_runner: LicensePlateModelRunner, detected_license_plates: dict[str, dict[str, any]], ): + self.requestor = requestor self.detected_license_plates = detected_license_plates self.model_runner = model_runner self.lpr_config = config.lpr diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index ba9b42f77..07115ca94 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -135,6 +135,7 @@ class EmbeddingMaintainer(threading.Thread): self.realtime_processors.append( LicensePlateRealTimeProcessor( self.config, + self.requestor, self.event_metadata_publisher, metrics, lpr_model_runner, @@ -149,6 +150,7 @@ class EmbeddingMaintainer(threading.Thread): self.post_processors.append( LicensePlatePostProcessor( self.config, + self.requestor, self.event_metadata_publisher, metrics, lpr_model_runner, @@ -583,6 +585,7 @@ class EmbeddingMaintainer(threading.Thread): "type": TrackedObjectUpdateTypesEnum.description, "id": event.id, "description": description, + "camera": event.camera, }, ) diff --git a/frigate/types.py b/frigate/types.py index 2422c5551..ee48cc02b 100644 --- a/frigate/types.py +++ b/frigate/types.py @@ -25,3 +25,5 @@ class ModelStatusTypesEnum(str, Enum): class TrackedObjectUpdateTypesEnum(str, Enum): description = "description" + face = "face" + lpr = "lpr" diff --git a/web/public/locales/en/views/faceLibrary.json b/web/public/locales/en/views/faceLibrary.json index cec5e8d7d..08ba2c673 100644 --- a/web/public/locales/en/views/faceLibrary.json +++ b/web/public/locales/en/views/faceLibrary.json @@ -65,6 +65,7 @@ "addFaceLibrary": "{{name}} has successfully been added to the Face Library!", "deletedFace_one": "Successfully deleted {{count}} face.", "deletedFace_other": "Successfully deleted {{count}} faces.", + "deletedName_zero": "Empty collection deleted successfully.", "deletedName_one": "{{count}} face has been successfully deleted.", "deletedName_other": "{{count}} faces have been successfully deleted.", "renamedFace": "Successfully renamed face to {{name}}", diff --git a/web/public/locales/yue_Hant/audio.json b/web/public/locales/yue-Hant/audio.json similarity index 100% rename from web/public/locales/yue_Hant/audio.json rename to web/public/locales/yue-Hant/audio.json diff --git a/web/public/locales/yue_Hant/components/auth.json b/web/public/locales/yue-Hant/components/auth.json similarity index 100% rename from web/public/locales/yue_Hant/components/auth.json rename to web/public/locales/yue-Hant/components/auth.json diff --git a/web/public/locales/yue_Hant/components/icons.json b/web/public/locales/yue-Hant/components/icons.json similarity index 100% rename from web/public/locales/yue_Hant/components/icons.json rename to web/public/locales/yue-Hant/components/icons.json diff --git a/web/public/locales/yue_Hant/components/input.json b/web/public/locales/yue-Hant/components/input.json similarity index 100% rename from web/public/locales/yue_Hant/components/input.json rename to web/public/locales/yue-Hant/components/input.json diff --git a/web/public/locales/yue_Hant/objects.json b/web/public/locales/yue-Hant/objects.json similarity index 100% rename from web/public/locales/yue_Hant/objects.json rename to web/public/locales/yue-Hant/objects.json diff --git a/web/public/locales/yue_Hant/views/configEditor.json b/web/public/locales/yue-Hant/views/configEditor.json similarity index 100% rename from web/public/locales/yue_Hant/views/configEditor.json rename to web/public/locales/yue-Hant/views/configEditor.json diff --git a/web/public/locales/yue_Hant/views/events.json b/web/public/locales/yue-Hant/views/events.json similarity index 100% rename from web/public/locales/yue_Hant/views/events.json rename to web/public/locales/yue-Hant/views/events.json diff --git a/web/public/locales/yue_Hant/views/recording.json b/web/public/locales/yue-Hant/views/recording.json similarity index 100% rename from web/public/locales/yue_Hant/views/recording.json rename to web/public/locales/yue-Hant/views/recording.json