Add face and lpr to tracked object update topic (#17940)

* Send tracked object updates for face and license_plate objects

* Update docs

* Add to type enum

* Add camera to object description update

* Formatting

* Consolidate yue-Hant

* Add missing

docs/docs/integrations/mqtt.md

@@ -104,7 +104,9 @@ Message published for each changed tracked object. The first message is publishe
 
 ### `frigate/tracked_object_update`
 
-Message published for updates to tracked object metadata, for example when GenAI runs and returns a tracked object description.
+Message published for updates to tracked object metadata, for example:
+
+#### Generative AI Description Update
 
 ```json
 {
@@ -114,6 +116,33 @@ Message published for updates to tracked object metadata, for example when GenAI
 }
 ```
 
+#### Face Recognition Update
+
+```json
+{
+  "type": "face",
+  "id": "1607123955.475377-mxklsc",
+  "name": "John",
+  "score": 0.95,
+  "camera": "front_door_cam",
+  "timestamp": 1607123958.748393,
+}
+```
+
+#### License Plate Recognition Update
+
+```json
+{
+  "type": "lpr",
+  "id": "1607123955.475377-mxklsc",
+  "name": "John's Car",
+  "plate": "123ABC",
+  "score": 0.95,
+  "camera": "driveway_cam",
+  "timestamp": 1607123958.748393,
+}
+```
+
 ### `frigate/reviews`
 
 Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. When additional objects are detected or when a zone change occurs, it will publish a, `update` message with the same id. When the review activity has ended a final `end` message is published.

frigate/comms/dispatcher.py

@@ -135,6 +135,7 @@ class Dispatcher:
                         "type": TrackedObjectUpdateTypesEnum.description,
                         "id": event.id,
                         "description": event.data["description"],
+                        "camera": event.camera,
                     }
                 ),
             )

frigate/data_processing/common/license_plate/mixin.py

@@ -2,6 +2,7 @@
 
 import base64
 import datetime
+import json
 import logging
 import math
 import os
@@ -23,6 +24,7 @@ from frigate.comms.event_metadata_updater import (
 )
 from frigate.const import CLIPS_DIR
 from frigate.embeddings.onnx.lpr_embedding import LPR_EMBEDDING_SIZE
+from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import EventsPerSecond
 from frigate.util.image import area
 
@@ -1510,6 +1512,20 @@ class LicensePlateProcessingMixin:
             )
 
         # always publish to recognized_license_plate field
+        self.requestor.send_data(
+            "tracked_object_update",
+            json.dumps(
+                {
+                    "type": TrackedObjectUpdateTypesEnum.lpr,
+                    "name": sub_label,
+                    "plate": top_plate,
+                    "score": avg_confidence,
+                    "id": id,
+                    "camera": camera,
+                    "timestamp": start,
+                }
+            ),
+        )
         self.sub_label_publisher.publish(
             EventMetadataTypeEnum.recognized_license_plate,
             (id, top_plate, avg_confidence),

frigate/data_processing/post/license_plate.py

@@ -9,6 +9,7 @@ from peewee import DoesNotExist
 
 from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
 from frigate.comms.event_metadata_updater import EventMetadataPublisher
+from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import FrigateConfig
 from frigate.data_processing.common.license_plate.mixin import (
     WRITE_DEBUG_IMAGES,
@@ -31,11 +32,13 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
     def __init__(
         self,
         config: FrigateConfig,
+        requestor: InterProcessRequestor,
         sub_label_publisher: EventMetadataPublisher,
         metrics: DataProcessorMetrics,
         model_runner: LicensePlateModelRunner,
         detected_license_plates: dict[str, dict[str, any]],
     ):
+        self.requestor = self.requestor
         self.detected_license_plates = detected_license_plates
         self.model_runner = model_runner
         self.lpr_config = config.lpr

frigate/data_processing/real_time/face.py

@@ -2,6 +2,7 @@
 
 import base64
 import datetime
+import json
 import logging
 import os
 import random
@@ -17,6 +18,7 @@ from frigate.comms.event_metadata_updater import (
     EventMetadataPublisher,
     EventMetadataTypeEnum,
 )
+from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import FrigateConfig
 from frigate.const import FACE_DIR, MODEL_CACHE_DIR
 from frigate.data_processing.common.face.model import (
@@ -24,6 +26,7 @@ from frigate.data_processing.common.face.model import (
     FaceNetRecognizer,
     FaceRecognizer,
 )
+from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import EventsPerSecond
 from frigate.util.image import area
 
@@ -42,11 +45,13 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
     def __init__(
         self,
         config: FrigateConfig,
+        requestor: InterProcessRequestor,
         sub_label_publisher: EventMetadataPublisher,
         metrics: DataProcessorMetrics,
     ):
         super().__init__(config, metrics)
         self.face_config = config.face_recognition
+        self.requestor = requestor
         self.sub_label_publisher = sub_label_publisher
         self.face_detector: cv2.FaceDetectorYN = None
         self.requires_face_detection = "face" not in self.config.objects.all_objects
@@ -157,8 +162,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
     def process_frame(self, obj_data: dict[str, any], frame: np.ndarray):
         """Look for faces in image."""
         self.metrics.face_rec_fps.value = self.faces_per_second.eps()
+        camera = obj_data["camera"]
 
-        if not self.config.cameras[obj_data["camera"]].face_recognition.enabled:
+        if not self.config.cameras[camera].face_recognition.enabled:
             return
 
         start = datetime.datetime.now().timestamp()
@@ -245,7 +251,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
             if (
                 not face_box
                 or area(face_box)
-                < self.config.cameras[obj_data["camera"]].face_recognition.min_area
+                < self.config.cameras[camera].face_recognition.min_area
             ):
                 logger.debug(f"Invalid face box {face}")
                 return
@@ -286,6 +292,20 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
             self.person_face_history[id]
         )
 
+        self.requestor.send_data(
+            "tracked_object_update",
+            json.dumps(
+                {
+                    "type": TrackedObjectUpdateTypesEnum.face,
+                    "name": weighted_sub_label,
+                    "score": weighted_score,
+                    "id": id,
+                    "camera": camera,
+                    "timestamp": start,
+                }
+            ),
+        )
+
         if weighted_score >= self.face_config.recognition_threshold:
             self.sub_label_publisher.publish(
                 EventMetadataTypeEnum.sub_label,

frigate/data_processing/real_time/license_plate.py

@@ -5,6 +5,7 @@ import logging
 import numpy as np
 
 from frigate.comms.event_metadata_updater import EventMetadataPublisher
+from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import FrigateConfig
 from frigate.data_processing.common.license_plate.mixin import (
     LicensePlateProcessingMixin,
@@ -23,11 +24,13 @@ class LicensePlateRealTimeProcessor(LicensePlateProcessingMixin, RealTimeProcess
     def __init__(
         self,
         config: FrigateConfig,
+        requestor: InterProcessRequestor,
         sub_label_publisher: EventMetadataPublisher,
         metrics: DataProcessorMetrics,
         model_runner: LicensePlateModelRunner,
         detected_license_plates: dict[str, dict[str, any]],
     ):
+        self.requestor = requestor
         self.detected_license_plates = detected_license_plates
         self.model_runner = model_runner
         self.lpr_config = config.lpr

frigate/embeddings/maintainer.py

@@ -135,6 +135,7 @@ class EmbeddingMaintainer(threading.Thread):
             self.realtime_processors.append(
                 LicensePlateRealTimeProcessor(
                     self.config,
+                    self.requestor,
                     self.event_metadata_publisher,
                     metrics,
                     lpr_model_runner,
@@ -149,6 +150,7 @@ class EmbeddingMaintainer(threading.Thread):
             self.post_processors.append(
                 LicensePlatePostProcessor(
                     self.config,
+                    self.requestor,
                     self.event_metadata_publisher,
                     metrics,
                     lpr_model_runner,
@@ -583,6 +585,7 @@ class EmbeddingMaintainer(threading.Thread):
                 "type": TrackedObjectUpdateTypesEnum.description,
                 "id": event.id,
                 "description": description,
+                "camera": event.camera,
             },
         )
 

frigate/types.py

@@ -25,3 +25,5 @@ class ModelStatusTypesEnum(str, Enum):
 
 class TrackedObjectUpdateTypesEnum(str, Enum):
     description = "description"
+    face = "face"
+    lpr = "lpr"

web/public/locales/en/views/faceLibrary.json

@@ -65,6 +65,7 @@
       "addFaceLibrary": "{{name}} has successfully been added to the Face Library!",
       "deletedFace_one": "Successfully deleted {{count}} face.",
       "deletedFace_other": "Successfully deleted {{count}} faces.",
+      "deletedName_zero": "Empty collection deleted successfully.",
       "deletedName_one": "{{count}} face has been successfully deleted.",
       "deletedName_other": "{{count}} faces have been successfully deleted.",
       "renamedFace": "Successfully renamed face to {{name}}",