Add face and lpr to tracked object update topic (#17940)

* Send tracked object updates for face and license_plate objects

* Update docs

* Add to type enum

* Add camera to object description update

* Formatting

* Consolidate yue-Hant

* Add missing
This commit is contained in:
Nicolas Mowen 2025-04-28 16:43:03 -06:00 committed by GitHub
parent 77ac3b6da0
commit 4b4053d174
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 81 additions and 3 deletions

View File

@ -104,7 +104,9 @@ Message published for each changed tracked object. The first message is publishe
### `frigate/tracked_object_update`
Message published for updates to tracked object metadata, for example when GenAI runs and returns a tracked object description.
Message published for updates to tracked object metadata, for example:
#### Generative AI Description Update
```json
{
@ -114,6 +116,33 @@ Message published for updates to tracked object metadata, for example when GenAI
}
```
#### Face Recognition Update
```json
{
"type": "face",
"id": "1607123955.475377-mxklsc",
"name": "John",
"score": 0.95,
"camera": "front_door_cam",
"timestamp": 1607123958.748393,
}
```
#### License Plate Recognition Update
```json
{
"type": "lpr",
"id": "1607123955.475377-mxklsc",
"name": "John's Car",
"plate": "123ABC",
"score": 0.95,
"camera": "driveway_cam",
"timestamp": 1607123958.748393,
}
```
### `frigate/reviews`
Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. When additional objects are detected or when a zone change occurs, it will publish a, `update` message with the same id. When the review activity has ended a final `end` message is published.

View File

@ -135,6 +135,7 @@ class Dispatcher:
"type": TrackedObjectUpdateTypesEnum.description,
"id": event.id,
"description": event.data["description"],
"camera": event.camera,
}
),
)

View File

@ -2,6 +2,7 @@
import base64
import datetime
import json
import logging
import math
import os
@ -23,6 +24,7 @@ from frigate.comms.event_metadata_updater import (
)
from frigate.const import CLIPS_DIR
from frigate.embeddings.onnx.lpr_embedding import LPR_EMBEDDING_SIZE
from frigate.types import TrackedObjectUpdateTypesEnum
from frigate.util.builtin import EventsPerSecond
from frigate.util.image import area
@ -1510,6 +1512,20 @@ class LicensePlateProcessingMixin:
)
# always publish to recognized_license_plate field
self.requestor.send_data(
"tracked_object_update",
json.dumps(
{
"type": TrackedObjectUpdateTypesEnum.lpr,
"name": sub_label,
"plate": top_plate,
"score": avg_confidence,
"id": id,
"camera": camera,
"timestamp": start,
}
),
)
self.sub_label_publisher.publish(
EventMetadataTypeEnum.recognized_license_plate,
(id, top_plate, avg_confidence),

View File

@ -9,6 +9,7 @@ from peewee import DoesNotExist
from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
from frigate.comms.event_metadata_updater import EventMetadataPublisher
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import FrigateConfig
from frigate.data_processing.common.license_plate.mixin import (
WRITE_DEBUG_IMAGES,
@ -31,11 +32,13 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
def __init__(
self,
config: FrigateConfig,
requestor: InterProcessRequestor,
sub_label_publisher: EventMetadataPublisher,
metrics: DataProcessorMetrics,
model_runner: LicensePlateModelRunner,
detected_license_plates: dict[str, dict[str, any]],
):
self.requestor = self.requestor
self.detected_license_plates = detected_license_plates
self.model_runner = model_runner
self.lpr_config = config.lpr

View File

@ -2,6 +2,7 @@
import base64
import datetime
import json
import logging
import os
import random
@ -17,6 +18,7 @@ from frigate.comms.event_metadata_updater import (
EventMetadataPublisher,
EventMetadataTypeEnum,
)
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import FrigateConfig
from frigate.const import FACE_DIR, MODEL_CACHE_DIR
from frigate.data_processing.common.face.model import (
@ -24,6 +26,7 @@ from frigate.data_processing.common.face.model import (
FaceNetRecognizer,
FaceRecognizer,
)
from frigate.types import TrackedObjectUpdateTypesEnum
from frigate.util.builtin import EventsPerSecond
from frigate.util.image import area
@ -42,11 +45,13 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
def __init__(
self,
config: FrigateConfig,
requestor: InterProcessRequestor,
sub_label_publisher: EventMetadataPublisher,
metrics: DataProcessorMetrics,
):
super().__init__(config, metrics)
self.face_config = config.face_recognition
self.requestor = requestor
self.sub_label_publisher = sub_label_publisher
self.face_detector: cv2.FaceDetectorYN = None
self.requires_face_detection = "face" not in self.config.objects.all_objects
@ -157,8 +162,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
def process_frame(self, obj_data: dict[str, any], frame: np.ndarray):
"""Look for faces in image."""
self.metrics.face_rec_fps.value = self.faces_per_second.eps()
camera = obj_data["camera"]
if not self.config.cameras[obj_data["camera"]].face_recognition.enabled:
if not self.config.cameras[camera].face_recognition.enabled:
return
start = datetime.datetime.now().timestamp()
@ -245,7 +251,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
if (
not face_box
or area(face_box)
< self.config.cameras[obj_data["camera"]].face_recognition.min_area
< self.config.cameras[camera].face_recognition.min_area
):
logger.debug(f"Invalid face box {face}")
return
@ -286,6 +292,20 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
self.person_face_history[id]
)
self.requestor.send_data(
"tracked_object_update",
json.dumps(
{
"type": TrackedObjectUpdateTypesEnum.face,
"name": weighted_sub_label,
"score": weighted_score,
"id": id,
"camera": camera,
"timestamp": start,
}
),
)
if weighted_score >= self.face_config.recognition_threshold:
self.sub_label_publisher.publish(
EventMetadataTypeEnum.sub_label,

View File

@ -5,6 +5,7 @@ import logging
import numpy as np
from frigate.comms.event_metadata_updater import EventMetadataPublisher
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import FrigateConfig
from frigate.data_processing.common.license_plate.mixin import (
LicensePlateProcessingMixin,
@ -23,11 +24,13 @@ class LicensePlateRealTimeProcessor(LicensePlateProcessingMixin, RealTimeProcess
def __init__(
self,
config: FrigateConfig,
requestor: InterProcessRequestor,
sub_label_publisher: EventMetadataPublisher,
metrics: DataProcessorMetrics,
model_runner: LicensePlateModelRunner,
detected_license_plates: dict[str, dict[str, any]],
):
self.requestor = requestor
self.detected_license_plates = detected_license_plates
self.model_runner = model_runner
self.lpr_config = config.lpr

View File

@ -135,6 +135,7 @@ class EmbeddingMaintainer(threading.Thread):
self.realtime_processors.append(
LicensePlateRealTimeProcessor(
self.config,
self.requestor,
self.event_metadata_publisher,
metrics,
lpr_model_runner,
@ -149,6 +150,7 @@ class EmbeddingMaintainer(threading.Thread):
self.post_processors.append(
LicensePlatePostProcessor(
self.config,
self.requestor,
self.event_metadata_publisher,
metrics,
lpr_model_runner,
@ -583,6 +585,7 @@ class EmbeddingMaintainer(threading.Thread):
"type": TrackedObjectUpdateTypesEnum.description,
"id": event.id,
"description": description,
"camera": event.camera,
},
)

View File

@ -25,3 +25,5 @@ class ModelStatusTypesEnum(str, Enum):
class TrackedObjectUpdateTypesEnum(str, Enum):
description = "description"
face = "face"
lpr = "lpr"

View File

@ -65,6 +65,7 @@
"addFaceLibrary": "{{name}} has successfully been added to the Face Library!",
"deletedFace_one": "Successfully deleted {{count}} face.",
"deletedFace_other": "Successfully deleted {{count}} faces.",
"deletedName_zero": "Empty collection deleted successfully.",
"deletedName_one": "{{count}} face has been successfully deleted.",
"deletedName_other": "{{count}} faces have been successfully deleted.",
"renamedFace": "Successfully renamed face to {{name}}",