Label attributes (#6829)

* pass attribute labels as attributes

* add label attrs to events and snapshots

* incorporate area of license_plate and face into snapshot selection

* populate sublabels for cars with logos

docs/docs/configuration/advanced.md

@@ -96,6 +96,16 @@ model:
 
 Note that if you rename objects in the labelmap, you will also need to update your `objects -> track` list as well.
 
+:::caution
+
+Some labels have special handling and modifications can disable functionality.
+
+`person` objects are associated with `face` and `amazon`
+
+`car` objects are associated with `license_plate`, `ups`, `fedex`, `amazon`
+
+:::
+
 ## Custom ffmpeg build
 
 Included with Frigate is a build of ffmpeg that works for the vast majority of users. However, there exists some hardware setups which have incompatibilities with the included build. In this case, a docker volume mapping can be used to overwrite the included ffmpeg build with an ffmpeg build that works for your specific hardware setup.

docs/docs/integrations/mqtt.md

@@ -62,7 +62,9 @@ Message published for each changed event. The first message is published when th
     "has_clip": false,
     "stationary": false, // whether or not the object is considered stationary
     "motionless_count": 0, // number of frames the object has been motionless
-    "position_changes": 2 // number of times the object has moved from a stationary position
+    "position_changes": 2, // number of times the object has moved from a stationary position
+    "attributes": [], // set of unique attributes that have been identified on the object
+    "current_attributes": [] // detailed data about the current attributes in this frame
   },
   "after": {
     "id": "1607123955.475377-mxklsc",
@@ -87,7 +89,16 @@ Message published for each changed event. The first message is published when th
     "has_clip": false,
     "stationary": false, // whether or not the object is considered stationary
     "motionless_count": 0, // number of frames the object has been motionless
-    "position_changes": 2 // number of times the object has changed position
+    "position_changes": 2, // number of times the object has changed position
+    "attributes": ["face"], // set of unique attributes that have been identified on the object
+    "current_attributes": [
+      // detailed data about the current attributes in this frame
+      {
+        "label": "face",
+        "box": [442, 506, 534, 524],
+        "score": 0.64
+      }
+    ]
   }
 }
 ```
@@ -163,9 +174,9 @@ Topic with current motion contour area for a camera. Published value is an integ
 
 Topic to send PTZ commands to camera.
 
-| Command                | Description                                                                             |
+| Command                | Description                                                                               |
-| ---------------------- | --------------------------------------------------------------------------------------- |
+| ---------------------- | ----------------------------------------------------------------------------------------- |
 | `preset-<preset_name>` | send command to move to preset with name `<preset_name>`                                  |
 | `MOVE_<dir>`           | send command to continuously move in `<dir>`, possible values are [UP, DOWN, LEFT, RIGHT] |
 | `ZOOM_<dir>`           | send command to continuously zoom `<dir>`, possible values are [IN, OUT]                  |
-| `STOP`                 | send command to stop moving                                                             |
+| `STOP`                 | send command to stop moving                                                               |

frigate/events/maintainer.py

@@ -147,6 +147,23 @@ class EventProcessor(threading.Thread):
                 )
             )
 
+            attributes = [
+                (
+                    None
+                    if event_data["snapshot"] is None
+                    else {
+                        "box": to_relative_box(
+                            width,
+                            height,
+                            a["box"],
+                        ),
+                        "label": a["label"],
+                        "score": a["score"],
+                    }
+                )
+                for a in event_data["snapshot"]["attributes"]
+            ]
+
             # keep these from being set back to false because the event
             # may have started while recordings and snapshots were enabled
             # this would be an issue for long running events
@@ -173,9 +190,14 @@ class EventProcessor(threading.Thread):
                     "region": region,
                     "score": score,
                     "top_score": event_data["top_score"],
+                    "attributes": attributes,
                 },
             }
 
+            # only overwrite the sub_label in the database if it's set
+            if event_data.get("sub_label") is not None:
+                event[Event.sub_label] = event_data["sub_label"]
+
             (
                 Event.insert(event)
                 .on_conflict(

frigate/object_processing.py

@@ -24,6 +24,7 @@ from frigate.const import CLIPS_DIR
 from frigate.events.maintainer import EventTypeEnum
 from frigate.util import (
     SharedMemoryFrameManager,
+    area,
     calculate_region,
     draw_box_with_label,
     draw_timestamp,
@@ -42,11 +43,45 @@ def on_edge(box, frame_shape):
         return True
 
 
-def is_better_thumbnail(current_thumb, new_obj, frame_shape) -> bool:
+def has_better_attr(current_thumb, new_obj, attr_label) -> bool:
+    max_new_attr = max(
+        [0]
+        + [area(a["box"]) for a in new_obj["attributes"] if a["label"] == attr_label]
+    )
+    max_current_attr = max(
+        [0]
+        + [
+            area(a["box"])
+            for a in current_thumb["attributes"]
+            if a["label"] == attr_label
+        ]
+    )
+
+    # if the thumb has a higher scoring attr
+    return max_new_attr > max_current_attr
+
+
+def is_better_thumbnail(label, current_thumb, new_obj, frame_shape) -> bool:
     # larger is better
     # cutoff images are less ideal, but they should also be smaller?
     # better scores are obviously better too
 
+    # check face on person
+    if label == "person":
+        if has_better_attr(current_thumb, new_obj, "face"):
+            return True
+        # if the current thumb has a face attr, dont update unless it gets better
+        if any([a["label"] == "face" for a in current_thumb["attributes"]]):
+            return False
+
+    # check license_plate on car
+    if label == "car":
+        if has_better_attr(current_thumb, new_obj, "license_plate"):
+            return True
+        # if the current thumb has a license_plate attr, dont update unless it gets better
+        if any([a["label"] == "license_plate" for a in current_thumb["attributes"]]):
+            return False
+
     # if the new_thumb is on an edge, and the current thumb is not
     if on_edge(new_obj["box"], frame_shape) and not on_edge(
         current_thumb["box"], frame_shape
@@ -76,6 +111,7 @@ class TrackedObject:
         self.zone_presence = {}
         self.current_zones = []
         self.entered_zones = []
+        self.attributes = set()
         self.false_positive = True
         self.has_clip = False
         self.has_snapshot = False
@@ -125,7 +161,10 @@ class TrackedObject:
         if not self.false_positive:
             # determine if this frame is a better thumbnail
             if self.thumbnail_data is None or is_better_thumbnail(
-                self.thumbnail_data, obj_data, self.camera_config.frame_shape
+                self.obj_data["label"],
+                self.thumbnail_data,
+                obj_data,
+                self.camera_config.frame_shape,
             ):
                 self.thumbnail_data = {
                     "frame_time": obj_data["frame_time"],
@@ -133,6 +172,7 @@ class TrackedObject:
                     "area": obj_data["area"],
                     "region": obj_data["region"],
                     "score": obj_data["score"],
+                    "attributes": obj_data["attributes"],
                 }
                 thumb_update = True
 
@@ -164,6 +204,19 @@ class TrackedObject:
                 if 0 < zone_score < zone.inertia:
                     self.zone_presence[name] = zone_score - 1
 
+        # maintain attributes
+        for attr in obj_data["attributes"]:
+            self.attributes.add(attr["label"])
+
+        # populate the sub_label for car with first logo if it exists
+        if self.obj_data["label"] == "car" and "sub_label" not in self.obj_data:
+            recognized_logos = self.attributes.intersection(
+                set(["ups", "fedex", "amazon"])
+            )
+            if len(recognized_logos) > 0:
+                self.obj_data["sub_label"] = recognized_logos.pop()
+
+        # check for significant change
         if not self.false_positive:
             # if the zones changed, signal an update
             if set(self.current_zones) != set(current_zones):
@@ -214,6 +267,8 @@ class TrackedObject:
             "entered_zones": self.entered_zones.copy(),
             "has_clip": self.has_clip,
             "has_snapshot": self.has_snapshot,
+            "attributes": list(self.attributes),
+            "current_attributes": self.obj_data["attributes"],
         }
 
         if include_thumbnail:
@@ -294,6 +349,21 @@ class TrackedObject:
                 color=color,
             )
 
+            # draw any attributes
+            for attribute in self.thumbnail_data["attributes"]:
+                box = attribute["box"]
+                draw_box_with_label(
+                    best_frame,
+                    box[0],
+                    box[1],
+                    box[2],
+                    box[3],
+                    attribute["label"],
+                    f"{attribute['score']:.0%}",
+                    thickness=thickness,
+                    color=color,
+                )
+
         if crop:
             box = self.thumbnail_data["box"]
             box_size = 300
@@ -421,6 +491,21 @@ class CameraState:
                     color=color,
                 )
 
+                # draw any attributes
+                for attribute in obj["current_attributes"]:
+                    box = attribute["box"]
+                    draw_box_with_label(
+                        frame_copy,
+                        box[0],
+                        box[1],
+                        box[2],
+                        box[3],
+                        attribute["label"],
+                        f"{attribute['score']:.0%}",
+                        thickness=thickness,
+                        color=color,
+                    )
+
         if draw_options.get("regions"):
             for region in regions:
                 cv2.rectangle(
@@ -553,6 +638,7 @@ class CameraState:
                 # or the current object is older than desired, use the new object
                 if (
                     is_better_thumbnail(
+                        object_type,
                         current_best.thumbnail_data,
                         obj.thumbnail_data,
                         self.camera_config.frame_shape,

frigate/video.py

@@ -723,6 +723,14 @@ def process_frames(
     stop_event,
     exit_on_empty: bool = False,
 ):
+    # attribute labels are not tracked and are not assigned regions
+    attribute_label_map = {
+        "person": ["face", "amazon"],
+        "car": ["ups", "fedex", "amazon", "license_plate"],
+    }
+    all_attribute_labels = [
+        item for sublist in attribute_label_map.values() for item in sublist
+    ]
     fps = process_info["process_fps"]
     detection_fps = process_info["detection_fps"]
     current_frame_time = process_info["detection_frame"]
@@ -758,6 +766,7 @@ def process_frames(
         motion_boxes = motion_detector.detect(frame) if motion_enabled.value else []
 
         regions = []
+        consolidated_detections = []
 
         # if detection is disabled
         if not detection_enabled.value:
@@ -894,12 +903,42 @@ def process_frames(
                 consolidated_detections = get_consolidated_object_detections(
                     detected_object_groups
                 )
+                tracked_detections = [
+                    d
+                    for d in consolidated_detections
+                    if d[0] not in all_attribute_labels
+                ]
                 # now that we have refined our detections, we need to track objects
-                object_tracker.match_and_update(frame_time, consolidated_detections)
+                object_tracker.match_and_update(frame_time, tracked_detections)
             # else, just update the frame times for the stationary objects
             else:
                 object_tracker.update_frame_times(frame_time)
 
+        # group the attribute detections based on what label they apply to
+        attribute_detections = {}
+        for label, attribute_labels in attribute_label_map.items():
+            attribute_detections[label] = [
+                d for d in consolidated_detections if d[0] in attribute_labels
+            ]
+
+        # build detections and add attributes
+        detections = {}
+        for obj in object_tracker.tracked_objects.values():
+            attributes = []
+            # if the objects label has associated attribute detections
+            if obj["label"] in attribute_detections.keys():
+                # add them to attributes if they intersect
+                for attribute_detection in attribute_detections[obj["label"]]:
+                    if box_inside(obj["box"], (attribute_detection[2])):
+                        attributes.append(
+                            {
+                                "label": attribute_detection[0],
+                                "score": attribute_detection[1],
+                                "box": attribute_detection[2],
+                            }
+                        )
+            detections[obj["id"]] = {**obj, "attributes": attributes}
+
         # debug object tracking
         if False:
             bgr_frame = cv2.cvtColor(
@@ -982,7 +1021,7 @@ def process_frames(
                 (
                     camera_name,
                     frame_time,
-                    object_tracker.tracked_objects,
+                    detections,
                     motion_boxes,
                     regions,
                 )