Attribute scores (#7100)

* rework attributes to get scores

* show sublabel score

* formatting

docs/docs/integrations/mqtt.md

@@ -63,7 +63,9 @@ Message published for each changed event. The first message is published when th
     "stationary": false, // whether or not the object is considered stationary
     "motionless_count": 0, // number of frames the object has been motionless
     "position_changes": 2, // number of times the object has moved from a stationary position
-    "attributes": [], // set of unique attributes that have been identified on the object
+    "attributes": {
+      "face": 0.64
+    }, // attributes with top score that have been identified on the object at any point
     "current_attributes": [] // detailed data about the current attributes in this frame
   },
   "after": {
@@ -90,13 +92,15 @@ Message published for each changed event. The first message is published when th
     "stationary": false, // whether or not the object is considered stationary
     "motionless_count": 0, // number of frames the object has been motionless
     "position_changes": 2, // number of times the object has changed position
-    "attributes": ["face"], // set of unique attributes that have been identified on the object
+    "attributes": {
+      "face": 0.86
+    }, // attributes with top score that have been identified on the object at any point
     "current_attributes": [
       // detailed data about the current attributes in this frame
       {
         "label": "face",
         "box": [442, 506, 534, 524],
-        "score": 0.64
+        "score": 0.86
       }
     ]
   }

frigate/events/maintainer.py

@@ -199,7 +199,8 @@ class EventProcessor(threading.Thread):
 
             # only overwrite the sub_label in the database if it's set
             if event_data.get("sub_label") is not None:
-                event[Event.sub_label] = event_data["sub_label"]
+                event[Event.sub_label] = event_data["sub_label"][0]
+                event[Event.data]["sub_label_score"] = event_data["sub_label"][1]
 
             (
                 Event.insert(event)

frigate/models.py

@@ -38,7 +38,9 @@ class Event(Model):  # type: ignore[misc]
         IntegerField()
     )  # TODO remove when columns can be dropped without rebuilding table
     retain_indefinitely = BooleanField(default=False)
-    ratio = FloatField(default=1.0)
+    ratio = FloatField(
+        default=1.0
+    )  # TODO remove when columns can be dropped without rebuilding table
     plus_id = CharField(max_length=30)
     model_hash = CharField(max_length=32)
     detector_type = CharField(max_length=32)

frigate/object_processing.py

@@ -112,7 +112,7 @@ class TrackedObject:
         self.zone_presence = {}
         self.current_zones = []
         self.entered_zones = []
-        self.attributes = set()
+        self.attributes = defaultdict(float)
         self.false_positive = True
         self.has_clip = False
         self.has_snapshot = False
@@ -207,15 +207,19 @@ class TrackedObject:
 
         # maintain attributes
         for attr in obj_data["attributes"]:
-            self.attributes.add(attr["label"])
+            if self.attributes[attr["label"]] < attr["score"]:
+                self.attributes[attr["label"]] = attr["score"]
 
-        # populate the sub_label for car with first logo if it exists
-        if self.obj_data["label"] == "car" and "sub_label" not in self.obj_data:
-            recognized_logos = self.attributes.intersection(
-                set(["ups", "fedex", "amazon"])
-            )
+        # populate the sub_label for car with highest scoring logo
+        if self.obj_data["label"] == "car":
+            recognized_logos = {
+                k: self.attributes[k]
+                for k in ["ups", "fedex", "amazon"]
+                if k in self.attributes
+            }
             if len(recognized_logos) > 0:
-                self.obj_data["sub_label"] = recognized_logos.pop()
+                max_logo = max(recognized_logos, key=recognized_logos.get)
+                self.obj_data["sub_label"] = (max_logo, recognized_logos[max_logo])
 
         # check for significant change
         if not self.false_positive:
@@ -274,7 +278,7 @@ class TrackedObject:
             "entered_zones": self.entered_zones.copy(),
             "has_clip": self.has_clip,
             "has_snapshot": self.has_snapshot,
-            "attributes": list(self.attributes),
+            "attributes": self.attributes,
             "current_attributes": self.obj_data["attributes"],
         }
 

web/src/icons/Score.jsx

@@ -0,0 +1,20 @@
+import { h } from 'preact';
+import { memo } from 'preact/compat';
+
+export function Score({ className = 'h-6 w-6', stroke = 'currentColor', fill = 'currentColor', onClick = () => {} }) {
+  return (
+    <svg
+      xmlns="http://www.w3.org/2000/svg"
+      className={className}
+      fill={fill}
+      viewBox="0 0 24 24"
+      stroke={stroke}
+      onClick={onClick}
+    >
+      <title>percent</title>
+      <path d="M12,9A3,3 0 0,0 9,12A3,3 0 0,0 12,15A3,3 0 0,0 15,12A3,3 0 0,0 12,9M19,19H15V21H19A2,2 0 0,0 21,19V15H19M19,3H15V5H19V9H21V5A2,2 0 0,0 19,3M5,5H9V3H5A2,2 0 0,0 3,5V9H5M5,15H3V19A2,2 0 0,0 5,21H9V19H5V15Z" />
+    </svg>
+  );
+}
+
+export default memo(Score);

web/src/routes/Events.jsx

@@ -30,6 +30,7 @@ import TimeAgo from '../components/TimeAgo';
 import Timepicker from '../components/TimePicker';
 import TimelineSummary from '../components/TimelineSummary';
 import TimelineEventOverlay from '../components/TimelineEventOverlay';
+import { Score } from '../icons/Score';
 
 const API_LIMIT = 25;
 
@@ -602,13 +603,10 @@ export default function Events({ path, ...props }) {
                     <div className="m-2 flex grow">
                       <div className="flex flex-col grow">
                         <div className="capitalize text-lg font-bold">
-                          {event.sub_label
-                            ? `${event.label.replaceAll('_', ' ')}: ${event.sub_label.replaceAll('_', ' ')}`
-                            : event.label.replaceAll('_', ' ')}
-                          {(event?.data?.top_score || event.top_score || 0) == 0
-                            ? null
-                            : ` (${((event?.data?.top_score || event.top_score) * 100).toFixed(0)}%)`}
+                          {event.label.replaceAll('_', ' ')}
+                          {event.sub_label ? `: ${event.sub_label.replaceAll('_', ' ')}` : null}
                         </div>
+
                         <div className="text-sm flex">
                           <Clock className="h-5 w-5 mr-2 inline" />
                           {formatUnixTimestampToDateTime(event.start_time, { ...config.ui })}
@@ -628,6 +626,15 @@ export default function Events({ path, ...props }) {
                           <Zone className="w-5 h-5 mr-2 inline" />
                           {event.zones.join(', ').replaceAll('_', ' ')}
                         </div>
+                        <div className="capitalize  text-sm flex align-center">
+                          <Score className="w-5 h-5 mr-2 inline" />
+                          {(event?.data?.top_score || event.top_score || 0) == 0
+                            ? null
+                            : `Label: ${((event?.data?.top_score || event.top_score) * 100).toFixed(0)}%`}
+                          {(event?.data?.sub_label_score || 0) == 0
+                            ? null
+                            : `, Sub Label: ${(event?.data?.sub_label_score * 100).toFixed(0)}%`}
+                        </div>
                       </div>
                       <div class="hidden sm:flex flex-col justify-end mr-2">
                         {event.end_time && event.has_snapshot && (