Add support for arbitrary sub labels in reviews (#10990)

* store arbitrary sub labels

* Include sub labels in tooltip

* Update tooltips on filmstrip

* Fix item display

* Fix bug with creating review segment

frigate/review/maintainer.py

@@ -46,8 +46,9 @@ class PendingReviewSegment:
         frame_time: float,
         severity: SeverityEnum,
         detections: dict[str, str],
-        zones: set[str] = set(),
+        sub_labels: set[str],
-        audio: set[str] = set(),
+        zones: set[str],
+        audio: set[str],
     ):
         rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6))
         self.id = f"{frame_time}-{rand_id}"
@@ -55,6 +56,7 @@ class PendingReviewSegment:
         self.start_time = frame_time
         self.severity = severity
         self.detections = detections
+        self.sub_labels = sub_labels
         self.zones = zones
         self.audio = audio
         self.last_update = frame_time
@@ -111,6 +113,7 @@ class PendingReviewSegment:
             ReviewSegment.data: {
                 "detections": list(set(self.detections.keys())),
                 "objects": list(set(self.detections.values())),
+                "sub_labels": list(self.sub_labels),
                 "zones": list(self.zones),
                 "audio": list(self.audio),
             },
@@ -181,6 +184,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                     segment.detections[object["id"]] = object["sub_label"][0]
                 else:
                     segment.detections[object["id"]] = f'{object["label"]}-verified'
+                    segment.sub_labels.add(object["sub_label"][0])
 
                 # if object is alert label
                 # and has entered required zones or required zones is not set
@@ -233,8 +237,8 @@ class ReviewSegmentMaintainer(threading.Thread):
         active_objects = get_active_objects(frame_time, camera_config, objects)
 
         if len(active_objects) > 0:
-            has_sig_object = False
             detections: dict[str, str] = {}
+            sub_labels = set()
             zones: set = set()
             severity = None
 
@@ -245,6 +249,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                     detections[object["id"]] = object["sub_label"][0]
                 else:
                     detections[object["id"]] = f'{object["label"]}-verified'
+                    sub_labels.add(object["sub_label"][0])
 
                 # if object is alert label
                 # and has entered required zones or required zones is not set
@@ -290,8 +295,9 @@ class ReviewSegmentMaintainer(threading.Thread):
                 self.active_review_segments[camera] = PendingReviewSegment(
                     camera,
                     frame_time,
-                    SeverityEnum.alert if has_sig_object else SeverityEnum.detection,
+                    severity,
                     detections,
+                    sub_labels=sub_labels,
                     audio=set(),
                     zones=zones,
                 )
@@ -435,6 +441,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                             severity,
                             {},
                             set(),
+                            set(),
                             detections,
                         )
                 elif topic == DetectionTypeEnum.api:
@@ -445,6 +452,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                         {manual_info["event_id"]: manual_info["label"]},
                         set(),
                         set(),
+                        set(),
                     )
 
                     if manual_info["state"] == ManualEventState.start:

web/src/components/card/AnimatedEventCard.tsx

@@ -83,7 +83,18 @@ export function AnimatedEventCard({ event }: AnimatedEventCardProps) {
         </div>
       </TooltipTrigger>
       <TooltipContent>
-        {`${[...event.data.objects, ...event.data.audio].join(", ").replaceAll("-verified", "")} detected`}
+        {`${[
+          ...new Set([
+            ...(event.data.objects || []),
+            ...(event.data.sub_labels || []),
+            ...(event.data.audio || []),
+          ]),
+        ]
+          .filter((item) => item !== undefined && !item.includes("-verified"))
+          .map((text) => text.charAt(0).toUpperCase() + text.substring(1))
+          .sort()
+          .join(", ")
+          .replaceAll("-verified", "")} detected`}
       </TooltipContent>
     </Tooltip>
   );

web/src/components/player/PreviewThumbnailPlayer.tsx

@@ -239,7 +239,7 @@ export default function PreviewThumbnailPlayer({
                       <Chip
                         className={`flex items-start justify-between space-x-1 ${playingBack ? "hidden" : ""} bg-gradient-to-br ${review.has_been_reviewed ? "from-green-600 to-green-700 bg-green-600" : "from-gray-400 to-gray-500 bg-gray-500"} z-0`}
                       >
-                        {review.data.objects.map((object) => {
+                        {review.data.objects.sort().map((object) => {
                           return getIconForLabel(object, "size-3 text-white");
                         })}
                         {review.data.audio.map((audio) => {
@@ -252,8 +252,18 @@ export default function PreviewThumbnailPlayer({
               </TooltipTrigger>
             </div>
             <TooltipContent className="capitalize">
-              {[...(review.data.objects || []), ...(review.data.audio || [])]
+              {[
-                .filter((item) => item !== undefined)
+                ...new Set([
+                  ...(review.data.objects || []),
+                  ...(review.data.sub_labels || []),
+                  ...(review.data.audio || []),
+                ]),
+              ]
+                .filter(
+                  (item) => item !== undefined && !item.includes("-verified"),
+                )
+                .map((text) => text.charAt(0).toUpperCase() + text.substring(1))
+                .sort()
                 .join(", ")
                 .replaceAll("-verified", "")}
             </TooltipContent>

web/src/types/review.ts

@@ -15,6 +15,7 @@ export type ReviewData = {
   audio: string[];
   detections: string[];
   objects: string[];
+  sub_labels?: string[];
   significant_motion_areas: number[];
   zones: string[];
 };