From ff823b87c8af09b18d315e1a07932d831dbd5665 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 16 Apr 2024 14:56:28 -0600 Subject: [PATCH] Add support for arbitrary sub labels in reviews (#10990) * store arbitrary sub labels * Include sub labels in tooltip * Update tooltips on filmstrip * Fix item display * Fix bug with creating review segment --- frigate/review/maintainer.py | 16 ++++++++++++---- web/src/components/card/AnimatedEventCard.tsx | 13 ++++++++++++- .../components/player/PreviewThumbnailPlayer.tsx | 16 +++++++++++++--- web/src/types/review.ts | 1 + 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/frigate/review/maintainer.py b/frigate/review/maintainer.py index ebc03a35e..86cac58a8 100644 --- a/frigate/review/maintainer.py +++ b/frigate/review/maintainer.py @@ -46,8 +46,9 @@ class PendingReviewSegment: frame_time: float, severity: SeverityEnum, detections: dict[str, str], - zones: set[str] = set(), - audio: set[str] = set(), + sub_labels: set[str], + zones: set[str], + audio: set[str], ): rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) self.id = f"{frame_time}-{rand_id}" @@ -55,6 +56,7 @@ class PendingReviewSegment: self.start_time = frame_time self.severity = severity self.detections = detections + self.sub_labels = sub_labels self.zones = zones self.audio = audio self.last_update = frame_time @@ -111,6 +113,7 @@ class PendingReviewSegment: ReviewSegment.data: { "detections": list(set(self.detections.keys())), "objects": list(set(self.detections.values())), + "sub_labels": list(self.sub_labels), "zones": list(self.zones), "audio": list(self.audio), }, @@ -181,6 +184,7 @@ class ReviewSegmentMaintainer(threading.Thread): segment.detections[object["id"]] = object["sub_label"][0] else: segment.detections[object["id"]] = f'{object["label"]}-verified' + segment.sub_labels.add(object["sub_label"][0]) # if object is alert label # and has entered required zones or required zones is not set @@ -233,8 +237,8 @@ class ReviewSegmentMaintainer(threading.Thread): active_objects = get_active_objects(frame_time, camera_config, objects) if len(active_objects) > 0: - has_sig_object = False detections: dict[str, str] = {} + sub_labels = set() zones: set = set() severity = None @@ -245,6 +249,7 @@ class ReviewSegmentMaintainer(threading.Thread): detections[object["id"]] = object["sub_label"][0] else: detections[object["id"]] = f'{object["label"]}-verified' + sub_labels.add(object["sub_label"][0]) # if object is alert label # and has entered required zones or required zones is not set @@ -290,8 +295,9 @@ class ReviewSegmentMaintainer(threading.Thread): self.active_review_segments[camera] = PendingReviewSegment( camera, frame_time, - SeverityEnum.alert if has_sig_object else SeverityEnum.detection, + severity, detections, + sub_labels=sub_labels, audio=set(), zones=zones, ) @@ -435,6 +441,7 @@ class ReviewSegmentMaintainer(threading.Thread): severity, {}, set(), + set(), detections, ) elif topic == DetectionTypeEnum.api: @@ -445,6 +452,7 @@ class ReviewSegmentMaintainer(threading.Thread): {manual_info["event_id"]: manual_info["label"]}, set(), set(), + set(), ) if manual_info["state"] == ManualEventState.start: diff --git a/web/src/components/card/AnimatedEventCard.tsx b/web/src/components/card/AnimatedEventCard.tsx index 85cfe3c3c..93ecbe919 100644 --- a/web/src/components/card/AnimatedEventCard.tsx +++ b/web/src/components/card/AnimatedEventCard.tsx @@ -83,7 +83,18 @@ export function AnimatedEventCard({ event }: AnimatedEventCardProps) { - {`${[...event.data.objects, ...event.data.audio].join(", ").replaceAll("-verified", "")} detected`} + {`${[ + ...new Set([ + ...(event.data.objects || []), + ...(event.data.sub_labels || []), + ...(event.data.audio || []), + ]), + ] + .filter((item) => item !== undefined && !item.includes("-verified")) + .map((text) => text.charAt(0).toUpperCase() + text.substring(1)) + .sort() + .join(", ") + .replaceAll("-verified", "")} detected`} ); diff --git a/web/src/components/player/PreviewThumbnailPlayer.tsx b/web/src/components/player/PreviewThumbnailPlayer.tsx index e640cfbb7..5d7cc6a15 100644 --- a/web/src/components/player/PreviewThumbnailPlayer.tsx +++ b/web/src/components/player/PreviewThumbnailPlayer.tsx @@ -239,7 +239,7 @@ export default function PreviewThumbnailPlayer({ - {review.data.objects.map((object) => { + {review.data.objects.sort().map((object) => { return getIconForLabel(object, "size-3 text-white"); })} {review.data.audio.map((audio) => { @@ -252,8 +252,18 @@ export default function PreviewThumbnailPlayer({ - {[...(review.data.objects || []), ...(review.data.audio || [])] - .filter((item) => item !== undefined) + {[ + ...new Set([ + ...(review.data.objects || []), + ...(review.data.sub_labels || []), + ...(review.data.audio || []), + ]), + ] + .filter( + (item) => item !== undefined && !item.includes("-verified"), + ) + .map((text) => text.charAt(0).toUpperCase() + text.substring(1)) + .sort() .join(", ") .replaceAll("-verified", "")} diff --git a/web/src/types/review.ts b/web/src/types/review.ts index ea5bbb250..b8e5254d9 100644 --- a/web/src/types/review.ts +++ b/web/src/types/review.ts @@ -15,6 +15,7 @@ export type ReviewData = { audio: string[]; detections: string[]; objects: string[]; + sub_labels?: string[]; significant_motion_areas: number[]; zones: string[]; };