From ec4d79eafcb929969c819e10248cce55dda56db9 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 12 Jun 2023 16:38:22 -0600 Subject: [PATCH] Fix bug in intersection logic (#6780) * Fix bug in intersection logic * Fix isort * Remove unrelated test * Formatting * Fix type in test --- frigate/test/test_video.py | 16 ++++++++- frigate/util.py | 14 +++++++- frigate/video.py | 70 ++++++++++++++++++++------------------ 3 files changed, 65 insertions(+), 35 deletions(-) diff --git a/frigate/test/test_video.py b/frigate/test/test_video.py index c27f4c801..e5f7e83fd 100644 --- a/frigate/test/test_video.py +++ b/frigate/test/test_video.py @@ -5,6 +5,7 @@ import numpy as np from norfair.drawing.color import Palette from norfair.drawing.drawer import Drawer +from frigate.util import intersection from frigate.video import ( get_cluster_boundary, get_cluster_candidates, @@ -63,7 +64,7 @@ def save_cluster_boundary_image(name, boxes, bounding_boxes): ) -class TestConfig(unittest.TestCase): +class TestRegion(unittest.TestCase): def setUp(self): self.frame_shape = (1000, 2000) self.min_region_size = 160 @@ -176,3 +177,16 @@ class TestConfig(unittest.TestCase): # save_clusters_image("dont_combine", boxes, cluster_candidates, regions) assert len(regions) == 2 + + +class TestObjectBoundingBoxes(unittest.TestCase): + def setUp(self) -> None: + pass + + def test_box_intersection(self): + box_a = [2012, 191, 2031, 205] + box_b = [887, 92, 985, 151] + box_c = [899, 128, 1080, 175] + + assert intersection(box_a, box_b) == None + assert intersection(box_b, box_c) == (899, 128, 985, 151) diff --git a/frigate/util.py b/frigate/util.py index 897aa8d21..396a0f874 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -572,7 +572,16 @@ def yuv_region_2_bgr(frame, region): raise -def intersection(box_a, box_b): +def intersection(box_a, box_b) -> Optional[list[int]]: + """Return intersection box or None if boxes do not intersect.""" + if ( + box_a[2] < box_b[0] + or box_a[0] > box_b[2] + or box_a[1] > box_b[3] + or box_a[3] < box_b[1] + ): + return None + return ( max(box_a[0], box_b[0]), max(box_a[1], box_b[1]), @@ -589,6 +598,9 @@ def intersection_over_union(box_a, box_b): # determine the (x, y)-coordinates of the intersection rectangle intersect = intersection(box_a, box_b) + if intersect is None: + return 0.0 + # compute the area of intersection rectangle inter_area = max(0, intersect[2] - intersect[0] + 1) * max( 0, intersect[3] - intersect[1] + 1 diff --git a/frigate/video.py b/frigate/video.py index 874490eae..291347ba3 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -669,6 +669,40 @@ def get_cluster_region(frame_shape, min_region, cluster, boxes): ) +def get_consolidated_object_detections(detected_object_groups): + """Drop detections that overlap too much""" + consolidated_detections = [] + for group in detected_object_groups.values(): + # if the group only has 1 item, skip + if len(group) == 1: + consolidated_detections.append(group[0]) + continue + + # sort smallest to largest by area + sorted_by_area = sorted(group, key=lambda g: g[3]) + + for current_detection_idx in range(0, len(sorted_by_area)): + current_detection = sorted_by_area[current_detection_idx][2] + overlap = 0 + for to_check_idx in range( + min(current_detection_idx + 1, len(sorted_by_area)), + len(sorted_by_area), + ): + to_check = sorted_by_area[to_check_idx][2] + intersect_box = intersection(current_detection, to_check) + # if 90% of smaller detection is inside of another detection, consolidate + if ( + intersect_box is not None + and area(intersect_box) / area(current_detection) > 0.9 + ): + overlap = 1 + break + if overlap == 0: + consolidated_detections.append(sorted_by_area[current_detection_idx]) + + return consolidated_detections + + def process_frames( camera_name: str, frame_queue: mp.Queue, @@ -849,9 +883,6 @@ def process_frames( # set the detections list to only include top objects detections = selected_objects - ## drop detections that overlap too much - consolidated_detections = [] - # if detection was run on this frame, consolidate if len(regions) > 0: # group by name @@ -859,36 +890,9 @@ def process_frames( for detection in detections: detected_object_groups[detection[0]].append(detection) - # loop over detections grouped by label - for group in detected_object_groups.values(): - # if the group only has 1 item, skip - if len(group) == 1: - consolidated_detections.append(group[0]) - continue - - # sort smallest to largest by area - sorted_by_area = sorted(group, key=lambda g: g[3]) - - for current_detection_idx in range(0, len(sorted_by_area)): - current_detection = sorted_by_area[current_detection_idx][2] - overlap = 0 - for to_check_idx in range( - min(current_detection_idx + 1, len(sorted_by_area)), - len(sorted_by_area), - ): - to_check = sorted_by_area[to_check_idx][2] - # if 90% of smaller detection is inside of another detection, consolidate - if ( - area(intersection(current_detection, to_check)) - / area(current_detection) - > 0.9 - ): - overlap = 1 - break - if overlap == 0: - consolidated_detections.append( - sorted_by_area[current_detection_idx] - ) + consolidated_detections = get_consolidated_object_detections( + detected_object_groups + ) # now that we have refined our detections, we need to track objects object_tracker.match_and_update(frame_time, consolidated_detections) # else, just update the frame times for the stationary objects