Fix bug in intersection logic (#6780)

* Fix bug in intersection logic

* Fix isort

* Remove unrelated test

* Formatting

* Fix type in test

frigate/test/test_video.py

@@ -5,6 +5,7 @@ import numpy as np
 from norfair.drawing.color import Palette
 from norfair.drawing.drawer import Drawer
 
+from frigate.util import intersection
 from frigate.video import (
     get_cluster_boundary,
     get_cluster_candidates,
@@ -63,7 +64,7 @@ def save_cluster_boundary_image(name, boxes, bounding_boxes):
     )
 
 
-class TestConfig(unittest.TestCase):
+class TestRegion(unittest.TestCase):
     def setUp(self):
         self.frame_shape = (1000, 2000)
         self.min_region_size = 160
@@ -176,3 +177,16 @@ class TestConfig(unittest.TestCase):
 
         # save_clusters_image("dont_combine", boxes, cluster_candidates, regions)
         assert len(regions) == 2
+
+
+class TestObjectBoundingBoxes(unittest.TestCase):
+    def setUp(self) -> None:
+        pass
+
+    def test_box_intersection(self):
+        box_a = [2012, 191, 2031, 205]
+        box_b = [887, 92, 985, 151]
+        box_c = [899, 128, 1080, 175]
+
+        assert intersection(box_a, box_b) == None
+        assert intersection(box_b, box_c) == (899, 128, 985, 151)

frigate/util.py

@@ -572,7 +572,16 @@ def yuv_region_2_bgr(frame, region):
         raise
 
 
-def intersection(box_a, box_b):
+def intersection(box_a, box_b) -> Optional[list[int]]:
+    """Return intersection box or None if boxes do not intersect."""
+    if (
+        box_a[2] < box_b[0]
+        or box_a[0] > box_b[2]
+        or box_a[1] > box_b[3]
+        or box_a[3] < box_b[1]
+    ):
+        return None
+
     return (
         max(box_a[0], box_b[0]),
         max(box_a[1], box_b[1]),
@@ -589,6 +598,9 @@ def intersection_over_union(box_a, box_b):
     # determine the (x, y)-coordinates of the intersection rectangle
     intersect = intersection(box_a, box_b)
 
+    if intersect is None:
+        return 0.0
+
     # compute the area of intersection rectangle
     inter_area = max(0, intersect[2] - intersect[0] + 1) * max(
         0, intersect[3] - intersect[1] + 1

frigate/video.py

@@ -669,6 +669,40 @@ def get_cluster_region(frame_shape, min_region, cluster, boxes):
     )
 
 
+def get_consolidated_object_detections(detected_object_groups):
+    """Drop detections that overlap too much"""
+    consolidated_detections = []
+    for group in detected_object_groups.values():
+        # if the group only has 1 item, skip
+        if len(group) == 1:
+            consolidated_detections.append(group[0])
+            continue
+
+        # sort smallest to largest by area
+        sorted_by_area = sorted(group, key=lambda g: g[3])
+
+        for current_detection_idx in range(0, len(sorted_by_area)):
+            current_detection = sorted_by_area[current_detection_idx][2]
+            overlap = 0
+            for to_check_idx in range(
+                min(current_detection_idx + 1, len(sorted_by_area)),
+                len(sorted_by_area),
+            ):
+                to_check = sorted_by_area[to_check_idx][2]
+                intersect_box = intersection(current_detection, to_check)
+                # if 90% of smaller detection is inside of another detection, consolidate
+                if (
+                    intersect_box is not None
+                    and area(intersect_box) / area(current_detection) > 0.9
+                ):
+                    overlap = 1
+                    break
+            if overlap == 0:
+                consolidated_detections.append(sorted_by_area[current_detection_idx])
+
+    return consolidated_detections
+
+
 def process_frames(
     camera_name: str,
     frame_queue: mp.Queue,
@@ -849,9 +883,6 @@ def process_frames(
             # set the detections list to only include top objects
             detections = selected_objects
 
-            ## drop detections that overlap too much
-            consolidated_detections = []
-
             # if detection was run on this frame, consolidate
             if len(regions) > 0:
                 # group by name
@@ -859,35 +890,8 @@ def process_frames(
                 for detection in detections:
                     detected_object_groups[detection[0]].append(detection)
 
-                # loop over detections grouped by label
+                consolidated_detections = get_consolidated_object_detections(
-                for group in detected_object_groups.values():
+                    detected_object_groups
-                    # if the group only has 1 item, skip
-                    if len(group) == 1:
-                        consolidated_detections.append(group[0])
-                        continue
-
-                    # sort smallest to largest by area
-                    sorted_by_area = sorted(group, key=lambda g: g[3])
-
-                    for current_detection_idx in range(0, len(sorted_by_area)):
-                        current_detection = sorted_by_area[current_detection_idx][2]
-                        overlap = 0
-                        for to_check_idx in range(
-                            min(current_detection_idx + 1, len(sorted_by_area)),
-                            len(sorted_by_area),
-                        ):
-                            to_check = sorted_by_area[to_check_idx][2]
-                            # if 90% of smaller detection is inside of another detection, consolidate
-                            if (
-                                area(intersection(current_detection, to_check))
-                                / area(current_detection)
-                                > 0.9
-                            ):
-                                overlap = 1
-                                break
-                        if overlap == 0:
-                            consolidated_detections.append(
-                                sorted_by_area[current_detection_idx]
                 )
                 # now that we have refined our detections, we need to track objects
                 object_tracker.match_and_update(frame_time, consolidated_detections)