Smarter Regions (#8194)

* Smarter Regions

* Formatting

* Cleanup

* Fix motion region checking logic

* Add database table and migration for regions

* Update region grid on startup

* Revert init delay change

* Fix mypy

* Move object related functions to util

* Remove unused

* Fix tests

* Remove log

* Update the region daily at 2

* Fix logic

* Formatting

* Initialize grid before starting processing frames

* Move back to creating grid in main process

* Formatting

* Fixes

* Formating

* Fix region check

* Accept all but true

* Use regions grid for startup scan

* Add clarifying comment

* Fix new grid requests

* Add tests

* Delete stale region grids from DB

frigate/app.py

@@ -36,7 +36,7 @@ from frigate.events.external import ExternalEventProcessor
 from frigate.events.maintainer import EventProcessor
 from frigate.http import create_app
 from frigate.log import log_process, root_configurer
-from frigate.models import Event, Recordings, RecordingsToDelete, Timeline
+from frigate.models import Event, Recordings, RecordingsToDelete, Regions, Timeline
 from frigate.object_detection import ObjectDetectProcess
 from frigate.object_processing import TrackedObjectProcessor
 from frigate.output import output_frames
@@ -49,6 +49,7 @@ from frigate.stats import StatsEmitter, stats_init
 from frigate.storage import StorageMaintainer
 from frigate.timeline import TimelineProcessor
 from frigate.types import CameraMetricsTypes, FeatureMetricsTypes, PTZMetricsTypes
+from frigate.util.object import get_camera_regions_grid
 from frigate.version import VERSION
 from frigate.video import capture_camera, track_camera
 from frigate.watchdog import FrigateWatchdog
@@ -69,6 +70,7 @@ class FrigateApp:
         self.feature_metrics: dict[str, FeatureMetricsTypes] = {}
         self.ptz_metrics: dict[str, PTZMetricsTypes] = {}
         self.processes: dict[str, int] = {}
+        self.region_grids: dict[str, list[list[dict[str, int]]]] = {}
 
     def set_environment_vars(self) -> None:
         for key, value in self.config.environment_vars.items():
@@ -161,6 +163,7 @@ class FrigateApp:
                 # issue https://github.com/python/typeshed/issues/8799
                 # from mypy 0.981 onwards
                 "frame_queue": mp.Queue(maxsize=2),
+                "region_grid_queue": mp.Queue(maxsize=1),
                 "capture_process": None,
                 "process": None,
                 "audio_rms": mp.Value("d", 0.0),  # type: ignore[typeddict-item]
@@ -327,7 +330,7 @@ class FrigateApp:
                 60, 10 * len([c for c in self.config.cameras.values() if c.enabled])
             ),
         )
-        models = [Event, Recordings, RecordingsToDelete, Timeline]
+        models = [Event, Recordings, RecordingsToDelete, Regions, Timeline]
         self.db.bind(models)
 
     def init_stats(self) -> None:
@@ -452,6 +455,17 @@ class FrigateApp:
         output_processor.start()
         logger.info(f"Output process started: {output_processor.pid}")
 
+    def init_historical_regions(self) -> None:
+        # delete region grids for removed or renamed cameras
+        cameras = list(self.config.cameras.keys())
+        Regions.delete().where(~(Regions.camera << cameras)).execute()
+
+        # create or update region grids for each camera
+        for camera in self.config.cameras.values():
+            self.region_grids[camera.name] = get_camera_regions_grid(
+                camera.name, camera.detect
+            )
+
     def start_camera_processors(self) -> None:
         for name, config in self.config.cameras.items():
             if not self.config.cameras[name].enabled:
@@ -469,8 +483,10 @@ class FrigateApp:
                     self.detection_queue,
                     self.detection_out_events[name],
                     self.detected_frames_queue,
+                    self.inter_process_queue,
                     self.camera_metrics[name],
                     self.ptz_metrics[name],
+                    self.region_grids[name],
                 ),
             )
             camera_process.daemon = True
@@ -611,6 +627,7 @@ class FrigateApp:
         self.start_detectors()
         self.start_video_output_processor()
         self.start_ptz_autotracker()
+        self.init_historical_regions()
         self.start_detected_frames_processor()
         self.start_camera_processors()
         self.start_camera_capture_processes()

frigate/comms/dispatcher.py

@@ -5,10 +5,11 @@ from abc import ABC, abstractmethod
 from typing import Any, Callable
 
 from frigate.config import FrigateConfig
-from frigate.const import INSERT_MANY_RECORDINGS
+from frigate.const import INSERT_MANY_RECORDINGS, REQUEST_REGION_GRID
 from frigate.models import Recordings
 from frigate.ptz.onvif import OnvifCommandEnum, OnvifController
 from frigate.types import CameraMetricsTypes, FeatureMetricsTypes, PTZMetricsTypes
+from frigate.util.object import get_camera_regions_grid
 from frigate.util.services import restart_frigate
 
 logger = logging.getLogger(__name__)
@@ -90,6 +91,11 @@ class Dispatcher:
             restart_frigate()
         elif topic == INSERT_MANY_RECORDINGS:
             Recordings.insert_many(payload).execute()
+        elif topic == REQUEST_REGION_GRID:
+            camera = payload
+            self.camera_metrics[camera]["region_grid_queue"].put(
+                get_camera_regions_grid(camera, self.config.cameras[camera].detect)
+            )
         else:
             self.publish(topic, payload, retain=False)
 

frigate/const.py

@@ -51,3 +51,4 @@ MAX_PLAYLIST_SECONDS = 7200  # support 2 hour segments for a single playlist to
 # Internal Comms Topics
 
 INSERT_MANY_RECORDINGS = "insert_many_recordings"
+REQUEST_REGION_GRID = "request_region_grid"

frigate/models.py

@@ -57,6 +57,12 @@ class Timeline(Model):  # type: ignore[misc]
     data = JSONField()  # ex: tracked object id, region, box, etc.
 
 
+class Regions(Model):  # type: ignore[misc]
+    camera = CharField(null=False, primary_key=True, max_length=20)
+    grid = JSONField()  # json blob of grid
+    last_update = DateTimeField()
+
+
 class Recordings(Model):  # type: ignore[misc]
     id = CharField(null=False, primary_key=True, max_length=30)
     camera = CharField(index=True, max_length=20)

frigate/test/test_reduce_boxes.py

@@ -1,6 +1,6 @@
 from unittest import TestCase, main
 
-from frigate.video import box_overlaps, reduce_boxes
+from frigate.util.object import box_overlaps, reduce_boxes
 
 
 class TestBoxOverlaps(TestCase):

frigate/test/test_video.py

@@ -6,10 +6,11 @@ from norfair.drawing.color import Palette
 from norfair.drawing.drawer import Drawer
 
 from frigate.util.image import intersection
-from frigate.video import (
+from frigate.util.object import (
     get_cluster_boundary,
     get_cluster_candidates,
     get_cluster_region,
+    get_region_from_grid,
 )
 
 
@@ -190,3 +191,36 @@ class TestObjectBoundingBoxes(unittest.TestCase):
 
         assert intersection(box_a, box_b) == None
         assert intersection(box_b, box_c) == (899, 128, 985, 151)
+
+
+class TestRegionGrid(unittest.TestCase):
+    def setUp(self) -> None:
+        pass
+
+    def test_region_in_range(self):
+        """Test that region is kept at minimal size when within std dev."""
+        frame_shape = (720, 1280)
+        box = [450, 450, 550, 550]
+        region_grid = [
+            [],
+            [],
+            [],
+            [{}, {}, {}, {}, {}, {"sizes": [0.25], "mean": 0.26, "std_dev": 0.01}],
+        ]
+
+        region = get_region_from_grid(frame_shape, box, 320, region_grid)
+        assert region[2] - region[0] == 320
+
+    def test_region_out_of_range(self):
+        """Test that region is upsized when outside of std dev."""
+        frame_shape = (720, 1280)
+        box = [450, 450, 550, 550]
+        region_grid = [
+            [],
+            [],
+            [],
+            [{}, {}, {}, {}, {}, {"sizes": [0.5], "mean": 0.5, "std_dev": 0.1}],
+        ]
+
+        region = get_region_from_grid(frame_shape, box, 320, region_grid)
+        assert region[2] - region[0] > 320

frigate/track/norfair_tracker.py

@@ -77,7 +77,7 @@ class NorfairTracker(ObjectTracker):
         self.tracker = Tracker(
             distance_function=frigate_distance,
             distance_threshold=2.5,
-            initialization_delay=config.detect.fps / 2,
+            initialization_delay=0,
             hit_counter_max=self.max_disappeared,
         )
         if self.ptz_autotracker_enabled.value:
@@ -106,6 +106,11 @@ class NorfairTracker(ObjectTracker):
             "ymax": self.detect_config.height,
         }
 
+        # start object with a hit count of `fps` to avoid quick detection -> loss
+        next(
+            (o for o in self.tracker.tracked_objects if o.global_id == track_id)
+        ).hit_counter = self.camera_config.detect.fps
+
     def deregister(self, id, track_id):
         del self.tracked_objects[id]
         del self.disappeared[id]

frigate/util/builtin.py

@@ -14,6 +14,7 @@ import numpy as np
 import pytz
 import yaml
 from ruamel.yaml import YAML
+from tzlocal import get_localzone
 
 from frigate.const import REGEX_HTTP_CAMERA_USER_PASS, REGEX_RTSP_CAMERA_USER_PASS
 
@@ -262,3 +263,10 @@ def find_by_key(dictionary, target_key):
                 if result is not None:
                     return result
     return None
+
+
+def get_tomorrow_at_2() -> datetime.datetime:
+    tomorrow = datetime.datetime.now(get_localzone()) + datetime.timedelta(days=1)
+    return tomorrow.replace(hour=2, minute=0, second=0).astimezone(
+        datetime.timezone.utc
+    )

frigate/util/object.py

@@ -0,0 +1,481 @@
+"""Utils for reading and writing object detection data."""
+
+import datetime
+import logging
+import math
+
+import cv2
+import numpy as np
+from peewee import DoesNotExist
+
+from frigate.config import DetectConfig, ModelConfig
+from frigate.detectors.detector_config import PixelFormatEnum
+from frigate.models import Event, Regions, Timeline
+from frigate.util.image import (
+    area,
+    calculate_region,
+    intersection,
+    intersection_over_union,
+    yuv_region_2_bgr,
+    yuv_region_2_rgb,
+    yuv_region_2_yuv,
+)
+
+logger = logging.getLogger(__name__)
+
+GRID_SIZE = 8
+
+
+def get_camera_regions_grid(
+    name: str, detect: DetectConfig
+) -> list[list[dict[str, any]]]:
+    """Build a grid of expected region sizes for a camera."""
+    # get grid from db if available
+    try:
+        regions: Regions = Regions.select().where(Regions.camera == name).get()
+        grid = regions.grid
+        last_update = regions.last_update
+    except DoesNotExist:
+        grid = []
+        for x in range(GRID_SIZE):
+            row = []
+            for y in range(GRID_SIZE):
+                row.append({"sizes": []})
+            grid.append(row)
+        last_update = 0
+
+    # get events for timeline entries
+    events = (
+        Event.select(Event.id)
+        .where(Event.camera == name)
+        .where((Event.false_positive == None) | (Event.false_positive == False))
+        .where(Event.start_time > last_update)
+    )
+    valid_event_ids = [e["id"] for e in events.dicts()]
+    logger.debug(f"Found {len(valid_event_ids)} new events for {name}")
+
+    # no new events, return as is
+    if not valid_event_ids:
+        return grid
+
+    new_update = datetime.datetime.now().timestamp()
+    timeline = (
+        Timeline.select(
+            *[
+                Timeline.camera,
+                Timeline.source,
+                Timeline.data,
+            ]
+        )
+        .where(Timeline.source_id << valid_event_ids)
+        .limit(10000)
+        .dicts()
+    )
+
+    logger.debug(f"Found {len(timeline)} new entries for {name}")
+
+    width = detect.width
+    height = detect.height
+
+    for t in timeline:
+        if t.get("source") != "tracked_object":
+            continue
+
+        box = t["data"]["box"]
+
+        # calculate centroid position
+        x = box[0] + (box[2] / 2)
+        y = box[1] + (box[3] / 2)
+
+        x_pos = int(x * GRID_SIZE)
+        y_pos = int(y * GRID_SIZE)
+
+        calculated_region = calculate_region(
+            (height, width),
+            box[0] * width,
+            box[1] * height,
+            (box[0] + box[2]) * width,
+            (box[1] + box[3]) * height,
+            320,
+            1.35,
+        )
+        # save width of region to grid as relative
+        grid[x_pos][y_pos]["sizes"].append(
+            (calculated_region[2] - calculated_region[0]) / width
+        )
+
+    for x in range(GRID_SIZE):
+        for y in range(GRID_SIZE):
+            cell = grid[x][y]
+
+            if len(cell["sizes"]) == 0:
+                continue
+
+            std_dev = np.std(cell["sizes"])
+            mean = np.mean(cell["sizes"])
+            logger.debug(f"std dev: {std_dev} mean: {mean}")
+            cell["x"] = x
+            cell["y"] = y
+            cell["std_dev"] = std_dev
+            cell["mean"] = mean
+
+    # update db with new grid
+    region = {
+        Regions.camera: name,
+        Regions.grid: grid,
+        Regions.last_update: new_update,
+    }
+    (
+        Regions.insert(region)
+        .on_conflict(
+            conflict_target=[Regions.camera],
+            update=region,
+        )
+        .execute()
+    )
+
+    return grid
+
+
+def get_cluster_region_from_grid(frame_shape, min_region, cluster, boxes, region_grid):
+    min_x = frame_shape[1]
+    min_y = frame_shape[0]
+    max_x = 0
+    max_y = 0
+    for b in cluster:
+        min_x = min(boxes[b][0], min_x)
+        min_y = min(boxes[b][1], min_y)
+        max_x = max(boxes[b][2], max_x)
+        max_y = max(boxes[b][3], max_y)
+    return get_region_from_grid(
+        frame_shape, [min_x, min_y, max_x, max_y], min_region, region_grid
+    )
+
+
+def get_region_from_grid(
+    frame_shape: tuple[int],
+    cluster: list[int],
+    min_region: int,
+    region_grid: list[list[dict[str, any]]],
+) -> list[int]:
+    """Get a region for a box based on the region grid."""
+    box = calculate_region(
+        frame_shape, cluster[0], cluster[1], cluster[2], cluster[3], min_region
+    )
+    centroid = (
+        box[0] + (min(frame_shape[1], box[2]) - box[0]) / 2,
+        box[1] + (min(frame_shape[0], box[3]) - box[1]) / 2,
+    )
+    grid_x = int(centroid[0] / frame_shape[1] * GRID_SIZE)
+    grid_y = int(centroid[1] / frame_shape[0] * GRID_SIZE)
+
+    cell = region_grid[grid_x][grid_y]
+
+    # if there is no known data, get standard region for motion box
+    if not cell or not cell["sizes"]:
+        return calculate_region(frame_shape, box[0], box[1], box[2], box[3], min_region)
+
+    # convert the calculated region size to relative
+    calc_size = (box[2] - box[0]) / frame_shape[1]
+
+    # if region is within expected size, don't resize
+    if (
+        (cell["mean"] - cell["std_dev"])
+        <= calc_size
+        <= (cell["mean"] + cell["std_dev"])
+    ):
+        return box
+    # TODO not sure how to handle case where cluster is larger than expected region
+    elif calc_size > (cell["mean"] + cell["std_dev"]):
+        return box
+
+    size = cell["mean"] * frame_shape[1]
+
+    # get region based on grid size
+    return calculate_region(
+        frame_shape,
+        max(0, centroid[0] - size / 2),
+        max(0, centroid[1] - size / 2),
+        min(frame_shape[1], centroid[0] + size / 2),
+        min(frame_shape[0], centroid[1] + size / 2),
+        min_region,
+    )
+
+
+def is_object_filtered(obj, objects_to_track, object_filters):
+    object_name = obj[0]
+    object_score = obj[1]
+    object_box = obj[2]
+    object_area = obj[3]
+    object_ratio = obj[4]
+
+    if object_name not in objects_to_track:
+        return True
+
+    if object_name in object_filters:
+        obj_settings = object_filters[object_name]
+
+        # if the min area is larger than the
+        # detected object, don't add it to detected objects
+        if obj_settings.min_area > object_area:
+            return True
+
+        # if the detected object is larger than the
+        # max area, don't add it to detected objects
+        if obj_settings.max_area < object_area:
+            return True
+
+        # if the score is lower than the min_score, skip
+        if obj_settings.min_score > object_score:
+            return True
+
+        # if the object is not proportionally wide enough
+        if obj_settings.min_ratio > object_ratio:
+            return True
+
+        # if the object is proportionally too wide
+        if obj_settings.max_ratio < object_ratio:
+            return True
+
+        if obj_settings.mask is not None:
+            # compute the coordinates of the object and make sure
+            # the location isn't outside the bounds of the image (can happen from rounding)
+            object_xmin = object_box[0]
+            object_xmax = object_box[2]
+            object_ymax = object_box[3]
+            y_location = min(int(object_ymax), len(obj_settings.mask) - 1)
+            x_location = min(
+                int((object_xmax + object_xmin) / 2.0),
+                len(obj_settings.mask[0]) - 1,
+            )
+
+            # if the object is in a masked location, don't add it to detected objects
+            if obj_settings.mask[y_location][x_location] == 0:
+                return True
+
+    return False
+
+
+def get_min_region_size(model_config: ModelConfig) -> int:
+    """Get the min region size."""
+    return max(model_config.height, model_config.width)
+
+
+def create_tensor_input(frame, model_config: ModelConfig, region):
+    if model_config.input_pixel_format == PixelFormatEnum.rgb:
+        cropped_frame = yuv_region_2_rgb(frame, region)
+    elif model_config.input_pixel_format == PixelFormatEnum.bgr:
+        cropped_frame = yuv_region_2_bgr(frame, region)
+    else:
+        cropped_frame = yuv_region_2_yuv(frame, region)
+
+    # Resize if needed
+    if cropped_frame.shape != (model_config.height, model_config.width, 3):
+        cropped_frame = cv2.resize(
+            cropped_frame,
+            dsize=(model_config.width, model_config.height),
+            interpolation=cv2.INTER_LINEAR,
+        )
+
+    # Expand dimensions since the model expects images to have shape: [1, height, width, 3]
+    return np.expand_dims(cropped_frame, axis=0)
+
+
+def box_overlaps(b1, b2):
+    if b1[2] < b2[0] or b1[0] > b2[2] or b1[1] > b2[3] or b1[3] < b2[1]:
+        return False
+    return True
+
+
+def box_inside(b1, b2):
+    # check if b2 is inside b1
+    if b2[0] >= b1[0] and b2[1] >= b1[1] and b2[2] <= b1[2] and b2[3] <= b1[3]:
+        return True
+    return False
+
+
+def reduce_boxes(boxes, iou_threshold=0.0):
+    clusters = []
+
+    for box in boxes:
+        matched = 0
+        for cluster in clusters:
+            if intersection_over_union(box, cluster) > iou_threshold:
+                matched = 1
+                cluster[0] = min(cluster[0], box[0])
+                cluster[1] = min(cluster[1], box[1])
+                cluster[2] = max(cluster[2], box[2])
+                cluster[3] = max(cluster[3], box[3])
+
+        if not matched:
+            clusters.append(list(box))
+
+    return [tuple(c) for c in clusters]
+
+
+def intersects_any(box_a, boxes):
+    for box in boxes:
+        if box_overlaps(box_a, box):
+            return True
+    return False
+
+
+def inside_any(box_a, boxes):
+    for box in boxes:
+        # check if box_a is inside of box
+        if box_inside(box, box_a):
+            return True
+    return False
+
+
+def get_cluster_boundary(box, min_region):
+    # compute the max region size for the current box (box is 10% of region)
+    box_width = box[2] - box[0]
+    box_height = box[3] - box[1]
+    max_region_area = abs(box_width * box_height) / 0.1
+    max_region_size = max(min_region, int(math.sqrt(max_region_area)))
+
+    centroid = (box_width / 2 + box[0], box_height / 2 + box[1])
+
+    max_x_dist = int(max_region_size - box_width / 2 * 1.1)
+    max_y_dist = int(max_region_size - box_height / 2 * 1.1)
+
+    return [
+        int(centroid[0] - max_x_dist),
+        int(centroid[1] - max_y_dist),
+        int(centroid[0] + max_x_dist),
+        int(centroid[1] + max_y_dist),
+    ]
+
+
+def get_cluster_candidates(frame_shape, min_region, boxes):
+    # and create a cluster of other boxes using it's max region size
+    # only include boxes where the region is an appropriate(except the region could possibly be smaller?)
+    # size in the cluster. in order to be in the cluster, the furthest corner needs to be within x,y offset
+    # determined by the max_region size minus half the box + 20%
+    # TODO: see if we can do this with numpy
+    cluster_candidates = []
+    used_boxes = []
+    # loop over each box
+    for current_index, b in enumerate(boxes):
+        if current_index in used_boxes:
+            continue
+        cluster = [current_index]
+        used_boxes.append(current_index)
+        cluster_boundary = get_cluster_boundary(b, min_region)
+        # find all other boxes that fit inside the boundary
+        for compare_index, compare_box in enumerate(boxes):
+            if compare_index in used_boxes:
+                continue
+
+            # if the box is not inside the potential cluster area, cluster them
+            if not box_inside(cluster_boundary, compare_box):
+                continue
+
+            # get the region if you were to add this box to the cluster
+            potential_cluster = cluster + [compare_index]
+            cluster_region = get_cluster_region(
+                frame_shape, min_region, potential_cluster, boxes
+            )
+            # if region could be smaller and either box would be too small
+            # for the resulting region, dont cluster
+            should_cluster = True
+            if (cluster_region[2] - cluster_region[0]) > min_region:
+                for b in potential_cluster:
+                    box = boxes[b]
+                    # boxes should be more than 5% of the area of the region
+                    if area(box) / area(cluster_region) < 0.05:
+                        should_cluster = False
+                        break
+
+            if should_cluster:
+                cluster.append(compare_index)
+                used_boxes.append(compare_index)
+        cluster_candidates.append(cluster)
+
+    # return the unique clusters only
+    unique = {tuple(sorted(c)) for c in cluster_candidates}
+    return [list(tup) for tup in unique]
+
+
+def get_cluster_region(frame_shape, min_region, cluster, boxes):
+    min_x = frame_shape[1]
+    min_y = frame_shape[0]
+    max_x = 0
+    max_y = 0
+    for b in cluster:
+        min_x = min(boxes[b][0], min_x)
+        min_y = min(boxes[b][1], min_y)
+        max_x = max(boxes[b][2], max_x)
+        max_y = max(boxes[b][3], max_y)
+    return calculate_region(
+        frame_shape, min_x, min_y, max_x, max_y, min_region, multiplier=1.2
+    )
+
+
+def get_consolidated_object_detections(detected_object_groups):
+    """Drop detections that overlap too much"""
+    consolidated_detections = []
+    for group in detected_object_groups.values():
+        # if the group only has 1 item, skip
+        if len(group) == 1:
+            consolidated_detections.append(group[0])
+            continue
+
+        # sort smallest to largest by area
+        sorted_by_area = sorted(group, key=lambda g: g[3])
+
+        for current_detection_idx in range(0, len(sorted_by_area)):
+            current_detection = sorted_by_area[current_detection_idx][2]
+            overlap = 0
+            for to_check_idx in range(
+                min(current_detection_idx + 1, len(sorted_by_area)),
+                len(sorted_by_area),
+            ):
+                to_check = sorted_by_area[to_check_idx][2]
+                intersect_box = intersection(current_detection, to_check)
+                # if 90% of smaller detection is inside of another detection, consolidate
+                if (
+                    intersect_box is not None
+                    and area(intersect_box) / area(current_detection) > 0.9
+                ):
+                    overlap = 1
+                    break
+            if overlap == 0:
+                consolidated_detections.append(sorted_by_area[current_detection_idx])
+
+    return consolidated_detections
+
+
+def get_startup_regions(
+    frame_shape: tuple[int],
+    region_min_size: int,
+    region_grid: list[list[dict[str, any]]],
+) -> list[list[int]]:
+    """Get a list of regions to run on startup."""
+    # return 8 most popular regions for the camera
+    all_cells = np.concatenate(region_grid).flat
+    startup_cells = sorted(all_cells, key=lambda c: len(c["sizes"]), reverse=True)[0:8]
+    regions = []
+
+    for cell in startup_cells:
+        # rest of the cells are empty
+        if not cell["sizes"]:
+            break
+
+        x = frame_shape[1] / GRID_SIZE * (0.5 + cell["x"])
+        y = frame_shape[0] / GRID_SIZE * (0.5 + cell["y"])
+        size = cell["mean"] * frame_shape[1]
+        regions.append(
+            calculate_region(
+                frame_shape,
+                x - size / 2,
+                y - size / 2,
+                x + size / 2,
+                y + size / 2,
+                region_min_size,
+                multiplier=1,
+            )
+        )
+
+    return regions

frigate/video.py

@@ -1,6 +1,5 @@
 import datetime
 import logging
-import math
 import multiprocessing as mp
 import os
 import queue
@@ -15,8 +14,12 @@ import numpy as np
 from setproctitle import setproctitle
 
 from frigate.config import CameraConfig, DetectConfig, ModelConfig
-from frigate.const import ALL_ATTRIBUTE_LABELS, ATTRIBUTE_LABEL_MAP, CACHE_DIR
+from frigate.const import (
-from frigate.detectors.detector_config import PixelFormatEnum
+    ALL_ATTRIBUTE_LABELS,
+    ATTRIBUTE_LABEL_MAP,
+    CACHE_DIR,
+    REQUEST_REGION_GRID,
+)
 from frigate.log import LogPipe
 from frigate.motion import MotionDetector
 from frigate.motion.improved_motion import ImprovedMotionDetector
@@ -24,103 +27,30 @@ from frigate.object_detection import RemoteObjectDetector
 from frigate.track import ObjectTracker
 from frigate.track.norfair_tracker import NorfairTracker
 from frigate.types import PTZMetricsTypes
-from frigate.util.builtin import EventsPerSecond
+from frigate.util.builtin import EventsPerSecond, get_tomorrow_at_2
 from frigate.util.image import (
     FrameManager,
     SharedMemoryFrameManager,
-    area,
-    calculate_region,
     draw_box_with_label,
-    intersection,
+)
-    intersection_over_union,
+from frigate.util.object import (
-    yuv_region_2_bgr,
+    box_inside,
-    yuv_region_2_rgb,
+    create_tensor_input,
-    yuv_region_2_yuv,
+    get_cluster_candidates,
+    get_cluster_region,
+    get_cluster_region_from_grid,
+    get_consolidated_object_detections,
+    get_min_region_size,
+    get_startup_regions,
+    inside_any,
+    intersects_any,
+    is_object_filtered,
 )
 from frigate.util.services import listen
 
 logger = logging.getLogger(__name__)
 
 
-def filtered(obj, objects_to_track, object_filters):
-    object_name = obj[0]
-    object_score = obj[1]
-    object_box = obj[2]
-    object_area = obj[3]
-    object_ratio = obj[4]
-
-    if object_name not in objects_to_track:
-        return True
-
-    if object_name in object_filters:
-        obj_settings = object_filters[object_name]
-
-        # if the min area is larger than the
-        # detected object, don't add it to detected objects
-        if obj_settings.min_area > object_area:
-            return True
-
-        # if the detected object is larger than the
-        # max area, don't add it to detected objects
-        if obj_settings.max_area < object_area:
-            return True
-
-        # if the score is lower than the min_score, skip
-        if obj_settings.min_score > object_score:
-            return True
-
-        # if the object is not proportionally wide enough
-        if obj_settings.min_ratio > object_ratio:
-            return True
-
-        # if the object is proportionally too wide
-        if obj_settings.max_ratio < object_ratio:
-            return True
-
-        if obj_settings.mask is not None:
-            # compute the coordinates of the object and make sure
-            # the location isn't outside the bounds of the image (can happen from rounding)
-            object_xmin = object_box[0]
-            object_xmax = object_box[2]
-            object_ymax = object_box[3]
-            y_location = min(int(object_ymax), len(obj_settings.mask) - 1)
-            x_location = min(
-                int((object_xmax + object_xmin) / 2.0),
-                len(obj_settings.mask[0]) - 1,
-            )
-
-            # if the object is in a masked location, don't add it to detected objects
-            if obj_settings.mask[y_location][x_location] == 0:
-                return True
-
-    return False
-
-
-def get_min_region_size(model_config: ModelConfig) -> int:
-    """Get the min region size."""
-    return max(model_config.height, model_config.width)
-
-
-def create_tensor_input(frame, model_config: ModelConfig, region):
-    if model_config.input_pixel_format == PixelFormatEnum.rgb:
-        cropped_frame = yuv_region_2_rgb(frame, region)
-    elif model_config.input_pixel_format == PixelFormatEnum.bgr:
-        cropped_frame = yuv_region_2_bgr(frame, region)
-    else:
-        cropped_frame = yuv_region_2_yuv(frame, region)
-
-    # Resize if needed
-    if cropped_frame.shape != (model_config.height, model_config.width, 3):
-        cropped_frame = cv2.resize(
-            cropped_frame,
-            dsize=(model_config.width, model_config.height),
-            interpolation=cv2.INTER_LINEAR,
-        )
-
-    # Expand dimensions since the model expects images to have shape: [1, height, width, 3]
-    return np.expand_dims(cropped_frame, axis=0)
-
-
 def stop_ffmpeg(ffmpeg_process, logger):
     logger.info("Terminating the existing ffmpeg process...")
     ffmpeg_process.terminate()
@@ -455,8 +385,10 @@ def track_camera(
     detection_queue,
     result_connection,
     detected_objects_queue,
+    inter_process_queue,
     process_info,
     ptz_metrics,
+    region_grid,
 ):
     stop_event = mp.Event()
 
@@ -471,6 +403,7 @@ def track_camera(
     listen()
 
     frame_queue = process_info["frame_queue"]
+    region_grid_queue = process_info["region_grid_queue"]
     detection_enabled = process_info["detection_enabled"]
     motion_enabled = process_info["motion_enabled"]
     improve_contrast_enabled = process_info["improve_contrast_enabled"]
@@ -499,7 +432,9 @@ def track_camera(
 
     process_frames(
         name,
+        inter_process_queue,
         frame_queue,
+        region_grid_queue,
         frame_shape,
         model_config,
         config.detect,
@@ -515,50 +450,12 @@ def track_camera(
         motion_enabled,
         stop_event,
         ptz_metrics,
+        region_grid,
     )
 
     logger.info(f"{name}: exiting subprocess")
 
 
-def box_overlaps(b1, b2):
-    if b1[2] < b2[0] or b1[0] > b2[2] or b1[1] > b2[3] or b1[3] < b2[1]:
-        return False
-    return True
-
-
-def box_inside(b1, b2):
-    # check if b2 is inside b1
-    if b2[0] >= b1[0] and b2[1] >= b1[1] and b2[2] <= b1[2] and b2[3] <= b1[3]:
-        return True
-    return False
-
-
-def reduce_boxes(boxes, iou_threshold=0.0):
-    clusters = []
-
-    for box in boxes:
-        matched = 0
-        for cluster in clusters:
-            if intersection_over_union(box, cluster) > iou_threshold:
-                matched = 1
-                cluster[0] = min(cluster[0], box[0])
-                cluster[1] = min(cluster[1], box[1])
-                cluster[2] = max(cluster[2], box[2])
-                cluster[3] = max(cluster[3], box[3])
-
-        if not matched:
-            clusters.append(list(box))
-
-    return [tuple(c) for c in clusters]
-
-
-def intersects_any(box_a, boxes):
-    for box in boxes:
-        if box_overlaps(box_a, box):
-            return True
-    return False
-
-
 def detect(
     detect_config: DetectConfig,
     object_detector,
@@ -597,134 +494,17 @@ def detect(
             region,
         )
         # apply object filters
-        if filtered(det, objects_to_track, object_filters):
+        if is_object_filtered(det, objects_to_track, object_filters):
             continue
         detections.append(det)
     return detections
 
 
-def get_cluster_boundary(box, min_region):
-    # compute the max region size for the current box (box is 10% of region)
-    box_width = box[2] - box[0]
-    box_height = box[3] - box[1]
-    max_region_area = abs(box_width * box_height) / 0.1
-    max_region_size = max(min_region, int(math.sqrt(max_region_area)))
-
-    centroid = (box_width / 2 + box[0], box_height / 2 + box[1])
-
-    max_x_dist = int(max_region_size - box_width / 2 * 1.1)
-    max_y_dist = int(max_region_size - box_height / 2 * 1.1)
-
-    return [
-        int(centroid[0] - max_x_dist),
-        int(centroid[1] - max_y_dist),
-        int(centroid[0] + max_x_dist),
-        int(centroid[1] + max_y_dist),
-    ]
-
-
-def get_cluster_candidates(frame_shape, min_region, boxes):
-    # and create a cluster of other boxes using it's max region size
-    # only include boxes where the region is an appropriate(except the region could possibly be smaller?)
-    # size in the cluster. in order to be in the cluster, the furthest corner needs to be within x,y offset
-    # determined by the max_region size minus half the box + 20%
-    # TODO: see if we can do this with numpy
-    cluster_candidates = []
-    used_boxes = []
-    # loop over each box
-    for current_index, b in enumerate(boxes):
-        if current_index in used_boxes:
-            continue
-        cluster = [current_index]
-        used_boxes.append(current_index)
-        cluster_boundary = get_cluster_boundary(b, min_region)
-        # find all other boxes that fit inside the boundary
-        for compare_index, compare_box in enumerate(boxes):
-            if compare_index in used_boxes:
-                continue
-
-            # if the box is not inside the potential cluster area, cluster them
-            if not box_inside(cluster_boundary, compare_box):
-                continue
-
-            # get the region if you were to add this box to the cluster
-            potential_cluster = cluster + [compare_index]
-            cluster_region = get_cluster_region(
-                frame_shape, min_region, potential_cluster, boxes
-            )
-            # if region could be smaller and either box would be too small
-            # for the resulting region, dont cluster
-            should_cluster = True
-            if (cluster_region[2] - cluster_region[0]) > min_region:
-                for b in potential_cluster:
-                    box = boxes[b]
-                    # boxes should be more than 5% of the area of the region
-                    if area(box) / area(cluster_region) < 0.05:
-                        should_cluster = False
-                        break
-
-            if should_cluster:
-                cluster.append(compare_index)
-                used_boxes.append(compare_index)
-        cluster_candidates.append(cluster)
-
-    # return the unique clusters only
-    unique = {tuple(sorted(c)) for c in cluster_candidates}
-    return [list(tup) for tup in unique]
-
-
-def get_cluster_region(frame_shape, min_region, cluster, boxes):
-    min_x = frame_shape[1]
-    min_y = frame_shape[0]
-    max_x = 0
-    max_y = 0
-    for b in cluster:
-        min_x = min(boxes[b][0], min_x)
-        min_y = min(boxes[b][1], min_y)
-        max_x = max(boxes[b][2], max_x)
-        max_y = max(boxes[b][3], max_y)
-    return calculate_region(
-        frame_shape, min_x, min_y, max_x, max_y, min_region, multiplier=1.2
-    )
-
-
-def get_consolidated_object_detections(detected_object_groups):
-    """Drop detections that overlap too much"""
-    consolidated_detections = []
-    for group in detected_object_groups.values():
-        # if the group only has 1 item, skip
-        if len(group) == 1:
-            consolidated_detections.append(group[0])
-            continue
-
-        # sort smallest to largest by area
-        sorted_by_area = sorted(group, key=lambda g: g[3])
-
-        for current_detection_idx in range(0, len(sorted_by_area)):
-            current_detection = sorted_by_area[current_detection_idx][2]
-            overlap = 0
-            for to_check_idx in range(
-                min(current_detection_idx + 1, len(sorted_by_area)),
-                len(sorted_by_area),
-            ):
-                to_check = sorted_by_area[to_check_idx][2]
-                intersect_box = intersection(current_detection, to_check)
-                # if 90% of smaller detection is inside of another detection, consolidate
-                if (
-                    intersect_box is not None
-                    and area(intersect_box) / area(current_detection) > 0.9
-                ):
-                    overlap = 1
-                    break
-            if overlap == 0:
-                consolidated_detections.append(sorted_by_area[current_detection_idx])
-
-    return consolidated_detections
-
-
 def process_frames(
     camera_name: str,
+    inter_process_queue: mp.Queue,
     frame_queue: mp.Queue,
+    region_grid_queue: mp.Queue,
     frame_shape,
     model_config: ModelConfig,
     detect_config: DetectConfig,
@@ -740,20 +520,35 @@ def process_frames(
     motion_enabled: mp.Value,
     stop_event,
     ptz_metrics: PTZMetricsTypes,
+    region_grid,
     exit_on_empty: bool = False,
 ):
     fps = process_info["process_fps"]
     detection_fps = process_info["detection_fps"]
     current_frame_time = process_info["detection_frame"]
+    next_region_update = get_tomorrow_at_2()
 
     fps_tracker = EventsPerSecond()
     fps_tracker.start()
 
-    startup_scan_counter = 0
+    startup_scan = True
 
     region_min_size = get_min_region_size(model_config)
 
     while not stop_event.is_set():
+        if (
+            datetime.datetime.now().astimezone(datetime.timezone.utc)
+            > next_region_update
+        ):
+            inter_process_queue.put((REQUEST_REGION_GRID, camera_name))
+
+            try:
+                region_grid = region_grid_queue.get(True, 10)
+            except queue.Empty:
+                logger.error(f"Unable to get updated region grid for {camera_name}")
+
+            next_region_update = get_tomorrow_at_2()
+
         try:
             if exit_on_empty:
                 frame_time = frame_queue.get(False)
@@ -815,40 +610,48 @@ def process_frames(
                 if obj["id"] not in stationary_object_ids
             ]
 
-            combined_boxes = tracked_object_boxes
+            # get consolidated regions for tracked objects
-            # only add in the motion boxes when not calibrating
-            if not motion_detector.is_calibrating():
-                combined_boxes += motion_boxes
-
-            cluster_candidates = get_cluster_candidates(
-                frame_shape, region_min_size, combined_boxes
-            )
-
             regions = [
                 get_cluster_region(
-                    frame_shape, region_min_size, candidate, combined_boxes
+                    frame_shape, region_min_size, candidate, tracked_object_boxes
+                )
+                for candidate in get_cluster_candidates(
+                    frame_shape, region_min_size, tracked_object_boxes
                 )
-                for candidate in cluster_candidates
             ]
 
-            # if starting up, get the next startup scan region
+            # only add in the motion boxes when not calibrating
-            if startup_scan_counter < 9:
+            if not motion_detector.is_calibrating():
-                ymin = int(frame_shape[0] / 3 * startup_scan_counter / 3)
+                # find motion boxes that are not inside tracked object regions
-                ymax = int(frame_shape[0] / 3 + ymin)
+                standalone_motion_boxes = [
-                xmin = int(frame_shape[1] / 3 * startup_scan_counter / 3)
+                    b for b in motion_boxes if not inside_any(b, regions)
-                xmax = int(frame_shape[1] / 3 + xmin)
+                ]
-                regions.append(
+
-                    calculate_region(
+                if standalone_motion_boxes:
+                    motion_clusters = get_cluster_candidates(
                         frame_shape,
-                        xmin,
-                        ymin,
-                        xmax,
-                        ymax,
                         region_min_size,
-                        multiplier=1.2,
+                        standalone_motion_boxes,
                     )
-                )
+                    motion_regions = [
-                startup_scan_counter += 1
+                        get_cluster_region_from_grid(
+                            frame_shape,
+                            region_min_size,
+                            candidate,
+                            standalone_motion_boxes,
+                            region_grid,
+                        )
+                        for candidate in motion_clusters
+                    ]
+                    regions += motion_regions
+
+            # if starting up, get the next startup scan region
+            if startup_scan:
+                for region in get_startup_regions(
+                    frame_shape, region_min_size, region_grid
+                ):
+                    regions.append(region)
+                startup_scan = False
 
             # resize regions and detect
             # seed with stationary objects

migrations/019_create_regions_table.py

@@ -0,0 +1,35 @@
+"""Peewee migrations -- 019_create_regions_table.py.
+
+Some examples (model - class or model name)::
+
+    > Model = migrator.orm['model_name']            # Return model in current state by name
+
+    > migrator.sql(sql)                             # Run custom SQL
+    > migrator.python(func, *args, **kwargs)        # Run python code
+    > migrator.create_model(Model)                  # Create a model (could be used as decorator)
+    > migrator.remove_model(model, cascade=True)    # Remove a model
+    > migrator.add_fields(model, **fields)          # Add fields to a model
+    > migrator.change_fields(model, **fields)       # Change fields
+    > migrator.remove_fields(model, *field_names, cascade=True)
+    > migrator.rename_field(model, old_field_name, new_field_name)
+    > migrator.rename_table(model, new_table_name)
+    > migrator.add_index(model, *col_names, unique=False)
+    > migrator.drop_index(model, *col_names)
+    > migrator.add_not_null(model, *field_names)
+    > migrator.drop_not_null(model, *field_names)
+    > migrator.add_default(model, field_name, default)
+
+"""
+import peewee as pw
+
+SQL = pw.SQL
+
+
+def migrate(migrator, database, fake=False, **kwargs):
+    migrator.sql(
+        'CREATE TABLE IF NOT EXISTS "regions" ("camera" VARCHAR(20) NOT NULL PRIMARY KEY, "last_update" DATETIME NOT NULL, "grid" JSON)'
+    )
+
+
+def rollback(migrator, database, fake=False, **kwargs):
+    pass