diff --git a/frigate/app.py b/frigate/app.py index a2e300526..1b807dd5b 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -36,7 +36,7 @@ from frigate.events.external import ExternalEventProcessor from frigate.events.maintainer import EventProcessor from frigate.http import create_app from frigate.log import log_process, root_configurer -from frigate.models import Event, Recordings, RecordingsToDelete, Timeline +from frigate.models import Event, Recordings, RecordingsToDelete, Regions, Timeline from frigate.object_detection import ObjectDetectProcess from frigate.object_processing import TrackedObjectProcessor from frigate.output import output_frames @@ -49,6 +49,7 @@ from frigate.stats import StatsEmitter, stats_init from frigate.storage import StorageMaintainer from frigate.timeline import TimelineProcessor from frigate.types import CameraMetricsTypes, FeatureMetricsTypes, PTZMetricsTypes +from frigate.util.object import get_camera_regions_grid from frigate.version import VERSION from frigate.video import capture_camera, track_camera from frigate.watchdog import FrigateWatchdog @@ -69,6 +70,7 @@ class FrigateApp: self.feature_metrics: dict[str, FeatureMetricsTypes] = {} self.ptz_metrics: dict[str, PTZMetricsTypes] = {} self.processes: dict[str, int] = {} + self.region_grids: dict[str, list[list[dict[str, int]]]] = {} def set_environment_vars(self) -> None: for key, value in self.config.environment_vars.items(): @@ -161,6 +163,7 @@ class FrigateApp: # issue https://github.com/python/typeshed/issues/8799 # from mypy 0.981 onwards "frame_queue": mp.Queue(maxsize=2), + "region_grid_queue": mp.Queue(maxsize=1), "capture_process": None, "process": None, "audio_rms": mp.Value("d", 0.0), # type: ignore[typeddict-item] @@ -327,7 +330,7 @@ class FrigateApp: 60, 10 * len([c for c in self.config.cameras.values() if c.enabled]) ), ) - models = [Event, Recordings, RecordingsToDelete, Timeline] + models = [Event, Recordings, RecordingsToDelete, Regions, Timeline] self.db.bind(models) def init_stats(self) -> None: @@ -452,6 +455,17 @@ class FrigateApp: output_processor.start() logger.info(f"Output process started: {output_processor.pid}") + def init_historical_regions(self) -> None: + # delete region grids for removed or renamed cameras + cameras = list(self.config.cameras.keys()) + Regions.delete().where(~(Regions.camera << cameras)).execute() + + # create or update region grids for each camera + for camera in self.config.cameras.values(): + self.region_grids[camera.name] = get_camera_regions_grid( + camera.name, camera.detect + ) + def start_camera_processors(self) -> None: for name, config in self.config.cameras.items(): if not self.config.cameras[name].enabled: @@ -469,8 +483,10 @@ class FrigateApp: self.detection_queue, self.detection_out_events[name], self.detected_frames_queue, + self.inter_process_queue, self.camera_metrics[name], self.ptz_metrics[name], + self.region_grids[name], ), ) camera_process.daemon = True @@ -611,6 +627,7 @@ class FrigateApp: self.start_detectors() self.start_video_output_processor() self.start_ptz_autotracker() + self.init_historical_regions() self.start_detected_frames_processor() self.start_camera_processors() self.start_camera_capture_processes() diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index e97095c8a..f3886a331 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -5,10 +5,11 @@ from abc import ABC, abstractmethod from typing import Any, Callable from frigate.config import FrigateConfig -from frigate.const import INSERT_MANY_RECORDINGS +from frigate.const import INSERT_MANY_RECORDINGS, REQUEST_REGION_GRID from frigate.models import Recordings from frigate.ptz.onvif import OnvifCommandEnum, OnvifController from frigate.types import CameraMetricsTypes, FeatureMetricsTypes, PTZMetricsTypes +from frigate.util.object import get_camera_regions_grid from frigate.util.services import restart_frigate logger = logging.getLogger(__name__) @@ -90,6 +91,11 @@ class Dispatcher: restart_frigate() elif topic == INSERT_MANY_RECORDINGS: Recordings.insert_many(payload).execute() + elif topic == REQUEST_REGION_GRID: + camera = payload + self.camera_metrics[camera]["region_grid_queue"].put( + get_camera_regions_grid(camera, self.config.cameras[camera].detect) + ) else: self.publish(topic, payload, retain=False) diff --git a/frigate/const.py b/frigate/const.py index c6912471b..56d0f4517 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -51,3 +51,4 @@ MAX_PLAYLIST_SECONDS = 7200 # support 2 hour segments for a single playlist to # Internal Comms Topics INSERT_MANY_RECORDINGS = "insert_many_recordings" +REQUEST_REGION_GRID = "request_region_grid" diff --git a/frigate/models.py b/frigate/models.py index b29ae91dc..65cbfbaac 100644 --- a/frigate/models.py +++ b/frigate/models.py @@ -57,6 +57,12 @@ class Timeline(Model): # type: ignore[misc] data = JSONField() # ex: tracked object id, region, box, etc. +class Regions(Model): # type: ignore[misc] + camera = CharField(null=False, primary_key=True, max_length=20) + grid = JSONField() # json blob of grid + last_update = DateTimeField() + + class Recordings(Model): # type: ignore[misc] id = CharField(null=False, primary_key=True, max_length=30) camera = CharField(index=True, max_length=20) diff --git a/frigate/test/test_reduce_boxes.py b/frigate/test/test_reduce_boxes.py index d26fcd40c..5ac913dfe 100644 --- a/frigate/test/test_reduce_boxes.py +++ b/frigate/test/test_reduce_boxes.py @@ -1,6 +1,6 @@ from unittest import TestCase, main -from frigate.video import box_overlaps, reduce_boxes +from frigate.util.object import box_overlaps, reduce_boxes class TestBoxOverlaps(TestCase): diff --git a/frigate/test/test_video.py b/frigate/test/test_video.py index 99736f658..9fd46a877 100644 --- a/frigate/test/test_video.py +++ b/frigate/test/test_video.py @@ -6,10 +6,11 @@ from norfair.drawing.color import Palette from norfair.drawing.drawer import Drawer from frigate.util.image import intersection -from frigate.video import ( +from frigate.util.object import ( get_cluster_boundary, get_cluster_candidates, get_cluster_region, + get_region_from_grid, ) @@ -190,3 +191,36 @@ class TestObjectBoundingBoxes(unittest.TestCase): assert intersection(box_a, box_b) == None assert intersection(box_b, box_c) == (899, 128, 985, 151) + + +class TestRegionGrid(unittest.TestCase): + def setUp(self) -> None: + pass + + def test_region_in_range(self): + """Test that region is kept at minimal size when within std dev.""" + frame_shape = (720, 1280) + box = [450, 450, 550, 550] + region_grid = [ + [], + [], + [], + [{}, {}, {}, {}, {}, {"sizes": [0.25], "mean": 0.26, "std_dev": 0.01}], + ] + + region = get_region_from_grid(frame_shape, box, 320, region_grid) + assert region[2] - region[0] == 320 + + def test_region_out_of_range(self): + """Test that region is upsized when outside of std dev.""" + frame_shape = (720, 1280) + box = [450, 450, 550, 550] + region_grid = [ + [], + [], + [], + [{}, {}, {}, {}, {}, {"sizes": [0.5], "mean": 0.5, "std_dev": 0.1}], + ] + + region = get_region_from_grid(frame_shape, box, 320, region_grid) + assert region[2] - region[0] > 320 diff --git a/frigate/track/norfair_tracker.py b/frigate/track/norfair_tracker.py index ff63ba563..42a2fde2f 100644 --- a/frigate/track/norfair_tracker.py +++ b/frigate/track/norfair_tracker.py @@ -77,7 +77,7 @@ class NorfairTracker(ObjectTracker): self.tracker = Tracker( distance_function=frigate_distance, distance_threshold=2.5, - initialization_delay=config.detect.fps / 2, + initialization_delay=0, hit_counter_max=self.max_disappeared, ) if self.ptz_autotracker_enabled.value: @@ -106,6 +106,11 @@ class NorfairTracker(ObjectTracker): "ymax": self.detect_config.height, } + # start object with a hit count of `fps` to avoid quick detection -> loss + next( + (o for o in self.tracker.tracked_objects if o.global_id == track_id) + ).hit_counter = self.camera_config.detect.fps + def deregister(self, id, track_id): del self.tracked_objects[id] del self.disappeared[id] diff --git a/frigate/util/builtin.py b/frigate/util/builtin.py index 5cb01e1c5..2a9b7053a 100644 --- a/frigate/util/builtin.py +++ b/frigate/util/builtin.py @@ -14,6 +14,7 @@ import numpy as np import pytz import yaml from ruamel.yaml import YAML +from tzlocal import get_localzone from frigate.const import REGEX_HTTP_CAMERA_USER_PASS, REGEX_RTSP_CAMERA_USER_PASS @@ -262,3 +263,10 @@ def find_by_key(dictionary, target_key): if result is not None: return result return None + + +def get_tomorrow_at_2() -> datetime.datetime: + tomorrow = datetime.datetime.now(get_localzone()) + datetime.timedelta(days=1) + return tomorrow.replace(hour=2, minute=0, second=0).astimezone( + datetime.timezone.utc + ) diff --git a/frigate/util/object.py b/frigate/util/object.py new file mode 100644 index 000000000..3fa98df59 --- /dev/null +++ b/frigate/util/object.py @@ -0,0 +1,481 @@ +"""Utils for reading and writing object detection data.""" + +import datetime +import logging +import math + +import cv2 +import numpy as np +from peewee import DoesNotExist + +from frigate.config import DetectConfig, ModelConfig +from frigate.detectors.detector_config import PixelFormatEnum +from frigate.models import Event, Regions, Timeline +from frigate.util.image import ( + area, + calculate_region, + intersection, + intersection_over_union, + yuv_region_2_bgr, + yuv_region_2_rgb, + yuv_region_2_yuv, +) + +logger = logging.getLogger(__name__) + +GRID_SIZE = 8 + + +def get_camera_regions_grid( + name: str, detect: DetectConfig +) -> list[list[dict[str, any]]]: + """Build a grid of expected region sizes for a camera.""" + # get grid from db if available + try: + regions: Regions = Regions.select().where(Regions.camera == name).get() + grid = regions.grid + last_update = regions.last_update + except DoesNotExist: + grid = [] + for x in range(GRID_SIZE): + row = [] + for y in range(GRID_SIZE): + row.append({"sizes": []}) + grid.append(row) + last_update = 0 + + # get events for timeline entries + events = ( + Event.select(Event.id) + .where(Event.camera == name) + .where((Event.false_positive == None) | (Event.false_positive == False)) + .where(Event.start_time > last_update) + ) + valid_event_ids = [e["id"] for e in events.dicts()] + logger.debug(f"Found {len(valid_event_ids)} new events for {name}") + + # no new events, return as is + if not valid_event_ids: + return grid + + new_update = datetime.datetime.now().timestamp() + timeline = ( + Timeline.select( + *[ + Timeline.camera, + Timeline.source, + Timeline.data, + ] + ) + .where(Timeline.source_id << valid_event_ids) + .limit(10000) + .dicts() + ) + + logger.debug(f"Found {len(timeline)} new entries for {name}") + + width = detect.width + height = detect.height + + for t in timeline: + if t.get("source") != "tracked_object": + continue + + box = t["data"]["box"] + + # calculate centroid position + x = box[0] + (box[2] / 2) + y = box[1] + (box[3] / 2) + + x_pos = int(x * GRID_SIZE) + y_pos = int(y * GRID_SIZE) + + calculated_region = calculate_region( + (height, width), + box[0] * width, + box[1] * height, + (box[0] + box[2]) * width, + (box[1] + box[3]) * height, + 320, + 1.35, + ) + # save width of region to grid as relative + grid[x_pos][y_pos]["sizes"].append( + (calculated_region[2] - calculated_region[0]) / width + ) + + for x in range(GRID_SIZE): + for y in range(GRID_SIZE): + cell = grid[x][y] + + if len(cell["sizes"]) == 0: + continue + + std_dev = np.std(cell["sizes"]) + mean = np.mean(cell["sizes"]) + logger.debug(f"std dev: {std_dev} mean: {mean}") + cell["x"] = x + cell["y"] = y + cell["std_dev"] = std_dev + cell["mean"] = mean + + # update db with new grid + region = { + Regions.camera: name, + Regions.grid: grid, + Regions.last_update: new_update, + } + ( + Regions.insert(region) + .on_conflict( + conflict_target=[Regions.camera], + update=region, + ) + .execute() + ) + + return grid + + +def get_cluster_region_from_grid(frame_shape, min_region, cluster, boxes, region_grid): + min_x = frame_shape[1] + min_y = frame_shape[0] + max_x = 0 + max_y = 0 + for b in cluster: + min_x = min(boxes[b][0], min_x) + min_y = min(boxes[b][1], min_y) + max_x = max(boxes[b][2], max_x) + max_y = max(boxes[b][3], max_y) + return get_region_from_grid( + frame_shape, [min_x, min_y, max_x, max_y], min_region, region_grid + ) + + +def get_region_from_grid( + frame_shape: tuple[int], + cluster: list[int], + min_region: int, + region_grid: list[list[dict[str, any]]], +) -> list[int]: + """Get a region for a box based on the region grid.""" + box = calculate_region( + frame_shape, cluster[0], cluster[1], cluster[2], cluster[3], min_region + ) + centroid = ( + box[0] + (min(frame_shape[1], box[2]) - box[0]) / 2, + box[1] + (min(frame_shape[0], box[3]) - box[1]) / 2, + ) + grid_x = int(centroid[0] / frame_shape[1] * GRID_SIZE) + grid_y = int(centroid[1] / frame_shape[0] * GRID_SIZE) + + cell = region_grid[grid_x][grid_y] + + # if there is no known data, get standard region for motion box + if not cell or not cell["sizes"]: + return calculate_region(frame_shape, box[0], box[1], box[2], box[3], min_region) + + # convert the calculated region size to relative + calc_size = (box[2] - box[0]) / frame_shape[1] + + # if region is within expected size, don't resize + if ( + (cell["mean"] - cell["std_dev"]) + <= calc_size + <= (cell["mean"] + cell["std_dev"]) + ): + return box + # TODO not sure how to handle case where cluster is larger than expected region + elif calc_size > (cell["mean"] + cell["std_dev"]): + return box + + size = cell["mean"] * frame_shape[1] + + # get region based on grid size + return calculate_region( + frame_shape, + max(0, centroid[0] - size / 2), + max(0, centroid[1] - size / 2), + min(frame_shape[1], centroid[0] + size / 2), + min(frame_shape[0], centroid[1] + size / 2), + min_region, + ) + + +def is_object_filtered(obj, objects_to_track, object_filters): + object_name = obj[0] + object_score = obj[1] + object_box = obj[2] + object_area = obj[3] + object_ratio = obj[4] + + if object_name not in objects_to_track: + return True + + if object_name in object_filters: + obj_settings = object_filters[object_name] + + # if the min area is larger than the + # detected object, don't add it to detected objects + if obj_settings.min_area > object_area: + return True + + # if the detected object is larger than the + # max area, don't add it to detected objects + if obj_settings.max_area < object_area: + return True + + # if the score is lower than the min_score, skip + if obj_settings.min_score > object_score: + return True + + # if the object is not proportionally wide enough + if obj_settings.min_ratio > object_ratio: + return True + + # if the object is proportionally too wide + if obj_settings.max_ratio < object_ratio: + return True + + if obj_settings.mask is not None: + # compute the coordinates of the object and make sure + # the location isn't outside the bounds of the image (can happen from rounding) + object_xmin = object_box[0] + object_xmax = object_box[2] + object_ymax = object_box[3] + y_location = min(int(object_ymax), len(obj_settings.mask) - 1) + x_location = min( + int((object_xmax + object_xmin) / 2.0), + len(obj_settings.mask[0]) - 1, + ) + + # if the object is in a masked location, don't add it to detected objects + if obj_settings.mask[y_location][x_location] == 0: + return True + + return False + + +def get_min_region_size(model_config: ModelConfig) -> int: + """Get the min region size.""" + return max(model_config.height, model_config.width) + + +def create_tensor_input(frame, model_config: ModelConfig, region): + if model_config.input_pixel_format == PixelFormatEnum.rgb: + cropped_frame = yuv_region_2_rgb(frame, region) + elif model_config.input_pixel_format == PixelFormatEnum.bgr: + cropped_frame = yuv_region_2_bgr(frame, region) + else: + cropped_frame = yuv_region_2_yuv(frame, region) + + # Resize if needed + if cropped_frame.shape != (model_config.height, model_config.width, 3): + cropped_frame = cv2.resize( + cropped_frame, + dsize=(model_config.width, model_config.height), + interpolation=cv2.INTER_LINEAR, + ) + + # Expand dimensions since the model expects images to have shape: [1, height, width, 3] + return np.expand_dims(cropped_frame, axis=0) + + +def box_overlaps(b1, b2): + if b1[2] < b2[0] or b1[0] > b2[2] or b1[1] > b2[3] or b1[3] < b2[1]: + return False + return True + + +def box_inside(b1, b2): + # check if b2 is inside b1 + if b2[0] >= b1[0] and b2[1] >= b1[1] and b2[2] <= b1[2] and b2[3] <= b1[3]: + return True + return False + + +def reduce_boxes(boxes, iou_threshold=0.0): + clusters = [] + + for box in boxes: + matched = 0 + for cluster in clusters: + if intersection_over_union(box, cluster) > iou_threshold: + matched = 1 + cluster[0] = min(cluster[0], box[0]) + cluster[1] = min(cluster[1], box[1]) + cluster[2] = max(cluster[2], box[2]) + cluster[3] = max(cluster[3], box[3]) + + if not matched: + clusters.append(list(box)) + + return [tuple(c) for c in clusters] + + +def intersects_any(box_a, boxes): + for box in boxes: + if box_overlaps(box_a, box): + return True + return False + + +def inside_any(box_a, boxes): + for box in boxes: + # check if box_a is inside of box + if box_inside(box, box_a): + return True + return False + + +def get_cluster_boundary(box, min_region): + # compute the max region size for the current box (box is 10% of region) + box_width = box[2] - box[0] + box_height = box[3] - box[1] + max_region_area = abs(box_width * box_height) / 0.1 + max_region_size = max(min_region, int(math.sqrt(max_region_area))) + + centroid = (box_width / 2 + box[0], box_height / 2 + box[1]) + + max_x_dist = int(max_region_size - box_width / 2 * 1.1) + max_y_dist = int(max_region_size - box_height / 2 * 1.1) + + return [ + int(centroid[0] - max_x_dist), + int(centroid[1] - max_y_dist), + int(centroid[0] + max_x_dist), + int(centroid[1] + max_y_dist), + ] + + +def get_cluster_candidates(frame_shape, min_region, boxes): + # and create a cluster of other boxes using it's max region size + # only include boxes where the region is an appropriate(except the region could possibly be smaller?) + # size in the cluster. in order to be in the cluster, the furthest corner needs to be within x,y offset + # determined by the max_region size minus half the box + 20% + # TODO: see if we can do this with numpy + cluster_candidates = [] + used_boxes = [] + # loop over each box + for current_index, b in enumerate(boxes): + if current_index in used_boxes: + continue + cluster = [current_index] + used_boxes.append(current_index) + cluster_boundary = get_cluster_boundary(b, min_region) + # find all other boxes that fit inside the boundary + for compare_index, compare_box in enumerate(boxes): + if compare_index in used_boxes: + continue + + # if the box is not inside the potential cluster area, cluster them + if not box_inside(cluster_boundary, compare_box): + continue + + # get the region if you were to add this box to the cluster + potential_cluster = cluster + [compare_index] + cluster_region = get_cluster_region( + frame_shape, min_region, potential_cluster, boxes + ) + # if region could be smaller and either box would be too small + # for the resulting region, dont cluster + should_cluster = True + if (cluster_region[2] - cluster_region[0]) > min_region: + for b in potential_cluster: + box = boxes[b] + # boxes should be more than 5% of the area of the region + if area(box) / area(cluster_region) < 0.05: + should_cluster = False + break + + if should_cluster: + cluster.append(compare_index) + used_boxes.append(compare_index) + cluster_candidates.append(cluster) + + # return the unique clusters only + unique = {tuple(sorted(c)) for c in cluster_candidates} + return [list(tup) for tup in unique] + + +def get_cluster_region(frame_shape, min_region, cluster, boxes): + min_x = frame_shape[1] + min_y = frame_shape[0] + max_x = 0 + max_y = 0 + for b in cluster: + min_x = min(boxes[b][0], min_x) + min_y = min(boxes[b][1], min_y) + max_x = max(boxes[b][2], max_x) + max_y = max(boxes[b][3], max_y) + return calculate_region( + frame_shape, min_x, min_y, max_x, max_y, min_region, multiplier=1.2 + ) + + +def get_consolidated_object_detections(detected_object_groups): + """Drop detections that overlap too much""" + consolidated_detections = [] + for group in detected_object_groups.values(): + # if the group only has 1 item, skip + if len(group) == 1: + consolidated_detections.append(group[0]) + continue + + # sort smallest to largest by area + sorted_by_area = sorted(group, key=lambda g: g[3]) + + for current_detection_idx in range(0, len(sorted_by_area)): + current_detection = sorted_by_area[current_detection_idx][2] + overlap = 0 + for to_check_idx in range( + min(current_detection_idx + 1, len(sorted_by_area)), + len(sorted_by_area), + ): + to_check = sorted_by_area[to_check_idx][2] + intersect_box = intersection(current_detection, to_check) + # if 90% of smaller detection is inside of another detection, consolidate + if ( + intersect_box is not None + and area(intersect_box) / area(current_detection) > 0.9 + ): + overlap = 1 + break + if overlap == 0: + consolidated_detections.append(sorted_by_area[current_detection_idx]) + + return consolidated_detections + + +def get_startup_regions( + frame_shape: tuple[int], + region_min_size: int, + region_grid: list[list[dict[str, any]]], +) -> list[list[int]]: + """Get a list of regions to run on startup.""" + # return 8 most popular regions for the camera + all_cells = np.concatenate(region_grid).flat + startup_cells = sorted(all_cells, key=lambda c: len(c["sizes"]), reverse=True)[0:8] + regions = [] + + for cell in startup_cells: + # rest of the cells are empty + if not cell["sizes"]: + break + + x = frame_shape[1] / GRID_SIZE * (0.5 + cell["x"]) + y = frame_shape[0] / GRID_SIZE * (0.5 + cell["y"]) + size = cell["mean"] * frame_shape[1] + regions.append( + calculate_region( + frame_shape, + x - size / 2, + y - size / 2, + x + size / 2, + y + size / 2, + region_min_size, + multiplier=1, + ) + ) + + return regions diff --git a/frigate/video.py b/frigate/video.py index 47e65811d..b961cc8a6 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -1,6 +1,5 @@ import datetime import logging -import math import multiprocessing as mp import os import queue @@ -15,8 +14,12 @@ import numpy as np from setproctitle import setproctitle from frigate.config import CameraConfig, DetectConfig, ModelConfig -from frigate.const import ALL_ATTRIBUTE_LABELS, ATTRIBUTE_LABEL_MAP, CACHE_DIR -from frigate.detectors.detector_config import PixelFormatEnum +from frigate.const import ( + ALL_ATTRIBUTE_LABELS, + ATTRIBUTE_LABEL_MAP, + CACHE_DIR, + REQUEST_REGION_GRID, +) from frigate.log import LogPipe from frigate.motion import MotionDetector from frigate.motion.improved_motion import ImprovedMotionDetector @@ -24,103 +27,30 @@ from frigate.object_detection import RemoteObjectDetector from frigate.track import ObjectTracker from frigate.track.norfair_tracker import NorfairTracker from frigate.types import PTZMetricsTypes -from frigate.util.builtin import EventsPerSecond +from frigate.util.builtin import EventsPerSecond, get_tomorrow_at_2 from frigate.util.image import ( FrameManager, SharedMemoryFrameManager, - area, - calculate_region, draw_box_with_label, - intersection, - intersection_over_union, - yuv_region_2_bgr, - yuv_region_2_rgb, - yuv_region_2_yuv, +) +from frigate.util.object import ( + box_inside, + create_tensor_input, + get_cluster_candidates, + get_cluster_region, + get_cluster_region_from_grid, + get_consolidated_object_detections, + get_min_region_size, + get_startup_regions, + inside_any, + intersects_any, + is_object_filtered, ) from frigate.util.services import listen logger = logging.getLogger(__name__) -def filtered(obj, objects_to_track, object_filters): - object_name = obj[0] - object_score = obj[1] - object_box = obj[2] - object_area = obj[3] - object_ratio = obj[4] - - if object_name not in objects_to_track: - return True - - if object_name in object_filters: - obj_settings = object_filters[object_name] - - # if the min area is larger than the - # detected object, don't add it to detected objects - if obj_settings.min_area > object_area: - return True - - # if the detected object is larger than the - # max area, don't add it to detected objects - if obj_settings.max_area < object_area: - return True - - # if the score is lower than the min_score, skip - if obj_settings.min_score > object_score: - return True - - # if the object is not proportionally wide enough - if obj_settings.min_ratio > object_ratio: - return True - - # if the object is proportionally too wide - if obj_settings.max_ratio < object_ratio: - return True - - if obj_settings.mask is not None: - # compute the coordinates of the object and make sure - # the location isn't outside the bounds of the image (can happen from rounding) - object_xmin = object_box[0] - object_xmax = object_box[2] - object_ymax = object_box[3] - y_location = min(int(object_ymax), len(obj_settings.mask) - 1) - x_location = min( - int((object_xmax + object_xmin) / 2.0), - len(obj_settings.mask[0]) - 1, - ) - - # if the object is in a masked location, don't add it to detected objects - if obj_settings.mask[y_location][x_location] == 0: - return True - - return False - - -def get_min_region_size(model_config: ModelConfig) -> int: - """Get the min region size.""" - return max(model_config.height, model_config.width) - - -def create_tensor_input(frame, model_config: ModelConfig, region): - if model_config.input_pixel_format == PixelFormatEnum.rgb: - cropped_frame = yuv_region_2_rgb(frame, region) - elif model_config.input_pixel_format == PixelFormatEnum.bgr: - cropped_frame = yuv_region_2_bgr(frame, region) - else: - cropped_frame = yuv_region_2_yuv(frame, region) - - # Resize if needed - if cropped_frame.shape != (model_config.height, model_config.width, 3): - cropped_frame = cv2.resize( - cropped_frame, - dsize=(model_config.width, model_config.height), - interpolation=cv2.INTER_LINEAR, - ) - - # Expand dimensions since the model expects images to have shape: [1, height, width, 3] - return np.expand_dims(cropped_frame, axis=0) - - def stop_ffmpeg(ffmpeg_process, logger): logger.info("Terminating the existing ffmpeg process...") ffmpeg_process.terminate() @@ -455,8 +385,10 @@ def track_camera( detection_queue, result_connection, detected_objects_queue, + inter_process_queue, process_info, ptz_metrics, + region_grid, ): stop_event = mp.Event() @@ -471,6 +403,7 @@ def track_camera( listen() frame_queue = process_info["frame_queue"] + region_grid_queue = process_info["region_grid_queue"] detection_enabled = process_info["detection_enabled"] motion_enabled = process_info["motion_enabled"] improve_contrast_enabled = process_info["improve_contrast_enabled"] @@ -499,7 +432,9 @@ def track_camera( process_frames( name, + inter_process_queue, frame_queue, + region_grid_queue, frame_shape, model_config, config.detect, @@ -515,50 +450,12 @@ def track_camera( motion_enabled, stop_event, ptz_metrics, + region_grid, ) logger.info(f"{name}: exiting subprocess") -def box_overlaps(b1, b2): - if b1[2] < b2[0] or b1[0] > b2[2] or b1[1] > b2[3] or b1[3] < b2[1]: - return False - return True - - -def box_inside(b1, b2): - # check if b2 is inside b1 - if b2[0] >= b1[0] and b2[1] >= b1[1] and b2[2] <= b1[2] and b2[3] <= b1[3]: - return True - return False - - -def reduce_boxes(boxes, iou_threshold=0.0): - clusters = [] - - for box in boxes: - matched = 0 - for cluster in clusters: - if intersection_over_union(box, cluster) > iou_threshold: - matched = 1 - cluster[0] = min(cluster[0], box[0]) - cluster[1] = min(cluster[1], box[1]) - cluster[2] = max(cluster[2], box[2]) - cluster[3] = max(cluster[3], box[3]) - - if not matched: - clusters.append(list(box)) - - return [tuple(c) for c in clusters] - - -def intersects_any(box_a, boxes): - for box in boxes: - if box_overlaps(box_a, box): - return True - return False - - def detect( detect_config: DetectConfig, object_detector, @@ -597,134 +494,17 @@ def detect( region, ) # apply object filters - if filtered(det, objects_to_track, object_filters): + if is_object_filtered(det, objects_to_track, object_filters): continue detections.append(det) return detections -def get_cluster_boundary(box, min_region): - # compute the max region size for the current box (box is 10% of region) - box_width = box[2] - box[0] - box_height = box[3] - box[1] - max_region_area = abs(box_width * box_height) / 0.1 - max_region_size = max(min_region, int(math.sqrt(max_region_area))) - - centroid = (box_width / 2 + box[0], box_height / 2 + box[1]) - - max_x_dist = int(max_region_size - box_width / 2 * 1.1) - max_y_dist = int(max_region_size - box_height / 2 * 1.1) - - return [ - int(centroid[0] - max_x_dist), - int(centroid[1] - max_y_dist), - int(centroid[0] + max_x_dist), - int(centroid[1] + max_y_dist), - ] - - -def get_cluster_candidates(frame_shape, min_region, boxes): - # and create a cluster of other boxes using it's max region size - # only include boxes where the region is an appropriate(except the region could possibly be smaller?) - # size in the cluster. in order to be in the cluster, the furthest corner needs to be within x,y offset - # determined by the max_region size minus half the box + 20% - # TODO: see if we can do this with numpy - cluster_candidates = [] - used_boxes = [] - # loop over each box - for current_index, b in enumerate(boxes): - if current_index in used_boxes: - continue - cluster = [current_index] - used_boxes.append(current_index) - cluster_boundary = get_cluster_boundary(b, min_region) - # find all other boxes that fit inside the boundary - for compare_index, compare_box in enumerate(boxes): - if compare_index in used_boxes: - continue - - # if the box is not inside the potential cluster area, cluster them - if not box_inside(cluster_boundary, compare_box): - continue - - # get the region if you were to add this box to the cluster - potential_cluster = cluster + [compare_index] - cluster_region = get_cluster_region( - frame_shape, min_region, potential_cluster, boxes - ) - # if region could be smaller and either box would be too small - # for the resulting region, dont cluster - should_cluster = True - if (cluster_region[2] - cluster_region[0]) > min_region: - for b in potential_cluster: - box = boxes[b] - # boxes should be more than 5% of the area of the region - if area(box) / area(cluster_region) < 0.05: - should_cluster = False - break - - if should_cluster: - cluster.append(compare_index) - used_boxes.append(compare_index) - cluster_candidates.append(cluster) - - # return the unique clusters only - unique = {tuple(sorted(c)) for c in cluster_candidates} - return [list(tup) for tup in unique] - - -def get_cluster_region(frame_shape, min_region, cluster, boxes): - min_x = frame_shape[1] - min_y = frame_shape[0] - max_x = 0 - max_y = 0 - for b in cluster: - min_x = min(boxes[b][0], min_x) - min_y = min(boxes[b][1], min_y) - max_x = max(boxes[b][2], max_x) - max_y = max(boxes[b][3], max_y) - return calculate_region( - frame_shape, min_x, min_y, max_x, max_y, min_region, multiplier=1.2 - ) - - -def get_consolidated_object_detections(detected_object_groups): - """Drop detections that overlap too much""" - consolidated_detections = [] - for group in detected_object_groups.values(): - # if the group only has 1 item, skip - if len(group) == 1: - consolidated_detections.append(group[0]) - continue - - # sort smallest to largest by area - sorted_by_area = sorted(group, key=lambda g: g[3]) - - for current_detection_idx in range(0, len(sorted_by_area)): - current_detection = sorted_by_area[current_detection_idx][2] - overlap = 0 - for to_check_idx in range( - min(current_detection_idx + 1, len(sorted_by_area)), - len(sorted_by_area), - ): - to_check = sorted_by_area[to_check_idx][2] - intersect_box = intersection(current_detection, to_check) - # if 90% of smaller detection is inside of another detection, consolidate - if ( - intersect_box is not None - and area(intersect_box) / area(current_detection) > 0.9 - ): - overlap = 1 - break - if overlap == 0: - consolidated_detections.append(sorted_by_area[current_detection_idx]) - - return consolidated_detections - - def process_frames( camera_name: str, + inter_process_queue: mp.Queue, frame_queue: mp.Queue, + region_grid_queue: mp.Queue, frame_shape, model_config: ModelConfig, detect_config: DetectConfig, @@ -740,20 +520,35 @@ def process_frames( motion_enabled: mp.Value, stop_event, ptz_metrics: PTZMetricsTypes, + region_grid, exit_on_empty: bool = False, ): fps = process_info["process_fps"] detection_fps = process_info["detection_fps"] current_frame_time = process_info["detection_frame"] + next_region_update = get_tomorrow_at_2() fps_tracker = EventsPerSecond() fps_tracker.start() - startup_scan_counter = 0 + startup_scan = True region_min_size = get_min_region_size(model_config) while not stop_event.is_set(): + if ( + datetime.datetime.now().astimezone(datetime.timezone.utc) + > next_region_update + ): + inter_process_queue.put((REQUEST_REGION_GRID, camera_name)) + + try: + region_grid = region_grid_queue.get(True, 10) + except queue.Empty: + logger.error(f"Unable to get updated region grid for {camera_name}") + + next_region_update = get_tomorrow_at_2() + try: if exit_on_empty: frame_time = frame_queue.get(False) @@ -815,40 +610,48 @@ def process_frames( if obj["id"] not in stationary_object_ids ] - combined_boxes = tracked_object_boxes - # only add in the motion boxes when not calibrating - if not motion_detector.is_calibrating(): - combined_boxes += motion_boxes - - cluster_candidates = get_cluster_candidates( - frame_shape, region_min_size, combined_boxes - ) - + # get consolidated regions for tracked objects regions = [ get_cluster_region( - frame_shape, region_min_size, candidate, combined_boxes + frame_shape, region_min_size, candidate, tracked_object_boxes + ) + for candidate in get_cluster_candidates( + frame_shape, region_min_size, tracked_object_boxes ) - for candidate in cluster_candidates ] - # if starting up, get the next startup scan region - if startup_scan_counter < 9: - ymin = int(frame_shape[0] / 3 * startup_scan_counter / 3) - ymax = int(frame_shape[0] / 3 + ymin) - xmin = int(frame_shape[1] / 3 * startup_scan_counter / 3) - xmax = int(frame_shape[1] / 3 + xmin) - regions.append( - calculate_region( + # only add in the motion boxes when not calibrating + if not motion_detector.is_calibrating(): + # find motion boxes that are not inside tracked object regions + standalone_motion_boxes = [ + b for b in motion_boxes if not inside_any(b, regions) + ] + + if standalone_motion_boxes: + motion_clusters = get_cluster_candidates( frame_shape, - xmin, - ymin, - xmax, - ymax, region_min_size, - multiplier=1.2, + standalone_motion_boxes, ) - ) - startup_scan_counter += 1 + motion_regions = [ + get_cluster_region_from_grid( + frame_shape, + region_min_size, + candidate, + standalone_motion_boxes, + region_grid, + ) + for candidate in motion_clusters + ] + regions += motion_regions + + # if starting up, get the next startup scan region + if startup_scan: + for region in get_startup_regions( + frame_shape, region_min_size, region_grid + ): + regions.append(region) + startup_scan = False # resize regions and detect # seed with stationary objects diff --git a/migrations/019_create_regions_table.py b/migrations/019_create_regions_table.py new file mode 100644 index 000000000..e1492581b --- /dev/null +++ b/migrations/019_create_regions_table.py @@ -0,0 +1,35 @@ +"""Peewee migrations -- 019_create_regions_table.py. + +Some examples (model - class or model name):: + + > Model = migrator.orm['model_name'] # Return model in current state by name + + > migrator.sql(sql) # Run custom SQL + > migrator.python(func, *args, **kwargs) # Run python code + > migrator.create_model(Model) # Create a model (could be used as decorator) + > migrator.remove_model(model, cascade=True) # Remove a model + > migrator.add_fields(model, **fields) # Add fields to a model + > migrator.change_fields(model, **fields) # Change fields + > migrator.remove_fields(model, *field_names, cascade=True) + > migrator.rename_field(model, old_field_name, new_field_name) + > migrator.rename_table(model, new_table_name) + > migrator.add_index(model, *col_names, unique=False) + > migrator.drop_index(model, *col_names) + > migrator.add_not_null(model, *field_names) + > migrator.drop_not_null(model, *field_names) + > migrator.add_default(model, field_name, default) + +""" +import peewee as pw + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + migrator.sql( + 'CREATE TABLE IF NOT EXISTS "regions" ("camera" VARCHAR(20) NOT NULL PRIMARY KEY, "last_update" DATETIME NOT NULL, "grid" JSON)' + ) + + +def rollback(migrator, database, fake=False, **kwargs): + pass