From 583912db9cdf9fca82c2c13a24f73b2718631f6a Mon Sep 17 00:00:00 2001
From: Blake Blackshear <blakeb@blakeshome.com>
Date: Sun, 6 Feb 2022 14:49:54 -0600
Subject: [PATCH] allow motion based retention when detect is disabled
---
frigate/video.py | 409 ++++++++++++++++++++++++-----------------------
1 file changed, 208 insertions(+), 201 deletions(-)
diff --git a/frigate/video.py b/frigate/video.py
index c7e4ef835..cb201fd22 100755
--- a/frigate/video.py
+++ b/frigate/video.py
@@ -491,212 +491,219 @@ def process_frames(
logger.info(f"{camera_name}: frame {frame_time} is not in memory store.")
continue
- if not detection_enabled.value:
- fps.value = fps_tracker.eps()
- object_tracker.match_and_update(frame_time, [])
- detected_objects_queue.put(
- (camera_name, frame_time, object_tracker.tracked_objects, [], [])
- )
- detection_fps.value = object_detector.fps.eps()
- frame_manager.close(f"{camera_name}{frame_time}")
- continue
-
# look for motion
motion_boxes = motion_detector.detect(frame)
- # get stationary object ids
- # check every Nth frame for stationary objects
- # disappeared objects are not stationary
- # also check for overlapping motion boxes
- stationary_object_ids = [
- obj["id"]
- for obj in object_tracker.tracked_objects.values()
- # if there hasn't been motion for 10 frames
- if obj["motionless_count"] >= 10
- # and it isn't due for a periodic check
- and (
- detect_config.stationary_interval == 0
- or obj["motionless_count"] % detect_config.stationary_interval != 0
- )
- # and it hasn't disappeared
- and object_tracker.disappeared[obj["id"]] == 0
- # and it doesn't overlap with any current motion boxes
- and not intersects_any(obj["box"], motion_boxes)
- ]
+ regions = []
- # get tracked object boxes that aren't stationary
- tracked_object_boxes = [
- obj["box"]
- for obj in object_tracker.tracked_objects.values()
- if not obj["id"] in stationary_object_ids
- ]
-
- # combine motion boxes with known locations of existing objects
- combined_boxes = reduce_boxes(motion_boxes + tracked_object_boxes)
-
- region_min_size = max(model_shape[0], model_shape[1])
- # compute regions
- regions = [
- calculate_region(
- frame_shape,
- a[0],
- a[1],
- a[2],
- a[3],
- region_min_size,
- multiplier=random.uniform(1.2, 1.5),
- )
- for a in combined_boxes
- ]
-
- # consolidate regions with heavy overlap
- regions = [
- calculate_region(
- frame_shape, a[0], a[1], a[2], a[3], region_min_size, multiplier=1.0
- )
- for a in reduce_boxes(regions, 0.4)
- ]
-
- # if starting up, get the next startup scan region
- if startup_scan_counter < 9:
- ymin = int(frame_shape[0] / 3 * startup_scan_counter / 3)
- ymax = int(frame_shape[0] / 3 + ymin)
- xmin = int(frame_shape[1] / 3 * startup_scan_counter / 3)
- xmax = int(frame_shape[1] / 3 + xmin)
- regions.append(
- calculate_region(
- frame_shape, xmin, ymin, xmax, ymax, region_min_size, multiplier=1.2
- )
- )
- startup_scan_counter += 1
-
- # resize regions and detect
- # seed with stationary objects
- detections = [
- (
- obj["label"],
- obj["score"],
- obj["box"],
- obj["area"],
- obj["region"],
- )
- for obj in object_tracker.tracked_objects.values()
- if obj["id"] in stationary_object_ids
- ]
-
- for region in regions:
- detections.extend(
- detect(
- object_detector,
- frame,
- model_shape,
- region,
- objects_to_track,
- object_filters,
- )
- )
-
- #########
- # merge objects, check for clipped objects and look again up to 4 times
- #########
- refining = len(regions) > 0
- refine_count = 0
- while refining and refine_count < 4:
- refining = False
-
- # group by name
- detected_object_groups = defaultdict(lambda: [])
- for detection in detections:
- detected_object_groups[detection[0]].append(detection)
-
- selected_objects = []
- for group in detected_object_groups.values():
-
- # apply non-maxima suppression to suppress weak, overlapping bounding boxes
- boxes = [
- (o[2][0], o[2][1], o[2][2] - o[2][0], o[2][3] - o[2][1])
- for o in group
- ]
- confidences = [o[1] for o in group]
- idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
-
- for index in idxs:
- obj = group[index[0]]
- if clipped(obj, frame_shape):
- box = obj[2]
- # calculate a new region that will hopefully get the entire object
- region = calculate_region(
- frame_shape, box[0], box[1], box[2], box[3], region_min_size
- )
-
- regions.append(region)
-
- selected_objects.extend(
- detect(
- object_detector,
- frame,
- model_shape,
- region,
- objects_to_track,
- object_filters,
- )
- )
-
- refining = True
- else:
- selected_objects.append(obj)
- # set the detections list to only include top, complete objects
- # and new detections
- detections = selected_objects
-
- if refining:
- refine_count += 1
-
- ## drop detections that overlap too much
- consolidated_detections = []
-
- # if detection was run on this frame, consolidate
- if len(regions) > 0:
- # group by name
- detected_object_groups = defaultdict(lambda: [])
- for detection in detections:
- detected_object_groups[detection[0]].append(detection)
-
- # loop over detections grouped by label
- for group in detected_object_groups.values():
- # if the group only has 1 item, skip
- if len(group) == 1:
- consolidated_detections.append(group[0])
- continue
-
- # sort smallest to largest by area
- sorted_by_area = sorted(group, key=lambda g: g[3])
-
- for current_detection_idx in range(0, len(sorted_by_area)):
- current_detection = sorted_by_area[current_detection_idx][2]
- overlap = 0
- for to_check_idx in range(
- min(current_detection_idx + 1, len(sorted_by_area)),
- len(sorted_by_area),
- ):
- to_check = sorted_by_area[to_check_idx][2]
- # if 90% of smaller detection is inside of another detection, consolidate
- if (
- area(intersection(current_detection, to_check))
- / area(current_detection)
- > 0.9
- ):
- overlap = 1
- break
- if overlap == 0:
- consolidated_detections.append(
- sorted_by_area[current_detection_idx]
- )
- # now that we have refined our detections, we need to track objects
- object_tracker.match_and_update(frame_time, consolidated_detections)
- # else, just update the frame times for the stationary objects
+ # if detection is disabled
+ if not detection_enabled.value:
+ object_tracker.match_and_update(frame_time, [])
else:
- object_tracker.update_frame_times(frame_time)
+ # get stationary object ids
+ # check every Nth frame for stationary objects
+ # disappeared objects are not stationary
+ # also check for overlapping motion boxes
+ stationary_object_ids = [
+ obj["id"]
+ for obj in object_tracker.tracked_objects.values()
+ # if there hasn't been motion for 10 frames
+ if obj["motionless_count"] >= 10
+ # and it isn't due for a periodic check
+ and (
+ detect_config.stationary_interval == 0
+ or obj["motionless_count"] % detect_config.stationary_interval != 0
+ )
+ # and it hasn't disappeared
+ and object_tracker.disappeared[obj["id"]] == 0
+ # and it doesn't overlap with any current motion boxes
+ and not intersects_any(obj["box"], motion_boxes)
+ ]
+
+ # get tracked object boxes that aren't stationary
+ tracked_object_boxes = [
+ obj["box"]
+ for obj in object_tracker.tracked_objects.values()
+ if not obj["id"] in stationary_object_ids
+ ]
+
+ # combine motion boxes with known locations of existing objects
+ combined_boxes = reduce_boxes(motion_boxes + tracked_object_boxes)
+
+ region_min_size = max(model_shape[0], model_shape[1])
+ # compute regions
+ regions = [
+ calculate_region(
+ frame_shape,
+ a[0],
+ a[1],
+ a[2],
+ a[3],
+ region_min_size,
+ multiplier=random.uniform(1.2, 1.5),
+ )
+ for a in combined_boxes
+ ]
+
+ # consolidate regions with heavy overlap
+ regions = [
+ calculate_region(
+ frame_shape, a[0], a[1], a[2], a[3], region_min_size, multiplier=1.0
+ )
+ for a in reduce_boxes(regions, 0.4)
+ ]
+
+ # if starting up, get the next startup scan region
+ if startup_scan_counter < 9:
+ ymin = int(frame_shape[0] / 3 * startup_scan_counter / 3)
+ ymax = int(frame_shape[0] / 3 + ymin)
+ xmin = int(frame_shape[1] / 3 * startup_scan_counter / 3)
+ xmax = int(frame_shape[1] / 3 + xmin)
+ regions.append(
+ calculate_region(
+ frame_shape,
+ xmin,
+ ymin,
+ xmax,
+ ymax,
+ region_min_size,
+ multiplier=1.2,
+ )
+ )
+ startup_scan_counter += 1
+
+ # resize regions and detect
+ # seed with stationary objects
+ detections = [
+ (
+ obj["label"],
+ obj["score"],
+ obj["box"],
+ obj["area"],
+ obj["region"],
+ )
+ for obj in object_tracker.tracked_objects.values()
+ if obj["id"] in stationary_object_ids
+ ]
+
+ for region in regions:
+ detections.extend(
+ detect(
+ object_detector,
+ frame,
+ model_shape,
+ region,
+ objects_to_track,
+ object_filters,
+ )
+ )
+
+ #########
+ # merge objects, check for clipped objects and look again up to 4 times
+ #########
+ refining = len(regions) > 0
+ refine_count = 0
+ while refining and refine_count < 4:
+ refining = False
+
+ # group by name
+ detected_object_groups = defaultdict(lambda: [])
+ for detection in detections:
+ detected_object_groups[detection[0]].append(detection)
+
+ selected_objects = []
+ for group in detected_object_groups.values():
+
+ # apply non-maxima suppression to suppress weak, overlapping bounding boxes
+ boxes = [
+ (o[2][0], o[2][1], o[2][2] - o[2][0], o[2][3] - o[2][1])
+ for o in group
+ ]
+ confidences = [o[1] for o in group]
+ idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
+
+ for index in idxs:
+ obj = group[index[0]]
+ if clipped(obj, frame_shape):
+ box = obj[2]
+ # calculate a new region that will hopefully get the entire object
+ region = calculate_region(
+ frame_shape,
+ box[0],
+ box[1],
+ box[2],
+ box[3],
+ region_min_size,
+ )
+
+ regions.append(region)
+
+ selected_objects.extend(
+ detect(
+ object_detector,
+ frame,
+ model_shape,
+ region,
+ objects_to_track,
+ object_filters,
+ )
+ )
+
+ refining = True
+ else:
+ selected_objects.append(obj)
+ # set the detections list to only include top, complete objects
+ # and new detections
+ detections = selected_objects
+
+ if refining:
+ refine_count += 1
+
+ ## drop detections that overlap too much
+ consolidated_detections = []
+
+ # if detection was run on this frame, consolidate
+ if len(regions) > 0:
+ # group by name
+ detected_object_groups = defaultdict(lambda: [])
+ for detection in detections:
+ detected_object_groups[detection[0]].append(detection)
+
+ # loop over detections grouped by label
+ for group in detected_object_groups.values():
+ # if the group only has 1 item, skip
+ if len(group) == 1:
+ consolidated_detections.append(group[0])
+ continue
+
+ # sort smallest to largest by area
+ sorted_by_area = sorted(group, key=lambda g: g[3])
+
+ for current_detection_idx in range(0, len(sorted_by_area)):
+ current_detection = sorted_by_area[current_detection_idx][2]
+ overlap = 0
+ for to_check_idx in range(
+ min(current_detection_idx + 1, len(sorted_by_area)),
+ len(sorted_by_area),
+ ):
+ to_check = sorted_by_area[to_check_idx][2]
+ # if 90% of smaller detection is inside of another detection, consolidate
+ if (
+ area(intersection(current_detection, to_check))
+ / area(current_detection)
+ > 0.9
+ ):
+ overlap = 1
+ break
+ if overlap == 0:
+ consolidated_detections.append(
+ sorted_by_area[current_detection_idx]
+ )
+ # now that we have refined our detections, we need to track objects
+ object_tracker.match_and_update(frame_time, consolidated_detections)
+ # else, just update the frame times for the stationary objects
+ else:
+ object_tracker.update_frame_times(frame_time)
# add to the queue if not full
if detected_objects_queue.full():