Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2024-11-21 19:07:46 +01:00
Code Issues Projects Releases Wiki Activity

Smarter Regions (#8194)

Browse Source 
* Smarter Regions

* Formatting

* Cleanup

* Fix motion region checking logic

* Add database table and migration for regions

* Update region grid on startup

* Revert init delay change

* Fix mypy

* Move object related functions to util

* Remove unused

* Fix tests

* Remove log

* Update the region daily at 2

* Fix logic

* Formatting

* Initialize grid before starting processing frames

* Move back to creating grid in main process

* Formatting

* Fixes

* Formating

* Fix region check

* Accept all but true

* Use regions grid for startup scan

* Add clarifying comment

* Fix new grid requests

* Add tests

* Delete stale region grids from DB
This commit is contained in:
Nicolas Mowen 2023-10-18 17:21:52 -06:00 committed by GitHub
parent 98200b7dda
commit 91f7d67c5e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 678 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
frigate/app.py
View File

@ -36,7 +36,7 @@ from frigate.events.external import ExternalEventProcessor
from frigate.events.maintainer import EventProcessor
from frigate.http import create_app
from frigate.log import log_process, root_configurer
from frigate.models import Event, Recordings, RecordingsToDelete, Timeline
from frigate.models import Event, Recordings, RecordingsToDelete, Regions, Timeline
from frigate.object_detection import ObjectDetectProcess
from frigate.object_processing import TrackedObjectProcessor
from frigate.output import output_frames
@ -49,6 +49,7 @@ from frigate.stats import StatsEmitter, stats_init
from frigate.storage import StorageMaintainer
from frigate.timeline import TimelineProcessor
from frigate.types import CameraMetricsTypes, FeatureMetricsTypes, PTZMetricsTypes
from frigate.util.object import get_camera_regions_grid
from frigate.version import VERSION
from frigate.video import capture_camera, track_camera
from frigate.watchdog import FrigateWatchdog
@ -69,6 +70,7 @@ class FrigateApp:
self.feature_metrics: dict[str, FeatureMetricsTypes] = {}
self.ptz_metrics: dict[str, PTZMetricsTypes] = {}
self.processes: dict[str, int] = {}
self.region_grids: dict[str, list[list[dict[str, int]]]] = {}
def set_environment_vars(self) -> None:
for key, value in self.config.environment_vars.items():
@ -161,6 +163,7 @@ class FrigateApp:
# issue https://github.com/python/typeshed/issues/8799
# from mypy 0.981 onwards
"frame_queue": mp.Queue(maxsize=2),
"region_grid_queue": mp.Queue(maxsize=1),
"capture_process": None,
"process": None,
"audio_rms": mp.Value("d", 0.0), # type: ignore[typeddict-item]
@ -327,7 +330,7 @@ class FrigateApp:
60, 10 * len([c for c in self.config.cameras.values() if c.enabled])
),
)
models = [Event, Recordings, RecordingsToDelete, Timeline]
models = [Event, Recordings, RecordingsToDelete, Regions, Timeline]
self.db.bind(models)
def init_stats(self) -> None:
@ -452,6 +455,17 @@ class FrigateApp:
output_processor.start()
logger.info(f"Output process started: {output_processor.pid}")
def init_historical_regions(self) -> None:
# delete region grids for removed or renamed cameras
cameras = list(self.config.cameras.keys())
Regions.delete().where(~(Regions.camera << cameras)).execute()
# create or update region grids for each camera
for camera in self.config.cameras.values():
self.region_grids[camera.name] = get_camera_regions_grid(
camera.name, camera.detect
)
def start_camera_processors(self) -> None:
for name, config in self.config.cameras.items():
if not self.config.cameras[name].enabled:
@ -469,8 +483,10 @@ class FrigateApp:
self.detection_queue,
self.detection_out_events[name],
self.detected_frames_queue,
self.inter_process_queue,
self.camera_metrics[name],
self.ptz_metrics[name],
self.region_grids[name],
),
)
camera_process.daemon = True
@ -611,6 +627,7 @@ class FrigateApp:
self.start_detectors()
self.start_video_output_processor()
self.start_ptz_autotracker()
self.init_historical_regions()
self.start_detected_frames_processor()
self.start_camera_processors()
self.start_camera_capture_processes()

8
frigate/comms/dispatcher.py
View File

@ -5,10 +5,11 @@ from abc import ABC, abstractmethod
from typing import Any, Callable
from frigate.config import FrigateConfig
from frigate.const import INSERT_MANY_RECORDINGS
from frigate.const import INSERT_MANY_RECORDINGS, REQUEST_REGION_GRID
from frigate.models import Recordings
from frigate.ptz.onvif import OnvifCommandEnum, OnvifController
from frigate.types import CameraMetricsTypes, FeatureMetricsTypes, PTZMetricsTypes
from frigate.util.object import get_camera_regions_grid
from frigate.util.services import restart_frigate
logger = logging.getLogger(__name__)
@ -90,6 +91,11 @@ class Dispatcher:
restart_frigate()
elif topic == INSERT_MANY_RECORDINGS:
Recordings.insert_many(payload).execute()
elif topic == REQUEST_REGION_GRID:
camera = payload
self.camera_metrics[camera]["region_grid_queue"].put(
get_camera_regions_grid(camera, self.config.cameras[camera].detect)
)
else:
self.publish(topic, payload, retain=False)

1
frigate/const.py
View File

@ -51,3 +51,4 @@ MAX_PLAYLIST_SECONDS = 7200 # support 2 hour segments for a single playlist to
# Internal Comms Topics
INSERT_MANY_RECORDINGS = "insert_many_recordings"
REQUEST_REGION_GRID = "request_region_grid"

6
frigate/models.py
View File

@ -57,6 +57,12 @@ class Timeline(Model): # type: ignore[misc]
data = JSONField() # ex: tracked object id, region, box, etc.
class Regions(Model): # type: ignore[misc]
camera = CharField(null=False, primary_key=True, max_length=20)
grid = JSONField() # json blob of grid
last_update = DateTimeField()
class Recordings(Model): # type: ignore[misc]
id = CharField(null=False, primary_key=True, max_length=30)
camera = CharField(index=True, max_length=20)

2
frigate/test/test_reduce_boxes.py
View File

@ -1,6 +1,6 @@
from unittest import TestCase, main
from frigate.video import box_overlaps, reduce_boxes
from frigate.util.object import box_overlaps, reduce_boxes
class TestBoxOverlaps(TestCase):

36
frigate/test/test_video.py
View File

@ -6,10 +6,11 @@ from norfair.drawing.color import Palette
from norfair.drawing.drawer import Drawer
from frigate.util.image import intersection
from frigate.video import (
from frigate.util.object import (
get_cluster_boundary,
get_cluster_candidates,
get_cluster_region,
get_region_from_grid,
)
@ -190,3 +191,36 @@ class TestObjectBoundingBoxes(unittest.TestCase):
assert intersection(box_a, box_b) == None
assert intersection(box_b, box_c) == (899, 128, 985, 151)
class TestRegionGrid(unittest.TestCase):
def setUp(self) -> None:
pass
def test_region_in_range(self):
"""Test that region is kept at minimal size when within std dev."""
frame_shape = (720, 1280)
box = [450, 450, 550, 550]
region_grid = [
[],
[],
[],
[{}, {}, {}, {}, {}, {"sizes": [0.25], "mean": 0.26, "std_dev": 0.01}],
]
region = get_region_from_grid(frame_shape, box, 320, region_grid)
assert region[2] - region[0] == 320
def test_region_out_of_range(self):
"""Test that region is upsized when outside of std dev."""
frame_shape = (720, 1280)
box = [450, 450, 550, 550]
region_grid = [
[],
[],
[],
[{}, {}, {}, {}, {}, {"sizes": [0.5], "mean": 0.5, "std_dev": 0.1}],
]
region = get_region_from_grid(frame_shape, box, 320, region_grid)
assert region[2] - region[0] > 320

7
frigate/track/norfair_tracker.py
View File

@ -77,7 +77,7 @@ class NorfairTracker(ObjectTracker):
self.tracker = Tracker(
distance_function=frigate_distance,
distance_threshold=2.5,
initialization_delay=config.detect.fps / 2,
initialization_delay=0,
hit_counter_max=self.max_disappeared,
)
if self.ptz_autotracker_enabled.value:
@ -106,6 +106,11 @@ class NorfairTracker(ObjectTracker):
"ymax": self.detect_config.height,
}
# start object with a hit count of `fps` to avoid quick detection -> loss
next(
(o for o in self.tracker.tracked_objects if o.global_id == track_id)
).hit_counter = self.camera_config.detect.fps
def deregister(self, id, track_id):
del self.tracked_objects[id]
del self.disappeared[id]

8
frigate/util/builtin.py
View File

@ -14,6 +14,7 @@ import numpy as np
import pytz
import yaml
from ruamel.yaml import YAML
from tzlocal import get_localzone
from frigate.const import REGEX_HTTP_CAMERA_USER_PASS, REGEX_RTSP_CAMERA_USER_PASS
@ -262,3 +263,10 @@ def find_by_key(dictionary, target_key):
if result is not None:
return result
return None
def get_tomorrow_at_2() -> datetime.datetime:
tomorrow = datetime.datetime.now(get_localzone()) + datetime.timedelta(days=1)
return tomorrow.replace(hour=2, minute=0, second=0).astimezone(
datetime.timezone.utc
)

481
frigate/util/object.py Normal file
View File

@ -0,0 +1,481 @@
"""Utils for reading and writing object detection data."""
import datetime
import logging
import math
import cv2
import numpy as np
from peewee import DoesNotExist
from frigate.config import DetectConfig, ModelConfig
from frigate.detectors.detector_config import PixelFormatEnum
from frigate.models import Event, Regions, Timeline
from frigate.util.image import (
area,
calculate_region,
intersection,
intersection_over_union,
yuv_region_2_bgr,
yuv_region_2_rgb,
yuv_region_2_yuv,
)
logger = logging.getLogger(__name__)
GRID_SIZE = 8
def get_camera_regions_grid(
name: str, detect: DetectConfig
) -> list[list[dict[str, any]]]:
"""Build a grid of expected region sizes for a camera."""
# get grid from db if available
try:
regions: Regions = Regions.select().where(Regions.camera == name).get()
grid = regions.grid
last_update = regions.last_update
except DoesNotExist:
grid = []
for x in range(GRID_SIZE):
row = []
for y in range(GRID_SIZE):
row.append({"sizes": []})
grid.append(row)
last_update = 0
# get events for timeline entries
events = (
Event.select(Event.id)
.where(Event.camera == name)
.where((Event.false_positive == None) | (Event.false_positive == False))
.where(Event.start_time > last_update)
)
valid_event_ids = [e["id"] for e in events.dicts()]
logger.debug(f"Found {len(valid_event_ids)} new events for {name}")
# no new events, return as is
if not valid_event_ids:
return grid
new_update = datetime.datetime.now().timestamp()
timeline = (
Timeline.select(
*[
Timeline.camera,
Timeline.source,
Timeline.data,
]
)
.where(Timeline.source_id << valid_event_ids)
.limit(10000)
.dicts()
)
logger.debug(f"Found {len(timeline)} new entries for {name}")
width = detect.width
height = detect.height
for t in timeline:
if t.get("source") != "tracked_object":
continue
box = t["data"]["box"]
# calculate centroid position
x = box[0] + (box[2] / 2)
y = box[1] + (box[3] / 2)
x_pos = int(x * GRID_SIZE)
y_pos = int(y * GRID_SIZE)
calculated_region = calculate_region(
(height, width),
box[0] * width,
box[1] * height,
(box[0] + box[2]) * width,
(box[1] + box[3]) * height,
320,
1.35,
)
# save width of region to grid as relative
grid[x_pos][y_pos]["sizes"].append(
(calculated_region[2] - calculated_region[0]) / width
)
for x in range(GRID_SIZE):
for y in range(GRID_SIZE):
cell = grid[x][y]
if len(cell["sizes"]) == 0:
continue
std_dev = np.std(cell["sizes"])
mean = np.mean(cell["sizes"])
logger.debug(f"std dev: {std_dev} mean: {mean}")
cell["x"] = x
cell["y"] = y
cell["std_dev"] = std_dev
cell["mean"] = mean
# update db with new grid
region = {
Regions.camera: name,
Regions.grid: grid,
Regions.last_update: new_update,
}
(
Regions.insert(region)
.on_conflict(
conflict_target=[Regions.camera],
update=region,
)
.execute()
)
return grid
def get_cluster_region_from_grid(frame_shape, min_region, cluster, boxes, region_grid):
min_x = frame_shape[1]
min_y = frame_shape[0]
max_x = 0
max_y = 0
for b in cluster:
min_x = min(boxes[b][0], min_x)
min_y = min(boxes[b][1], min_y)
max_x = max(boxes[b][2], max_x)
max_y = max(boxes[b][3], max_y)
return get_region_from_grid(
frame_shape, [min_x, min_y, max_x, max_y], min_region, region_grid
)
def get_region_from_grid(
frame_shape: tuple[int],
cluster: list[int],
min_region: int,
region_grid: list[list[dict[str, any]]],
) -> list[int]:
"""Get a region for a box based on the region grid."""
box = calculate_region(
frame_shape, cluster[0], cluster[1], cluster[2], cluster[3], min_region
)
centroid = (
box[0] + (min(frame_shape[1], box[2]) - box[0]) / 2,
box[1] + (min(frame_shape[0], box[3]) - box[1]) / 2,
)
grid_x = int(centroid[0] / frame_shape[1] * GRID_SIZE)
grid_y = int(centroid[1] / frame_shape[0] * GRID_SIZE)
cell = region_grid[grid_x][grid_y]
# if there is no known data, get standard region for motion box
if not cell or not cell["sizes"]:
return calculate_region(frame_shape, box[0], box[1], box[2], box[3], min_region)
# convert the calculated region size to relative
calc_size = (box[2] - box[0]) / frame_shape[1]
# if region is within expected size, don't resize
if (
(cell["mean"] - cell["std_dev"])
<= calc_size
<= (cell["mean"] + cell["std_dev"])
):
return box
# TODO not sure how to handle case where cluster is larger than expected region
elif calc_size > (cell["mean"] + cell["std_dev"]):
return box
size = cell["mean"] * frame_shape[1]
# get region based on grid size
return calculate_region(
frame_shape,
max(0, centroid[0] - size / 2),
max(0, centroid[1] - size / 2),
min(frame_shape[1], centroid[0] + size / 2),
min(frame_shape[0], centroid[1] + size / 2),
min_region,
)
def is_object_filtered(obj, objects_to_track, object_filters):
object_name = obj[0]
object_score = obj[1]
object_box = obj[2]
object_area = obj[3]
object_ratio = obj[4]
if object_name not in objects_to_track:
return True
if object_name in object_filters:
obj_settings = object_filters[object_name]
# if the min area is larger than the
# detected object, don't add it to detected objects
if obj_settings.min_area > object_area:
return True
# if the detected object is larger than the
# max area, don't add it to detected objects
if obj_settings.max_area < object_area:
return True
# if the score is lower than the min_score, skip
if obj_settings.min_score > object_score:
return True
# if the object is not proportionally wide enough
if obj_settings.min_ratio > object_ratio:
return True
# if the object is proportionally too wide
if obj_settings.max_ratio < object_ratio:
return True
if obj_settings.mask is not None:
# compute the coordinates of the object and make sure
# the location isn't outside the bounds of the image (can happen from rounding)
object_xmin = object_box[0]
object_xmax = object_box[2]
object_ymax = object_box[3]
y_location = min(int(object_ymax), len(obj_settings.mask) - 1)
x_location = min(
int((object_xmax + object_xmin) / 2.0),
len(obj_settings.mask[0]) - 1,
)
# if the object is in a masked location, don't add it to detected objects
if obj_settings.mask[y_location][x_location] == 0:
return True
return False
def get_min_region_size(model_config: ModelConfig) -> int:
"""Get the min region size."""
return max(model_config.height, model_config.width)
def create_tensor_input(frame, model_config: ModelConfig, region):
if model_config.input_pixel_format == PixelFormatEnum.rgb:
cropped_frame = yuv_region_2_rgb(frame, region)
elif model_config.input_pixel_format == PixelFormatEnum.bgr:
cropped_frame = yuv_region_2_bgr(frame, region)
else:
cropped_frame = yuv_region_2_yuv(frame, region)
# Resize if needed
if cropped_frame.shape != (model_config.height, model_config.width, 3):
cropped_frame = cv2.resize(
cropped_frame,
dsize=(model_config.width, model_config.height),
interpolation=cv2.INTER_LINEAR,
)
# Expand dimensions since the model expects images to have shape: [1, height, width, 3]
return np.expand_dims(cropped_frame, axis=0)
def box_overlaps(b1, b2):
if b1[2] < b2[0] or b1[0] > b2[2] or b1[1] > b2[3] or b1[3] < b2[1]:
return False
return True
def box_inside(b1, b2):
# check if b2 is inside b1
if b2[0] >= b1[0] and b2[1] >= b1[1] and b2[2] <= b1[2] and b2[3] <= b1[3]:
return True
return False
def reduce_boxes(boxes, iou_threshold=0.0):
clusters = []
for box in boxes:
matched = 0
for cluster in clusters:
if intersection_over_union(box, cluster) > iou_threshold:
matched = 1
cluster[0] = min(cluster[0], box[0])
cluster[1] = min(cluster[1], box[1])
cluster[2] = max(cluster[2], box[2])
cluster[3] = max(cluster[3], box[3])
if not matched:
clusters.append(list(box))
return [tuple(c) for c in clusters]
def intersects_any(box_a, boxes):
for box in boxes:
if box_overlaps(box_a, box):
return True
return False
def inside_any(box_a, boxes):
for box in boxes:
# check if box_a is inside of box
if box_inside(box, box_a):
return True
return False
def get_cluster_boundary(box, min_region):
# compute the max region size for the current box (box is 10% of region)
box_width = box[2] - box[0]
box_height = box[3] - box[1]
max_region_area = abs(box_width * box_height) / 0.1
max_region_size = max(min_region, int(math.sqrt(max_region_area)))
centroid = (box_width / 2 + box[0], box_height / 2 + box[1])
max_x_dist = int(max_region_size - box_width / 2 * 1.1)
max_y_dist = int(max_region_size - box_height / 2 * 1.1)
return [
int(centroid[0] - max_x_dist),
int(centroid[1] - max_y_dist),
int(centroid[0] + max_x_dist),
int(centroid[1] + max_y_dist),
]
def get_cluster_candidates(frame_shape, min_region, boxes):
# and create a cluster of other boxes using it's max region size
# only include boxes where the region is an appropriate(except the region could possibly be smaller?)
# size in the cluster. in order to be in the cluster, the furthest corner needs to be within x,y offset
# determined by the max_region size minus half the box + 20%
# TODO: see if we can do this with numpy
cluster_candidates = []
used_boxes = []
# loop over each box
for current_index, b in enumerate(boxes):
if current_index in used_boxes:
continue
cluster = [current_index]
used_boxes.append(current_index)
cluster_boundary = get_cluster_boundary(b, min_region)
# find all other boxes that fit inside the boundary
for compare_index, compare_box in enumerate(boxes):
if compare_index in used_boxes:
continue
# if the box is not inside the potential cluster area, cluster them
if not box_inside(cluster_boundary, compare_box):
continue
# get the region if you were to add this box to the cluster
potential_cluster = cluster + [compare_index]
cluster_region = get_cluster_region(
frame_shape, min_region, potential_cluster, boxes
)
# if region could be smaller and either box would be too small
# for the resulting region, dont cluster
should_cluster = True
if (cluster_region[2] - cluster_region[0]) > min_region:
for b in potential_cluster:
box = boxes[b]
# boxes should be more than 5% of the area of the region
if area(box) / area(cluster_region) < 0.05:
should_cluster = False
break
if should_cluster:
cluster.append(compare_index)
used_boxes.append(compare_index)
cluster_candidates.append(cluster)
# return the unique clusters only
unique = {tuple(sorted(c)) for c in cluster_candidates}
return [list(tup) for tup in unique]
def get_cluster_region(frame_shape, min_region, cluster, boxes):
min_x = frame_shape[1]
min_y = frame_shape[0]
max_x = 0
max_y = 0
for b in cluster:
min_x = min(boxes[b][0], min_x)
min_y = min(boxes[b][1], min_y)
max_x = max(boxes[b][2], max_x)
max_y = max(boxes[b][3], max_y)
return calculate_region(
frame_shape, min_x, min_y, max_x, max_y, min_region, multiplier=1.2
)
def get_consolidated_object_detections(detected_object_groups):
"""Drop detections that overlap too much"""
consolidated_detections = []
for group in detected_object_groups.values():
# if the group only has 1 item, skip
if len(group) == 1:
consolidated_detections.append(group[0])
continue
# sort smallest to largest by area
sorted_by_area = sorted(group, key=lambda g: g[3])
for current_detection_idx in range(0, len(sorted_by_area)):
current_detection = sorted_by_area[current_detection_idx][2]
overlap = 0
for to_check_idx in range(
min(current_detection_idx + 1, len(sorted_by_area)),
len(sorted_by_area),
):
to_check = sorted_by_area[to_check_idx][2]
intersect_box = intersection(current_detection, to_check)
# if 90% of smaller detection is inside of another detection, consolidate
if (
intersect_box is not None
and area(intersect_box) / area(current_detection) > 0.9
):
overlap = 1
break
if overlap == 0:
consolidated_detections.append(sorted_by_area[current_detection_idx])
return consolidated_detections
def get_startup_regions(
frame_shape: tuple[int],
region_min_size: int,
region_grid: list[list[dict[str, any]]],
) -> list[list[int]]:
"""Get a list of regions to run on startup."""
# return 8 most popular regions for the camera
all_cells = np.concatenate(region_grid).flat
startup_cells = sorted(all_cells, key=lambda c: len(c["sizes"]), reverse=True)[0:8]
regions = []
for cell in startup_cells:
# rest of the cells are empty
if not cell["sizes"]:
break
x = frame_shape[1] / GRID_SIZE * (0.5 + cell["x"])
y = frame_shape[0] / GRID_SIZE * (0.5 + cell["y"])
size = cell["mean"] * frame_shape[1]
regions.append(
calculate_region(
frame_shape,
x - size / 2,
y - size / 2,
x + size / 2,
y + size / 2,
region_min_size,
multiplier=1,
)
)
return regions

355
frigate/video.py
View File

@ -1,6 +1,5 @@
import datetime
import logging
import math
import multiprocessing as mp
import os
import queue
@ -15,8 +14,12 @@ import numpy as np
from setproctitle import setproctitle
from frigate.config import CameraConfig, DetectConfig, ModelConfig
from frigate.const import ALL_ATTRIBUTE_LABELS, ATTRIBUTE_LABEL_MAP, CACHE_DIR
from frigate.detectors.detector_config import PixelFormatEnum
from frigate.const import (
ALL_ATTRIBUTE_LABELS,
ATTRIBUTE_LABEL_MAP,
CACHE_DIR,
REQUEST_REGION_GRID,
)
from frigate.log import LogPipe
from frigate.motion import MotionDetector
from frigate.motion.improved_motion import ImprovedMotionDetector
@ -24,103 +27,30 @@ from frigate.object_detection import RemoteObjectDetector
from frigate.track import ObjectTracker
from frigate.track.norfair_tracker import NorfairTracker
from frigate.types import PTZMetricsTypes
from frigate.util.builtin import EventsPerSecond
from frigate.util.builtin import EventsPerSecond, get_tomorrow_at_2
from frigate.util.image import (
FrameManager,
SharedMemoryFrameManager,
area,
calculate_region,
draw_box_with_label,
intersection,
intersection_over_union,
yuv_region_2_bgr,
yuv_region_2_rgb,
yuv_region_2_yuv,
)
from frigate.util.object import (
box_inside,
create_tensor_input,
get_cluster_candidates,
get_cluster_region,
get_cluster_region_from_grid,
get_consolidated_object_detections,
get_min_region_size,
get_startup_regions,
inside_any,
intersects_any,
is_object_filtered,
)
from frigate.util.services import listen
logger = logging.getLogger(__name__)
def filtered(obj, objects_to_track, object_filters):
object_name = obj[0]
object_score = obj[1]
object_box = obj[2]
object_area = obj[3]
object_ratio = obj[4]
if object_name not in objects_to_track:
return True
if object_name in object_filters:
obj_settings = object_filters[object_name]
# if the min area is larger than the
# detected object, don't add it to detected objects
if obj_settings.min_area > object_area:
return True
# if the detected object is larger than the
# max area, don't add it to detected objects
if obj_settings.max_area < object_area:
return True
# if the score is lower than the min_score, skip
if obj_settings.min_score > object_score:
return True
# if the object is not proportionally wide enough
if obj_settings.min_ratio > object_ratio:
return True
# if the object is proportionally too wide
if obj_settings.max_ratio < object_ratio:
return True
if obj_settings.mask is not None:
# compute the coordinates of the object and make sure
# the location isn't outside the bounds of the image (can happen from rounding)
object_xmin = object_box[0]
object_xmax = object_box[2]
object_ymax = object_box[3]
y_location = min(int(object_ymax), len(obj_settings.mask) - 1)
x_location = min(
int((object_xmax + object_xmin) / 2.0),
len(obj_settings.mask[0]) - 1,
)
# if the object is in a masked location, don't add it to detected objects
if obj_settings.mask[y_location][x_location] == 0:
return True
return False
def get_min_region_size(model_config: ModelConfig) -> int:
"""Get the min region size."""
return max(model_config.height, model_config.width)
def create_tensor_input(frame, model_config: ModelConfig, region):
if model_config.input_pixel_format == PixelFormatEnum.rgb:
cropped_frame = yuv_region_2_rgb(frame, region)
elif model_config.input_pixel_format == PixelFormatEnum.bgr:
cropped_frame = yuv_region_2_bgr(frame, region)
else:
cropped_frame = yuv_region_2_yuv(frame, region)
# Resize if needed
if cropped_frame.shape != (model_config.height, model_config.width, 3):
cropped_frame = cv2.resize(
cropped_frame,
dsize=(model_config.width, model_config.height),
interpolation=cv2.INTER_LINEAR,
)
# Expand dimensions since the model expects images to have shape: [1, height, width, 3]
return np.expand_dims(cropped_frame, axis=0)
def stop_ffmpeg(ffmpeg_process, logger):
logger.info("Terminating the existing ffmpeg process...")
ffmpeg_process.terminate()
@ -455,8 +385,10 @@ def track_camera(
detection_queue,
result_connection,
detected_objects_queue,
inter_process_queue,
process_info,
ptz_metrics,
region_grid,
):
stop_event = mp.Event()
@ -471,6 +403,7 @@ def track_camera(
listen()
frame_queue = process_info["frame_queue"]
region_grid_queue = process_info["region_grid_queue"]
detection_enabled = process_info["detection_enabled"]
motion_enabled = process_info["motion_enabled"]
improve_contrast_enabled = process_info["improve_contrast_enabled"]
@ -499,7 +432,9 @@ def track_camera(
process_frames(
name,
inter_process_queue,
frame_queue,
region_grid_queue,
frame_shape,
model_config,
config.detect,
@ -515,50 +450,12 @@ def track_camera(
motion_enabled,
stop_event,
ptz_metrics,
region_grid,
)
logger.info(f"{name}: exiting subprocess")
def box_overlaps(b1, b2):
if b1[2] < b2[0] or b1[0] > b2[2] or b1[1] > b2[3] or b1[3] < b2[1]:
return False
return True
def box_inside(b1, b2):
# check if b2 is inside b1
if b2[0] >= b1[0] and b2[1] >= b1[1] and b2[2] <= b1[2] and b2[3] <= b1[3]:
return True
return False
def reduce_boxes(boxes, iou_threshold=0.0):
clusters = []
for box in boxes:
matched = 0
for cluster in clusters:
if intersection_over_union(box, cluster) > iou_threshold:
matched = 1
cluster[0] = min(cluster[0], box[0])
cluster[1] = min(cluster[1], box[1])
cluster[2] = max(cluster[2], box[2])
cluster[3] = max(cluster[3], box[3])
if not matched:
clusters.append(list(box))
return [tuple(c) for c in clusters]
def intersects_any(box_a, boxes):
for box in boxes:
if box_overlaps(box_a, box):
return True
return False
def detect(
detect_config: DetectConfig,
object_detector,
@ -597,134 +494,17 @@ def detect(
region,
)
# apply object filters
if filtered(det, objects_to_track, object_filters):
if is_object_filtered(det, objects_to_track, object_filters):
continue
detections.append(det)
return detections
def get_cluster_boundary(box, min_region):
# compute the max region size for the current box (box is 10% of region)
box_width = box[2] - box[0]
box_height = box[3] - box[1]
max_region_area = abs(box_width * box_height) / 0.1
max_region_size = max(min_region, int(math.sqrt(max_region_area)))
centroid = (box_width / 2 + box[0], box_height / 2 + box[1])
max_x_dist = int(max_region_size - box_width / 2 * 1.1)
max_y_dist = int(max_region_size - box_height / 2 * 1.1)
return [
int(centroid[0] - max_x_dist),
int(centroid[1] - max_y_dist),
int(centroid[0] + max_x_dist),
int(centroid[1] + max_y_dist),
]
def get_cluster_candidates(frame_shape, min_region, boxes):
# and create a cluster of other boxes using it's max region size
# only include boxes where the region is an appropriate(except the region could possibly be smaller?)
# size in the cluster. in order to be in the cluster, the furthest corner needs to be within x,y offset
# determined by the max_region size minus half the box + 20%
# TODO: see if we can do this with numpy
cluster_candidates = []
used_boxes = []
# loop over each box
for current_index, b in enumerate(boxes):
if current_index in used_boxes:
continue
cluster = [current_index]
used_boxes.append(current_index)
cluster_boundary = get_cluster_boundary(b, min_region)
# find all other boxes that fit inside the boundary
for compare_index, compare_box in enumerate(boxes):
if compare_index in used_boxes:
continue
# if the box is not inside the potential cluster area, cluster them
if not box_inside(cluster_boundary, compare_box):
continue
# get the region if you were to add this box to the cluster
potential_cluster = cluster + [compare_index]
cluster_region = get_cluster_region(
frame_shape, min_region, potential_cluster, boxes
)
# if region could be smaller and either box would be too small
# for the resulting region, dont cluster
should_cluster = True
if (cluster_region[2] - cluster_region[0]) > min_region:
for b in potential_cluster:
box = boxes[b]
# boxes should be more than 5% of the area of the region
if area(box) / area(cluster_region) < 0.05:
should_cluster = False
break
if should_cluster:
cluster.append(compare_index)
used_boxes.append(compare_index)
cluster_candidates.append(cluster)
# return the unique clusters only
unique = {tuple(sorted(c)) for c in cluster_candidates}
return [list(tup) for tup in unique]
def get_cluster_region(frame_shape, min_region, cluster, boxes):
min_x = frame_shape[1]
min_y = frame_shape[0]
max_x = 0
max_y = 0
for b in cluster:
min_x = min(boxes[b][0], min_x)
min_y = min(boxes[b][1], min_y)
max_x = max(boxes[b][2], max_x)
max_y = max(boxes[b][3], max_y)
return calculate_region(
frame_shape, min_x, min_y, max_x, max_y, min_region, multiplier=1.2
)
def get_consolidated_object_detections(detected_object_groups):
"""Drop detections that overlap too much"""
consolidated_detections = []
for group in detected_object_groups.values():
# if the group only has 1 item, skip
if len(group) == 1:
consolidated_detections.append(group[0])
continue
# sort smallest to largest by area
sorted_by_area = sorted(group, key=lambda g: g[3])
for current_detection_idx in range(0, len(sorted_by_area)):
current_detection = sorted_by_area[current_detection_idx][2]
overlap = 0
for to_check_idx in range(
min(current_detection_idx + 1, len(sorted_by_area)),
len(sorted_by_area),
):
to_check = sorted_by_area[to_check_idx][2]
intersect_box = intersection(current_detection, to_check)
# if 90% of smaller detection is inside of another detection, consolidate
if (
intersect_box is not None
and area(intersect_box) / area(current_detection) > 0.9
):
overlap = 1
break
if overlap == 0:
consolidated_detections.append(sorted_by_area[current_detection_idx])
return consolidated_detections
def process_frames(
camera_name: str,
inter_process_queue: mp.Queue,
frame_queue: mp.Queue,
region_grid_queue: mp.Queue,
frame_shape,
model_config: ModelConfig,
detect_config: DetectConfig,
@ -740,20 +520,35 @@ def process_frames(
motion_enabled: mp.Value,
stop_event,
ptz_metrics: PTZMetricsTypes,
region_grid,
exit_on_empty: bool = False,
):
fps = process_info["process_fps"]
detection_fps = process_info["detection_fps"]
current_frame_time = process_info["detection_frame"]
next_region_update = get_tomorrow_at_2()
fps_tracker = EventsPerSecond()
fps_tracker.start()
startup_scan_counter = 0
startup_scan = True
region_min_size = get_min_region_size(model_config)
while not stop_event.is_set():
if (
datetime.datetime.now().astimezone(datetime.timezone.utc)
> next_region_update
):
inter_process_queue.put((REQUEST_REGION_GRID, camera_name))
try:
region_grid = region_grid_queue.get(True, 10)
except queue.Empty:
logger.error(f"Unable to get updated region grid for {camera_name}")
next_region_update = get_tomorrow_at_2()
try:
if exit_on_empty:
frame_time = frame_queue.get(False)
@ -815,40 +610,48 @@ def process_frames(
if obj["id"] not in stationary_object_ids
]
combined_boxes = tracked_object_boxes
# only add in the motion boxes when not calibrating
if not motion_detector.is_calibrating():
combined_boxes += motion_boxes
cluster_candidates = get_cluster_candidates(
frame_shape, region_min_size, combined_boxes
)
# get consolidated regions for tracked objects
regions = [
get_cluster_region(
frame_shape, region_min_size, candidate, combined_boxes
frame_shape, region_min_size, candidate, tracked_object_boxes
)
for candidate in get_cluster_candidates(
frame_shape, region_min_size, tracked_object_boxes
)
for candidate in cluster_candidates
]
# if starting up, get the next startup scan region
if startup_scan_counter < 9:
ymin = int(frame_shape[0] / 3 * startup_scan_counter / 3)
ymax = int(frame_shape[0] / 3 + ymin)
xmin = int(frame_shape[1] / 3 * startup_scan_counter / 3)
xmax = int(frame_shape[1] / 3 + xmin)
regions.append(
calculate_region(
# only add in the motion boxes when not calibrating
if not motion_detector.is_calibrating():
# find motion boxes that are not inside tracked object regions
standalone_motion_boxes = [
b for b in motion_boxes if not inside_any(b, regions)
]
if standalone_motion_boxes:
motion_clusters = get_cluster_candidates(
frame_shape,
xmin,
ymin,
xmax,
ymax,
region_min_size,
multiplier=1.2,
standalone_motion_boxes,
)
)
startup_scan_counter += 1
motion_regions = [
get_cluster_region_from_grid(
frame_shape,
region_min_size,
candidate,
standalone_motion_boxes,
region_grid,
)
for candidate in motion_clusters
]
regions += motion_regions
# if starting up, get the next startup scan region
if startup_scan:
for region in get_startup_regions(
frame_shape, region_min_size, region_grid
):
regions.append(region)
startup_scan = False
# resize regions and detect
# seed with stationary objects

35
migrations/019_create_regions_table.py Normal file
View File

@ -0,0 +1,35 @@
"""Peewee migrations -- 019_create_regions_table.py.
Some examples (model - class or model name)::
> Model = migrator.orm['model_name'] # Return model in current state by name
> migrator.sql(sql) # Run custom SQL
> migrator.python(func, *args, **kwargs) # Run python code
> migrator.create_model(Model) # Create a model (could be used as decorator)
> migrator.remove_model(model, cascade=True) # Remove a model
> migrator.add_fields(model, **fields) # Add fields to a model
> migrator.change_fields(model, **fields) # Change fields
> migrator.remove_fields(model, *field_names, cascade=True)
> migrator.rename_field(model, old_field_name, new_field_name)
> migrator.rename_table(model, new_table_name)
> migrator.add_index(model, *col_names, unique=False)
> migrator.drop_index(model, *col_names)
> migrator.add_not_null(model, *field_names)
> migrator.drop_not_null(model, *field_names)
> migrator.add_default(model, field_name, default)
"""
import peewee as pw
SQL = pw.SQL
def migrate(migrator, database, fake=False, **kwargs):
migrator.sql(
'CREATE TABLE IF NOT EXISTS "regions" ("camera" VARCHAR(20) NOT NULL PRIMARY KEY, "last_update" DATETIME NOT NULL, "grid" JSON)'
)
def rollback(migrator, database, fake=False, **kwargs):
pass