mirror of
https://github.com/blakeblackshear/frigate.git
synced 2024-11-21 19:07:46 +01:00
implement filtering and switch to NMS with OpenCV
This commit is contained in:
Blake Blackshear
parent
5d0c12fbd4
commit
7b1da388d9
@ -90,38 +90,6 @@ class DetectedObjectsProcessor(threading.Thread):
|
||||
# Compute the area
|
||||
obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin'])
|
||||
|
||||
# find the matching region
|
||||
# region = self.camera.regions[frame['region_id']]
|
||||
|
||||
|
||||
# object_name = obj['name']
|
||||
# TODO: move all this to wherever we manage "tracked objects"
|
||||
# if object_name in region['objects']:
|
||||
# obj_settings = region['objects'][object_name]
|
||||
|
||||
# # if the min area is larger than the
|
||||
# # detected object, don't add it to detected objects
|
||||
# if obj_settings.get('min_area',-1) > obj['area']:
|
||||
# continue
|
||||
|
||||
# # if the detected object is larger than the
|
||||
# # max area, don't add it to detected objects
|
||||
# if obj_settings.get('max_area', region['size']**2) < obj['area']:
|
||||
# continue
|
||||
|
||||
# # if the score is lower than the threshold, skip
|
||||
# if obj_settings.get('threshold', 0) > obj['score']:
|
||||
# continue
|
||||
|
||||
# # compute the coordinates of the object and make sure
|
||||
# # the location isnt outside the bounds of the image (can happen from rounding)
|
||||
# y_location = min(int(obj['ymax']), len(self.mask)-1)
|
||||
# x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1)
|
||||
|
||||
# # if the object is in a masked location, don't add it to detected objects
|
||||
# if self.camera.mask[y_location][x_location] == [0]:
|
||||
# continue
|
||||
|
||||
self.camera.detected_objects[frame['frame_time']].append(obj)
|
||||
|
||||
with self.camera.regions_in_process_lock:
|
||||
@ -143,47 +111,38 @@ class RegionRefiner(threading.Thread):
|
||||
def run(self):
|
||||
prctl.set_name(self.__class__.__name__)
|
||||
while True:
|
||||
# TODO: I need to process the frames in order for tracking...
|
||||
frame_time = self.camera.finished_frame_queue.get()
|
||||
|
||||
detected_objects = self.camera.detected_objects[frame_time].copy()
|
||||
# print(f"{frame_time} finished")
|
||||
|
||||
object_groups = []
|
||||
# apply non-maxima suppression to suppress weak, overlapping bounding boxes
|
||||
boxes = [(o['box']['xmin'], o['box']['ymin'], o['box']['xmax']-o['box']['xmin'], o['box']['ymax']-o['box']['ymin'])
|
||||
for o in detected_objects]
|
||||
confidences = [o['score'] for o in detected_objects]
|
||||
idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
|
||||
|
||||
# group all the duplicate objects together
|
||||
# TODO: should I be grouping by object type too? also, the order can determine how well they group...
|
||||
for new_obj in self.camera.detected_objects[frame_time]:
|
||||
matching_group = self.find_group(new_obj, object_groups)
|
||||
if matching_group is None:
|
||||
object_groups.append([new_obj])
|
||||
else:
|
||||
object_groups[matching_group].append(new_obj)
|
||||
# print(f"{frame_time} - NMS reduced objects from {len(detected_objects)} to {len(idxs)}")
|
||||
|
||||
# just keep the unclipped objects
|
||||
self.camera.detected_objects[frame_time] = [obj for obj in self.camera.detected_objects[frame_time] if obj['clipped'] == False]
|
||||
|
||||
# print(f"{frame_time} found {len(object_groups)} groups")
|
||||
look_again = False
|
||||
# find the largest unclipped object in each group
|
||||
for group in object_groups:
|
||||
unclipped_objects = [obj for obj in group if obj['clipped'] == False]
|
||||
# if no unclipped objects, we need to look again
|
||||
if len(unclipped_objects) == 0:
|
||||
# print(f"{frame_time} no unclipped objects in group")
|
||||
# get selected objects
|
||||
selected_objects = []
|
||||
for index in idxs:
|
||||
obj = detected_objects[index[0]]
|
||||
selected_objects.append(obj)
|
||||
if obj['clipped']:
|
||||
box = obj['box']
|
||||
# calculate a new region that will hopefully get the entire object
|
||||
(size, x_offset, y_offset) = calculate_region(self.camera.frame_shape,
|
||||
box['xmin'], box['ymin'],
|
||||
box['xmax'], box['ymax'])
|
||||
# print(f"{frame_time} new region: {size} {x_offset} {y_offset}")
|
||||
|
||||
with self.camera.regions_in_process_lock:
|
||||
if not frame_time in self.camera.regions_in_process:
|
||||
self.camera.regions_in_process[frame_time] = 1
|
||||
else:
|
||||
self.camera.regions_in_process[frame_time] += 1
|
||||
xmin = min([obj['box']['xmin'] for obj in group])
|
||||
ymin = min([obj['box']['ymin'] for obj in group])
|
||||
xmax = max([obj['box']['xmax'] for obj in group])
|
||||
ymax = max([obj['box']['ymax'] for obj in group])
|
||||
# calculate a new region that will hopefully get the entire object
|
||||
(size, x_offset, y_offset) = calculate_region(self.camera.frame_shape,
|
||||
xmin, ymin,
|
||||
xmax, ymax)
|
||||
# print(f"{frame_time} new region: {size} {x_offset} {y_offset}")
|
||||
|
||||
# add it to the queue
|
||||
self.camera.resize_queue.put({
|
||||
@ -201,26 +160,14 @@ class RegionRefiner(threading.Thread):
|
||||
|
||||
# if we are looking again, then this frame is not ready for processing
|
||||
if look_again:
|
||||
# remove the clipped objects
|
||||
self.camera.detected_objects[frame_time] = [o for o in selected_objects if not o['clipped']]
|
||||
continue
|
||||
|
||||
# dedupe the unclipped objects
|
||||
deduped_objects = []
|
||||
for obj in self.camera.detected_objects[frame_time]:
|
||||
duplicate = None
|
||||
for index, deduped_obj in enumerate(deduped_objects):
|
||||
# if the IOU is more than 0.7, consider it a duplicate
|
||||
if self.has_overlap(obj, deduped_obj, .5):
|
||||
duplicate = index
|
||||
break
|
||||
# filter objects based on camera settings
|
||||
selected_objects = [o for o in selected_objects if not self.filtered(o)]
|
||||
|
||||
# get the higher scoring object
|
||||
if duplicate is None:
|
||||
deduped_objects.append(obj)
|
||||
else:
|
||||
if deduped_objects[duplicate]['score'] < obj['score']:
|
||||
deduped_objects[duplicate] = obj
|
||||
|
||||
self.camera.detected_objects[frame_time] = deduped_objects
|
||||
self.camera.detected_objects[frame_time] = selected_objects
|
||||
|
||||
with self.camera.objects_parsed:
|
||||
self.camera.objects_parsed.notify_all()
|
||||
@ -233,6 +180,37 @@ class RegionRefiner(threading.Thread):
|
||||
self.camera.last_processed_frame = self.camera.frame_queue.get()
|
||||
self.camera.refined_frame_queue.put(self.camera.last_processed_frame)
|
||||
|
||||
def filtered(self, obj):
|
||||
object_name = obj['name']
|
||||
|
||||
if object_name in self.camera.object_filters:
|
||||
obj_settings = self.camera.object_filters[object_name]
|
||||
|
||||
# if the min area is larger than the
|
||||
# detected object, don't add it to detected objects
|
||||
if obj_settings.get('min_area',-1) > obj['area']:
|
||||
return True
|
||||
|
||||
# if the detected object is larger than the
|
||||
# max area, don't add it to detected objects
|
||||
if obj_settings.get('max_area', self.camera.frame_shape[0]*self.camera.frame_shape[1]) < obj['area']:
|
||||
return True
|
||||
|
||||
# if the score is lower than the threshold, skip
|
||||
if obj_settings.get('threshold', 0) > obj['score']:
|
||||
return True
|
||||
|
||||
# compute the coordinates of the object and make sure
|
||||
# the location isnt outside the bounds of the image (can happen from rounding)
|
||||
y_location = min(int(obj['ymax']), len(self.camera.mask)-1)
|
||||
x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.camera.mask[0])-1)
|
||||
|
||||
# if the object is in a masked location, don't add it to detected objects
|
||||
if self.camera.mask[y_location][x_location] == [0]:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def has_overlap(self, new_obj, obj, overlap=.7):
|
||||
# compute intersection rectangle with existing object and new objects region
|
||||
existing_obj_current_region = compute_intersection_rectangle(obj['box'], new_obj['region'])
|
||||
|
||||
@ -172,17 +172,21 @@ class Camera:
|
||||
self.capture_thread = None
|
||||
self.fps = EventsPerSecond()
|
||||
|
||||
# for each region, merge the object config
|
||||
self.detection_prep_threads = []
|
||||
for region in self.config['regions']:
|
||||
region_objects = region.get('objects', {})
|
||||
# build objects config for region
|
||||
objects_with_config = set().union(global_objects_config.keys(), camera_objects_config.keys(), region_objects.keys())
|
||||
merged_objects_config = defaultdict(lambda: {})
|
||||
# merge object filter config
|
||||
objects_with_config = set().union(global_objects_config.keys(), camera_objects_config.keys())
|
||||
for obj in objects_with_config:
|
||||
merged_objects_config[obj] = {**global_objects_config.get(obj,{}), **camera_objects_config.get(obj, {}), **region_objects.get(obj, {})}
|
||||
self.object_filters = {**global_objects_config.get(obj,{}), **camera_objects_config.get(obj, {})}
|
||||
|
||||
region['objects'] = merged_objects_config
|
||||
# # for each region, merge the object config
|
||||
# for region in self.config['regions']:
|
||||
# region_objects = region.get('objects', {})
|
||||
# # build objects config for region
|
||||
# objects_with_config = set().union(global_objects_config.keys(), camera_objects_config.keys(), region_objects.keys())
|
||||
# merged_objects_config = defaultdict(lambda: {})
|
||||
# for obj in objects_with_config:
|
||||
# merged_objects_config[obj] = {**global_objects_config.get(obj,{}), **camera_objects_config.get(obj, {}), **region_objects.get(obj, {})}
|
||||
|
||||
# region['objects'] = merged_objects_config
|
||||
|
||||
# start a thread to queue resize requests for regions
|
||||
self.region_requester = RegionRequester(self)
|
||||
@ -275,9 +279,6 @@ class Camera:
|
||||
|
||||
def start(self):
|
||||
self.start_or_restart_capture()
|
||||
# start the object detection prep threads
|
||||
for detection_prep_thread in self.detection_prep_threads:
|
||||
detection_prep_thread.start()
|
||||
self.watchdog.start()
|
||||
|
||||
def join(self):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user