implement filtering and switch to NMS with OpenCV

This commit is contained in:
Blake Blackshear 2020-01-04 12:02:06 -06:00
parent f5a2252b29
commit 0c6717090c
2 changed files with 70 additions and 91 deletions

View File

@ -90,38 +90,6 @@ class DetectedObjectsProcessor(threading.Thread):
# Compute the area # Compute the area
obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin']) obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin'])
# find the matching region
# region = self.camera.regions[frame['region_id']]
# object_name = obj['name']
# TODO: move all this to wherever we manage "tracked objects"
# if object_name in region['objects']:
# obj_settings = region['objects'][object_name]
# # if the min area is larger than the
# # detected object, don't add it to detected objects
# if obj_settings.get('min_area',-1) > obj['area']:
# continue
# # if the detected object is larger than the
# # max area, don't add it to detected objects
# if obj_settings.get('max_area', region['size']**2) < obj['area']:
# continue
# # if the score is lower than the threshold, skip
# if obj_settings.get('threshold', 0) > obj['score']:
# continue
# # compute the coordinates of the object and make sure
# # the location isnt outside the bounds of the image (can happen from rounding)
# y_location = min(int(obj['ymax']), len(self.mask)-1)
# x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1)
# # if the object is in a masked location, don't add it to detected objects
# if self.camera.mask[y_location][x_location] == [0]:
# continue
self.camera.detected_objects[frame['frame_time']].append(obj) self.camera.detected_objects[frame['frame_time']].append(obj)
with self.camera.regions_in_process_lock: with self.camera.regions_in_process_lock:
@ -143,47 +111,38 @@ class RegionRefiner(threading.Thread):
def run(self): def run(self):
prctl.set_name(self.__class__.__name__) prctl.set_name(self.__class__.__name__)
while True: while True:
# TODO: I need to process the frames in order for tracking...
frame_time = self.camera.finished_frame_queue.get() frame_time = self.camera.finished_frame_queue.get()
detected_objects = self.camera.detected_objects[frame_time].copy()
# print(f"{frame_time} finished") # print(f"{frame_time} finished")
object_groups = [] # apply non-maxima suppression to suppress weak, overlapping bounding boxes
boxes = [(o['box']['xmin'], o['box']['ymin'], o['box']['xmax']-o['box']['xmin'], o['box']['ymax']-o['box']['ymin'])
for o in detected_objects]
confidences = [o['score'] for o in detected_objects]
idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
# group all the duplicate objects together # print(f"{frame_time} - NMS reduced objects from {len(detected_objects)} to {len(idxs)}")
# TODO: should I be grouping by object type too? also, the order can determine how well they group...
for new_obj in self.camera.detected_objects[frame_time]:
matching_group = self.find_group(new_obj, object_groups)
if matching_group is None:
object_groups.append([new_obj])
else:
object_groups[matching_group].append(new_obj)
# just keep the unclipped objects
self.camera.detected_objects[frame_time] = [obj for obj in self.camera.detected_objects[frame_time] if obj['clipped'] == False]
# print(f"{frame_time} found {len(object_groups)} groups")
look_again = False look_again = False
# find the largest unclipped object in each group # get selected objects
for group in object_groups: selected_objects = []
unclipped_objects = [obj for obj in group if obj['clipped'] == False] for index in idxs:
# if no unclipped objects, we need to look again obj = detected_objects[index[0]]
if len(unclipped_objects) == 0: selected_objects.append(obj)
# print(f"{frame_time} no unclipped objects in group") if obj['clipped']:
box = obj['box']
# calculate a new region that will hopefully get the entire object
(size, x_offset, y_offset) = calculate_region(self.camera.frame_shape,
box['xmin'], box['ymin'],
box['xmax'], box['ymax'])
# print(f"{frame_time} new region: {size} {x_offset} {y_offset}")
with self.camera.regions_in_process_lock: with self.camera.regions_in_process_lock:
if not frame_time in self.camera.regions_in_process: if not frame_time in self.camera.regions_in_process:
self.camera.regions_in_process[frame_time] = 1 self.camera.regions_in_process[frame_time] = 1
else: else:
self.camera.regions_in_process[frame_time] += 1 self.camera.regions_in_process[frame_time] += 1
xmin = min([obj['box']['xmin'] for obj in group])
ymin = min([obj['box']['ymin'] for obj in group])
xmax = max([obj['box']['xmax'] for obj in group])
ymax = max([obj['box']['ymax'] for obj in group])
# calculate a new region that will hopefully get the entire object
(size, x_offset, y_offset) = calculate_region(self.camera.frame_shape,
xmin, ymin,
xmax, ymax)
# print(f"{frame_time} new region: {size} {x_offset} {y_offset}")
# add it to the queue # add it to the queue
self.camera.resize_queue.put({ self.camera.resize_queue.put({
@ -201,26 +160,14 @@ class RegionRefiner(threading.Thread):
# if we are looking again, then this frame is not ready for processing # if we are looking again, then this frame is not ready for processing
if look_again: if look_again:
# remove the clipped objects
self.camera.detected_objects[frame_time] = [o for o in selected_objects if not o['clipped']]
continue continue
# dedupe the unclipped objects # filter objects based on camera settings
deduped_objects = [] selected_objects = [o for o in selected_objects if not self.filtered(o)]
for obj in self.camera.detected_objects[frame_time]:
duplicate = None
for index, deduped_obj in enumerate(deduped_objects):
# if the IOU is more than 0.7, consider it a duplicate
if self.has_overlap(obj, deduped_obj, .5):
duplicate = index
break
# get the higher scoring object
if duplicate is None:
deduped_objects.append(obj)
else:
if deduped_objects[duplicate]['score'] < obj['score']:
deduped_objects[duplicate] = obj
self.camera.detected_objects[frame_time] = deduped_objects self.camera.detected_objects[frame_time] = selected_objects
with self.camera.objects_parsed: with self.camera.objects_parsed:
self.camera.objects_parsed.notify_all() self.camera.objects_parsed.notify_all()
@ -232,6 +179,37 @@ class RegionRefiner(threading.Thread):
while self.camera.frame_queue.qsize() > 0 and self.camera.frame_queue.queue[0] not in self.camera.regions_in_process: while self.camera.frame_queue.qsize() > 0 and self.camera.frame_queue.queue[0] not in self.camera.regions_in_process:
self.camera.last_processed_frame = self.camera.frame_queue.get() self.camera.last_processed_frame = self.camera.frame_queue.get()
self.camera.refined_frame_queue.put(self.camera.last_processed_frame) self.camera.refined_frame_queue.put(self.camera.last_processed_frame)
def filtered(self, obj):
object_name = obj['name']
if object_name in self.camera.object_filters:
obj_settings = self.camera.object_filters[object_name]
# if the min area is larger than the
# detected object, don't add it to detected objects
if obj_settings.get('min_area',-1) > obj['area']:
return True
# if the detected object is larger than the
# max area, don't add it to detected objects
if obj_settings.get('max_area', self.camera.frame_shape[0]*self.camera.frame_shape[1]) < obj['area']:
return True
# if the score is lower than the threshold, skip
if obj_settings.get('threshold', 0) > obj['score']:
return True
# compute the coordinates of the object and make sure
# the location isnt outside the bounds of the image (can happen from rounding)
y_location = min(int(obj['ymax']), len(self.camera.mask)-1)
x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.camera.mask[0])-1)
# if the object is in a masked location, don't add it to detected objects
if self.camera.mask[y_location][x_location] == [0]:
return True
return False
def has_overlap(self, new_obj, obj, overlap=.7): def has_overlap(self, new_obj, obj, overlap=.7):
# compute intersection rectangle with existing object and new objects region # compute intersection rectangle with existing object and new objects region

View File

@ -172,17 +172,21 @@ class Camera:
self.capture_thread = None self.capture_thread = None
self.fps = EventsPerSecond() self.fps = EventsPerSecond()
# for each region, merge the object config # merge object filter config
self.detection_prep_threads = [] objects_with_config = set().union(global_objects_config.keys(), camera_objects_config.keys())
for region in self.config['regions']: for obj in objects_with_config:
region_objects = region.get('objects', {}) self.object_filters = {**global_objects_config.get(obj,{}), **camera_objects_config.get(obj, {})}
# build objects config for region
objects_with_config = set().union(global_objects_config.keys(), camera_objects_config.keys(), region_objects.keys()) # # for each region, merge the object config
merged_objects_config = defaultdict(lambda: {}) # for region in self.config['regions']:
for obj in objects_with_config: # region_objects = region.get('objects', {})
merged_objects_config[obj] = {**global_objects_config.get(obj,{}), **camera_objects_config.get(obj, {}), **region_objects.get(obj, {})} # # build objects config for region
# objects_with_config = set().union(global_objects_config.keys(), camera_objects_config.keys(), region_objects.keys())
# merged_objects_config = defaultdict(lambda: {})
# for obj in objects_with_config:
# merged_objects_config[obj] = {**global_objects_config.get(obj,{}), **camera_objects_config.get(obj, {}), **region_objects.get(obj, {})}
region['objects'] = merged_objects_config # region['objects'] = merged_objects_config
# start a thread to queue resize requests for regions # start a thread to queue resize requests for regions
self.region_requester = RegionRequester(self) self.region_requester = RegionRequester(self)
@ -275,9 +279,6 @@ class Camera:
def start(self): def start(self):
self.start_or_restart_capture() self.start_or_restart_capture()
# start the object detection prep threads
for detection_prep_thread in self.detection_prep_threads:
detection_prep_thread.start()
self.watchdog.start() self.watchdog.start()
def join(self): def join(self):