From 0c6717090c30385205e94708b7c521706850a619 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 4 Jan 2020 12:02:06 -0600 Subject: [PATCH] implement filtering and switch to NMS with OpenCV --- frigate/objects.py | 134 +++++++++++++++++++-------------------------- frigate/video.py | 27 ++++----- 2 files changed, 70 insertions(+), 91 deletions(-) diff --git a/frigate/objects.py b/frigate/objects.py index f35274bc1..37eea7956 100644 --- a/frigate/objects.py +++ b/frigate/objects.py @@ -90,38 +90,6 @@ class DetectedObjectsProcessor(threading.Thread): # Compute the area obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin']) - # find the matching region - # region = self.camera.regions[frame['region_id']] - - - # object_name = obj['name'] - # TODO: move all this to wherever we manage "tracked objects" - # if object_name in region['objects']: - # obj_settings = region['objects'][object_name] - - # # if the min area is larger than the - # # detected object, don't add it to detected objects - # if obj_settings.get('min_area',-1) > obj['area']: - # continue - - # # if the detected object is larger than the - # # max area, don't add it to detected objects - # if obj_settings.get('max_area', region['size']**2) < obj['area']: - # continue - - # # if the score is lower than the threshold, skip - # if obj_settings.get('threshold', 0) > obj['score']: - # continue - - # # compute the coordinates of the object and make sure - # # the location isnt outside the bounds of the image (can happen from rounding) - # y_location = min(int(obj['ymax']), len(self.mask)-1) - # x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1) - - # # if the object is in a masked location, don't add it to detected objects - # if self.camera.mask[y_location][x_location] == [0]: - # continue - self.camera.detected_objects[frame['frame_time']].append(obj) with self.camera.regions_in_process_lock: @@ -143,47 +111,38 @@ class RegionRefiner(threading.Thread): def run(self): prctl.set_name(self.__class__.__name__) while True: - # TODO: I need to process the frames in order for tracking... frame_time = self.camera.finished_frame_queue.get() + detected_objects = self.camera.detected_objects[frame_time].copy() # print(f"{frame_time} finished") - object_groups = [] + # apply non-maxima suppression to suppress weak, overlapping bounding boxes + boxes = [(o['box']['xmin'], o['box']['ymin'], o['box']['xmax']-o['box']['xmin'], o['box']['ymax']-o['box']['ymin']) + for o in detected_objects] + confidences = [o['score'] for o in detected_objects] + idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4) - # group all the duplicate objects together - # TODO: should I be grouping by object type too? also, the order can determine how well they group... - for new_obj in self.camera.detected_objects[frame_time]: - matching_group = self.find_group(new_obj, object_groups) - if matching_group is None: - object_groups.append([new_obj]) - else: - object_groups[matching_group].append(new_obj) - - # just keep the unclipped objects - self.camera.detected_objects[frame_time] = [obj for obj in self.camera.detected_objects[frame_time] if obj['clipped'] == False] + # print(f"{frame_time} - NMS reduced objects from {len(detected_objects)} to {len(idxs)}") - # print(f"{frame_time} found {len(object_groups)} groups") look_again = False - # find the largest unclipped object in each group - for group in object_groups: - unclipped_objects = [obj for obj in group if obj['clipped'] == False] - # if no unclipped objects, we need to look again - if len(unclipped_objects) == 0: - # print(f"{frame_time} no unclipped objects in group") + # get selected objects + selected_objects = [] + for index in idxs: + obj = detected_objects[index[0]] + selected_objects.append(obj) + if obj['clipped']: + box = obj['box'] + # calculate a new region that will hopefully get the entire object + (size, x_offset, y_offset) = calculate_region(self.camera.frame_shape, + box['xmin'], box['ymin'], + box['xmax'], box['ymax']) + # print(f"{frame_time} new region: {size} {x_offset} {y_offset}") + with self.camera.regions_in_process_lock: if not frame_time in self.camera.regions_in_process: self.camera.regions_in_process[frame_time] = 1 else: self.camera.regions_in_process[frame_time] += 1 - xmin = min([obj['box']['xmin'] for obj in group]) - ymin = min([obj['box']['ymin'] for obj in group]) - xmax = max([obj['box']['xmax'] for obj in group]) - ymax = max([obj['box']['ymax'] for obj in group]) - # calculate a new region that will hopefully get the entire object - (size, x_offset, y_offset) = calculate_region(self.camera.frame_shape, - xmin, ymin, - xmax, ymax) - # print(f"{frame_time} new region: {size} {x_offset} {y_offset}") # add it to the queue self.camera.resize_queue.put({ @@ -201,26 +160,14 @@ class RegionRefiner(threading.Thread): # if we are looking again, then this frame is not ready for processing if look_again: + # remove the clipped objects + self.camera.detected_objects[frame_time] = [o for o in selected_objects if not o['clipped']] continue - # dedupe the unclipped objects - deduped_objects = [] - for obj in self.camera.detected_objects[frame_time]: - duplicate = None - for index, deduped_obj in enumerate(deduped_objects): - # if the IOU is more than 0.7, consider it a duplicate - if self.has_overlap(obj, deduped_obj, .5): - duplicate = index - break - - # get the higher scoring object - if duplicate is None: - deduped_objects.append(obj) - else: - if deduped_objects[duplicate]['score'] < obj['score']: - deduped_objects[duplicate] = obj + # filter objects based on camera settings + selected_objects = [o for o in selected_objects if not self.filtered(o)] - self.camera.detected_objects[frame_time] = deduped_objects + self.camera.detected_objects[frame_time] = selected_objects with self.camera.objects_parsed: self.camera.objects_parsed.notify_all() @@ -232,6 +179,37 @@ class RegionRefiner(threading.Thread): while self.camera.frame_queue.qsize() > 0 and self.camera.frame_queue.queue[0] not in self.camera.regions_in_process: self.camera.last_processed_frame = self.camera.frame_queue.get() self.camera.refined_frame_queue.put(self.camera.last_processed_frame) + + def filtered(self, obj): + object_name = obj['name'] + + if object_name in self.camera.object_filters: + obj_settings = self.camera.object_filters[object_name] + + # if the min area is larger than the + # detected object, don't add it to detected objects + if obj_settings.get('min_area',-1) > obj['area']: + return True + + # if the detected object is larger than the + # max area, don't add it to detected objects + if obj_settings.get('max_area', self.camera.frame_shape[0]*self.camera.frame_shape[1]) < obj['area']: + return True + + # if the score is lower than the threshold, skip + if obj_settings.get('threshold', 0) > obj['score']: + return True + + # compute the coordinates of the object and make sure + # the location isnt outside the bounds of the image (can happen from rounding) + y_location = min(int(obj['ymax']), len(self.camera.mask)-1) + x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.camera.mask[0])-1) + + # if the object is in a masked location, don't add it to detected objects + if self.camera.mask[y_location][x_location] == [0]: + return True + + return False def has_overlap(self, new_obj, obj, overlap=.7): # compute intersection rectangle with existing object and new objects region diff --git a/frigate/video.py b/frigate/video.py index 08ce987c4..417a97a39 100644 --- a/frigate/video.py +++ b/frigate/video.py @@ -172,17 +172,21 @@ class Camera: self.capture_thread = None self.fps = EventsPerSecond() - # for each region, merge the object config - self.detection_prep_threads = [] - for region in self.config['regions']: - region_objects = region.get('objects', {}) - # build objects config for region - objects_with_config = set().union(global_objects_config.keys(), camera_objects_config.keys(), region_objects.keys()) - merged_objects_config = defaultdict(lambda: {}) - for obj in objects_with_config: - merged_objects_config[obj] = {**global_objects_config.get(obj,{}), **camera_objects_config.get(obj, {}), **region_objects.get(obj, {})} + # merge object filter config + objects_with_config = set().union(global_objects_config.keys(), camera_objects_config.keys()) + for obj in objects_with_config: + self.object_filters = {**global_objects_config.get(obj,{}), **camera_objects_config.get(obj, {})} + + # # for each region, merge the object config + # for region in self.config['regions']: + # region_objects = region.get('objects', {}) + # # build objects config for region + # objects_with_config = set().union(global_objects_config.keys(), camera_objects_config.keys(), region_objects.keys()) + # merged_objects_config = defaultdict(lambda: {}) + # for obj in objects_with_config: + # merged_objects_config[obj] = {**global_objects_config.get(obj,{}), **camera_objects_config.get(obj, {}), **region_objects.get(obj, {})} - region['objects'] = merged_objects_config + # region['objects'] = merged_objects_config # start a thread to queue resize requests for regions self.region_requester = RegionRequester(self) @@ -275,9 +279,6 @@ class Camera: def start(self): self.start_or_restart_capture() - # start the object detection prep threads - for detection_prep_thread in self.detection_prep_threads: - detection_prep_thread.start() self.watchdog.start() def join(self):