From be1673b00a1e3b630fc2e0b822ebd78c6b7a5679 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Mon, 23 Dec 2019 06:40:48 -0600 Subject: [PATCH] process detected objects in a queue --- frigate/object_detection.py | 2 +- frigate/objects.py | 96 ++++++++++++++++++++++++++++++++++++- frigate/video.py | 92 +++-------------------------------- 3 files changed, 102 insertions(+), 88 deletions(-) diff --git a/frigate/object_detection.py b/frigate/object_detection.py index 4db24d8b3..4a510c1c1 100644 --- a/frigate/object_detection.py +++ b/frigate/object_detection.py @@ -35,7 +35,7 @@ class PreppedQueueProcessor(threading.Thread): self.fps.update() self.avg_inference_speed = (self.avg_inference_speed*9 + self.engine.get_inference_time())/10 - self.cameras[frame['camera_name']].add_objects(frame) + self.cameras[frame['camera_name']].detected_objects_queue.put(frame) class RegionRequester(threading.Thread): def __init__(self, camera): diff --git a/frigate/objects.py b/frigate/objects.py index 8b8b8af6b..e635a466d 100644 --- a/frigate/objects.py +++ b/frigate/objects.py @@ -4,7 +4,7 @@ import threading import cv2 import prctl import numpy as np -from . util import draw_box_with_label +from . util import draw_box_with_label, LABELS class ObjectCleaner(threading.Thread): def __init__(self, objects_parsed, detected_objects): @@ -37,6 +37,100 @@ class ObjectCleaner(threading.Thread): with self._objects_parsed: self._objects_parsed.notify_all() +class DetectedObjectsProcessor(threading.Thread): + def __init__(self, camera): + threading.Thread.__init__(self) + self.camera = camera + + def run(self): + prctl.set_name(self.__class__.__name__) + while True: + frame = self.camera.detected_objects_queue.get() + + objects = frame['detected_objects'] + + if len(objects) == 0: + return + + for raw_obj in objects: + obj = { + 'score': float(raw_obj.score), + 'box': raw_obj.bounding_box.flatten().tolist(), + 'name': str(LABELS[raw_obj.label_id]), + 'frame_time': frame['frame_time'], + 'region_id': frame['region_id'] + } + + # find the matching region + region = self.camera.regions[frame['region_id']] + + # Compute some extra properties + obj.update({ + 'xmin': int((obj['box'][0] * frame['size']) + frame['x_offset']), + 'ymin': int((obj['box'][1] * frame['size']) + frame['y_offset']), + 'xmax': int((obj['box'][2] * frame['size']) + frame['x_offset']), + 'ymax': int((obj['box'][3] * frame['size']) + frame['y_offset']) + }) + + # Compute the area + obj['area'] = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin']) + + object_name = obj['name'] + + if object_name in region['objects']: + obj_settings = region['objects'][object_name] + + # if the min area is larger than the + # detected object, don't add it to detected objects + if obj_settings.get('min_area',-1) > obj['area']: + continue + + # if the detected object is larger than the + # max area, don't add it to detected objects + if obj_settings.get('max_area', region['size']**2) < obj['area']: + continue + + # if the score is lower than the threshold, skip + if obj_settings.get('threshold', 0) > obj['score']: + continue + + # compute the coordinates of the object and make sure + # the location isnt outside the bounds of the image (can happen from rounding) + y_location = min(int(obj['ymax']), len(self.mask)-1) + x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1) + + # if the object is in a masked location, don't add it to detected objects + if self.camera.mask[y_location][x_location] == [0]: + continue + + # look to see if the bounding box is too close to the region border and the region border is not the edge of the frame + # if ((frame['x_offset'] > 0 and obj['box'][0] < 0.01) or + # (frame['y_offset'] > 0 and obj['box'][1] < 0.01) or + # (frame['x_offset']+frame['size'] < self.frame_shape[1] and obj['box'][2] > 0.99) or + # (frame['y_offset']+frame['size'] < self.frame_shape[0] and obj['box'][3] > 0.99)): + + # size, x_offset, y_offset = calculate_region(self.frame_shape, obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax']) + # This triggers WAY too often with stationary objects on the edge of a region. + # Every frame triggers it and fills the queue... + # I need to create a new region and add it to the list of regions, but + # it needs to check for a duplicate region first. + + # self.resize_queue.put({ + # 'camera_name': self.name, + # 'frame_time': frame['frame_time'], + # 'region_id': frame['region_id'], + # 'size': size, + # 'x_offset': x_offset, + # 'y_offset': y_offset + # }) + # print('object too close to region border') + #continue + + self.camera.detected_objects.append(obj) + + with self.camera.objects_parsed: + self.camera.objects_parsed.notify_all() + # Maintains the frame and object with the highest score class BestFrames(threading.Thread): diff --git a/frigate/video.py b/frigate/video.py index 14e0a5323..242b56298 100644 --- a/frigate/video.py +++ b/frigate/video.py @@ -12,7 +12,7 @@ import prctl from collections import defaultdict from . util import tonumpyarray, LABELS, draw_box_with_label, calculate_region, EventsPerSecond from . object_detection import RegionPrepper, RegionRequester -from . objects import ObjectCleaner, BestFrames +from . objects import ObjectCleaner, BestFrames, DetectedObjectsProcessor from . mqtt import MqttObjectPublisher # Stores 2 seconds worth of frames so they can be used for other threads @@ -144,6 +144,11 @@ class Camera: # Queue for prepped frames, max size set to (number of regions * 5) max_queue_size = len(self.config['regions'])*5 self.resize_queue = queue.Queue(max_queue_size) + + # Queue for raw detected objects + self.detected_objects_queue = queue.Queue() + self.detected_objects_processor = DetectedObjectsProcessor(self) + self.detected_objects_processor.start() # initialize the frame cache self.cached_frame_with_objects = { @@ -259,91 +264,6 @@ class Camera: def get_capture_pid(self): return self.ffmpeg_process.pid - def add_objects(self, frame): - objects = frame['detected_objects'] - - if len(objects) == 0: - return - - for raw_obj in objects: - obj = { - 'score': float(raw_obj.score), - 'box': raw_obj.bounding_box.flatten().tolist(), - 'name': str(LABELS[raw_obj.label_id]), - 'frame_time': frame['frame_time'], - 'region_id': frame['region_id'] - } - - # find the matching region - region = self.regions[frame['region_id']] - - # Compute some extra properties - obj.update({ - 'xmin': int((obj['box'][0] * frame['size']) + frame['x_offset']), - 'ymin': int((obj['box'][1] * frame['size']) + frame['y_offset']), - 'xmax': int((obj['box'][2] * frame['size']) + frame['x_offset']), - 'ymax': int((obj['box'][3] * frame['size']) + frame['y_offset']) - }) - - # Compute the area - obj['area'] = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin']) - - object_name = obj['name'] - - if object_name in region['objects']: - obj_settings = region['objects'][object_name] - - # if the min area is larger than the - # detected object, don't add it to detected objects - if obj_settings.get('min_area',-1) > obj['area']: - continue - - # if the detected object is larger than the - # max area, don't add it to detected objects - if obj_settings.get('max_area', region['size']**2) < obj['area']: - continue - - # if the score is lower than the threshold, skip - if obj_settings.get('threshold', 0) > obj['score']: - continue - - # compute the coordinates of the object and make sure - # the location isnt outside the bounds of the image (can happen from rounding) - y_location = min(int(obj['ymax']), len(self.mask)-1) - x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1) - - # if the object is in a masked location, don't add it to detected objects - if self.mask[y_location][x_location] == [0]: - continue - - # look to see if the bounding box is too close to the region border and the region border is not the edge of the frame - # if ((frame['x_offset'] > 0 and obj['box'][0] < 0.01) or - # (frame['y_offset'] > 0 and obj['box'][1] < 0.01) or - # (frame['x_offset']+frame['size'] < self.frame_shape[1] and obj['box'][2] > 0.99) or - # (frame['y_offset']+frame['size'] < self.frame_shape[0] and obj['box'][3] > 0.99)): - - # size, x_offset, y_offset = calculate_region(self.frame_shape, obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax']) - # This triggers WAY too often with stationary objects on the edge of a region. - # Every frame triggers it and fills the queue... - # I need to create a new region and add it to the list of regions, but - # it needs to check for a duplicate region first. - - # self.resize_queue.put({ - # 'camera_name': self.name, - # 'frame_time': frame['frame_time'], - # 'region_id': frame['region_id'], - # 'size': size, - # 'x_offset': x_offset, - # 'y_offset': y_offset - # }) - # print('object too close to region border') - #continue - - self.detected_objects.append(obj) - - with self.objects_parsed: - self.objects_parsed.notify_all() - def get_best(self, label): return self.best_frames.best_frames.get(label)