process detected objects in a queue

This commit is contained in:
Blake Blackshear 2019-12-23 06:40:48 -06:00
parent b6130e77ff
commit be1673b00a
3 changed files with 102 additions and 88 deletions

View File

@ -35,7 +35,7 @@ class PreppedQueueProcessor(threading.Thread):
self.fps.update()
self.avg_inference_speed = (self.avg_inference_speed*9 + self.engine.get_inference_time())/10
self.cameras[frame['camera_name']].add_objects(frame)
self.cameras[frame['camera_name']].detected_objects_queue.put(frame)
class RegionRequester(threading.Thread):
def __init__(self, camera):

View File

@ -4,7 +4,7 @@ import threading
import cv2
import prctl
import numpy as np
from . util import draw_box_with_label
from . util import draw_box_with_label, LABELS
class ObjectCleaner(threading.Thread):
def __init__(self, objects_parsed, detected_objects):
@ -37,6 +37,100 @@ class ObjectCleaner(threading.Thread):
with self._objects_parsed:
self._objects_parsed.notify_all()
class DetectedObjectsProcessor(threading.Thread):
def __init__(self, camera):
threading.Thread.__init__(self)
self.camera = camera
def run(self):
prctl.set_name(self.__class__.__name__)
while True:
frame = self.camera.detected_objects_queue.get()
objects = frame['detected_objects']
if len(objects) == 0:
return
for raw_obj in objects:
obj = {
'score': float(raw_obj.score),
'box': raw_obj.bounding_box.flatten().tolist(),
'name': str(LABELS[raw_obj.label_id]),
'frame_time': frame['frame_time'],
'region_id': frame['region_id']
}
# find the matching region
region = self.camera.regions[frame['region_id']]
# Compute some extra properties
obj.update({
'xmin': int((obj['box'][0] * frame['size']) + frame['x_offset']),
'ymin': int((obj['box'][1] * frame['size']) + frame['y_offset']),
'xmax': int((obj['box'][2] * frame['size']) + frame['x_offset']),
'ymax': int((obj['box'][3] * frame['size']) + frame['y_offset'])
})
# Compute the area
obj['area'] = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
object_name = obj['name']
if object_name in region['objects']:
obj_settings = region['objects'][object_name]
# if the min area is larger than the
# detected object, don't add it to detected objects
if obj_settings.get('min_area',-1) > obj['area']:
continue
# if the detected object is larger than the
# max area, don't add it to detected objects
if obj_settings.get('max_area', region['size']**2) < obj['area']:
continue
# if the score is lower than the threshold, skip
if obj_settings.get('threshold', 0) > obj['score']:
continue
# compute the coordinates of the object and make sure
# the location isnt outside the bounds of the image (can happen from rounding)
y_location = min(int(obj['ymax']), len(self.mask)-1)
x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1)
# if the object is in a masked location, don't add it to detected objects
if self.camera.mask[y_location][x_location] == [0]:
continue
# look to see if the bounding box is too close to the region border and the region border is not the edge of the frame
# if ((frame['x_offset'] > 0 and obj['box'][0] < 0.01) or
# (frame['y_offset'] > 0 and obj['box'][1] < 0.01) or
# (frame['x_offset']+frame['size'] < self.frame_shape[1] and obj['box'][2] > 0.99) or
# (frame['y_offset']+frame['size'] < self.frame_shape[0] and obj['box'][3] > 0.99)):
# size, x_offset, y_offset = calculate_region(self.frame_shape, obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'])
# This triggers WAY too often with stationary objects on the edge of a region.
# Every frame triggers it and fills the queue...
# I need to create a new region and add it to the list of regions, but
# it needs to check for a duplicate region first.
# self.resize_queue.put({
# 'camera_name': self.name,
# 'frame_time': frame['frame_time'],
# 'region_id': frame['region_id'],
# 'size': size,
# 'x_offset': x_offset,
# 'y_offset': y_offset
# })
# print('object too close to region border')
#continue
self.camera.detected_objects.append(obj)
with self.camera.objects_parsed:
self.camera.objects_parsed.notify_all()
# Maintains the frame and object with the highest score
class BestFrames(threading.Thread):

View File

@ -12,7 +12,7 @@ import prctl
from collections import defaultdict
from . util import tonumpyarray, LABELS, draw_box_with_label, calculate_region, EventsPerSecond
from . object_detection import RegionPrepper, RegionRequester
from . objects import ObjectCleaner, BestFrames
from . objects import ObjectCleaner, BestFrames, DetectedObjectsProcessor
from . mqtt import MqttObjectPublisher
# Stores 2 seconds worth of frames so they can be used for other threads
@ -144,6 +144,11 @@ class Camera:
# Queue for prepped frames, max size set to (number of regions * 5)
max_queue_size = len(self.config['regions'])*5
self.resize_queue = queue.Queue(max_queue_size)
# Queue for raw detected objects
self.detected_objects_queue = queue.Queue()
self.detected_objects_processor = DetectedObjectsProcessor(self)
self.detected_objects_processor.start()
# initialize the frame cache
self.cached_frame_with_objects = {
@ -259,91 +264,6 @@ class Camera:
def get_capture_pid(self):
return self.ffmpeg_process.pid
def add_objects(self, frame):
objects = frame['detected_objects']
if len(objects) == 0:
return
for raw_obj in objects:
obj = {
'score': float(raw_obj.score),
'box': raw_obj.bounding_box.flatten().tolist(),
'name': str(LABELS[raw_obj.label_id]),
'frame_time': frame['frame_time'],
'region_id': frame['region_id']
}
# find the matching region
region = self.regions[frame['region_id']]
# Compute some extra properties
obj.update({
'xmin': int((obj['box'][0] * frame['size']) + frame['x_offset']),
'ymin': int((obj['box'][1] * frame['size']) + frame['y_offset']),
'xmax': int((obj['box'][2] * frame['size']) + frame['x_offset']),
'ymax': int((obj['box'][3] * frame['size']) + frame['y_offset'])
})
# Compute the area
obj['area'] = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
object_name = obj['name']
if object_name in region['objects']:
obj_settings = region['objects'][object_name]
# if the min area is larger than the
# detected object, don't add it to detected objects
if obj_settings.get('min_area',-1) > obj['area']:
continue
# if the detected object is larger than the
# max area, don't add it to detected objects
if obj_settings.get('max_area', region['size']**2) < obj['area']:
continue
# if the score is lower than the threshold, skip
if obj_settings.get('threshold', 0) > obj['score']:
continue
# compute the coordinates of the object and make sure
# the location isnt outside the bounds of the image (can happen from rounding)
y_location = min(int(obj['ymax']), len(self.mask)-1)
x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1)
# if the object is in a masked location, don't add it to detected objects
if self.mask[y_location][x_location] == [0]:
continue
# look to see if the bounding box is too close to the region border and the region border is not the edge of the frame
# if ((frame['x_offset'] > 0 and obj['box'][0] < 0.01) or
# (frame['y_offset'] > 0 and obj['box'][1] < 0.01) or
# (frame['x_offset']+frame['size'] < self.frame_shape[1] and obj['box'][2] > 0.99) or
# (frame['y_offset']+frame['size'] < self.frame_shape[0] and obj['box'][3] > 0.99)):
# size, x_offset, y_offset = calculate_region(self.frame_shape, obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'])
# This triggers WAY too often with stationary objects on the edge of a region.
# Every frame triggers it and fills the queue...
# I need to create a new region and add it to the list of regions, but
# it needs to check for a duplicate region first.
# self.resize_queue.put({
# 'camera_name': self.name,
# 'frame_time': frame['frame_time'],
# 'region_id': frame['region_id'],
# 'size': size,
# 'x_offset': x_offset,
# 'y_offset': y_offset
# })
# print('object too close to region border')
#continue
self.detected_objects.append(obj)
with self.objects_parsed:
self.objects_parsed.notify_all()
def get_best(self, label):
return self.best_frames.best_frames.get(label)