process detected objects in a queue

frigate/object_detection.py

@@ -35,7 +35,7 @@ class PreppedQueueProcessor(threading.Thread):
             self.fps.update()
             self.avg_inference_speed = (self.avg_inference_speed*9 + self.engine.get_inference_time())/10
 
-            self.cameras[frame['camera_name']].add_objects(frame)
+            self.cameras[frame['camera_name']].detected_objects_queue.put(frame)
 
 class RegionRequester(threading.Thread):
     def __init__(self, camera):

frigate/objects.py

@@ -4,7 +4,7 @@ import threading
 import cv2
 import prctl
 import numpy as np
-from . util import draw_box_with_label
+from . util import draw_box_with_label, LABELS
 
 class ObjectCleaner(threading.Thread):
     def __init__(self, objects_parsed, detected_objects):
@@ -37,6 +37,100 @@ class ObjectCleaner(threading.Thread):
                 with self._objects_parsed:
                     self._objects_parsed.notify_all()
 
+class DetectedObjectsProcessor(threading.Thread):
+    def __init__(self, camera):
+        threading.Thread.__init__(self)
+        self.camera = camera
+
+    def run(self):
+        prctl.set_name(self.__class__.__name__)
+        while True:
+            frame = self.camera.detected_objects_queue.get()
+
+            objects = frame['detected_objects']
+
+            if len(objects) == 0:
+                return
+
+            for raw_obj in objects:
+                obj = {
+                    'score': float(raw_obj.score),
+                    'box': raw_obj.bounding_box.flatten().tolist(),
+                    'name': str(LABELS[raw_obj.label_id]),
+                    'frame_time': frame['frame_time'],
+                    'region_id': frame['region_id']
+                }
+
+                # find the matching region
+                region = self.camera.regions[frame['region_id']]
+
+                # Compute some extra properties
+                obj.update({
+                    'xmin': int((obj['box'][0] * frame['size']) + frame['x_offset']),
+                    'ymin': int((obj['box'][1] * frame['size']) + frame['y_offset']),
+                    'xmax': int((obj['box'][2] * frame['size']) + frame['x_offset']),
+                    'ymax': int((obj['box'][3] * frame['size']) + frame['y_offset'])
+                })
+                
+                # Compute the area
+                obj['area'] = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
+
+                object_name = obj['name']
+
+                if object_name in region['objects']:
+                    obj_settings = region['objects'][object_name]
+
+                    # if the min area is larger than the
+                    # detected object, don't add it to detected objects
+                    if obj_settings.get('min_area',-1) > obj['area']:
+                        continue
+                    
+                    # if the detected object is larger than the
+                    # max area, don't add it to detected objects
+                    if obj_settings.get('max_area', region['size']**2) < obj['area']:
+                        continue
+
+                    # if the score is lower than the threshold, skip
+                    if obj_settings.get('threshold', 0) > obj['score']:
+                        continue
+                
+                    # compute the coordinates of the object and make sure
+                    # the location isnt outside the bounds of the image (can happen from rounding)
+                    y_location = min(int(obj['ymax']), len(self.mask)-1)
+                    x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1)
+
+                    # if the object is in a masked location, don't add it to detected objects
+                    if self.camera.mask[y_location][x_location] == [0]:
+                        continue
+                
+                # look to see if the bounding box is too close to the region border and the region border is not the edge of the frame
+                # if ((frame['x_offset'] > 0 and obj['box'][0] < 0.01) or 
+                #     (frame['y_offset'] > 0 and obj['box'][1] < 0.01) or
+                #     (frame['x_offset']+frame['size'] < self.frame_shape[1] and obj['box'][2] > 0.99) or
+                #     (frame['y_offset']+frame['size'] < self.frame_shape[0] and obj['box'][3] > 0.99)):
+
+                #     size, x_offset, y_offset = calculate_region(self.frame_shape, obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'])
+                    # This triggers WAY too often with stationary objects on the edge of a region. 
+                    # Every frame triggers it and fills the queue...
+                    # I need to create a new region and add it to the list of regions, but 
+                    # it needs to check for a duplicate region first.
+
+                    # self.resize_queue.put({
+                    #     'camera_name': self.name,
+                    #     'frame_time': frame['frame_time'],
+                    #     'region_id': frame['region_id'],
+                    #     'size': size,
+                    #     'x_offset': x_offset,
+                    #     'y_offset': y_offset
+                    # })
+                    # print('object too close to region border')
+                    #continue
+
+                self.camera.detected_objects.append(obj)
+
+            with self.camera.objects_parsed:
+                self.camera.objects_parsed.notify_all()
+
 
 # Maintains the frame and object with the highest score
 class BestFrames(threading.Thread):

frigate/video.py

@@ -12,7 +12,7 @@ import prctl
 from collections import defaultdict
 from . util import tonumpyarray, LABELS, draw_box_with_label, calculate_region, EventsPerSecond
 from . object_detection import RegionPrepper, RegionRequester
-from . objects import ObjectCleaner, BestFrames
+from . objects import ObjectCleaner, BestFrames, DetectedObjectsProcessor
 from . mqtt import MqttObjectPublisher
 
 # Stores 2 seconds worth of frames so they can be used for other threads
@@ -145,6 +145,11 @@ class Camera:
         max_queue_size = len(self.config['regions'])*5
         self.resize_queue = queue.Queue(max_queue_size)
 
+        # Queue for raw detected objects
+        self.detected_objects_queue = queue.Queue()
+        self.detected_objects_processor = DetectedObjectsProcessor(self)
+        self.detected_objects_processor.start()
+        
         # initialize the frame cache
         self.cached_frame_with_objects = {
             'frame_bytes': [],
@@ -259,91 +264,6 @@ class Camera:
     def get_capture_pid(self):
         return self.ffmpeg_process.pid
     
-    def add_objects(self, frame):
-        objects = frame['detected_objects']
-
-        if len(objects) == 0:
-            return
-
-        for raw_obj in objects:
-            obj = {
-                'score': float(raw_obj.score),
-                'box': raw_obj.bounding_box.flatten().tolist(),
-                'name': str(LABELS[raw_obj.label_id]),
-                'frame_time': frame['frame_time'],
-                'region_id': frame['region_id']
-            }
-
-            # find the matching region
-            region = self.regions[frame['region_id']]
-
-            # Compute some extra properties
-            obj.update({
-                'xmin': int((obj['box'][0] * frame['size']) + frame['x_offset']),
-                'ymin': int((obj['box'][1] * frame['size']) + frame['y_offset']),
-                'xmax': int((obj['box'][2] * frame['size']) + frame['x_offset']),
-                'ymax': int((obj['box'][3] * frame['size']) + frame['y_offset'])
-            })
-            
-            # Compute the area
-            obj['area'] = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
-
-            object_name = obj['name']
-
-            if object_name in region['objects']:
-                obj_settings = region['objects'][object_name]
-
-                # if the min area is larger than the
-                # detected object, don't add it to detected objects
-                if obj_settings.get('min_area',-1) > obj['area']:
-                    continue
-                
-                # if the detected object is larger than the
-                # max area, don't add it to detected objects
-                if obj_settings.get('max_area', region['size']**2) < obj['area']:
-                    continue
-
-                # if the score is lower than the threshold, skip
-                if obj_settings.get('threshold', 0) > obj['score']:
-                    continue
-            
-                # compute the coordinates of the object and make sure
-                # the location isnt outside the bounds of the image (can happen from rounding)
-                y_location = min(int(obj['ymax']), len(self.mask)-1)
-                x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1)
-
-                # if the object is in a masked location, don't add it to detected objects
-                if self.mask[y_location][x_location] == [0]:
-                    continue
-            
-            # look to see if the bounding box is too close to the region border and the region border is not the edge of the frame
-            # if ((frame['x_offset'] > 0 and obj['box'][0] < 0.01) or 
-            #     (frame['y_offset'] > 0 and obj['box'][1] < 0.01) or
-            #     (frame['x_offset']+frame['size'] < self.frame_shape[1] and obj['box'][2] > 0.99) or
-            #     (frame['y_offset']+frame['size'] < self.frame_shape[0] and obj['box'][3] > 0.99)):
-
-            #     size, x_offset, y_offset = calculate_region(self.frame_shape, obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'])
-                # This triggers WAY too often with stationary objects on the edge of a region. 
-                # Every frame triggers it and fills the queue...
-                # I need to create a new region and add it to the list of regions, but 
-                # it needs to check for a duplicate region first.
-
-                # self.resize_queue.put({
-                #     'camera_name': self.name,
-                #     'frame_time': frame['frame_time'],
-                #     'region_id': frame['region_id'],
-                #     'size': size,
-                #     'x_offset': x_offset,
-                #     'y_offset': y_offset
-                # })
-                # print('object too close to region border')
-                #continue
-
-            self.detected_objects.append(obj)
-
-        with self.objects_parsed:
-            self.objects_parsed.notify_all()
-    
     def get_best(self, label):
         return self.best_frames.best_frames.get(label)