diff --git a/detect_objects.py b/detect_objects.py index 50acd5a7d..5ac5f5299 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -37,22 +37,16 @@ DEBUG = (os.getenv('DEBUG') == '1') def main(): DETECTED_OBJECTS = [] - recent_motion_frames = {} + recent_frames = {} # Parse selected regions regions = [] for region_string in REGIONS.split(':'): region_parts = region_string.split(',') - region_mask_image = cv2.imread("/config/{}".format(region_parts[5]), cv2.IMREAD_GRAYSCALE) - region_mask = np.where(region_mask_image==[0]) regions.append({ 'size': int(region_parts[0]), 'x_offset': int(region_parts[1]), 'y_offset': int(region_parts[2]), 'min_person_area': int(region_parts[3]), - 'min_object_size': int(region_parts[4]), - 'mask': region_mask, - # Event for motion detection signaling - 'motion_detected': mp.Event(), # array for prepped frame with shape (1, 300, 300, 3) 'prepped_frame_array': mp.Array(ctypes.c_uint8, 300*300*3), # shared value for storing the prepped_frame_time @@ -81,14 +75,13 @@ def main(): frame_lock = mp.Lock() # Condition for notifying that a new frame is ready frame_ready = mp.Condition() - # Condition for notifying that motion status changed globally - motion_changed = mp.Condition() - + # Shared memory array for passing prepped frame to tensorflow prepped_frame_array = mp.Array(ctypes.c_uint8, 300*300*3) # create shared value for storing the frame_time prepped_frame_time = mp.Value('d', 0.0) # Event for notifying that object detection needs a new frame prepped_frame_grabbed = mp.Event() + # Event for notifying that new frame is ready for detection prepped_frame_ready = mp.Event() # Condition for notifying that objects were parsed objects_parsed = mp.Condition() @@ -96,6 +89,7 @@ def main(): object_queue = mp.Queue() # Queue for prepped frames prepped_frame_queue = queue.Queue(len(regions)*2) + # Array for passing original region box to compute object bounding box prepped_frame_box = mp.Array(ctypes.c_uint16, 3) # shape current frame so it can be treated as an image @@ -106,32 +100,18 @@ def main(): shared_frame_time, frame_lock, frame_ready, frame_shape, RTSP_URL)) capture_process.daemon = True - # for each region, start a separate process for motion detection and object detection + # for each region, start a separate thread to resize the region and prep for detection detection_prep_threads = [] - motion_processes = [] for region in regions: detection_prep_threads.append(FramePrepper( frame_arr, shared_frame_time, frame_ready, frame_lock, - region['motion_detected'], region['size'], region['x_offset'], region['y_offset'], prepped_frame_queue )) - motion_process = mp.Process(target=detect_motion, args=(shared_arr, - shared_frame_time, - frame_lock, frame_ready, - region['motion_detected'], - motion_changed, - frame_shape, - region['size'], region['x_offset'], region['y_offset'], - region['min_object_size'], region['mask'], - DEBUG)) - motion_process.daemon = True - motion_processes.append(motion_process) - prepped_queue_processor = PreppedQueueProcessor( prepped_frame_array, prepped_frame_time, @@ -157,24 +137,22 @@ def main(): # start a thread to store recent motion frames for processing frame_tracker = FrameTracker(frame_arr, shared_frame_time, frame_ready, frame_lock, - recent_motion_frames, motion_changed, [region['motion_detected'] for region in regions]) + recent_frames) frame_tracker.start() # start a thread to store the highest scoring recent person frame - best_person_frame = BestPersonFrame(objects_parsed, recent_motion_frames, DETECTED_OBJECTS, - motion_changed, [region['motion_detected'] for region in regions]) + best_person_frame = BestPersonFrame(objects_parsed, recent_frames, DETECTED_OBJECTS) best_person_frame.start() # start a thread to parse objects from the queue object_parser = ObjectParser(object_queue, objects_parsed, DETECTED_OBJECTS) object_parser.start() # start a thread to expire objects from the detected objects list - object_cleaner = ObjectCleaner(objects_parsed, DETECTED_OBJECTS, - motion_changed, [region['motion_detected'] for region in regions]) + object_cleaner = ObjectCleaner(objects_parsed, DETECTED_OBJECTS) object_cleaner.start() # connect to mqtt and setup last will - def on_connect(client, userdata, flags, rc): + def on_connect(client, userdata, flags, rc): print("On connect called") # publish a message to signal that the service is running client.publish(MQTT_TOPIC_PREFIX+'/available', 'online', retain=True) @@ -191,32 +169,16 @@ def main(): mqtt_publisher = MqttObjectPublisher(client, MQTT_TOPIC_PREFIX, objects_parsed, DETECTED_OBJECTS) mqtt_publisher.start() - # start thread to publish motion status - mqtt_motion_publisher = MqttMotionPublisher(client, MQTT_TOPIC_PREFIX, motion_changed, - [region['motion_detected'] for region in regions]) - mqtt_motion_publisher.start() - # start the process of capturing frames capture_process.start() print("capture_process pid ", capture_process.pid) - # start the object detection prep processes + # start the object detection prep threads for detection_prep_thread in detection_prep_threads: detection_prep_thread.start() detection_process.start() print("detection_process pid ", detection_process.pid) - - # start the motion detection processes - # for motion_process in motion_processes: - # motion_process.start() - # print("motion_process pid ", motion_process.pid) - - # TEMP: short circuit the motion detection - for region in regions: - region['motion_detected'].set() - with motion_changed: - motion_changed.notify_all() # create a flask app that encodes frames a mjpeg on demand app = Flask(__name__) @@ -259,8 +221,6 @@ def main(): for region in regions: color = (255,255,255) - if region['motion_detected'].is_set(): - color = (0,255,0) cv2.rectangle(frame, (region['x_offset'], region['y_offset']), (region['x_offset']+region['size'], region['y_offset']+region['size']), color, 2) @@ -277,8 +237,6 @@ def main(): capture_process.join() for detection_prep_thread in detection_prep_threads: detection_prep_thread.join() - for motion_process in motion_processes: - motion_process.join() detection_process.join() frame_tracker.join() best_person_frame.join() diff --git a/frigate/object_detection.py b/frigate/object_detection.py index 235739d7a..0b8099247 100644 --- a/frigate/object_detection.py +++ b/frigate/object_detection.py @@ -47,7 +47,7 @@ def detect_objects(prepped_frame_array, prepped_frame_time, objects = engine.DetectWithInputTensor(prepped_frame_copy, threshold=0.5, top_k=3) # time.sleep(0.1) # objects = [] - print(engine.get_inference_time()) + # print(engine.get_inference_time()) # put detected objects in the queue if objects: for obj in objects: @@ -109,7 +109,7 @@ class PreppedQueueProcessor(threading.Thread): # should this be a region class? class FramePrepper(threading.Thread): def __init__(self, shared_frame, frame_time, frame_ready, - frame_lock, motion_detected, + frame_lock, region_size, region_x_offset, region_y_offset, prepped_frame_queue): @@ -118,7 +118,6 @@ class FramePrepper(threading.Thread): self.frame_time = frame_time self.frame_ready = frame_ready self.frame_lock = frame_lock - self.motion_detected = motion_detected self.region_size = region_size self.region_x_offset = region_x_offset self.region_y_offset = region_y_offset @@ -129,9 +128,6 @@ class FramePrepper(threading.Thread): while True: now = datetime.datetime.now().timestamp() - # wait until motion is detected - self.motion_detected.wait() - with self.frame_ready: # if there isnt a frame ready for processing or it is old, wait for a new frame if self.frame_time.value == frame_time or (now - self.frame_time.value) > 0.5: diff --git a/frigate/objects.py b/frigate/objects.py index 9c602430b..605e7254d 100644 --- a/frigate/objects.py +++ b/frigate/objects.py @@ -30,114 +30,92 @@ class ObjectParser(threading.Thread): self._objects_parsed.notify_all() class ObjectCleaner(threading.Thread): - def __init__(self, objects_parsed, detected_objects, motion_changed, motion_regions): + def __init__(self, objects_parsed, detected_objects): threading.Thread.__init__(self) self._objects_parsed = objects_parsed self._detected_objects = detected_objects - self.motion_changed = motion_changed - self.motion_regions = motion_regions def run(self): while True: - # while there is motion - while len([r for r in self.motion_regions if r.is_set()]) > 0: - # wait a bit before checking for expired frames - time.sleep(0.2) + # wait a bit before checking for expired frames + time.sleep(0.2) - # expire the objects that are more than 1 second old - now = datetime.datetime.now().timestamp() - # look for the first object found within the last second - # (newest objects are appended to the end) - detected_objects = self._detected_objects.copy() + # expire the objects that are more than 1 second old + now = datetime.datetime.now().timestamp() + # look for the first object found within the last second + # (newest objects are appended to the end) + detected_objects = self._detected_objects.copy() - #print([round(now-obj['frame_time'],2) for obj in detected_objects]) - num_to_delete = 0 - for obj in detected_objects: - if now-obj['frame_time']<2: - break - num_to_delete += 1 - if num_to_delete > 0: - del self._detected_objects[:num_to_delete] + #print([round(now-obj['frame_time'],2) for obj in detected_objects]) + num_to_delete = 0 + for obj in detected_objects: + if now-obj['frame_time']<2: + break + num_to_delete += 1 + if num_to_delete > 0: + del self._detected_objects[:num_to_delete] - # notify that parsed objects were changed - with self._objects_parsed: - self._objects_parsed.notify_all() + # notify that parsed objects were changed + with self._objects_parsed: + self._objects_parsed.notify_all() - # wait for the global motion flag to change - with self.motion_changed: - self.motion_changed.wait() # Maintains the frame and person with the highest score from the most recent # motion event class BestPersonFrame(threading.Thread): - def __init__(self, objects_parsed, recent_frames, detected_objects, motion_changed, motion_regions): + def __init__(self, objects_parsed, recent_frames, detected_objects): threading.Thread.__init__(self) self.objects_parsed = objects_parsed self.recent_frames = recent_frames self.detected_objects = detected_objects - self.motion_changed = motion_changed - self.motion_regions = motion_regions self.best_person = None self.best_frame = None def run(self): - motion_start = 0.0 - motion_end = 0.0 - while True: - # while there is motion - while len([r for r in self.motion_regions if r.is_set()]) > 0: - # wait until objects have been parsed - with self.objects_parsed: - self.objects_parsed.wait() + # wait until objects have been parsed + with self.objects_parsed: + self.objects_parsed.wait() - # make a copy of detected objects - detected_objects = self.detected_objects.copy() - detected_people = [obj for obj in detected_objects if obj['name'] == 'person'] - # make a copy of the recent frames - recent_frames = self.recent_frames.copy() + # make a copy of detected objects + detected_objects = self.detected_objects.copy() + detected_people = [obj for obj in detected_objects if obj['name'] == 'person'] + # make a copy of the recent frames + recent_frames = self.recent_frames.copy() - # get the highest scoring person - new_best_person = max(detected_people, key=lambda x:x['score'], default=self.best_person) + # get the highest scoring person + new_best_person = max(detected_people, key=lambda x:x['score'], default=self.best_person) - # if there isnt a person, continue - if new_best_person is None: - continue + # if there isnt a person, continue + if new_best_person is None: + continue - # if there is no current best_person - if self.best_person is None: + # if there is no current best_person + if self.best_person is None: + self.best_person = new_best_person + # if there is already a best_person + else: + now = datetime.datetime.now().timestamp() + # if the new best person is a higher score than the current best person + # or the current person is more than 1 minute old, use the new best person + if new_best_person['score'] > self.best_person['score'] or (now - self.best_person['frame_time']) > 60: self.best_person = new_best_person - # if there is already a best_person - else: - now = datetime.datetime.now().timestamp() - # if the new best person is a higher score than the current best person - # or the current person is more than 1 minute old, use the new best person - if new_best_person['score'] > self.best_person['score'] or (now - self.best_person['frame_time']) > 60: - self.best_person = new_best_person - if not self.best_person is None and self.best_person['frame_time'] in recent_frames: - best_frame = recent_frames[self.best_person['frame_time']] - best_frame = cv2.cvtColor(best_frame, cv2.COLOR_BGR2RGB) - # draw the bounding box on the frame - vis_util.draw_bounding_box_on_image_array(best_frame, - self.best_person['ymin'], - self.best_person['xmin'], - self.best_person['ymax'], - self.best_person['xmax'], - color='red', - thickness=2, - display_str_list=["{}: {}%".format(self.best_person['name'],int(self.best_person['score']*100))], - use_normalized_coordinates=False) + if not self.best_person is None and self.best_person['frame_time'] in recent_frames: + best_frame = recent_frames[self.best_person['frame_time']] + best_frame = cv2.cvtColor(best_frame, cv2.COLOR_BGR2RGB) + # draw the bounding box on the frame + vis_util.draw_bounding_box_on_image_array(best_frame, + self.best_person['ymin'], + self.best_person['xmin'], + self.best_person['ymax'], + self.best_person['xmax'], + color='red', + thickness=2, + display_str_list=["{}: {}%".format(self.best_person['name'],int(self.best_person['score']*100))], + use_normalized_coordinates=False) - # convert back to BGR - self.best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR) - - motion_end = datetime.datetime.now().timestamp() - - # wait for the global motion flag to change - with self.motion_changed: - self.motion_changed.wait() - - motion_start = datetime.datetime.now().timestamp() \ No newline at end of file + # convert back to BGR + self.best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR) diff --git a/frigate/video.py b/frigate/video.py index 0edcc6989..a4eeef8e5 100644 --- a/frigate/video.py +++ b/frigate/video.py @@ -54,42 +54,34 @@ def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_s # Stores 2 seconds worth of frames when motion is detected so they can be used for other threads class FrameTracker(threading.Thread): - def __init__(self, shared_frame, frame_time, frame_ready, frame_lock, recent_frames, motion_changed, motion_regions): + def __init__(self, shared_frame, frame_time, frame_ready, frame_lock, recent_frames): threading.Thread.__init__(self) self.shared_frame = shared_frame self.frame_time = frame_time self.frame_ready = frame_ready self.frame_lock = frame_lock self.recent_frames = recent_frames - self.motion_changed = motion_changed - self.motion_regions = motion_regions def run(self): frame_time = 0.0 while True: - # while there is motion - while len([r for r in self.motion_regions if r.is_set()]) > 0: - now = datetime.datetime.now().timestamp() - # wait for a frame - with self.frame_ready: - # if there isnt a frame ready for processing or it is old, wait for a signal - if self.frame_time.value == frame_time or (now - self.frame_time.value) > 0.5: - self.frame_ready.wait() - - # lock and make a copy of the frame - with self.frame_lock: - frame = self.shared_frame.copy() - frame_time = self.frame_time.value - - # add the frame to recent frames - self.recent_frames[frame_time] = frame + now = datetime.datetime.now().timestamp() + # wait for a frame + with self.frame_ready: + # if there isnt a frame ready for processing or it is old, wait for a signal + if self.frame_time.value == frame_time or (now - self.frame_time.value) > 0.5: + self.frame_ready.wait() + + # lock and make a copy of the frame + with self.frame_lock: + frame = self.shared_frame.copy() + frame_time = self.frame_time.value + + # add the frame to recent frames + self.recent_frames[frame_time] = frame - # delete any old frames - stored_frame_times = list(self.recent_frames.keys()) - for k in stored_frame_times: - if (now - k) > 2: - del self.recent_frames[k] - - # wait for the global motion flag to change - with self.motion_changed: - self.motion_changed.wait() \ No newline at end of file + # delete any old frames + stored_frame_times = list(self.recent_frames.keys()) + for k in stored_frame_times: + if (now - k) > 2: + del self.recent_frames[k]