From f7c8b742e8de342eeafe109f49a3c78168886ab0 Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 07:26:09 -0600 Subject: [PATCH 01/15] increase min confidence to .5 --- detect_objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detect_objects.py b/detect_objects.py index 3cd222a06..5861c5a4d 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -66,7 +66,7 @@ def detect_objects(cropped_frame, sess, detection_graph, region_size, region_x_o objects = [] for index, value in enumerate(classes[0]): score = scores[0, index] - if score > 0.1: + if score > 0.5: box = boxes[0, index].tolist() box[0] = (box[0] * region_size) + region_y_offset box[1] = (box[1] * region_size) + region_x_offset From 7d4cfe43ad35039d6dd6fad60b3ec8a1fc1c3612 Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 07:27:11 -0600 Subject: [PATCH 02/15] WIP: debug images --- detect_objects.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/detect_objects.py b/detect_objects.py index 5861c5a4d..e3e8fd2e1 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -43,7 +43,7 @@ categories = label_map_util.convert_label_map_to_categories(label_map, max_num_c use_display_name=True) category_index = label_map_util.create_category_index(categories) -def detect_objects(cropped_frame, sess, detection_graph, region_size, region_x_offset, region_y_offset): +def detect_objects(cropped_frame, sess, detection_graph, region_size, region_x_offset, region_y_offset, debug): # Expand dimensions since the model expects images to have shape: [1, None, None, 3] image_np_expanded = np.expand_dims(cropped_frame, axis=0) image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') @@ -62,6 +62,19 @@ def detect_objects(cropped_frame, sess, detection_graph, region_size, region_x_o [boxes, scores, classes, num_detections], feed_dict={image_tensor: image_np_expanded}) + if debug: + if len([category_index.get(value) for index,value in enumerate(classes[0]) if scores[0,index] > 0.5]) > 0: + vis_util.visualize_boxes_and_labels_on_image_array( + cropped_frame, + np.squeeze(boxes), + np.squeeze(classes).astype(np.int32), + np.squeeze(scores), + category_index, + use_normalized_coordinates=True, + line_thickness=4) + cv2.imwrite("/lab/debug/obj-{}-{}-{}.jpg".format(region_x_offset, region_y_offset, datetime.datetime.now().timestamp()), cropped_frame) + + # build an array of detected objects objects = [] for index, value in enumerate(classes[0]): @@ -212,7 +225,8 @@ def main(): region['motion_detected'], frame_shape, region['size'], region['x_offset'], region['y_offset'], - region['min_object_size'])) + region['min_object_size'], + True)) motion_process.daemon = True motion_processes.append(motion_process) @@ -330,6 +344,7 @@ def fetch_frames(shared_arr, shared_frame_time, ready_for_frame_flags, frame_sha # do the actual object detection def process_frames(shared_arr, shared_output_arr, shared_frame_time, shared_motion, frame_shape, region_size, region_x_offset, region_y_offset): + debug = True # shape shared input array into frame for processing arr = tonumpyarray(shared_arr).reshape(frame_shape) @@ -383,12 +398,12 @@ def process_frames(shared_arr, shared_output_arr, shared_frame_time, shared_moti # convert to RGB cropped_frame_rgb = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2RGB) # do the object detection - objects = detect_objects(cropped_frame_rgb, sess, detection_graph, region_size, region_x_offset, region_y_offset) + objects = detect_objects(cropped_frame_rgb, sess, detection_graph, region_size, region_x_offset, region_y_offset, True) # copy the detected objects to the output array, filling the array when needed shared_output_arr[:] = objects + [0.0] * (60-len(objects)) # do the actual motion detection -def detect_motion(shared_arr, shared_frame_time, ready_for_frame, shared_motion, frame_shape, region_size, region_x_offset, region_y_offset, min_motion_area): +def detect_motion(shared_arr, shared_frame_time, ready_for_frame, shared_motion, frame_shape, region_size, region_x_offset, region_y_offset, min_motion_area, debug): # shape shared input array into frame for processing arr = tonumpyarray(shared_arr).reshape(frame_shape) @@ -463,6 +478,10 @@ def detect_motion(shared_arr, shared_frame_time, ready_for_frame, shared_motion, # if the contour is big enough, count it as motion contour_area = cv2.contourArea(c) if contour_area > min_motion_area: + if debug: + (x, y, w, h) = cv2.boundingRect(c) + cv2.rectangle(thresh, (x, y), (x + w, y + h), (0, 255, 0), 2) + motion_frames += 1 # if there have been enough consecutive motion frames, report motion if motion_frames >= 3: @@ -470,6 +489,8 @@ def detect_motion(shared_arr, shared_frame_time, ready_for_frame, shared_motion, last_motion = now break motion_frames = 0 + if debug and motion_frames > 0: + cv2.imwrite("/lab/debug/motion-{}-{}-{}.jpg".format(region_x_offset, region_y_offset, datetime.datetime.now().timestamp()), thresh) if __name__ == '__main__': mp.freeze_support() From d9d3937a0f05310bb4768b5035226ca97525e0f4 Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 08:00:35 -0600 Subject: [PATCH 03/15] reset motion counter when no contours exist --- detect_objects.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/detect_objects.py b/detect_objects.py index e3e8fd2e1..d0e6052fa 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -473,6 +473,11 @@ def detect_motion(shared_arr, shared_frame_time, ready_for_frame, shared_motion, cv2.CHAIN_APPROX_SIMPLE) cnts = imutils.grab_contours(cnts) + # if there are no contours, there is no motion + if len(cnts) < 1: + motion_frames = 0 + continue + # loop over the contours for c in cnts: # if the contour is big enough, count it as motion @@ -489,6 +494,7 @@ def detect_motion(shared_arr, shared_frame_time, ready_for_frame, shared_motion, last_motion = now break motion_frames = 0 + if debug and motion_frames > 0: cv2.imwrite("/lab/debug/motion-{}-{}-{}.jpg".format(region_x_offset, region_y_offset, datetime.datetime.now().timestamp()), thresh) From 2b0ebfa3541d5eaad36e6d20e5f2ee3c0fa3f6be Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 10:34:48 -0600 Subject: [PATCH 04/15] draw contours and area on image for motion debugging --- detect_objects.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/detect_objects.py b/detect_objects.py index d0e6052fa..f14155cf6 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -478,25 +478,33 @@ def detect_motion(shared_arr, shared_frame_time, ready_for_frame, shared_motion, motion_frames = 0 continue + motion_found = False + # loop over the contours for c in cnts: # if the contour is big enough, count it as motion contour_area = cv2.contourArea(c) if contour_area > min_motion_area: + motion_found = True if debug: - (x, y, w, h) = cv2.boundingRect(c) - cv2.rectangle(thresh, (x, y), (x + w, y + h), (0, 255, 0), 2) - - motion_frames += 1 - # if there have been enough consecutive motion frames, report motion - if motion_frames >= 3: - shared_motion.value = 1 - last_motion = now - break + cv2.drawContours(cropped_frame, [c], -1, (0, 255, 0), 2) + x, y, w, h = cv2.boundingRect(c) + cv2.putText(cropped_frame, str(contour_area), (x, y), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 100, 0), 2) + else: + break + + if motion_found: + motion_frames += 1 + # if there have been enough consecutive motion frames, report motion + if motion_frames >= 3: + shared_motion.value = 1 + last_motion = now + else: motion_frames = 0 if debug and motion_frames > 0: - cv2.imwrite("/lab/debug/motion-{}-{}-{}.jpg".format(region_x_offset, region_y_offset, datetime.datetime.now().timestamp()), thresh) + cv2.imwrite("/lab/debug/motion-{}-{}-{}.jpg".format(region_x_offset, region_y_offset, datetime.datetime.now().timestamp()), cropped_frame) if __name__ == '__main__': mp.freeze_support() From a84f88c5a58ba8049398e3095e06802e010d34f7 Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 10:36:09 -0600 Subject: [PATCH 05/15] comments --- detect_objects.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/detect_objects.py b/detect_objects.py index f14155cf6..95286d4e8 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -145,11 +145,13 @@ class MqttPublisher(threading.Thread): if obj['name'] in self.object_classes: payload[obj['name']].append(obj) + # send message for objects new_payload = json.dumps(payload, sort_keys=True) if new_payload != last_sent_payload: last_sent_payload = new_payload self.client.publish(self.topic_prefix+'/objects', new_payload, retain=False) - + + # send message for motion motion_status = 'OFF' if any(obj.value == 1 for obj in self.motion_flags): motion_status = 'ON' @@ -157,7 +159,6 @@ class MqttPublisher(threading.Thread): if motion_status != last_motion: last_motion = motion_status self.client.publish(self.topic_prefix+'/motion', motion_status, retain=False) - time.sleep(0.1) From 284a96d5c7597a9ec1e6e41004d2326a71da21c2 Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 10:37:11 -0600 Subject: [PATCH 06/15] update readme --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b888bc5c..ed57f1dba 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,13 @@ Access the mjpeg stream at http://localhost:5000 - [x] Switch to MQTT prefix - [x] Add last will and availability for MQTT - [ ] Add ability to turn detection on and off via MQTT -- [ ] Add a max size for motion and objects +- [ ] Add a max size for motion and objects (height/width > 1.5, total area > 1500 and < 100,000) +- [ ] Make motion less sensitive to rain +- [ ] Use Events or Conditions to signal between threads rather than polling a value +- [ ] Implement a debug option to save images with detected objects +- [ ] Only report if x% of the recent frames have a person to avoid single frame false positives (maybe take an average of the person scores in the past x frames?) - [ ] Filter out detected objects that are not the right size +- [ ] Make resilient to network drop outs - [ ] Merge bounding boxes that span multiple regions - [ ] Switch to a config file - [ ] Allow motion regions to be different than object detection regions From b67f323215bbff361cd03d231032ff8f47eaabed Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 11:52:56 -0600 Subject: [PATCH 07/15] use a lock and condition to signal to motion detection processes --- detect_objects.py | 73 +++++++++++++++++------------------------------ 1 file changed, 26 insertions(+), 47 deletions(-) diff --git a/detect_objects.py b/detect_objects.py index 95286d4e8..b0e32f306 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -200,11 +200,15 @@ def main(): shared_arr = mp.Array(ctypes.c_uint16, flat_array_length) # create shared value for storing the frame_time shared_frame_time = mp.Value('d', 0.0) + # Lock to control access to the frame while writing + frame_lock = mp.Lock() + # Condition for notifying that a new frame is ready + frame_ready = mp.Condition() # shape current frame so it can be treated as an image frame_arr = tonumpyarray(shared_arr).reshape(frame_shape) capture_process = mp.Process(target=fetch_frames, args=(shared_arr, - shared_frame_time, [region['ready_for_frame'] for region in regions], frame_shape)) + shared_frame_time, frame_lock, frame_ready, frame_shape)) capture_process.daemon = True detection_processes = [] @@ -222,7 +226,7 @@ def main(): for index, region in enumerate(regions): motion_process = mp.Process(target=detect_motion, args=(shared_arr, shared_frame_time, - region['ready_for_frame'], + frame_lock, frame_ready, region['motion_detected'], frame_shape, region['size'], region['x_offset'], region['y_offset'], @@ -311,7 +315,7 @@ def tonumpyarray(mp_arr): # fetch the frames as fast a possible, only decoding the frames when the # detection_process has consumed the current frame -def fetch_frames(shared_arr, shared_frame_time, ready_for_frame_flags, frame_shape): +def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_shape): # convert shared memory array into numpy and shape into image array arr = tonumpyarray(shared_arr).reshape(frame_shape) @@ -326,20 +330,17 @@ def fetch_frames(shared_arr, shared_frame_time, ready_for_frame_flags, frame_sha # snapshot the time the frame was grabbed frame_time = datetime.datetime.now() if ret: - # if the anyone is ready for the next frame decode it - # otherwise skip this frame and move onto the next one - if any(flag.value == 1 for flag in ready_for_frame_flags): - # go ahead and decode the current frame - ret, frame = video.retrieve() - if ret: - arr[:] = frame - shared_frame_time.value = frame_time.timestamp() - # signal to the detection_processes by setting the shared_frame_time - for flag in ready_for_frame_flags: - flag.value = 0 - else: - # sleep a little to reduce CPU usage - time.sleep(0.1) + # go ahead and decode the current frame + ret, frame = video.retrieve() + if ret: + # Lock access and update frame + frame_lock.acquire() + arr[:] = frame + shared_frame_time.value = frame_time.timestamp() + frame_lock.release() + # Notify with the condition that a new frame is ready + with frame_ready: + frame_ready.notify_all() video.release() @@ -404,7 +405,7 @@ def process_frames(shared_arr, shared_output_arr, shared_frame_time, shared_moti shared_output_arr[:] = objects + [0.0] * (60-len(objects)) # do the actual motion detection -def detect_motion(shared_arr, shared_frame_time, ready_for_frame, shared_motion, frame_shape, region_size, region_x_offset, region_y_offset, min_motion_area, debug): +def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, shared_motion, frame_shape, region_size, region_x_offset, region_y_offset, min_motion_area, debug): # shape shared input array into frame for processing arr = tonumpyarray(shared_arr).reshape(frame_shape) @@ -419,39 +420,17 @@ def detect_motion(shared_arr, shared_frame_time, ready_for_frame, shared_motion, if last_motion > 0 and (now - last_motion) > 2: last_motion = -1 shared_motion.value = 0 - # if there isnt a frame ready for processing - if shared_frame_time.value == frame_time: - # save the first time there were no frames available - if no_frames_available == -1: - no_frames_available = now - # if there havent been any frames available in 30 seconds, - # sleep to avoid using so much cpu if the camera feed is down - if no_frames_available > 0 and (now - no_frames_available) > 30: - time.sleep(1) - print("sleeping because no frames have been available in a while") - else: - # rest a little bit to avoid maxing out the CPU - time.sleep(0.1) - if ready_for_frame.value == 0: - ready_for_frame.value = 1 - continue - # we got a valid frame, so reset the timer - no_frames_available = -1 - - # if the frame is more than 0.5 second old, discard it - if (now - shared_frame_time.value) > 0.5: - # signal that we need a new frame - ready_for_frame.value = 1 - # rest a little bit to avoid maxing out the CPU - time.sleep(0.1) - continue + with frame_ready: + # if there isnt a frame ready for processing or it is old, wait for a signal + if shared_frame_time.value == frame_time or (now - shared_frame_time.value) > 0.5: + frame_ready.wait() - # make a copy of the cropped frame + # lock and make a copy of the cropped frame + frame_lock.acquire() cropped_frame = arr[region_y_offset:region_y_offset+region_size, region_x_offset:region_x_offset+region_size].copy().astype('uint8') frame_time = shared_frame_time.value - # signal that the frame has been used so a new one will be ready - ready_for_frame.value = 1 + frame_lock.release() # convert to grayscale gray = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2GRAY) From 8525f05f291246c6dc205036a04a65ec4c3b2bc1 Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 11:54:40 -0600 Subject: [PATCH 08/15] get a lock when copying frame for mjpeg stream --- detect_objects.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/detect_objects.py b/detect_objects.py index b0e32f306..7a8e435cc 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -266,8 +266,10 @@ def main(): time.sleep(0.2) # make a copy of the current detected objects detected_objects = DETECTED_OBJECTS.copy() - # make a copy of the current frame + # lock and make a copy of the current frame + frame_lock.aquire() frame = frame_arr.copy() + frame_lock.release() # convert to RGB for drawing frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # draw the bounding boxes on the screen From 957bd2adeb38735ccb91a13e8fc50ad576e1cac2 Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 12:12:48 -0600 Subject: [PATCH 09/15] use events to signal when motion is detected --- detect_objects.py | 83 ++++++++++++++++------------------------------- 1 file changed, 28 insertions(+), 55 deletions(-) diff --git a/detect_objects.py b/detect_objects.py index 7a8e435cc..a77bd8e44 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -153,7 +153,7 @@ class MqttPublisher(threading.Thread): # send message for motion motion_status = 'OFF' - if any(obj.value == 1 for obj in self.motion_flags): + if any(obj.is_set() for obj in self.motion_flags): motion_status = 'ON' if motion_status != last_motion: @@ -172,11 +172,8 @@ def main(): 'x_offset': int(region_parts[1]), 'y_offset': int(region_parts[2]), 'min_object_size': int(region_parts[3]), - # shared value for signaling to the capture process that we are ready for the next frame - # (1 for ready 0 for not ready) - 'ready_for_frame': mp.Value('i', 1), - # shared value for motion detection signal (1 for motion 0 for no motion) - 'motion_detected': mp.Value('i', 0), + # Event for motion detection signaling + 'motion_detected': mp.Event(), # create shared array for storing 10 detected objects # note: this must be a double even though the value you are storing # is a float. otherwise it stops updating the value in shared @@ -212,18 +209,18 @@ def main(): capture_process.daemon = True detection_processes = [] - for index, region in enumerate(regions): + motion_processes = [] + for region in regions: detection_process = mp.Process(target=process_frames, args=(shared_arr, region['output_array'], shared_frame_time, + frame_lock, frame_ready, region['motion_detected'], frame_shape, region['size'], region['x_offset'], region['y_offset'])) detection_process.daemon = True detection_processes.append(detection_process) - motion_processes = [] - for index, region in enumerate(regions): motion_process = mp.Process(target=detect_motion, args=(shared_arr, shared_frame_time, frame_lock, frame_ready, @@ -267,9 +264,8 @@ def main(): # make a copy of the current detected objects detected_objects = DETECTED_OBJECTS.copy() # lock and make a copy of the current frame - frame_lock.aquire() - frame = frame_arr.copy() - frame_lock.release() + with frame_lock: + frame = frame_arr.copy() # convert to RGB for drawing frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # draw the bounding boxes on the screen @@ -286,7 +282,7 @@ def main(): for region in regions: color = (255,255,255) - if region['motion_detected'].value == 1: + if region['motion_detected'].is_set(): color = (0,255,0) cv2.rectangle(frame, (region['x_offset'], region['y_offset']), (region['x_offset']+region['size'], region['y_offset']+region['size']), @@ -336,10 +332,9 @@ def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_s ret, frame = video.retrieve() if ret: # Lock access and update frame - frame_lock.acquire() - arr[:] = frame - shared_frame_time.value = frame_time.timestamp() - frame_lock.release() + with frame_lock: + arr[:] = frame + shared_frame_time.value = frame_time.timestamp() # Notify with the condition that a new frame is ready with frame_ready: frame_ready.notify_all() @@ -347,7 +342,7 @@ def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_s video.release() # do the actual object detection -def process_frames(shared_arr, shared_output_arr, shared_frame_time, shared_motion, frame_shape, region_size, region_x_offset, region_y_offset): +def process_frames(shared_arr, shared_output_arr, shared_frame_time, frame_lock, frame_ready, motion_detected, frame_shape, region_size, region_x_offset, region_y_offset): debug = True # shape shared input array into frame for processing arr = tonumpyarray(shared_arr).reshape(frame_shape) @@ -362,42 +357,22 @@ def process_frames(shared_arr, shared_output_arr, shared_frame_time, shared_moti tf.import_graph_def(od_graph_def, name='') sess = tf.Session(graph=detection_graph) - no_frames_available = -1 frame_time = 0.0 while True: now = datetime.datetime.now().timestamp() - # if there is no motion detected - if shared_motion.value == 0: - time.sleep(0.1) - continue - # if there isnt a new frame ready for processing - if shared_frame_time.value == frame_time: - # save the first time there were no frames available - if no_frames_available == -1: - no_frames_available = now - # if there havent been any frames available in 30 seconds, - # sleep to avoid using so much cpu if the camera feed is down - if no_frames_available > 0 and (now - no_frames_available) > 30: - time.sleep(1) - print("sleeping because no frames have been available in a while") - else: - # rest a little bit to avoid maxing out the CPU - time.sleep(0.1) - continue - - # we got a valid frame, so reset the timer - no_frames_available = -1 + # wait until motion is detected + motion_detected.wait() - # if the frame is more than 0.5 second old, ignore it - if (now - shared_frame_time.value) > 0.5: - # rest a little bit to avoid maxing out the CPU - time.sleep(0.1) - continue + with frame_ready: + # if there isnt a frame ready for processing or it is old, wait for a signal + if shared_frame_time.value == frame_time or (now - shared_frame_time.value) > 0.5: + frame_ready.wait() # make a copy of the cropped frame - cropped_frame = arr[region_y_offset:region_y_offset+region_size, region_x_offset:region_x_offset+region_size].copy() - frame_time = shared_frame_time.value + with frame_lock: + cropped_frame = arr[region_y_offset:region_y_offset+region_size, region_x_offset:region_x_offset+region_size].copy() + frame_time = shared_frame_time.value # convert to RGB cropped_frame_rgb = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2RGB) @@ -407,11 +382,10 @@ def process_frames(shared_arr, shared_output_arr, shared_frame_time, shared_moti shared_output_arr[:] = objects + [0.0] * (60-len(objects)) # do the actual motion detection -def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, shared_motion, frame_shape, region_size, region_x_offset, region_y_offset, min_motion_area, debug): +def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, motion_detected, frame_shape, region_size, region_x_offset, region_y_offset, min_motion_area, debug): # shape shared input array into frame for processing arr = tonumpyarray(shared_arr).reshape(frame_shape) - no_frames_available = -1 avg_frame = None last_motion = -1 frame_time = 0.0 @@ -421,7 +395,7 @@ def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, shared # if it has been long enough since the last motion, clear the flag if last_motion > 0 and (now - last_motion) > 2: last_motion = -1 - shared_motion.value = 0 + motion_detected.clear() with frame_ready: # if there isnt a frame ready for processing or it is old, wait for a signal @@ -429,10 +403,9 @@ def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, shared frame_ready.wait() # lock and make a copy of the cropped frame - frame_lock.acquire() - cropped_frame = arr[region_y_offset:region_y_offset+region_size, region_x_offset:region_x_offset+region_size].copy().astype('uint8') - frame_time = shared_frame_time.value - frame_lock.release() + with frame_lock: + cropped_frame = arr[region_y_offset:region_y_offset+region_size, region_x_offset:region_x_offset+region_size].copy().astype('uint8') + frame_time = shared_frame_time.value # convert to grayscale gray = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2GRAY) @@ -480,7 +453,7 @@ def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, shared motion_frames += 1 # if there have been enough consecutive motion frames, report motion if motion_frames >= 3: - shared_motion.value = 1 + motion_detected.set() last_motion = now else: motion_frames = 0 From 92e1833defefbba25e721c699ed65b5f8989490a Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 12:32:19 -0600 Subject: [PATCH 10/15] use a condition to notify when object detection has run --- detect_objects.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/detect_objects.py b/detect_objects.py index a77bd8e44..bc1b2f778 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -93,14 +93,19 @@ def detect_objects(cropped_frame, sess, detection_graph, region_size, region_x_o return objects class ObjectParser(threading.Thread): - def __init__(self, object_arrays): + def __init__(self, objects_changed, object_arrays): threading.Thread.__init__(self) + self._objects_changed = objects_changed self._object_arrays = object_arrays def run(self): global DETECTED_OBJECTS while True: detected_objects = [] + # wait until object detection has run + with self._objects_changed: + self._objects_changed.wait() + for object_array in self._object_arrays: object_index = 0 while(object_index < 60 and object_array[object_index] > 0): @@ -115,7 +120,6 @@ class ObjectParser(threading.Thread): }) object_index += 6 DETECTED_OBJECTS = detected_objects - time.sleep(0.1) class MqttPublisher(threading.Thread): def __init__(self, host, topic_prefix, object_classes, motion_flags): threading.Thread.__init__(self) @@ -201,6 +205,8 @@ def main(): frame_lock = mp.Lock() # Condition for notifying that a new frame is ready frame_ready = mp.Condition() + # Condition for notifying that object detection ran + objects_changed = mp.Condition() # shape current frame so it can be treated as an image frame_arr = tonumpyarray(shared_arr).reshape(frame_shape) @@ -216,6 +222,7 @@ def main(): shared_frame_time, frame_lock, frame_ready, region['motion_detected'], + objects_changed, frame_shape, region['size'], region['x_offset'], region['y_offset'])) detection_process.daemon = True @@ -232,7 +239,7 @@ def main(): motion_process.daemon = True motion_processes.append(motion_process) - object_parser = ObjectParser([region['output_array'] for region in regions]) + object_parser = ObjectParser(objects_changed, [region['output_array'] for region in regions]) object_parser.start() mqtt_publisher = MqttPublisher(MQTT_HOST, MQTT_TOPIC_PREFIX, @@ -342,7 +349,8 @@ def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_s video.release() # do the actual object detection -def process_frames(shared_arr, shared_output_arr, shared_frame_time, frame_lock, frame_ready, motion_detected, frame_shape, region_size, region_x_offset, region_y_offset): +def process_frames(shared_arr, shared_output_arr, shared_frame_time, frame_lock, frame_ready, + motion_detected, objects_changed, frame_shape, region_size, region_x_offset, region_y_offset): debug = True # shape shared input array into frame for processing arr = tonumpyarray(shared_arr).reshape(frame_shape) @@ -380,6 +388,8 @@ def process_frames(shared_arr, shared_output_arr, shared_frame_time, frame_lock, objects = detect_objects(cropped_frame_rgb, sess, detection_graph, region_size, region_x_offset, region_y_offset, True) # copy the detected objects to the output array, filling the array when needed shared_output_arr[:] = objects + [0.0] * (60-len(objects)) + with objects_changed: + objects_changed.notify_all() # do the actual motion detection def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, motion_detected, frame_shape, region_size, region_x_offset, region_y_offset, min_motion_area, debug): From 42ff739cea76a253a68c1033eb64ba606ec083a4 Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 12:59:51 -0600 Subject: [PATCH 11/15] move mqtt motion publishing into separate thread --- detect_objects.py | 70 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/detect_objects.py b/detect_objects.py index bc1b2f778..097068f40 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -120,23 +120,41 @@ class ObjectParser(threading.Thread): }) object_index += 6 DETECTED_OBJECTS = detected_objects -class MqttPublisher(threading.Thread): - def __init__(self, host, topic_prefix, object_classes, motion_flags): + +class MqttMotionPublisher(threading.Thread): + def __init__(self, client, topic_prefix, motion_changed, motion_flags): threading.Thread.__init__(self) - self.client = mqtt.Client() - self.client.will_set(topic_prefix+'/available', payload='offline', qos=1, retain=True) - self.client.connect(host, 1883, 60) - self.client.loop_start() - self.client.publish(topic_prefix+'/available', 'online', retain=True) + self.client = client + self.topic_prefix = topic_prefix + self.motion_changed = motion_changed + self.motion_flags = motion_flags + + def run(self): + last_sent_motion = "" + while True: + with self.motion_changed: + self.motion_changed.wait() + + # send message for motion + motion_status = 'OFF' + if any(obj.is_set() for obj in self.motion_flags): + motion_status = 'ON' + + if last_sent_motion != motion_status: + last_sent_motion = motion_status + self.client.publish(self.topic_prefix+'/motion', motion_status, retain=False) + +class MqttPublisher(threading.Thread): + def __init__(self, client, topic_prefix, object_classes): + threading.Thread.__init__(self) + self.client = client self.topic_prefix = topic_prefix self.object_classes = object_classes - self.motion_flags = motion_flags def run(self): global DETECTED_OBJECTS last_sent_payload = "" - last_motion = "" while True: # initialize the payload payload = {} @@ -154,15 +172,6 @@ class MqttPublisher(threading.Thread): if new_payload != last_sent_payload: last_sent_payload = new_payload self.client.publish(self.topic_prefix+'/objects', new_payload, retain=False) - - # send message for motion - motion_status = 'OFF' - if any(obj.is_set() for obj in self.motion_flags): - motion_status = 'ON' - - if motion_status != last_motion: - last_motion = motion_status - self.client.publish(self.topic_prefix+'/motion', motion_status, retain=False) time.sleep(0.1) @@ -205,6 +214,8 @@ def main(): frame_lock = mp.Lock() # Condition for notifying that a new frame is ready frame_ready = mp.Condition() + # Condition for notifying that motion status changed globally + motion_changed = mp.Condition() # Condition for notifying that object detection ran objects_changed = mp.Condition() # shape current frame so it can be treated as an image @@ -232,6 +243,7 @@ def main(): shared_frame_time, frame_lock, frame_ready, region['motion_detected'], + motion_changed, frame_shape, region['size'], region['x_offset'], region['y_offset'], region['min_object_size'], @@ -242,11 +254,20 @@ def main(): object_parser = ObjectParser(objects_changed, [region['output_array'] for region in regions]) object_parser.start() - mqtt_publisher = MqttPublisher(MQTT_HOST, MQTT_TOPIC_PREFIX, - MQTT_OBJECT_CLASSES.split(','), - [region['motion_detected'] for region in regions]) + client = mqtt.Client() + client.will_set(MQTT_TOPIC_PREFIX+'/available', payload='offline', qos=1, retain=True) + client.connect(MQTT_HOST, 1883, 60) + client.loop_start() + client.publish(MQTT_TOPIC_PREFIX+'/available', 'online', retain=True) + + mqtt_publisher = MqttPublisher(client, MQTT_TOPIC_PREFIX, + MQTT_OBJECT_CLASSES.split(',')) mqtt_publisher.start() + mqtt_motion_publisher = MqttMotionPublisher(client, MQTT_TOPIC_PREFIX, motion_changed, + [region['motion_detected'] for region in regions]) + mqtt_motion_publisher.start() + capture_process.start() print("capture_process pid ", capture_process.pid) for detection_process in detection_processes: @@ -392,7 +413,8 @@ def process_frames(shared_arr, shared_output_arr, shared_frame_time, frame_lock, objects_changed.notify_all() # do the actual motion detection -def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, motion_detected, frame_shape, region_size, region_x_offset, region_y_offset, min_motion_area, debug): +def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, motion_detected, motion_changed, + frame_shape, region_size, region_x_offset, region_y_offset, min_motion_area, debug): # shape shared input array into frame for processing arr = tonumpyarray(shared_arr).reshape(frame_shape) @@ -406,6 +428,8 @@ def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, motion if last_motion > 0 and (now - last_motion) > 2: last_motion = -1 motion_detected.clear() + with motion_changed: + motion_changed.notify_all() with frame_ready: # if there isnt a frame ready for processing or it is old, wait for a signal @@ -464,6 +488,8 @@ def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, motion # if there have been enough consecutive motion frames, report motion if motion_frames >= 3: motion_detected.set() + with motion_changed: + motion_changed.notify_all() last_motion = now else: motion_frames = 0 From 03c57bf67d0ddd02570b122f471a7d133c3b08a0 Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 13:12:27 -0600 Subject: [PATCH 12/15] use a condition to signal to the mqtt publisher that objects were parsed --- detect_objects.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/detect_objects.py b/detect_objects.py index 097068f40..d69ff4c9e 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -93,9 +93,10 @@ def detect_objects(cropped_frame, sess, detection_graph, region_size, region_x_o return objects class ObjectParser(threading.Thread): - def __init__(self, objects_changed, object_arrays): + def __init__(self, objects_changed, objects_parsed, object_arrays): threading.Thread.__init__(self) self._objects_changed = objects_changed + self._objects_parsed = objects_parsed self._object_arrays = object_arrays def run(self): @@ -103,6 +104,7 @@ class ObjectParser(threading.Thread): while True: detected_objects = [] # wait until object detection has run + # TODO: what if something else changed while I was processing??? with self._objects_changed: self._objects_changed.wait() @@ -120,6 +122,9 @@ class ObjectParser(threading.Thread): }) object_index += 6 DETECTED_OBJECTS = detected_objects + # notify that objects were parsed + with self._objects_parsed: + self._objects_parsed.notify_all() class MqttMotionPublisher(threading.Thread): def __init__(self, client, topic_prefix, motion_changed, motion_flags): @@ -139,16 +144,17 @@ class MqttMotionPublisher(threading.Thread): motion_status = 'OFF' if any(obj.is_set() for obj in self.motion_flags): motion_status = 'ON' - + if last_sent_motion != motion_status: last_sent_motion = motion_status self.client.publish(self.topic_prefix+'/motion', motion_status, retain=False) -class MqttPublisher(threading.Thread): - def __init__(self, client, topic_prefix, object_classes): +class MqttObjectPublisher(threading.Thread): + def __init__(self, client, topic_prefix, objects_parsed, object_classes): threading.Thread.__init__(self) self.client = client self.topic_prefix = topic_prefix + self.objects_parsed = objects_parsed self.object_classes = object_classes def run(self): @@ -156,10 +162,16 @@ class MqttPublisher(threading.Thread): last_sent_payload = "" while True: + # initialize the payload payload = {} for obj in self.object_classes: payload[obj] = [] + + # wait until objects have been parsed + with self.objects_parsed: + self.objects_parsed.wait() + # loop over detected objects and populate # the payload detected_objects = DETECTED_OBJECTS.copy() @@ -167,14 +179,12 @@ class MqttPublisher(threading.Thread): if obj['name'] in self.object_classes: payload[obj['name']].append(obj) - # send message for objects + # send message for objects if different new_payload = json.dumps(payload, sort_keys=True) if new_payload != last_sent_payload: last_sent_payload = new_payload self.client.publish(self.topic_prefix+'/objects', new_payload, retain=False) - time.sleep(0.1) - def main(): # Parse selected regions regions = [] @@ -218,6 +228,8 @@ def main(): motion_changed = mp.Condition() # Condition for notifying that object detection ran objects_changed = mp.Condition() + # Condition for notifying that objects were parsed + objects_parsed = mp.Condition() # shape current frame so it can be treated as an image frame_arr = tonumpyarray(shared_arr).reshape(frame_shape) @@ -251,7 +263,7 @@ def main(): motion_process.daemon = True motion_processes.append(motion_process) - object_parser = ObjectParser(objects_changed, [region['output_array'] for region in regions]) + object_parser = ObjectParser(objects_changed, objects_parsed, [region['output_array'] for region in regions]) object_parser.start() client = mqtt.Client() @@ -260,7 +272,7 @@ def main(): client.loop_start() client.publish(MQTT_TOPIC_PREFIX+'/available', 'online', retain=True) - mqtt_publisher = MqttPublisher(client, MQTT_TOPIC_PREFIX, + mqtt_publisher = MqttObjectPublisher(client, MQTT_TOPIC_PREFIX, objects_parsed, MQTT_OBJECT_CLASSES.split(',')) mqtt_publisher.start() From 290150603ecea84e801f099883de2dc682adac2c Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Feb 2019 20:36:01 -0600 Subject: [PATCH 13/15] remove debug time --- detect_objects.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/detect_objects.py b/detect_objects.py index d69ff4c9e..939f514d8 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -328,8 +328,6 @@ def main(): (region['x_offset']+region['size'], region['y_offset']+region['size']), color, 2) - cv2.putText(frame, datetime.datetime.now().strftime("%H:%M:%S"), (1125, 20), - cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2) # convert back to BGR frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) # encode the image into a jpg From b6547de82c11e96f59a9650b54ee99eeabb1f3dd Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Tue, 19 Feb 2019 06:47:00 -0600 Subject: [PATCH 14/15] adjust debugging params and alpha for background averaging --- detect_objects.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/detect_objects.py b/detect_objects.py index 939f514d8..6e9f30605 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -247,7 +247,8 @@ def main(): region['motion_detected'], objects_changed, frame_shape, - region['size'], region['x_offset'], region['y_offset'])) + region['size'], region['x_offset'], region['y_offset'], + False)) detection_process.daemon = True detection_processes.append(detection_process) @@ -381,7 +382,8 @@ def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_s # do the actual object detection def process_frames(shared_arr, shared_output_arr, shared_frame_time, frame_lock, frame_ready, - motion_detected, objects_changed, frame_shape, region_size, region_x_offset, region_y_offset): + motion_detected, objects_changed, frame_shape, region_size, region_x_offset, region_y_offset, + debug): debug = True # shape shared input array into frame for processing arr = tonumpyarray(shared_arr).reshape(frame_shape) @@ -416,7 +418,7 @@ def process_frames(shared_arr, shared_output_arr, shared_frame_time, frame_lock, # convert to RGB cropped_frame_rgb = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2RGB) # do the object detection - objects = detect_objects(cropped_frame_rgb, sess, detection_graph, region_size, region_x_offset, region_y_offset, True) + objects = detect_objects(cropped_frame_rgb, sess, detection_graph, region_size, region_x_offset, region_y_offset, debug) # copy the detected objects to the output array, filling the array when needed shared_output_arr[:] = objects + [0.0] * (60-len(objects)) with objects_changed: @@ -461,7 +463,7 @@ def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, motion continue # look at the delta from the avg_frame - cv2.accumulateWeighted(gray, avg_frame, 0.5) + cv2.accumulateWeighted(gray, avg_frame, 0.01) frameDelta = cv2.absdiff(gray, cv2.convertScaleAbs(avg_frame)) thresh = cv2.threshold(frameDelta, 25, 255, cv2.THRESH_BINARY)[1] @@ -504,7 +506,7 @@ def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, motion else: motion_frames = 0 - if debug and motion_frames > 0: + if debug and motion_frames >= 3: cv2.imwrite("/lab/debug/motion-{}-{}-{}.jpg".format(region_x_offset, region_y_offset, datetime.datetime.now().timestamp()), cropped_frame) if __name__ == '__main__': From c43d83be200f22e676bd3d193b0bc445abfcf0a3 Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Tue, 19 Feb 2019 06:47:37 -0600 Subject: [PATCH 15/15] update readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ed57f1dba..70379be36 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Access the mjpeg stream at http://localhost:5000 - [ ] Add ability to turn detection on and off via MQTT - [ ] Add a max size for motion and objects (height/width > 1.5, total area > 1500 and < 100,000) - [ ] Make motion less sensitive to rain -- [ ] Use Events or Conditions to signal between threads rather than polling a value +- [x] Use Events or Conditions to signal between threads rather than polling a value - [ ] Implement a debug option to save images with detected objects - [ ] Only report if x% of the recent frames have a person to avoid single frame false positives (maybe take an average of the person scores in the past x frames?) - [ ] Filter out detected objects that are not the right size @@ -53,6 +53,7 @@ Access the mjpeg stream at http://localhost:5000 - [ ] Merge bounding boxes that span multiple regions - [ ] Switch to a config file - [ ] Allow motion regions to be different than object detection regions +- [ ] Add motion detection masking - [x] Change color of bounding box if motion detected - [x] Look for a subset of object types - [ ] Try and reduce CPU usage by simplifying the tensorflow model to just include the objects we care about