From 8bae05cfe26f58c06df36eddd49ed4ea1b441de0 Mon Sep 17 00:00:00 2001 From: blakeblackshear Date: Sun, 17 Mar 2019 09:03:52 -0500 Subject: [PATCH] first working version, single region and motion detection disabled --- Dockerfile | 35 +++++++++++----- detect_objects.py | 13 ++++-- frigate/motion.py | 2 +- frigate/object_detection.py | 83 +++++++++++++------------------------ frigate/util.py | 2 +- frigate/video.py | 2 +- 6 files changed, 65 insertions(+), 72 deletions(-) diff --git a/Dockerfile b/Dockerfile index 80bca9645..9d8713cf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,20 +26,25 @@ RUN apt-get -qq update && apt-get -qq install --no-install-recommends -y python3 vim \ ffmpeg \ unzip \ + libusb-1.0-0-dev \ + python3-setuptools \ + python3-numpy \ + zlib1g-dev \ + libgoogle-glog-dev \ + swig \ + libunwind-dev \ + libc++-dev \ + libc++abi-dev \ + build-essential \ && rm -rf /var/lib/apt/lists/* # Install core packages RUN wget -q -O /tmp/get-pip.py --no-check-certificate https://bootstrap.pypa.io/get-pip.py && python3 /tmp/get-pip.py RUN pip install -U pip \ numpy \ + pillow \ matplotlib \ notebook \ - jupyter \ - pandas \ - moviepy \ - tensorflow \ - keras \ - autovizwidget \ Flask \ imutils \ paho-mqtt @@ -59,9 +64,6 @@ RUN cd /usr/local/src/ \ && ldconfig \ && rm -rf /usr/local/src/protobuf-3.5.1/ -# Add dataframe display widget -RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension - # Download & build OpenCV RUN wget -q -P /usr/local/src/ --no-check-certificate https://github.com/opencv/opencv/archive/4.0.1.zip RUN cd /usr/local/src/ \ @@ -75,6 +77,16 @@ RUN cd /usr/local/src/ \ && make install \ && rm -rf /usr/local/src/opencv-4.0.1 +# Download and install EdgeTPU libraries +RUN wget -q -O edgetpu_api.tar.gz --no-check-certificate http://storage.googleapis.com/cloud-iot-edge-pretrained-models/edgetpu_api.tar.gz + +RUN tar xzf edgetpu_api.tar.gz \ + && cd python-tflite-source \ + && cp -p libedgetpu/libedgetpu_arm32_throttled.so /lib/arm-linux-gnueabihf/libedgetpu.so \ + && cp edgetpu/swig/compiled_so/_edgetpu_cpp_wrapper_arm32.so edgetpu/swig/_edgetpu_cpp_wrapper.so \ + && cp edgetpu/swig/compiled_so/edgetpu_cpp_wrapper.py edgetpu/swig/ \ + && python3 setup.py develop --user + # Minimize image size RUN (apt-get autoremove -y; \ apt-get autoclean -y) @@ -87,4 +99,7 @@ WORKDIR /opt/frigate/ ADD frigate frigate/ COPY detect_objects.py . -CMD ["python3", "-u", "detect_objects.py"] \ No newline at end of file +CMD ["python3", "-u", "detect_objects.py"] + +# WORKDIR /python-tflite-source/edgetpu/ +# CMD ["python3", "-u", "demo/classify_image.py", "--model", "test_data/mobilenet_v2_1.0_224_inat_bird_quant_edgetpu.tflite", "--label", "test_data/inat_bird_labels.txt", "--image", "test_data/parrot.jpg"] \ No newline at end of file diff --git a/detect_objects.py b/detect_objects.py index 643859c63..405a86a75 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -72,7 +72,7 @@ def main(): # compute the flattened array length from the array shape flat_array_length = frame_shape[0] * frame_shape[1] * frame_shape[2] # create shared array for storing the full frame image data - shared_arr = mp.Array(ctypes.c_uint16, flat_array_length) + shared_arr = mp.Array(ctypes.c_uint8, flat_array_length) # create shared value for storing the frame_time shared_frame_time = mp.Value('d', 0.0) # Lock to control access to the frame @@ -173,9 +173,14 @@ def main(): print("detection_process pid ", detection_process.pid) # start the motion detection processes - for motion_process in motion_processes: - motion_process.start() - print("motion_process pid ", motion_process.pid) + # for motion_process in motion_processes: + # motion_process.start() + # print("motion_process pid ", motion_process.pid) + + for region in regions: + region['motion_detected'].set() + with motion_changed: + motion_changed.notify_all() # create a flask app that encodes frames a mjpeg on demand app = Flask(__name__) diff --git a/frigate/motion.py b/frigate/motion.py index afe0e8fb7..b09c8afc7 100644 --- a/frigate/motion.py +++ b/frigate/motion.py @@ -34,7 +34,7 @@ def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, motion # lock and make a copy of the cropped frame with frame_lock: - cropped_frame = arr[region_y_offset:region_y_offset+region_size, region_x_offset:region_x_offset+region_size].copy().astype('uint8') + cropped_frame = arr[region_y_offset:region_y_offset+region_size, region_x_offset:region_x_offset+region_size].copy() frame_time = shared_frame_time.value # convert to grayscale diff --git a/frigate/object_detection.py b/frigate/object_detection.py index 2c68f2b40..dfc3d81b5 100644 --- a/frigate/object_detection.py +++ b/frigate/object_detection.py @@ -1,9 +1,8 @@ import datetime import cv2 import numpy as np -import tensorflow as tf -from object_detection.utils import label_map_util -from object_detection.utils import visualization_utils as vis_util +from edgetpu.detection.engine import DetectionEngine +from PIL import Image from . util import tonumpyarray # TODO: make dynamic? @@ -13,58 +12,38 @@ PATH_TO_CKPT = '/frozen_inference_graph.pb' # List of the strings that is used to add correct label for each box. PATH_TO_LABELS = '/label_map.pbtext' -# Loading label map -label_map = label_map_util.load_labelmap(PATH_TO_LABELS) -categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, - use_display_name=True) -category_index = label_map_util.create_category_index(categories) +# Function to read labels from text files. +def ReadLabelFile(file_path): + with open(file_path, 'r') as f: + lines = f.readlines() + ret = {} + for line in lines: + pair = line.strip().split(maxsplit=1) + ret[int(pair[0])] = pair[1].strip() + return ret # do the actual object detection -def tf_detect_objects(cropped_frame, sess, detection_graph, region_size, region_x_offset, region_y_offset, debug): +def tf_detect_objects(cropped_frame, engine, labels, region_size, region_x_offset, region_y_offset, debug): + # Resize to 300x300 + cropped_frame = cv2.resize(cropped_frame, dsize=(300, 300), interpolation=cv2.INTER_LINEAR) # Expand dimensions since the model expects images to have shape: [1, None, None, 3] image_np_expanded = np.expand_dims(cropped_frame, axis=0) - image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') - - # Each box represents a part of the image where a particular object was detected. - boxes = detection_graph.get_tensor_by_name('detection_boxes:0') - - # Each score represent how level of confidence for each of the objects. - # Score is shown on the result image, together with the class label. - scores = detection_graph.get_tensor_by_name('detection_scores:0') - classes = detection_graph.get_tensor_by_name('detection_classes:0') - num_detections = detection_graph.get_tensor_by_name('num_detections:0') # Actual detection. - (boxes, scores, classes, num_detections) = sess.run( - [boxes, scores, classes, num_detections], - feed_dict={image_tensor: image_np_expanded}) - - if debug: - if len([value for index,value in enumerate(classes[0]) if str(category_index.get(value).get('name')) == 'person' and scores[0,index] > 0.5]) > 0: - vis_util.visualize_boxes_and_labels_on_image_array( - cropped_frame, - np.squeeze(boxes), - np.squeeze(classes).astype(np.int32), - np.squeeze(scores), - category_index, - use_normalized_coordinates=True, - line_thickness=4) - cv2.imwrite("/lab/debug/obj-{}-{}-{}.jpg".format(region_x_offset, region_y_offset, datetime.datetime.now().timestamp()), cropped_frame) - + ans = engine.DetectWithInputTensor(image_np_expanded.flatten(), threshold=0.5, top_k=3) # build an array of detected objects objects = [] - for index, value in enumerate(classes[0]): - score = scores[0, index] - if score > 0.5: - box = boxes[0, index].tolist() + if ans: + for obj in ans: + box = obj.bounding_box.flatten().tolist() objects.append({ - 'name': str(category_index.get(value).get('name')), - 'score': float(score), - 'ymin': int((box[0] * region_size) + region_y_offset), - 'xmin': int((box[1] * region_size) + region_x_offset), - 'ymax': int((box[2] * region_size) + region_y_offset), - 'xmax': int((box[3] * region_size) + region_x_offset) + 'name': str(labels[obj.label_id]), + 'score': float(obj.score), + 'xmin': int((box[0] * region_size) + region_x_offset), + 'ymin': int((box[1] * region_size) + region_y_offset), + 'xmax': int((box[2] * region_size) + region_x_offset), + 'ymax': int((box[3] * region_size) + region_y_offset) }) return objects @@ -75,15 +54,9 @@ def detect_objects(shared_arr, object_queue, shared_frame_time, frame_lock, fram # shape shared input array into frame for processing arr = tonumpyarray(shared_arr).reshape(frame_shape) - # Load a (frozen) Tensorflow model into memory before the processing loop - detection_graph = tf.Graph() - with detection_graph.as_default(): - od_graph_def = tf.GraphDef() - with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid: - serialized_graph = fid.read() - od_graph_def.ParseFromString(serialized_graph) - tf.import_graph_def(od_graph_def, name='') - sess = tf.Session(graph=detection_graph) + # Load the edgetpu engine and labels + engine = DetectionEngine(PATH_TO_CKPT) + labels = ReadLabelFile(PATH_TO_LABELS) frame_time = 0.0 while True: @@ -105,7 +78,7 @@ def detect_objects(shared_arr, object_queue, shared_frame_time, frame_lock, fram # convert to RGB cropped_frame_rgb = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2RGB) # do the object detection - objects = tf_detect_objects(cropped_frame_rgb, sess, detection_graph, region_size, region_x_offset, region_y_offset, debug) + objects = tf_detect_objects(cropped_frame_rgb, engine, labels, region_size, region_x_offset, region_y_offset, debug) for obj in objects: # ignore persons below the size threshold if obj['name'] == 'person' and (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin']) < min_person_area: diff --git a/frigate/util.py b/frigate/util.py index 984d37193..fa174f195 100644 --- a/frigate/util.py +++ b/frigate/util.py @@ -2,4 +2,4 @@ import numpy as np # convert shared memory array into numpy array def tonumpyarray(mp_arr): - return np.frombuffer(mp_arr.get_obj(), dtype=np.uint16) \ No newline at end of file + return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8) \ No newline at end of file diff --git a/frigate/video.py b/frigate/video.py index 783bbff13..0edcc6989 100644 --- a/frigate/video.py +++ b/frigate/video.py @@ -78,7 +78,7 @@ class FrameTracker(threading.Thread): # lock and make a copy of the frame with self.frame_lock: - frame = self.shared_frame.copy().astype('uint8') + frame = self.shared_frame.copy() frame_time = self.frame_time.value # add the frame to recent frames