From 569e07949fbf9c08072985b98696ba561cac4dcf Mon Sep 17 00:00:00 2001
From: Blake Blackshear <blakeb@blakeshome.com>
Date: Sat, 15 Feb 2020 21:07:54 -0600
Subject: [PATCH] split into separate processes

---
 Dockerfile                   |   9 +-
 detect_objects.py            |  89 ++--
 frigate/edgetpu.py           |  18 +-
 frigate/motion.py            |   9 +-
 frigate/mqtt.py              |  54 ---
 frigate/object_detection.py  |   2 +-
 frigate/object_processing.py | 146 +++++++
 frigate/objects.py           | 515 ++++++++++++-----------
 frigate/util.py              | 160 ++++---
 frigate/video.py             | 792 +++++++++++++++++++++++++++++------
 10 files changed, 1234 insertions(+), 560 deletions(-)
 delete mode 100644 frigate/mqtt.py
 create mode 100644 frigate/object_processing.py
 mode change 100644 => 100755 frigate/util.py
 mode change 100644 => 100755 frigate/video.py

diff --git a/Dockerfile b/Dockerfile
index d1abe96f3..5289b5a89 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -26,10 +26,11 @@ RUN apt -qq update && apt -qq install --no-install-recommends -y \
         scipy \
     && python3.7 -m pip install -U \
         SharedArray \
-        # Flask \
-        # paho-mqtt \
-        # PyYAML \
-        # matplotlib \
+        Flask \
+        paho-mqtt \
+        PyYAML \
+        matplotlib \
+        pyarrow \
     && echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" > /etc/apt/sources.list.d/coral-edgetpu.list \
     && wget -q -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
     && apt -qq update \
diff --git a/detect_objects.py b/detect_objects.py
index e6f685175..447bffb59 100644
--- a/detect_objects.py
+++ b/detect_objects.py
@@ -2,13 +2,16 @@ import cv2
 import time
 import queue
 import yaml
+import multiprocessing as mp
+import subprocess as sp
 import numpy as np
 from flask import Flask, Response, make_response, jsonify
 import paho.mqtt.client as mqtt
 
-from frigate.video import Camera
-from frigate.object_detection import PreppedQueueProcessor
+from frigate.video import track_camera
+from frigate.object_processing import TrackedObjectProcessor
 from frigate.util import EventsPerSecond
+from frigate.edgetpu import EdgeTPUProcess
 
 with open('/config/config.yml') as f:
     CONFIG = yaml.safe_load(f)
@@ -38,8 +41,7 @@ FFMPEG_DEFAULT_CONFIG = {
          '-stimeout', '5000000',
          '-use_wallclock_as_timestamps', '1']),
     'output_args': FFMPEG_CONFIG.get('output_args',
-        ['-vf', 'mpdecimate',
-         '-f', 'rawvideo',
+        ['-f', 'rawvideo',
          '-pix_fmt', 'rgb24'])
 }
 
@@ -48,6 +50,10 @@ GLOBAL_OBJECT_CONFIG = CONFIG.get('objects', {})
 WEB_PORT = CONFIG.get('web_port', 5000)
 DEBUG = (CONFIG.get('debug', '0') == '1')
 
+# MODEL_PATH = CONFIG.get('tflite_model', '/lab/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite')
+MODEL_PATH = CONFIG.get('tflite_model', '/lab/detect.tflite')
+LABEL_MAP = CONFIG.get('label_map', '/lab/labelmap.txt')
+
 def main():
     # connect to mqtt and setup last will
     def on_connect(client, userdata, flags, rc):
@@ -70,28 +76,44 @@ def main():
         client.username_pw_set(MQTT_USER, password=MQTT_PASS)
     client.connect(MQTT_HOST, MQTT_PORT, 60)
     client.loop_start()
-    
-    # Queue for prepped frames, max size set to number of regions * 3
-    prepped_frame_queue = queue.Queue()
 
-    cameras = {}
+    # start plasma store
+    plasma_cmd = ['plasma_store', '-m', '400000000', '-s', '/tmp/plasma']
+    plasma_process = sp.Popen(plasma_cmd, stdout=sp.DEVNULL, stderr=sp.DEVNULL)
+
+    ##
+    # Setup config defaults for cameras
+    ##
     for name, config in CONFIG['cameras'].items():
-        cameras[name] = Camera(name, FFMPEG_DEFAULT_CONFIG, GLOBAL_OBJECT_CONFIG, config, 
-            prepped_frame_queue, client, MQTT_TOPIC_PREFIX)
+        config['snapshots'] = {
+            'show_timestamp': config.get('snapshots', {}).get('show_timestamp', True)
+        }
 
-    fps_tracker = EventsPerSecond()
+    # Queue for cameras to push tracked objects to
+    tracked_objects_queue = mp.Queue()
+    
+    # Start the shared tflite process
+    tflite_process = EdgeTPUProcess(MODEL_PATH)
 
-    prepped_queue_processor = PreppedQueueProcessor(
-        cameras,
-        prepped_frame_queue,
-        fps_tracker
-    )
-    prepped_queue_processor.start()
-    fps_tracker.start()
+    camera_processes = []
+    camera_stats_values = {}
+    for name, config in CONFIG['cameras'].items():
+        camera_stats_values[name] = {
+            'fps': mp.Value('d', 10.0),
+            'avg_wait': mp.Value('d', 0.0)
+        }
+        camera_process = mp.Process(target=track_camera, args=(name, config, FFMPEG_DEFAULT_CONFIG, GLOBAL_OBJECT_CONFIG, 
+            tflite_process.detect_lock, tflite_process.detect_ready, tflite_process.frame_ready, tracked_objects_queue, 
+            camera_stats_values[name]['fps'], camera_stats_values[name]['avg_wait']))
+        camera_process.daemon = True
+        camera_processes.append(camera_process)
 
-    for name, camera in cameras.items():
-        camera.start()
-        print("Capture process for {}: {}".format(name, camera.get_capture_pid()))
+    for camera_process in camera_processes:
+        camera_process.start()
+        print(f"Camera_process started {camera_process.pid}")
+    
+    object_processor = TrackedObjectProcessor(CONFIG['cameras'], client, MQTT_TOPIC_PREFIX, tracked_objects_queue)
+    object_processor.start()
 
     # create a flask app that encodes frames a mjpeg on demand
     app = Flask(__name__)
@@ -105,21 +127,23 @@ def main():
     def stats():
         stats = {
             'coral': {
-                'fps': fps_tracker.eps(),
-                'inference_speed': prepped_queue_processor.avg_inference_speed,
-                'queue_length': prepped_frame_queue.qsize()
+                'fps': tflite_process.fps.value,
+                'inference_speed': tflite_process.avg_inference_speed.value
             }
         }
 
-        for name, camera in cameras.items():
-            stats[name] = camera.stats()
+        for name, camera_stats in camera_stats_values.items():
+            stats[name] = {
+                'fps': camera_stats['fps'].value,
+                'avg_wait': camera_stats['avg_wait'].value
+            }
 
         return jsonify(stats)
 
     @app.route('/<camera_name>/<label>/best.jpg')
     def best(camera_name, label):
-        if camera_name in cameras:
-            best_frame = cameras[camera_name].get_best(label)
+        if camera_name in CONFIG['cameras']:
+            best_frame = object_processor.get_best(camera_name, label)
             if best_frame is None:
                 best_frame = np.zeros((720,1280,3), np.uint8)
             best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)
@@ -132,7 +156,7 @@ def main():
 
     @app.route('/<camera_name>')
     def mjpeg_feed(camera_name):
-        if camera_name in cameras:
+        if camera_name in CONFIG['cameras']:
             # return a multipart response
             return Response(imagestream(camera_name),
                             mimetype='multipart/x-mixed-replace; boundary=frame')
@@ -143,13 +167,16 @@ def main():
         while True:
             # max out at 1 FPS
             time.sleep(1)
-            frame = cameras[camera_name].get_current_frame_with_objects()
+            frame = object_processor.current_frame_with_objects(camera_name)
             yield (b'--frame\r\n'
                 b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n\r\n')
 
     app.run(host='0.0.0.0', port=WEB_PORT, debug=False)
 
-    camera.join()
+    for camera_process in camera_processes:
+        camera_process.join()
+    
+    plasma_process.terminate()
 
 if __name__ == '__main__':
     main()
diff --git a/frigate/edgetpu.py b/frigate/edgetpu.py
index 6e09ef3ca..1aecaeb48 100644
--- a/frigate/edgetpu.py
+++ b/frigate/edgetpu.py
@@ -1,8 +1,11 @@
+import os
+import datetime
 import multiprocessing as mp
 import numpy as np
 import SharedArray as sa
 import tflite_runtime.interpreter as tflite
 from tflite_runtime.interpreter import load_delegate
+from frigate.util import EventsPerSecond
 
 def load_labels(path, encoding='utf-8'):
   """Loads labels from file (with or without index numbers).
@@ -59,6 +62,7 @@ class ObjectDetector():
 
 class EdgeTPUProcess():
     def __init__(self, model):
+        # TODO: see if we can use the plasma store with a queue and maintain the same speeds
         try:
             sa.delete("frame")
         except:
@@ -74,22 +78,32 @@ class EdgeTPUProcess():
         self.detect_lock = mp.Lock()
         self.detect_ready = mp.Event()
         self.frame_ready = mp.Event()
+        self.fps = mp.Value('d', 0.0)
+        self.avg_inference_speed = mp.Value('d', 10.0)
 
-        def run_detector(model, detect_ready, frame_ready):
+        def run_detector(model, detect_ready, frame_ready, fps, avg_speed):
+            print(f"Starting detection process: {os.getpid()}")
             object_detector = ObjectDetector(model)
             input_frame = sa.attach("frame")
             detections = sa.attach("detections")
+            fps_tracker = EventsPerSecond()
+            fps_tracker.start()
 
             while True:
                 # wait until a frame is ready
                 frame_ready.wait()
+                start = datetime.datetime.now().timestamp()
                 # signal that the process is busy
                 frame_ready.clear()
                 detections[:] = object_detector.detect_raw(input_frame)
                 # signal that the process is ready to detect
                 detect_ready.set()
+                fps_tracker.update()
+                fps.value = fps_tracker.eps()
+                duration = datetime.datetime.now().timestamp()-start
+                avg_speed.value = (avg_speed.value*9 + duration)/10
 
-        self.detect_process = mp.Process(target=run_detector, args=(model, self.detect_ready, self.frame_ready))
+        self.detect_process = mp.Process(target=run_detector, args=(model, self.detect_ready, self.frame_ready, self.fps, self.avg_inference_speed))
         self.detect_process.daemon = True
         self.detect_process.start()
 
diff --git a/frigate/motion.py b/frigate/motion.py
index 8022d9469..57f27afa6 100644
--- a/frigate/motion.py
+++ b/frigate/motion.py
@@ -3,14 +3,15 @@ import imutils
 import numpy as np
 
 class MotionDetector():
-    # TODO: add motion masking
-    def __init__(self, frame_shape, resize_factor=4):
+    def __init__(self, frame_shape, mask, resize_factor=4):
         self.resize_factor = resize_factor
         self.motion_frame_size = (int(frame_shape[0]/resize_factor), int(frame_shape[1]/resize_factor))
         self.avg_frame = np.zeros(self.motion_frame_size, np.float)
         self.avg_delta = np.zeros(self.motion_frame_size, np.float)
         self.motion_frame_count = 0
         self.frame_counter = 0
+        resized_mask = cv2.resize(mask, dsize=(self.motion_frame_size[1], self.motion_frame_size[0]), interpolation=cv2.INTER_LINEAR)
+        self.mask = np.where(resized_mask==[0])
 
     def detect(self, frame):
         motion_boxes = []
@@ -21,6 +22,9 @@ class MotionDetector():
         # convert to grayscale
         gray = cv2.cvtColor(resized_frame, cv2.COLOR_BGR2GRAY)
 
+        # mask frame
+        gray[self.mask] = [255]
+
         # it takes ~30 frames to establish a baseline
         # dont bother looking for motion
         if self.frame_counter < 30:
@@ -58,7 +62,6 @@ class MotionDetector():
                 # if the contour is big enough, count it as motion
                 contour_area = cv2.contourArea(c)
                 if contour_area > 100:
-                    # cv2.drawContours(resized_frame, [c], -1, (255,255,255), 2)
                     x, y, w, h = cv2.boundingRect(c)
                     motion_boxes.append((x*self.resize_factor, y*self.resize_factor, (x+w)*self.resize_factor, (y+h)*self.resize_factor))
         
diff --git a/frigate/mqtt.py b/frigate/mqtt.py
deleted file mode 100644
index 6c09abbb1..000000000
--- a/frigate/mqtt.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import json
-import cv2
-import threading
-import prctl
-from collections import Counter, defaultdict
-import itertools
-
-class MqttObjectPublisher(threading.Thread):
-    def __init__(self, client, topic_prefix, camera):
-        threading.Thread.__init__(self)
-        self.client = client
-        self.topic_prefix = topic_prefix
-        self.camera = camera
-
-    def run(self):
-        prctl.set_name(self.__class__.__name__)
-        current_object_status = defaultdict(lambda: 'OFF')
-        while True:
-            # wait until objects have been tracked
-            with self.camera.objects_tracked:
-                self.camera.objects_tracked.wait()
-
-            # count objects with more than 2 entries in history by type
-            obj_counter = Counter()
-            for obj in self.camera.object_tracker.tracked_objects.values():
-                if len(obj['history']) > 1:
-                    obj_counter[obj['name']] += 1
-            
-            # report on detected objects
-            for obj_name, count in obj_counter.items():
-                new_status = 'ON' if count > 0 else 'OFF'
-                if new_status != current_object_status[obj_name]:
-                    current_object_status[obj_name] = new_status
-                    self.client.publish(self.topic_prefix+'/'+obj_name, new_status, retain=False)
-                    # send the snapshot over mqtt if we have it as well
-                    if obj_name in self.camera.best_frames.best_frames:
-                        best_frame = cv2.cvtColor(self.camera.best_frames.best_frames[obj_name], cv2.COLOR_RGB2BGR)
-                        ret, jpg = cv2.imencode('.jpg', best_frame)
-                        if ret:
-                            jpg_bytes = jpg.tobytes()
-                            self.client.publish(self.topic_prefix+'/'+obj_name+'/snapshot', jpg_bytes, retain=True)
-
-            # expire any objects that are ON and no longer detected
-            expired_objects = [obj_name for obj_name, status in current_object_status.items() if status == 'ON' and not obj_name in obj_counter]
-            for obj_name in expired_objects:
-                current_object_status[obj_name] = 'OFF'
-                self.client.publish(self.topic_prefix+'/'+obj_name, 'OFF', retain=False)
-                # send updated snapshot snapshot over mqtt if we have it as well
-                if obj_name in self.camera.best_frames.best_frames:
-                    best_frame = cv2.cvtColor(self.camera.best_frames.best_frames[obj_name], cv2.COLOR_RGB2BGR)
-                    ret, jpg = cv2.imencode('.jpg', best_frame)
-                    if ret:
-                        jpg_bytes = jpg.tobytes()
-                        self.client.publish(self.topic_prefix+'/'+obj_name+'/snapshot', jpg_bytes, retain=True)
\ No newline at end of file
diff --git a/frigate/object_detection.py b/frigate/object_detection.py
index 4a04094c3..b190f84c3 100644
--- a/frigate/object_detection.py
+++ b/frigate/object_detection.py
@@ -3,7 +3,7 @@ import time
 import cv2
 import threading
 import copy
-import prctl
+# import prctl
 import numpy as np
 from edgetpu.detection.engine import DetectionEngine
 
diff --git a/frigate/object_processing.py b/frigate/object_processing.py
new file mode 100644
index 000000000..c881d4a91
--- /dev/null
+++ b/frigate/object_processing.py
@@ -0,0 +1,146 @@
+import json
+import hashlib
+import datetime
+import copy
+import cv2
+import threading
+import numpy as np
+from collections import Counter, defaultdict
+import itertools
+import pyarrow.plasma as plasma
+import SharedArray as sa
+import matplotlib.pyplot as plt
+from frigate.util import draw_box_with_label, ReadLabelFile
+
+PATH_TO_LABELS = '/lab/labelmap.txt'
+
+LABELS = ReadLabelFile(PATH_TO_LABELS)
+cmap = plt.cm.get_cmap('tab10', len(LABELS.keys()))
+
+COLOR_MAP = {}
+for key, val in LABELS.items():
+    COLOR_MAP[val] = tuple(int(round(255 * c)) for c in cmap(key)[:3])
+
+class TrackedObjectProcessor(threading.Thread):
+    def __init__(self, config, client, topic_prefix, tracked_objects_queue):
+        threading.Thread.__init__(self)
+        self.config = config
+        self.client = client
+        self.topic_prefix = topic_prefix
+        self.tracked_objects_queue = tracked_objects_queue
+        self.plasma_client = plasma.connect("/tmp/plasma")
+        self.camera_data = defaultdict(lambda: {
+            'best_objects': {},
+            'object_status': defaultdict(lambda: defaultdict(lambda: 'OFF')),
+            'tracked_objects': {}
+        })
+        
+    def get_best(self, camera, label):
+        if label in self.camera_data[camera]['best_objects']:
+            return self.camera_data[camera]['best_objects'][label]['frame']
+        else:
+            return None
+
+    def get_frame(self, config, camera, obj):
+        object_id_hash = hashlib.sha1(str.encode(f"{camera}{obj['frame_time']}"))
+        object_id_bytes = object_id_hash.digest()
+        object_id = plasma.ObjectID(object_id_bytes)
+        best_frame = self.plasma_client.get(object_id)
+        box = obj['box']
+        draw_box_with_label(best_frame, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}")
+        # print a timestamp
+        if config['snapshots']['show_timestamp']:
+            time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
+            cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
+        return best_frame
+    
+    def current_frame_with_objects(self, camera):
+        frame_time = self.camera_data[camera]['current_frame']
+        object_id_hash = hashlib.sha1(str.encode(f"{camera}{frame_time}"))
+        object_id_bytes = object_id_hash.digest()
+        object_id = plasma.ObjectID(object_id_bytes)
+        current_frame = self.plasma_client.get(object_id)
+            
+        tracked_objects = copy.deepcopy(self.camera_data[camera]['tracked_objects'])
+
+        # draw the bounding boxes on the screen
+        for obj in tracked_objects.values():
+            thickness = 2
+            color = COLOR_MAP[obj['label']]
+            
+            if obj['frame_time'] != frame_time:
+                thickness = 1
+                color = (255,0,0)
+
+            box = obj['box']
+            draw_box_with_label(current_frame, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}", thickness=thickness, color=color)
+        
+        # # print fps
+        # cv2.putText(frame, str(self.fps.eps())+'FPS', (10, 60), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
+
+        # convert to BGR
+        frame = cv2.cvtColor(current_frame, cv2.COLOR_RGB2BGR)
+
+        # encode the image into a jpg
+        ret, jpg = cv2.imencode('.jpg', frame)
+
+        return jpg.tobytes()
+
+    def run(self):
+        while True:
+            camera, frame_time, tracked_objects = self.tracked_objects_queue.get()
+
+            config = self.config[camera]
+            best_objects = self.camera_data[camera]['best_objects']
+            current_object_status = self.camera_data[camera]['object_status']
+            self.camera_data[camera]['tracked_objects'] = tracked_objects
+            self.camera_data[camera]['current_frame'] = frame_time
+            
+            ###
+            # Maintain the highest scoring recent object and frame for each label
+            ###
+            for obj in tracked_objects.values():
+                if obj['label'] in best_objects:
+                    now = datetime.datetime.now().timestamp()
+                    # if the object is a higher score than the current best score 
+                    # or the current object is more than 1 minute old, use the new object
+                    if obj['score'] > best_objects[obj['label']]['score'] or (now - best_objects[obj['label']]['frame_time']) > 60:
+                        obj['frame'] = self.get_frame(config, camera, obj)
+                        best_objects[obj['label']] = obj
+                else:
+                    obj['frame'] = self.get_frame(config, camera, obj)
+                    best_objects[obj['label']] = obj
+
+            ###
+            # Report over MQTT
+            ###
+            # count objects with more than 2 entries in history by type
+            obj_counter = Counter()
+            for obj in tracked_objects.values():
+                if len(obj['history']) > 1:
+                    obj_counter[obj['label']] += 1
+                    
+            # report on detected objects
+            for obj_name, count in obj_counter.items():
+                new_status = 'ON' if count > 0 else 'OFF'
+                if new_status != current_object_status[obj_name]:
+                    current_object_status[obj_name] = new_status
+                    self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}", new_status, retain=False)
+                    # send the best snapshot over mqtt
+                    best_frame = cv2.cvtColor(best_objects[obj_name]['frame'], cv2.COLOR_RGB2BGR)
+                    ret, jpg = cv2.imencode('.jpg', best_frame)
+                    if ret:
+                        jpg_bytes = jpg.tobytes()
+                        self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}/snapshot", jpg_bytes, retain=True)
+
+            # expire any objects that are ON and no longer detected
+            expired_objects = [obj_name for obj_name, status in current_object_status.items() if status == 'ON' and not obj_name in obj_counter]
+            for obj_name in expired_objects:
+                current_object_status[obj_name] = 'OFF'
+                self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}", 'OFF', retain=False)
+                # send updated snapshot over mqtt
+                best_frame = cv2.cvtColor(best_objects[obj_name]['frame'], cv2.COLOR_RGB2BGR)
+                ret, jpg = cv2.imencode('.jpg', best_frame)
+                if ret:
+                    jpg_bytes = jpg.tobytes()
+                    self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}/snapshot", jpg_bytes, retain=True)
\ No newline at end of file
diff --git a/frigate/objects.py b/frigate/objects.py
index 6c6f0226e..4e99fc6c0 100644
--- a/frigate/objects.py
+++ b/frigate/objects.py
@@ -2,277 +2,266 @@ import time
 import datetime
 import threading
 import cv2
-import prctl
+# import prctl
 import itertools
 import copy
 import numpy as np
 import multiprocessing as mp
 from collections import defaultdict
 from scipy.spatial import distance as dist
-from frigate.util import draw_box_with_label, LABELS, compute_intersection_rectangle, compute_intersection_over_union, calculate_region
+from frigate.util import draw_box_with_label, LABELS, calculate_region
 
-class ObjectCleaner(threading.Thread):
-    def __init__(self, camera):
-        threading.Thread.__init__(self)
-        self.camera = camera
+# class ObjectCleaner(threading.Thread):
+#     def __init__(self, camera):
+#         threading.Thread.__init__(self)
+#         self.camera = camera
 
-    def run(self):
-        prctl.set_name("ObjectCleaner")
-        while True:
+#     def run(self):
+#         prctl.set_name("ObjectCleaner")
+#         while True:
 
-            # wait a bit before checking for expired frames
-            time.sleep(0.2)
+#             # wait a bit before checking for expired frames
+#             time.sleep(0.2)
 
-            for frame_time in list(self.camera.detected_objects.keys()).copy():
-                if not frame_time in self.camera.frame_cache:
-                    del self.camera.detected_objects[frame_time]
+#             for frame_time in list(self.camera.detected_objects.keys()).copy():
+#                 if not frame_time in self.camera.frame_cache:
+#                     del self.camera.detected_objects[frame_time]
             
-            objects_deregistered = False
-            with self.camera.object_tracker.tracked_objects_lock:
-                now = datetime.datetime.now().timestamp()
-                for id, obj in list(self.camera.object_tracker.tracked_objects.items()):
-                    # if the object is more than 10 seconds old
-                    # and not in the most recent frame, deregister
-                    if (now - obj['frame_time']) > 10 and self.camera.object_tracker.most_recent_frame_time > obj['frame_time']:
-                        self.camera.object_tracker.deregister(id)
-                        objects_deregistered = True
+#             objects_deregistered = False
+#             with self.camera.object_tracker.tracked_objects_lock:
+#                 now = datetime.datetime.now().timestamp()
+#                 for id, obj in list(self.camera.object_tracker.tracked_objects.items()):
+#                     # if the object is more than 10 seconds old
+#                     # and not in the most recent frame, deregister
+#                     if (now - obj['frame_time']) > 10 and self.camera.object_tracker.most_recent_frame_time > obj['frame_time']:
+#                         self.camera.object_tracker.deregister(id)
+#                         objects_deregistered = True
             
-            if objects_deregistered:
-                with self.camera.objects_tracked:
-                    self.camera.objects_tracked.notify_all()
+#             if objects_deregistered:
+#                 with self.camera.objects_tracked:
+#                     self.camera.objects_tracked.notify_all()
 
-class DetectedObjectsProcessor(threading.Thread):
-    def __init__(self, camera):
-        threading.Thread.__init__(self)
-        self.camera = camera
+# class DetectedObjectsProcessor(threading.Thread):
+#     def __init__(self, camera):
+#         threading.Thread.__init__(self)
+#         self.camera = camera
 
-    def run(self):
-        prctl.set_name(self.__class__.__name__)
-        while True:
-            frame = self.camera.detected_objects_queue.get()
+#     def run(self):
+#         prctl.set_name(self.__class__.__name__)
+#         while True:
+#             frame = self.camera.detected_objects_queue.get()
 
-            objects = frame['detected_objects']
+#             objects = frame['detected_objects']
 
-            for raw_obj in objects:
-                name = str(LABELS[raw_obj.label_id])
+#             for raw_obj in objects:
+#                 name = str(LABELS[raw_obj.label_id])
 
-                if not name in self.camera.objects_to_track:
-                    continue
+#                 if not name in self.camera.objects_to_track:
+#                     continue
 
-                obj = {
-                    'name': name,
-                    'score': float(raw_obj.score),
-                    'box': {
-                        'xmin': int((raw_obj.bounding_box[0][0] * frame['size']) + frame['x_offset']),
-                        'ymin': int((raw_obj.bounding_box[0][1] * frame['size']) + frame['y_offset']),
-                        'xmax': int((raw_obj.bounding_box[1][0] * frame['size']) + frame['x_offset']),
-                        'ymax': int((raw_obj.bounding_box[1][1] * frame['size']) + frame['y_offset'])
-                    },
-                    'region': {
-                        'xmin': frame['x_offset'],
-                        'ymin': frame['y_offset'],
-                        'xmax': frame['x_offset']+frame['size'],
-                        'ymax': frame['y_offset']+frame['size']
-                    },
-                    'frame_time': frame['frame_time'],
-                    'region_id': frame['region_id']
-                }
+#                 obj = {
+#                     'name': name,
+#                     'score': float(raw_obj.score),
+#                     'box': {
+#                         'xmin': int((raw_obj.bounding_box[0][0] * frame['size']) + frame['x_offset']),
+#                         'ymin': int((raw_obj.bounding_box[0][1] * frame['size']) + frame['y_offset']),
+#                         'xmax': int((raw_obj.bounding_box[1][0] * frame['size']) + frame['x_offset']),
+#                         'ymax': int((raw_obj.bounding_box[1][1] * frame['size']) + frame['y_offset'])
+#                     },
+#                     'region': {
+#                         'xmin': frame['x_offset'],
+#                         'ymin': frame['y_offset'],
+#                         'xmax': frame['x_offset']+frame['size'],
+#                         'ymax': frame['y_offset']+frame['size']
+#                     },
+#                     'frame_time': frame['frame_time'],
+#                     'region_id': frame['region_id']
+#                 }
                 
-                # if the object is within 5 pixels of the region border, and the region is not on the edge
-                # consider the object to be clipped
-                obj['clipped'] = False
-                if ((obj['region']['xmin'] > 5 and obj['box']['xmin']-obj['region']['xmin'] <= 5) or 
-                    (obj['region']['ymin'] > 5 and obj['box']['ymin']-obj['region']['ymin'] <= 5) or
-                    (self.camera.frame_shape[1]-obj['region']['xmax'] > 5 and obj['region']['xmax']-obj['box']['xmax'] <= 5) or
-                    (self.camera.frame_shape[0]-obj['region']['ymax'] > 5 and obj['region']['ymax']-obj['box']['ymax'] <= 5)):
-                    obj['clipped'] = True
+#                 # if the object is within 5 pixels of the region border, and the region is not on the edge
+#                 # consider the object to be clipped
+#                 obj['clipped'] = False
+#                 if ((obj['region']['xmin'] > 5 and obj['box']['xmin']-obj['region']['xmin'] <= 5) or 
+#                     (obj['region']['ymin'] > 5 and obj['box']['ymin']-obj['region']['ymin'] <= 5) or
+#                     (self.camera.frame_shape[1]-obj['region']['xmax'] > 5 and obj['region']['xmax']-obj['box']['xmax'] <= 5) or
+#                     (self.camera.frame_shape[0]-obj['region']['ymax'] > 5 and obj['region']['ymax']-obj['box']['ymax'] <= 5)):
+#                     obj['clipped'] = True
                 
-                # Compute the area
-                # TODO: +1 right?
-                obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin'])
+#                 # Compute the area
+#                 # TODO: +1 right?
+#                 obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin'])
 
-                self.camera.detected_objects[frame['frame_time']].append(obj)
+#                 self.camera.detected_objects[frame['frame_time']].append(obj)
             
-            # TODO: use in_process and processed counts instead to avoid lock 
-            with self.camera.regions_in_process_lock:
-                if frame['frame_time'] in self.camera.regions_in_process:
-                    self.camera.regions_in_process[frame['frame_time']] -= 1
-                # print(f"{frame['frame_time']} remaining regions {self.camera.regions_in_process[frame['frame_time']]}")
+#             # TODO: use in_process and processed counts instead to avoid lock 
+#             with self.camera.regions_in_process_lock:
+#                 if frame['frame_time'] in self.camera.regions_in_process:
+#                     self.camera.regions_in_process[frame['frame_time']] -= 1
+#                 # print(f"{frame['frame_time']} remaining regions {self.camera.regions_in_process[frame['frame_time']]}")
 
-                    if self.camera.regions_in_process[frame['frame_time']] == 0:
-                        del self.camera.regions_in_process[frame['frame_time']]
-                        # print(f"{frame['frame_time']} no remaining regions")
-                        self.camera.finished_frame_queue.put(frame['frame_time'])
-                else:
-                    self.camera.finished_frame_queue.put(frame['frame_time'])
+#                     if self.camera.regions_in_process[frame['frame_time']] == 0:
+#                         del self.camera.regions_in_process[frame['frame_time']]
+#                         # print(f"{frame['frame_time']} no remaining regions")
+#                         self.camera.finished_frame_queue.put(frame['frame_time'])
+#                 else:
+#                     self.camera.finished_frame_queue.put(frame['frame_time'])
 
-# Thread that checks finished frames for clipped objects and sends back
-# for processing if needed
-# TODO: evaluate whether or not i really need separate threads/queues for each step
-#       given that only 1 thread will really be able to run at a time. you need a 
-#       separate process to actually do things in parallel for when you are CPU bound. 
-#       threads are good when you are waiting and could be processing while you wait
-class RegionRefiner(threading.Thread):
-    def __init__(self, camera):
-        threading.Thread.__init__(self)
-        self.camera = camera
+# # Thread that checks finished frames for clipped objects and sends back
+# # for processing if needed
+# # TODO: evaluate whether or not i really need separate threads/queues for each step
+# #       given that only 1 thread will really be able to run at a time. you need a 
+# #       separate process to actually do things in parallel for when you are CPU bound. 
+# #       threads are good when you are waiting and could be processing while you wait
+# class RegionRefiner(threading.Thread):
+#     def __init__(self, camera):
+#         threading.Thread.__init__(self)
+#         self.camera = camera
 
-    def run(self):
-        prctl.set_name(self.__class__.__name__)
-        while True:
-            frame_time = self.camera.finished_frame_queue.get()
+#     def run(self):
+#         prctl.set_name(self.__class__.__name__)
+#         while True:
+#             frame_time = self.camera.finished_frame_queue.get()
 
-            detected_objects = self.camera.detected_objects[frame_time].copy()
-            # print(f"{frame_time} finished")
+#             detected_objects = self.camera.detected_objects[frame_time].copy()
+#             # print(f"{frame_time} finished")
 
-            # group by name
-            detected_object_groups = defaultdict(lambda: [])
-            for obj in detected_objects:
-                detected_object_groups[obj['name']].append(obj)
+#             # group by name
+#             detected_object_groups = defaultdict(lambda: [])
+#             for obj in detected_objects:
+#                 detected_object_groups[obj['name']].append(obj)
 
-            look_again = False
-            selected_objects = []
-            for group in detected_object_groups.values():
+#             look_again = False
+#             selected_objects = []
+#             for group in detected_object_groups.values():
 
-                # apply non-maxima suppression to suppress weak, overlapping bounding boxes
-                boxes = [(o['box']['xmin'], o['box']['ymin'], o['box']['xmax']-o['box']['xmin'], o['box']['ymax']-o['box']['ymin'])
-                    for o in group]
-                confidences = [o['score'] for o in group]
-                idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
+#                 # apply non-maxima suppression to suppress weak, overlapping bounding boxes
+#                 boxes = [(o['box']['xmin'], o['box']['ymin'], o['box']['xmax']-o['box']['xmin'], o['box']['ymax']-o['box']['ymin'])
+#                     for o in group]
+#                 confidences = [o['score'] for o in group]
+#                 idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
 
-                for index in idxs:
-                    obj = group[index[0]]
-                    selected_objects.append(obj)
-                    if obj['clipped']:
-                        box = obj['box']
-                        # calculate a new region that will hopefully get the entire object
-                        (size, x_offset, y_offset) = calculate_region(self.camera.frame_shape, 
-                            box['xmin'], box['ymin'],
-                            box['xmax'], box['ymax'])
-                        # print(f"{frame_time} new region: {size} {x_offset} {y_offset}")
+#                 for index in idxs:
+#                     obj = group[index[0]]
+#                     selected_objects.append(obj)
+#                     if obj['clipped']:
+#                         box = obj['box']
+#                         # calculate a new region that will hopefully get the entire object
+#                         (size, x_offset, y_offset) = calculate_region(self.camera.frame_shape, 
+#                             box['xmin'], box['ymin'],
+#                             box['xmax'], box['ymax'])
+#                         # print(f"{frame_time} new region: {size} {x_offset} {y_offset}")
 
-                        with self.camera.regions_in_process_lock:
-                            if not frame_time in self.camera.regions_in_process:
-                                self.camera.regions_in_process[frame_time] = 1
-                            else:
-                                self.camera.regions_in_process[frame_time] += 1
+#                         with self.camera.regions_in_process_lock:
+#                             if not frame_time in self.camera.regions_in_process:
+#                                 self.camera.regions_in_process[frame_time] = 1
+#                             else:
+#                                 self.camera.regions_in_process[frame_time] += 1
 
-                        # add it to the queue
-                        self.camera.resize_queue.put({
-                            'camera_name': self.camera.name,
-                            'frame_time': frame_time,
-                            'region_id': -1,
-                            'size': size,
-                            'x_offset': x_offset,
-                            'y_offset': y_offset
-                        })
-                        self.camera.dynamic_region_fps.update()
-                        look_again = True
+#                         # add it to the queue
+#                         self.camera.resize_queue.put({
+#                             'camera_name': self.camera.name,
+#                             'frame_time': frame_time,
+#                             'region_id': -1,
+#                             'size': size,
+#                             'x_offset': x_offset,
+#                             'y_offset': y_offset
+#                         })
+#                         self.camera.dynamic_region_fps.update()
+#                         look_again = True
 
-            # if we are looking again, then this frame is not ready for processing
-            if look_again:
-                # remove the clipped objects
-                self.camera.detected_objects[frame_time] = [o for o in selected_objects if not o['clipped']]
-                continue
+#             # if we are looking again, then this frame is not ready for processing
+#             if look_again:
+#                 # remove the clipped objects
+#                 self.camera.detected_objects[frame_time] = [o for o in selected_objects if not o['clipped']]
+#                 continue
 
-            # filter objects based on camera settings
-            selected_objects = [o for o in selected_objects if not self.filtered(o)]
+#             # filter objects based on camera settings
+#             selected_objects = [o for o in selected_objects if not self.filtered(o)]
 
-            self.camera.detected_objects[frame_time] = selected_objects
+#             self.camera.detected_objects[frame_time] = selected_objects
             
-            # print(f"{frame_time} is actually finished")
+#             # print(f"{frame_time} is actually finished")
 
-            # keep adding frames to the refined queue as long as they are finished
-            with self.camera.regions_in_process_lock:
-                while self.camera.frame_queue.qsize() > 0 and self.camera.frame_queue.queue[0] not in self.camera.regions_in_process:
-                    self.camera.last_processed_frame = self.camera.frame_queue.get()
-                    self.camera.refined_frame_queue.put(self.camera.last_processed_frame)
+#             # keep adding frames to the refined queue as long as they are finished
+#             with self.camera.regions_in_process_lock:
+#                 while self.camera.frame_queue.qsize() > 0 and self.camera.frame_queue.queue[0] not in self.camera.regions_in_process:
+#                     self.camera.last_processed_frame = self.camera.frame_queue.get()
+#                     self.camera.refined_frame_queue.put(self.camera.last_processed_frame)
 
-    def filtered(self, obj):
-        object_name = obj['name']
+#     def filtered(self, obj):
+#         object_name = obj['name']
         
-        if object_name in self.camera.object_filters:
-            obj_settings = self.camera.object_filters[object_name]
+#         if object_name in self.camera.object_filters:
+#             obj_settings = self.camera.object_filters[object_name]
 
-            # if the min area is larger than the
-            # detected object, don't add it to detected objects
-            if obj_settings.get('min_area',-1) > obj['area']:
-                return True
+#             # if the min area is larger than the
+#             # detected object, don't add it to detected objects
+#             if obj_settings.get('min_area',-1) > obj['area']:
+#                 return True
             
-            # if the detected object is larger than the
-            # max area, don't add it to detected objects
-            if obj_settings.get('max_area', self.camera.frame_shape[0]*self.camera.frame_shape[1]) < obj['area']:
-                return True
+#             # if the detected object is larger than the
+#             # max area, don't add it to detected objects
+#             if obj_settings.get('max_area', self.camera.frame_shape[0]*self.camera.frame_shape[1]) < obj['area']:
+#                 return True
 
-            # if the score is lower than the threshold, skip
-            if obj_settings.get('threshold', 0) > obj['score']:
-                return True
+#             # if the score is lower than the threshold, skip
+#             if obj_settings.get('threshold', 0) > obj['score']:
+#                 return True
         
-            # compute the coordinates of the object and make sure
-            # the location isnt outside the bounds of the image (can happen from rounding)
-            y_location = min(int(obj['box']['ymax']), len(self.camera.mask)-1)
-            x_location = min(int((obj['box']['xmax']-obj['box']['xmin'])/2.0)+obj['box']['xmin'], len(self.camera.mask[0])-1)
+#             # compute the coordinates of the object and make sure
+#             # the location isnt outside the bounds of the image (can happen from rounding)
+#             y_location = min(int(obj['box']['ymax']), len(self.camera.mask)-1)
+#             x_location = min(int((obj['box']['xmax']-obj['box']['xmin'])/2.0)+obj['box']['xmin'], len(self.camera.mask[0])-1)
 
-            # if the object is in a masked location, don't add it to detected objects
-            if self.camera.mask[y_location][x_location] == [0]:
-                return True
+#             # if the object is in a masked location, don't add it to detected objects
+#             if self.camera.mask[y_location][x_location] == [0]:
+#                 return True
             
-            return False
+#             return False
              
-    def has_overlap(self, new_obj, obj, overlap=.7):
-        # compute intersection rectangle with existing object and new objects region
-        existing_obj_current_region = compute_intersection_rectangle(obj['box'], new_obj['region'])
+#     def has_overlap(self, new_obj, obj, overlap=.7):
+#         # compute intersection rectangle with existing object and new objects region
+#         existing_obj_current_region = compute_intersection_rectangle(obj['box'], new_obj['region'])
 
-        # compute intersection rectangle with new object and existing objects region
-        new_obj_existing_region = compute_intersection_rectangle(new_obj['box'], obj['region'])
+#         # compute intersection rectangle with new object and existing objects region
+#         new_obj_existing_region = compute_intersection_rectangle(new_obj['box'], obj['region'])
 
-        # compute iou for the two intersection rectangles that were just computed
-        iou = compute_intersection_over_union(existing_obj_current_region, new_obj_existing_region)
+#         # compute iou for the two intersection rectangles that were just computed
+#         iou = compute_intersection_over_union(existing_obj_current_region, new_obj_existing_region)
 
-        # if intersection is greater than overlap
-        if iou > overlap:
-            return True
-        else:
-            return False
+#         # if intersection is greater than overlap
+#         if iou > overlap:
+#             return True
+#         else:
+#             return False
     
-    def find_group(self, new_obj, groups):
-        for index, group in enumerate(groups):
-            for obj in group:
-                if self.has_overlap(new_obj, obj):
-                    return index
-        return None
+#     def find_group(self, new_obj, groups):
+#         for index, group in enumerate(groups):
+#             for obj in group:
+#                 if self.has_overlap(new_obj, obj):
+#                     return index
+#         return None
 
-class ObjectTracker(threading.Thread):
-    def __init__(self, camera, max_disappeared):
-        threading.Thread.__init__(self)
-        self.camera = camera
+class ObjectTracker():
+    def __init__(self, max_disappeared):
         self.tracked_objects = {}
-        self.tracked_objects_lock = mp.Lock()
-        self.most_recent_frame_time = None
-    
-    def run(self):
-        prctl.set_name(self.__class__.__name__)
-        while True:
-            frame_time = self.camera.refined_frame_queue.get()
-            with self.tracked_objects_lock:
-                self.match_and_update(self.camera.detected_objects[frame_time])
-                self.most_recent_frame_time = frame_time
-                self.camera.frame_output_queue.put((frame_time, copy.deepcopy(self.tracked_objects)))
-            if len(self.tracked_objects) > 0:
-                with self.camera.objects_tracked:
-                    self.camera.objects_tracked.notify_all()
+        self.disappeared = {}
+        self.max_disappeared = max_disappeared
 
     def register(self, index, obj):
-        id = "{}-{}".format(str(obj['frame_time']), index)
+        id = f"{obj['frame_time']}-{index}"
         obj['id'] = id
         obj['top_score'] = obj['score']
         self.add_history(obj)
         self.tracked_objects[id] = obj
+        self.disappeared[id] = 0
 
     def deregister(self, id):
         del self.tracked_objects[id]
+        del self.disappeared[id]
     
     def update(self, id, new_obj):
+        self.disappeared[id] = 0
         self.tracked_objects[id].update(new_obj)
         self.add_history(self.tracked_objects[id])
         if self.tracked_objects[id]['score'] > self.tracked_objects[id]['top_score']:
@@ -291,25 +280,37 @@ class ObjectTracker(threading.Thread):
         else:
             obj['history'] = [entry]
 
-    def match_and_update(self, new_objects):
+    def match_and_update(self, frame_time, new_objects):
         if len(new_objects) == 0:
+            for id in list(self.tracked_objects.keys()):
+                if self.disappeared[id] >= self.max_disappeared:
+                    self.deregister(id)
+                else:
+                    self.disappeared[id] += 1
             return
             
         # group by name
         new_object_groups = defaultdict(lambda: [])
         for obj in new_objects:
-            new_object_groups[obj['name']].append(obj)
+            new_object_groups[obj[0]].append({
+                'label': obj[0],
+                'score': obj[1],
+                'box': obj[2],
+                'area': obj[3],
+                'region': obj[4],
+                'frame_time': frame_time
+            })
         
         # track objects for each label type
         for label, group in new_object_groups.items():
-            current_objects = [o for o in self.tracked_objects.values() if o['name'] == label]
+            current_objects = [o for o in self.tracked_objects.values() if o['label'] == label]
             current_ids = [o['id'] for o in current_objects]
             current_centroids = np.array([o['centroid'] for o in current_objects])
 
             # compute centroids of new objects
             for obj in group:
-                centroid_x = int((obj['box']['xmin']+obj['box']['xmax']) / 2.0)
-                centroid_y = int((obj['box']['ymin']+obj['box']['ymax']) / 2.0)
+                centroid_x = int((obj['box'][0]+obj['box'][2]) / 2.0)
+                centroid_y = int((obj['box'][1]+obj['box'][3]) / 2.0)
                 obj['centroid'] = (centroid_x, centroid_y)
 
             if len(current_objects) == 0:
@@ -363,56 +364,66 @@ class ObjectTracker(threading.Thread):
                 usedCols.add(col)
 
             # compute the column index we have NOT yet examined
+            unusedRows = set(range(0, D.shape[0])).difference(usedRows)
             unusedCols = set(range(0, D.shape[1])).difference(usedCols)
 
+            # in the event that the number of object centroids is
+			# equal or greater than the number of input centroids
+			# we need to check and see if some of these objects have
+			# potentially disappeared
+            if D.shape[0] >= D.shape[1]:
+                for row in unusedRows:
+                    id = current_ids[row]
+
+                    if self.disappeared[id] >= self.max_disappeared:
+                        self.deregister(id)
+                    else:
+                        self.disappeared[id] += 1
             # if the number of input centroids is greater
             # than the number of existing object centroids we need to
             # register each new input centroid as a trackable object
-            # if D.shape[0] < D.shape[1]:
-            # TODO: rather than assuming these are new objects, we could
-            # look to see if any of the remaining boxes have a large amount
-            # of overlap...
-            for col in unusedCols:
-                self.register(col, group[col])
+            else:
+                for col in unusedCols:
+                    self.register(col, group[col])
 
 # Maintains the frame and object with the highest score
-class BestFrames(threading.Thread):
-    def __init__(self, camera):
-        threading.Thread.__init__(self)
-        self.camera = camera
-        self.best_objects = {}
-        self.best_frames = {}
+# class BestFrames(threading.Thread):
+#     def __init__(self, camera):
+#         threading.Thread.__init__(self)
+#         self.camera = camera
+#         self.best_objects = {}
+#         self.best_frames = {}
 
-    def run(self):
-        prctl.set_name(self.__class__.__name__)
-        while True:
-            # wait until objects have been tracked
-            with self.camera.objects_tracked:
-                self.camera.objects_tracked.wait()
+#     def run(self):
+#         prctl.set_name(self.__class__.__name__)
+#         while True:
+#             # wait until objects have been tracked
+#             with self.camera.objects_tracked:
+#                 self.camera.objects_tracked.wait()
 
-            # make a copy of tracked objects
-            tracked_objects = list(self.camera.object_tracker.tracked_objects.values())
+#             # make a copy of tracked objects
+#             tracked_objects = list(self.camera.object_tracker.tracked_objects.values())
 
-            for obj in tracked_objects:
-                if obj['name'] in self.best_objects:
-                    now = datetime.datetime.now().timestamp()
-                    # if the object is a higher score than the current best score 
-                    # or the current object is more than 1 minute old, use the new object
-                    if obj['score'] > self.best_objects[obj['name']]['score'] or (now - self.best_objects[obj['name']]['frame_time']) > 60:
-                        self.best_objects[obj['name']] = copy.deepcopy(obj)
-                else:
-                    self.best_objects[obj['name']] = copy.deepcopy(obj)
+#             for obj in tracked_objects:
+#                 if obj['name'] in self.best_objects:
+#                     now = datetime.datetime.now().timestamp()
+#                     # if the object is a higher score than the current best score 
+#                     # or the current object is more than 1 minute old, use the new object
+#                     if obj['score'] > self.best_objects[obj['name']]['score'] or (now - self.best_objects[obj['name']]['frame_time']) > 60:
+#                         self.best_objects[obj['name']] = copy.deepcopy(obj)
+#                 else:
+#                     self.best_objects[obj['name']] = copy.deepcopy(obj)
             
-            for name, obj in self.best_objects.items():
-                if obj['frame_time'] in self.camera.frame_cache:
-                    best_frame = self.camera.frame_cache[obj['frame_time']]
+#             for name, obj in self.best_objects.items():
+#                 if obj['frame_time'] in self.camera.frame_cache:
+#                     best_frame = self.camera.frame_cache[obj['frame_time']]
 
-                    draw_box_with_label(best_frame, obj['box']['xmin'], obj['box']['ymin'], 
-                        obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']))
+#                     draw_box_with_label(best_frame, obj['box']['xmin'], obj['box']['ymin'], 
+#                         obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']))
                     
-                    # print a timestamp
-                    if self.camera.snapshot_config['show_timestamp']:
-                        time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
-                        cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
+#                     # print a timestamp
+#                     if self.camera.snapshot_config['show_timestamp']:
+#                         time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
+#                         cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
                     
-                    self.best_frames[name] = best_frame
\ No newline at end of file
+#                     self.best_frames[name] = best_frame
\ No newline at end of file
diff --git a/frigate/util.py b/frigate/util.py
old mode 100644
new mode 100755
index e441f39e1..b0ccf1f6f
--- a/frigate/util.py
+++ b/frigate/util.py
@@ -15,73 +15,11 @@ def ReadLabelFile(file_path):
         ret[int(pair[0])] = pair[1].strip()
     return ret
 
-def calculate_region(frame_shape, xmin, ymin, xmax, ymax):    
-    # size is larger than longest edge
-    size = int(max(xmax-xmin, ymax-ymin)*2)
-    # if the size is too big to fit in the frame
-    if size > min(frame_shape[0], frame_shape[1]):
-        size = min(frame_shape[0], frame_shape[1])
-
-    # x_offset is midpoint of bounding box minus half the size
-    x_offset = int((xmax-xmin)/2.0+xmin-size/2.0)
-    # if outside the image
-    if x_offset < 0:
-        x_offset = 0
-    elif x_offset > (frame_shape[1]-size):
-        x_offset = (frame_shape[1]-size)
-
-    # y_offset is midpoint of bounding box minus half the size
-    y_offset = int((ymax-ymin)/2.0+ymin-size/2.0)
-    # if outside the image
-    if y_offset < 0:
-        y_offset = 0
-    elif y_offset > (frame_shape[0]-size):
-        y_offset = (frame_shape[0]-size)
-
-    return (size, x_offset, y_offset)
-
-def compute_intersection_rectangle(box_a, box_b):
-    return {
-        'xmin': max(box_a['xmin'], box_b['xmin']),
-        'ymin': max(box_a['ymin'], box_b['ymin']),
-        'xmax': min(box_a['xmax'], box_b['xmax']),
-        'ymax': min(box_a['ymax'], box_b['ymax'])
-    }
-    
-def compute_intersection_over_union(box_a, box_b):
-    # determine the (x, y)-coordinates of the intersection rectangle
-    intersect = compute_intersection_rectangle(box_a, box_b)
-
-    # compute the area of intersection rectangle
-    inter_area = max(0, intersect['xmax'] - intersect['xmin'] + 1) * max(0, intersect['ymax'] - intersect['ymin'] + 1)
-
-    if inter_area == 0:
-        return 0.0
-    
-    # compute the area of both the prediction and ground-truth
-    # rectangles
-    box_a_area = (box_a['xmax'] - box_a['xmin'] + 1) * (box_a['ymax'] - box_a['ymin'] + 1)
-    box_b_area = (box_b['xmax'] - box_b['xmin'] + 1) * (box_b['ymax'] - box_b['ymin'] + 1)
-
-    # compute the intersection over union by taking the intersection
-    # area and dividing it by the sum of prediction + ground-truth
-    # areas - the interesection area
-    iou = inter_area / float(box_a_area + box_b_area - inter_area)
-
-    # return the intersection over union value
-    return iou
-
-# convert shared memory array into numpy array
-def tonumpyarray(mp_arr):
-    return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)
-
 def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label, info, thickness=2, color=None, position='ul'):
     if color is None:
-        color = COLOR_MAP[label]
+        color = (0,0,255)
     display_text = "{}: {}".format(label, info)
-    cv2.rectangle(frame, (x_min, y_min), 
-        (x_max, y_max), 
-        color, thickness)
+    cv2.rectangle(frame, (x_min, y_min), (x_max, y_max), color, thickness)
     font_scale = 0.5
     font = cv2.FONT_HERSHEY_SIMPLEX
     # get the width and height of the text box
@@ -107,37 +45,81 @@ def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label, info, thicknes
     cv2.rectangle(frame, textbox_coords[0], textbox_coords[1], color, cv2.FILLED)
     cv2.putText(frame, display_text, (text_offset_x, text_offset_y + line_height - 3), font, fontScale=font_scale, color=(0, 0, 0), thickness=2)
 
-# Path to frozen detection graph. This is the actual model that is used for the object detection.
-PATH_TO_CKPT = '/frozen_inference_graph.pb'
-# List of the strings that is used to add correct label for each box.
-PATH_TO_LABELS = '/label_map.pbtext'
+def calculate_region(frame_shape, xmin, ymin, xmax, ymax, multiplier=2):    
+    # size is larger than longest edge
+    size = int(max(xmax-xmin, ymax-ymin)*multiplier)
+    # if the size is too big to fit in the frame
+    if size > min(frame_shape[0], frame_shape[1]):
+        size = min(frame_shape[0], frame_shape[1])
 
-LABELS = ReadLabelFile(PATH_TO_LABELS)
-cmap = plt.cm.get_cmap('tab10', len(LABELS.keys()))
+    # x_offset is midpoint of bounding box minus half the size
+    x_offset = int((xmax-xmin)/2.0+xmin-size/2.0)
+    # if outside the image
+    if x_offset < 0:
+        x_offset = 0
+    elif x_offset > (frame_shape[1]-size):
+        x_offset = (frame_shape[1]-size)
 
-COLOR_MAP = {}
-for key, val in LABELS.items():
-    COLOR_MAP[val] = tuple(int(round(255 * c)) for c in cmap(key)[:3])
+    # y_offset is midpoint of bounding box minus half the size
+    y_offset = int((ymax-ymin)/2.0+ymin-size/2.0)
+    # if outside the image
+    if y_offset < 0:
+        y_offset = 0
+    elif y_offset > (frame_shape[0]-size):
+        y_offset = (frame_shape[0]-size)
 
-class QueueMerger():
-    def __init__(self, from_queues, to_queue):
-        self.from_queues = from_queues
-        self.to_queue = to_queue
-        self.merge_threads = []
+    return (x_offset, y_offset, x_offset+size, y_offset+size)
 
-    def start(self):
-        for from_q in self.from_queues:
-            self.merge_threads.append(QueueTransfer(from_q,self.to_queue))
+def intersection(box_a, box_b):
+    return (
+        max(box_a[0], box_b[0]),
+        max(box_a[1], box_b[1]),
+        min(box_a[2], box_b[2]),
+        min(box_a[3], box_b[3])
+    )
 
-class QueueTransfer(threading.Thread):
-    def __init__(self, from_queue, to_queue):
-        threading.Thread.__init__(self)
-        self.from_queue = from_queue
-        self.to_queue = to_queue
+def area(box):
+    return (box[2]-box[0] + 1)*(box[3]-box[1] + 1)
+    
+def intersection_over_union(box_a, box_b):
+    # determine the (x, y)-coordinates of the intersection rectangle
+    intersect = intersection(box_a, box_b)
 
-    def run(self):
-        while True:
-            self.to_queue.put(self.from_queue.get())
+    # compute the area of intersection rectangle
+    inter_area = max(0, intersect[2] - intersect[0] + 1) * max(0, intersect[3] - intersect[1] + 1)
+
+    if inter_area == 0:
+        return 0.0
+    
+    # compute the area of both the prediction and ground-truth
+    # rectangles
+    box_a_area = (box_a[2] - box_a[0] + 1) * (box_a[3] - box_a[1] + 1)
+    box_b_area = (box_b[2] - box_b[0] + 1) * (box_b[3] - box_b[1] + 1)
+
+    # compute the intersection over union by taking the intersection
+    # area and dividing it by the sum of prediction + ground-truth
+    # areas - the interesection area
+    iou = inter_area / float(box_a_area + box_b_area - inter_area)
+
+    # return the intersection over union value
+    return iou
+
+def clipped(obj, frame_shape):
+    # if the object is within 5 pixels of the region border, and the region is not on the edge
+    # consider the object to be clipped
+    box = obj[2]
+    region = obj[4]
+    if ((region[0] > 5 and box[0]-region[0] <= 5) or 
+        (region[1] > 5 and box[1]-region[1] <= 5) or
+        (frame_shape[1]-region[2] > 5 and region[2]-box[2] <= 5) or
+        (frame_shape[0]-region[3] > 5 and region[3]-box[3] <= 5)):
+        return True
+    else:
+        return False
+
+# convert shared memory array into numpy array
+def tonumpyarray(mp_arr):
+    return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)
 
 class EventsPerSecond:
     def __init__(self, max_events=1000):
diff --git a/frigate/video.py b/frigate/video.py
old mode 100644
new mode 100755
index 1fc8d2e3f..8b1b65a34
--- a/frigate/video.py
+++ b/frigate/video.py
@@ -8,40 +8,47 @@ import ctypes
 import multiprocessing as mp
 import subprocess as sp
 import numpy as np
-import prctl
+import hashlib
+import pyarrow.plasma as plasma
+import SharedArray as sa
+# import prctl
 import copy
 import itertools
 import json
 from collections import defaultdict
-from frigate.util import tonumpyarray, LABELS, draw_box_with_label, calculate_region, EventsPerSecond
-from frigate.object_detection import RegionPrepper, RegionRequester
-from frigate.objects import ObjectCleaner, BestFrames, DetectedObjectsProcessor, RegionRefiner, ObjectTracker
-from frigate.mqtt import MqttObjectPublisher
+from frigate.util import tonumpyarray, LABELS, draw_box_with_label, area, calculate_region, clipped, intersection_over_union, intersection, EventsPerSecond
+# from frigate.object_detection import RegionPrepper, RegionRequester
+from frigate.objects import ObjectTracker
+# from frigate.mqtt import MqttObjectPublisher
+from frigate.edgetpu import RemoteObjectDetector
+from frigate.motion import MotionDetector
 
 # Stores 2 seconds worth of frames so they can be used for other threads
-class FrameTracker(threading.Thread):
-    def __init__(self, frame_time, frame_ready, frame_lock, recent_frames):
-        threading.Thread.__init__(self)
-        self.frame_time = frame_time
-        self.frame_ready = frame_ready
-        self.frame_lock = frame_lock
-        self.recent_frames = recent_frames
+# TODO: we do actually know when these frames are no longer needed
+# class FrameTracker(threading.Thread):
+#     def __init__(self, frame_time, frame_ready, frame_lock, recent_frames):
+#         threading.Thread.__init__(self)
+#         self.frame_time = frame_time
+#         self.frame_ready = frame_ready
+#         self.frame_lock = frame_lock
+#         self.recent_frames = recent_frames
     
-    def run(self):
-        prctl.set_name(self.__class__.__name__)
-        while True:
-            # wait for a frame
-            with self.frame_ready:
-                self.frame_ready.wait()
+#     def run(self):
+#         prctl.set_name(self.__class__.__name__)
+#         while True:
+#             # wait for a frame
+#             with self.frame_ready:
+#                 self.frame_ready.wait()
 
-            # delete any old frames
-            stored_frame_times = list(self.recent_frames.keys())
-            stored_frame_times.sort(reverse=True)
-            if len(stored_frame_times) > 100:
-                frames_to_delete = stored_frame_times[50:]
-                for k in frames_to_delete:
-                    del self.recent_frames[k]
+#             # delete any old frames
+#             stored_frame_times = list(self.recent_frames.keys())
+#             stored_frame_times.sort(reverse=True)
+#             if len(stored_frame_times) > 100:
+#                 frames_to_delete = stored_frame_times[50:]
+#                 for k in frames_to_delete:
+#                     del self.recent_frames[k]
 
+# TODO: add back opencv fallback
 def get_frame_shape(source):
     ffprobe_cmd = " ".join([
         'ffprobe',
@@ -76,6 +83,7 @@ def get_ffmpeg_input(ffmpeg_input):
     frigate_vars = {k: v for k, v in os.environ.items() if k.startswith('FRIGATE_')}
     return ffmpeg_input.format(**frigate_vars)
 
+<<<<<<< HEAD
 class CameraWatchdog(threading.Thread):
     def __init__(self, camera):
         threading.Thread.__init__(self)
@@ -294,112 +302,648 @@ class Camera:
             self.capture_thread.join()
             self.ffmpeg_process = None
             self.capture_thread = None
+=======
+# class CameraWatchdog(threading.Thread):
+#     def __init__(self, camera):
+#         threading.Thread.__init__(self)
+#         self.camera = camera
+
+#     def run(self):
+#         prctl.set_name(self.__class__.__name__)
+#         while True:
+#             # wait a bit before checking
+#             time.sleep(10)
+
+#             if self.camera.frame_time.value != 0.0 and (datetime.datetime.now().timestamp() - self.camera.frame_time.value) > self.camera.watchdog_timeout:
+#                 print(self.camera.name + ": last frame is more than 5 minutes old, restarting camera capture...")
+#                 self.camera.start_or_restart_capture()
+#                 time.sleep(5)
+
+# # Thread to read the stdout of the ffmpeg process and update the current frame
+# class CameraCapture(threading.Thread):
+#     def __init__(self, camera):
+#         threading.Thread.__init__(self)
+#         self.camera = camera
+
+#     def run(self):
+#         prctl.set_name(self.__class__.__name__)
+#         frame_num = 0
+#         while True:
+#             if self.camera.ffmpeg_process.poll() != None:
+#                 print(self.camera.name + ": ffmpeg process is not running. exiting capture thread...")
+#                 break
+
+#             raw_image = self.camera.ffmpeg_process.stdout.read(self.camera.frame_size)
+
+#             if len(raw_image) == 0:
+#                 print(self.camera.name + ": ffmpeg didnt return a frame. something is wrong. exiting capture thread...")
+#                 break
+
+#             frame_num += 1
+#             if (frame_num % self.camera.take_frame) != 0:
+#                 continue
+
+#             with self.camera.frame_lock:
+#                 # TODO: use frame_queue instead
+#                 self.camera.frame_time.value = datetime.datetime.now().timestamp()
+#                 self.camera.frame_cache[self.camera.frame_time.value] = (
+#                     np
+#                     .frombuffer(raw_image, np.uint8)
+#                     .reshape(self.camera.frame_shape)
+#                 )
+#                 self.camera.frame_queue.put(self.camera.frame_time.value)
+#             # Notify with the condition that a new frame is ready
+#             with self.camera.frame_ready:
+#                 self.camera.frame_ready.notify_all()
+
+#             self.camera.fps.update()
+
+# class VideoWriter(threading.Thread):
+#     def __init__(self, camera):
+#         threading.Thread.__init__(self)
+#         self.camera = camera
+
+#     def run(self):
+#         prctl.set_name(self.__class__.__name__)
+#         while True:
+#             (frame_time, tracked_objects) = self.camera.frame_output_queue.get()
+#             # if len(tracked_objects) == 0:
+#             #     continue
+#             # f = open(f"/debug/output/{self.camera.name}-{str(format(frame_time, '.8f'))}.jpg", 'wb')
+#             # f.write(self.camera.frame_with_objects(frame_time, tracked_objects))
+#             # f.close()
+
+# class Camera:
+#     def __init__(self, name, ffmpeg_config, global_objects_config, config, tflite_process, mqtt_client, mqtt_prefix):
+#         self.name = name
+#         self.config = config
+#         self.detected_objects = defaultdict(lambda: [])
+#         self.frame_cache = {}
+#         self.last_processed_frame = None
+#         # queue for re-assembling frames in order
+#         self.frame_queue = queue.Queue()
+#         # track how many regions have been requested for a frame so we know when a frame is complete
+#         self.regions_in_process = {}
+#         # Lock to control access
+#         self.regions_in_process_lock = mp.Lock()
+#         self.finished_frame_queue = queue.Queue()
+#         self.refined_frame_queue = queue.Queue()
+#         self.frame_output_queue = queue.Queue()
+
+#         self.ffmpeg = config.get('ffmpeg', {})
+#         self.ffmpeg_input = get_ffmpeg_input(self.ffmpeg['input'])
+#         self.ffmpeg_global_args = self.ffmpeg.get('global_args', ffmpeg_config['global_args'])
+#         self.ffmpeg_hwaccel_args = self.ffmpeg.get('hwaccel_args', ffmpeg_config['hwaccel_args'])
+#         self.ffmpeg_input_args = self.ffmpeg.get('input_args', ffmpeg_config['input_args'])
+#         self.ffmpeg_output_args = self.ffmpeg.get('output_args', ffmpeg_config['output_args'])
+
+#         camera_objects_config = config.get('objects', {})
+
+#         self.take_frame = self.config.get('take_frame', 1)
+#         self.watchdog_timeout = self.config.get('watchdog_timeout', 300)
+#         self.snapshot_config = {
+#             'show_timestamp': self.config.get('snapshots', {}).get('show_timestamp', True)
+#         }
+#         self.regions = self.config['regions']
+#         self.frame_shape = get_frame_shape(self.ffmpeg_input)
+#         self.frame_size = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
+#         self.mqtt_client = mqtt_client
+#         self.mqtt_topic_prefix = '{}/{}'.format(mqtt_prefix, self.name)
+
+#         # create shared value for storing the frame_time
+#         self.frame_time = mp.Value('d', 0.0)
+#         # Lock to control access to the frame
+#         self.frame_lock = mp.Lock()
+#         # Condition for notifying that a new frame is ready
+#         self.frame_ready = mp.Condition()
+#         # Condition for notifying that objects were tracked
+#         self.objects_tracked = mp.Condition()
+
+#         # Queue for prepped frames, max size set to (number of regions * 5)
+#         self.resize_queue = queue.Queue()
+
+#         # Queue for raw detected objects
+#         self.detected_objects_queue = queue.Queue()
+#         self.detected_objects_processor = DetectedObjectsProcessor(self)
+#         self.detected_objects_processor.start()
+
+#         # initialize the frame cache
+#         self.cached_frame_with_objects = {
+#             'frame_bytes': [],
+#             'frame_time': 0
+#         }
+
+#         self.ffmpeg_process = None
+#         self.capture_thread = None
+#         self.fps = EventsPerSecond()
+#         self.skipped_region_tracker = EventsPerSecond()
+
+#         # combine tracked objects lists
+#         self.objects_to_track = set().union(global_objects_config.get('track', ['person', 'car', 'truck']), camera_objects_config.get('track', []))
+
+#         # merge object filters
+#         global_object_filters = global_objects_config.get('filters', {})
+#         camera_object_filters = camera_objects_config.get('filters', {})
+#         objects_with_config = set().union(global_object_filters.keys(), camera_object_filters.keys())
+#         self.object_filters = {}
+#         for obj in objects_with_config:
+#             self.object_filters[obj] = {**global_object_filters.get(obj, {}), **camera_object_filters.get(obj, {})}
+
+#         # start a thread to track objects
+#         self.object_tracker = ObjectTracker(self, 10)
+#         self.object_tracker.start()
+
+#         # start a thread to write tracked frames to disk
+#         self.video_writer = VideoWriter(self)
+#         self.video_writer.start()
+
+#         # start a thread to queue resize requests for regions
+#         self.region_requester = RegionRequester(self)
+#         self.region_requester.start()
+
+#         # start a thread to cache recent frames for processing
+#         self.frame_tracker = FrameTracker(self.frame_time, 
+#             self.frame_ready, self.frame_lock, self.frame_cache)
+#         self.frame_tracker.start()
+
+#         # start a thread to resize regions
+#         self.region_prepper = RegionPrepper(self, self.frame_cache, self.resize_queue, prepped_frame_queue)
+#         self.region_prepper.start()
+
+#         # start a thread to store the highest scoring recent frames for monitored object types
+#         self.best_frames = BestFrames(self)
+#         self.best_frames.start()
+
+#         # start a thread to expire objects from the detected objects list
+#         self.object_cleaner = ObjectCleaner(self)
+#         self.object_cleaner.start()
+
+#         # start a thread to refine regions when objects are clipped
+#         self.dynamic_region_fps = EventsPerSecond()
+#         self.region_refiner = RegionRefiner(self)
+#         self.region_refiner.start()
+#         self.dynamic_region_fps.start()
+
+#         # start a thread to publish object scores
+#         mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self)
+#         mqtt_publisher.start()
+
+#         # create a watchdog thread for capture process
+#         self.watchdog = CameraWatchdog(self)
+
+#         # load in the mask for object detection
+#         if 'mask' in self.config:
+#             self.mask = cv2.imread("/config/{}".format(self.config['mask']), cv2.IMREAD_GRAYSCALE)
+#         else:
+#             self.mask = None
+
+#         if self.mask is None:
+#             self.mask = np.zeros((self.frame_shape[0], self.frame_shape[1], 1), np.uint8)
+#             self.mask[:] = 255
+
+
+#     def start_or_restart_capture(self):
+#         if not self.ffmpeg_process is None:
+#             print("Terminating the existing ffmpeg process...")
+#             self.ffmpeg_process.terminate()
+#             try:
+#                 print("Waiting for ffmpeg to exit gracefully...")
+#                 self.ffmpeg_process.wait(timeout=30)
+#             except sp.TimeoutExpired:
+#                 print("FFmpeg didnt exit. Force killing...")
+#                 self.ffmpeg_process.kill()
+#                 self.ffmpeg_process.wait()
+
+#             print("Waiting for the capture thread to exit...")
+#             self.capture_thread.join()
+#             self.ffmpeg_process = None
+#             self.capture_thread = None
+>>>>>>> 9b1c7e9... split into separate processes
             
-        # create the process to capture frames from the input stream and store in a shared array
-        print("Creating a new ffmpeg process...")
-        self.start_ffmpeg()
+#         # create the process to capture frames from the input stream and store in a shared array
+#         print("Creating a new ffmpeg process...")
+#         self.start_ffmpeg()
         
-        print("Creating a new capture thread...")
-        self.capture_thread = CameraCapture(self)
-        print("Starting a new capture thread...")
-        self.capture_thread.start()
-        self.fps.start()
-        self.skipped_region_tracker.start()
+#         print("Creating a new capture thread...")
+#         self.capture_thread = CameraCapture(self)
+#         print("Starting a new capture thread...")
+#         self.capture_thread.start()
+#         self.fps.start()
+#         self.skipped_region_tracker.start()
     
-    def start_ffmpeg(self):
-        ffmpeg_cmd = (['ffmpeg'] +
-            self.ffmpeg_global_args +
-            self.ffmpeg_hwaccel_args +
-            self.ffmpeg_input_args +
-            ['-i', self.ffmpeg_input] +
-            self.ffmpeg_output_args +
+#     def start_ffmpeg(self):
+#         ffmpeg_cmd = (['ffmpeg'] +
+#             self.ffmpeg_global_args +
+#             self.ffmpeg_hwaccel_args +
+#             self.ffmpeg_input_args +
+#             ['-i', self.ffmpeg_input] +
+#             self.ffmpeg_output_args +
+#             ['pipe:'])
+
+#         print(" ".join(ffmpeg_cmd))
+        
+#         self.ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=self.frame_size)
+    
+#     def start(self):
+#         self.start_or_restart_capture()
+#         self.watchdog.start()
+    
+#     def join(self):
+#         self.capture_thread.join()
+    
+#     def get_capture_pid(self):
+#         return self.ffmpeg_process.pid
+    
+#     def get_best(self, label):
+#         return self.best_frames.best_frames.get(label)
+
+#     def stats(self):
+#         # TODO: anything else?
+#         return {
+#             'camera_fps': self.fps.eps(60),
+#             'resize_queue': self.resize_queue.qsize(),
+#             'frame_queue': self.frame_queue.qsize(),
+#             'finished_frame_queue': self.finished_frame_queue.qsize(),
+#             'refined_frame_queue': self.refined_frame_queue.qsize(),
+#             'regions_in_process': self.regions_in_process,
+#             'dynamic_regions_per_sec': self.dynamic_region_fps.eps(),
+#             'skipped_regions_per_sec': self.skipped_region_tracker.eps(60)
+#         }
+    
+#     def frame_with_objects(self, frame_time, tracked_objects=None):
+#         if not frame_time in self.frame_cache:
+#             frame = np.zeros(self.frame_shape, np.uint8)
+#         else:
+#             frame = self.frame_cache[frame_time].copy()
+            
+#         detected_objects = self.detected_objects[frame_time].copy()
+
+#         for region in self.regions:
+#             color = (255,255,255)
+#             cv2.rectangle(frame, (region['x_offset'], region['y_offset']), 
+#                 (region['x_offset']+region['size'], region['y_offset']+region['size']), 
+#                 color, 2)
+
+#         # draw the bounding boxes on the screen
+
+#         if tracked_objects is None:
+#             with self.object_tracker.tracked_objects_lock:
+#                 tracked_objects = copy.deepcopy(self.object_tracker.tracked_objects)
+
+#         for obj in detected_objects:
+#             draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']), thickness=3)
+        
+#         for id, obj in tracked_objects.items():
+#             color = (0, 255,0) if obj['frame_time'] == frame_time else (255, 0, 0)
+#             draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], id, color=color, thickness=1, position='bl')
+
+#         # print a timestamp
+#         time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
+#         cv2.putText(frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
+        
+#         # print fps
+#         cv2.putText(frame, str(self.fps.eps())+'FPS', (10, 60), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
+
+#         # convert to BGR
+#         frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
+
+#         # encode the image into a jpg
+#         ret, jpg = cv2.imencode('.jpg', frame)
+
+#         return jpg.tobytes()
+
+#     def get_current_frame_with_objects(self):
+#         frame_time = self.last_processed_frame
+#         if frame_time == self.cached_frame_with_objects['frame_time']:
+#             return self.cached_frame_with_objects['frame_bytes']
+
+#         frame_bytes = self.frame_with_objects(frame_time)
+
+#         self.cached_frame_with_objects = {
+#             'frame_bytes': frame_bytes,
+#             'frame_time': frame_time
+#         }
+
+#         return frame_bytes
+
+def filtered(obj, objects_to_track, object_filters, mask):
+    object_name = obj[0]
+
+    if not object_name in objects_to_track:
+        return True
+    
+    if object_name in object_filters:
+        obj_settings = object_filters[object_name]
+
+        # if the min area is larger than the
+        # detected object, don't add it to detected objects
+        if obj_settings.get('min_area',-1) > obj[3]:
+            return True
+        
+        # if the detected object is larger than the
+        # max area, don't add it to detected objects
+        if obj_settings.get('max_area', 24000000) < obj[3]:
+            return True
+
+        # if the score is lower than the threshold, skip
+        if obj_settings.get('threshold', 0) > obj[1]:
+            return True
+    
+        # compute the coordinates of the object and make sure
+        # the location isnt outside the bounds of the image (can happen from rounding)
+        y_location = min(int(obj[2][3]), len(mask)-1)
+        x_location = min(int((obj[2][2]-obj[2][0])/2.0)+obj[2][0], len(mask[0])-1)
+
+        # if the object is in a masked location, don't add it to detected objects
+        if mask[y_location][x_location] == [0]:
+            return True
+        
+        return False
+
+def create_tensor_input(frame, region):
+    cropped_frame = frame[region[1]:region[3], region[0]:region[2]]
+
+    # Resize to 300x300 if needed
+    if cropped_frame.shape != (300, 300, 3):
+        cropped_frame = cv2.resize(cropped_frame, dsize=(300, 300), interpolation=cv2.INTER_LINEAR)
+    
+    # Expand dimensions since the model expects images to have shape: [1, 300, 300, 3]
+    return np.expand_dims(cropped_frame, axis=0)
+
+def track_camera(name, config, ffmpeg_global_config, global_objects_config, detect_lock, detect_ready, frame_ready, detected_objects_queue, fps, avg_wait):
+    print(f"Starting process for {name}: {os.getpid()}")
+
+    # Merge the ffmpeg config with the global config
+    ffmpeg = config.get('ffmpeg', {})
+    ffmpeg_input = get_ffmpeg_input(ffmpeg['input'])
+    ffmpeg_global_args = ffmpeg.get('global_args', ffmpeg_global_config['global_args'])
+    ffmpeg_hwaccel_args = ffmpeg.get('hwaccel_args', ffmpeg_global_config['hwaccel_args'])
+    ffmpeg_input_args = ffmpeg.get('input_args', ffmpeg_global_config['input_args'])
+    ffmpeg_output_args = ffmpeg.get('output_args', ffmpeg_global_config['output_args'])
+
+    # Merge the tracked object config with the global config
+    camera_objects_config = config.get('objects', {})    
+    # combine tracked objects lists
+    objects_to_track = set().union(global_objects_config.get('track', ['person', 'car', 'truck']), camera_objects_config.get('track', []))
+    # merge object filters
+    global_object_filters = global_objects_config.get('filters', {})
+    camera_object_filters = camera_objects_config.get('filters', {})
+    objects_with_config = set().union(global_object_filters.keys(), camera_object_filters.keys())
+    object_filters = {}
+    for obj in objects_with_config:
+        object_filters[obj] = {**global_object_filters.get(obj, {}), **camera_object_filters.get(obj, {})}
+
+    take_frame = config.get('take_frame', 1)
+
+    # watchdog_timeout = config.get('watchdog_timeout', 300)
+
+    frame_shape = get_frame_shape(ffmpeg_input)
+    frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2]
+
+    try:
+        sa.delete(name)
+    except:
+        pass
+
+    frame = sa.create(name, shape=frame_shape, dtype=np.uint8)
+
+    # load in the mask for object detection
+    if 'mask' in config:
+        mask = cv2.imread("/config/{}".format(config['mask']), cv2.IMREAD_GRAYSCALE)
+    else:
+        mask = None
+
+    if mask is None:
+        mask = np.zeros((frame_shape[0], frame_shape[1], 1), np.uint8)
+        mask[:] = 255
+
+    motion_detector = MotionDetector(frame_shape, mask, resize_factor=6)
+    object_detector = RemoteObjectDetector('/lab/labelmap.txt', detect_lock, detect_ready, frame_ready)
+
+    object_tracker = ObjectTracker(10)
+
+    ffmpeg_cmd = (['ffmpeg'] +
+            ffmpeg_global_args +
+            ffmpeg_hwaccel_args +
+            ffmpeg_input_args +
+            ['-i', ffmpeg_input] +
+            ffmpeg_output_args +
             ['pipe:'])
 
-        print(" ".join(ffmpeg_cmd))
+    print(" ".join(ffmpeg_cmd))
+    
+    ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=frame_size)
+    
+    plasma_client = plasma.connect("/tmp/plasma")
+    frame_num = 0
+    fps_tracker = EventsPerSecond()
+    fps_tracker.start()
+    while True:
+        # TODO: implement something to determine if it had to wait for a frame at all
+        # to determine if it might be behind and the buffer is filling up
+        start = datetime.datetime.now().timestamp()
+        frame_bytes = ffmpeg_process.stdout.read(frame_size)
+        duration = datetime.datetime.now().timestamp()-start
+        avg_wait.value = (avg_wait.value*9 + duration)/10
+
+        if not frame_bytes:
+            # TODO: restart the ffmpeg process and track number of restarts
+            break
+
+        # limit frame rate
+        frame_num += 1
+        if (frame_num % take_frame) != 0:
+            continue
+
+        fps_tracker.update()
+        fps.value = fps_tracker.eps()
+
+        frame_time = datetime.datetime.now().timestamp()
         
-        self.ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=self.frame_size)
-    
-    def start(self):
-        self.start_or_restart_capture()
-        self.watchdog.start()
-    
-    def join(self):
-        self.capture_thread.join()
-    
-    def get_capture_pid(self):
-        return self.ffmpeg_process.pid
-    
-    def get_best(self, label):
-        return self.best_frames.best_frames.get(label)
-
-    def stats(self):
-        return {
-            'camera_fps': self.fps.eps(60),
-            'resize_queue': self.resize_queue.qsize(),
-            'frame_queue': self.frame_queue.qsize(),
-            'finished_frame_queue': self.finished_frame_queue.qsize(),
-            'refined_frame_queue': self.refined_frame_queue.qsize(),
-            'regions_in_process': self.regions_in_process,
-            'dynamic_regions_per_sec': self.dynamic_region_fps.eps(),
-            'skipped_regions_per_sec': self.skipped_region_tracker.eps(60)
-        }
-    
-    def frame_with_objects(self, frame_time, tracked_objects=None):
-        if not frame_time in self.frame_cache:
-            frame = np.zeros(self.frame_shape, np.uint8)
-        else:
-            frame = self.frame_cache[frame_time].copy()
-            
-        detected_objects = self.detected_objects[frame_time].copy()
-
-        for region in self.regions:
-            color = (255,255,255)
-            cv2.rectangle(frame, (region['x_offset'], region['y_offset']), 
-                (region['x_offset']+region['size'], region['y_offset']+region['size']), 
-                color, 2)
-
-        # draw the bounding boxes on the screen
-
-        if tracked_objects is None:
-            with self.object_tracker.tracked_objects_lock:
-                tracked_objects = copy.deepcopy(self.object_tracker.tracked_objects)
-
-        for obj in detected_objects:
-            draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']), thickness=3)
+        # Store frame in numpy array
+        frame[:] = (np
+                    .frombuffer(frame_bytes, np.uint8)
+                    .reshape(frame_shape))
         
-        for id, obj in tracked_objects.items():
-            color = (0, 255,0) if obj['frame_time'] == frame_time else (255, 0, 0)
-            draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], id, color=color, thickness=1, position='bl')
+        # look for motion
+        motion_boxes = motion_detector.detect(frame)
 
-        # print a timestamp
-        time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
-        cv2.putText(frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
+        tracked_objects = object_tracker.tracked_objects.values()
+
+        # merge areas of motion that intersect with a known tracked object into a single area to look at
+        areas_of_interest = []
+        used_motion_boxes = []
+        for obj in tracked_objects:
+            x_min, y_min, x_max, y_max = obj['box']
+            for m_index, motion_box in enumerate(motion_boxes):
+                if area(intersection(obj['box'], motion_box))/area(motion_box) > .5:
+                    used_motion_boxes.append(m_index)
+                    x_min = min(obj['box'][0], motion_box[0])
+                    y_min = min(obj['box'][1], motion_box[1])
+                    x_max = max(obj['box'][2], motion_box[2])
+                    y_max = max(obj['box'][3], motion_box[3])
+            areas_of_interest.append((x_min, y_min, x_max, y_max))
+        unused_motion_boxes = set(range(0, len(motion_boxes))).difference(used_motion_boxes)
         
-        # print fps
-        cv2.putText(frame, str(self.fps.eps())+'FPS', (10, 60), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
-
-        # convert to BGR
-        frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
-
-        # encode the image into a jpg
-        ret, jpg = cv2.imencode('.jpg', frame)
-
-        return jpg.tobytes()
-
-    def get_current_frame_with_objects(self):
-        frame_time = self.last_processed_frame
-        if frame_time == self.cached_frame_with_objects['frame_time']:
-            return self.cached_frame_with_objects['frame_bytes']
-
-        frame_bytes = self.frame_with_objects(frame_time)
-
-        self.cached_frame_with_objects = {
-            'frame_bytes': frame_bytes,
-            'frame_time': frame_time
-        }
-
-        return frame_bytes
-
-
-    
+        # compute motion regions
+        motion_regions = [calculate_region(frame_shape, motion_boxes[i][0], motion_boxes[i][1], motion_boxes[i][2], motion_boxes[i][3], 1.2)
+            for i in unused_motion_boxes]
         
+        # compute tracked object regions
+        object_regions = [calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.2)
+            for a in areas_of_interest]
+        
+        # merge regions with high IOU
+        merged_regions = motion_regions+object_regions
+        while True:
+            max_iou = 0.0
+            max_indices = None
+            region_indices = range(len(merged_regions))
+            for a, b in itertools.combinations(region_indices, 2):
+                iou = intersection_over_union(merged_regions[a], merged_regions[b])
+                if iou > max_iou:
+                    max_iou = iou
+                    max_indices = (a, b)
+            if max_iou > 0.1:
+                a = merged_regions[max_indices[0]]
+                b = merged_regions[max_indices[1]]
+                merged_regions.append(calculate_region(frame_shape,
+                    min(a[0], b[0]),
+                    min(a[1], b[1]),
+                    max(a[2], b[2]),
+                    max(a[3], b[3]),
+                    1
+                ))
+                del merged_regions[max(max_indices[0], max_indices[1])]
+                del merged_regions[min(max_indices[0], max_indices[1])]
+            else:
+                break
+
+        # resize regions and detect
+        detections = []
+        for region in merged_regions:
+
+            tensor_input = create_tensor_input(frame, region)
+
+            region_detections = object_detector.detect(tensor_input)
+
+            for d in region_detections:
+                box = d[2]
+                size = region[2]-region[0]
+                x_min = int((box[1] * size) + region[0])
+                y_min = int((box[0] * size) + region[1])
+                x_max = int((box[3] * size) + region[0])
+                y_max = int((box[2] * size) + region[1])
+                det = (d[0],
+                    d[1],
+                    (x_min, y_min, x_max, y_max),
+                    (x_max-x_min)*(y_max-y_min),
+                    region)
+                if filtered(det, objects_to_track, object_filters, mask):
+                    continue
+                detections.append(det)
+
+        #########
+        # merge objects, check for clipped objects and look again up to N times
+        #########
+        refining = True
+        refine_count = 0
+        while refining and refine_count < 4:
+            refining = False
+
+            # group by name
+            detected_object_groups = defaultdict(lambda: [])
+            for detection in detections:
+                detected_object_groups[detection[0]].append(detection)
+
+            selected_objects = []
+            for group in detected_object_groups.values():
+
+                # apply non-maxima suppression to suppress weak, overlapping bounding boxes
+                boxes = [(o[2][0], o[2][1], o[2][2]-o[2][0], o[2][3]-o[2][1])
+                    for o in group]
+                confidences = [o[1] for o in group]
+                idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
+
+                for index in idxs:
+                    obj = group[index[0]]
+                    if clipped(obj, frame_shape): #obj['clipped']:
+                        box = obj[2]
+                        # calculate a new region that will hopefully get the entire object
+                        region = calculate_region(frame_shape, 
+                            box[0], box[1],
+                            box[2], box[3])
+                        
+                        tensor_input = create_tensor_input(frame, region)
+                        # run detection on new region
+                        refined_detections = object_detector.detect(tensor_input)
+                        for d in refined_detections:
+                            box = d[2]
+                            size = region[2]-region[0]
+                            x_min = int((box[1] * size) + region[0])
+                            y_min = int((box[0] * size) + region[1])
+                            x_max = int((box[3] * size) + region[0])
+                            y_max = int((box[2] * size) + region[1])
+                            det = (d[0],
+                                d[1],
+                                (x_min, y_min, x_max, y_max),
+                                (x_max-x_min)*(y_max-y_min),
+                                region)
+                            if filtered(det, objects_to_track, object_filters, mask):
+                                continue
+                            selected_objects.append(det)
+
+                        refining = True
+                    else:
+                        selected_objects.append(obj)
+                
+            # set the detections list to only include top, complete objects
+            # and new detections
+            detections = selected_objects
+
+            if refining:
+                refine_count += 1
+        
+        # now that we have refined our detections, we need to track objects
+        object_tracker.match_and_update(frame_time, detections)
+
+        # put the frame in the plasma store
+        object_id = hashlib.sha1(str.encode(f"{name}{frame_time}")).digest()
+        plasma_client.put(frame, plasma.ObjectID(object_id))
+        # add to the queue
+        detected_objects_queue.put((name, frame_time, object_tracker.tracked_objects))
+
+        # if (frames >= 700 and frames <= 1635) or (frames >= 2500):
+        # if (frames >= 300 and frames <= 600):
+        # if (frames >= 0):
+            # row1 = cv2.hconcat([gray, cv2.convertScaleAbs(avg_frame)])
+            # row2 = cv2.hconcat([frameDelta, thresh])
+            # cv2.imwrite(f"/lab/debug/output/{frames}.jpg", cv2.vconcat([row1, row2]))
+            # # cv2.imwrite(f"/lab/debug/output/resized-frame-{frames}.jpg", resized_frame)
+            # for region in motion_regions:
+            #     cv2.rectangle(frame, (region[0], region[1]), (region[2], region[3]), (255,128,0), 2)
+            # for region in object_regions:
+            #     cv2.rectangle(frame, (region[0], region[1]), (region[2], region[3]), (0,128,255), 2)
+            # for region in merged_regions:
+            #     cv2.rectangle(frame, (region[0], region[1]), (region[2], region[3]), (0,255,0), 2)
+            # for box in motion_boxes:
+            #     cv2.rectangle(frame, (box[0], box[1]), (box[2], box[3]), (255,0,0), 2)
+            # for detection in detections:
+            #     box = detection[2]
+            #     draw_box_with_label(frame, box[0], box[1], box[2], box[3], detection[0], f"{detection[1]*100}%")
+            # for obj in object_tracker.tracked_objects.values():
+            #     box = obj['box']
+            #     draw_box_with_label(frame, box[0], box[1], box[2], box[3], obj['label'], obj['id'], thickness=1, color=(0,0,255), position='bl')
+            # cv2.putText(frame, str(total_detections), (10, 10), cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.5, color=(0, 0, 0), thickness=2)
+            # cv2.putText(frame, str(frame_detections), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.5, color=(0, 0, 0), thickness=2)
+            # cv2.imwrite(f"/lab/debug/output/frame-{frames}.jpg", frame)
+            # break
+
+    # start a thread to publish object scores
+    # mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self)
+    # mqtt_publisher.start()
+
+    # create a watchdog thread for capture process
+    # self.watchdog = CameraWatchdog(self)
+
+