improve watchdog and coral fps tracking

detect_objects.py

@@ -1,5 +1,6 @@
 import cv2
 import time
+import datetime
 import queue
 import yaml
 import threading
@@ -53,12 +54,13 @@ WEB_PORT = CONFIG.get('web_port', 5000)
 DEBUG = (CONFIG.get('debug', '0') == '1')
 
 class CameraWatchdog(threading.Thread):
-    def __init__(self, camera_processes, config, tflite_process, tracked_objects_queue):
+    def __init__(self, camera_processes, config, tflite_process, tracked_objects_queue, object_processor):
         threading.Thread.__init__(self)
         self.camera_processes = camera_processes
         self.config = config
         self.tflite_process = tflite_process
         self.tracked_objects_queue = tracked_objects_queue
+        self.object_processor = object_processor
 
     def run(self):
         time.sleep(10)
@@ -68,6 +70,17 @@ class CameraWatchdog(threading.Thread):
 
             for name, camera_process in self.camera_processes.items():
                 process = camera_process['process']
+                if (datetime.datetime.now().timestamp() - self.object_processor.get_current_frame_time(name)) > 30:
+                    print(f"Last frame for {name} is more than 30 seconds old...")
+                    if process.is_alive():
+                        process.terminate()
+                        try:
+                            print("Waiting for process to exit gracefully...")
+                            process.wait(timeout=30)
+                        except sp.TimeoutExpired:
+                            print("Process didnt exit. Force killing...")
+                            process.kill()
+                            process.wait()
                 if not process.is_alive():
                     print(f"Process for {name} is not alive. Starting again...")
                     camera_process['fps'].value = float(self.config[name]['fps'])
@@ -131,11 +144,13 @@ def main():
     for name, config in CONFIG['cameras'].items():
         camera_processes[name] = {
             'fps': mp.Value('d', float(config['fps'])),
-            'skipped_fps': mp.Value('d', 0.0)
+            'skipped_fps': mp.Value('d', 0.0),
+            'detection_fps': mp.Value('d', 0.0),
+            'last_frame': datetime.datetime.now().timestamp()
         }
         camera_process = mp.Process(target=track_camera, args=(name, config, FFMPEG_DEFAULT_CONFIG, GLOBAL_OBJECT_CONFIG, 
             tflite_process.detect_lock, tflite_process.detect_ready, tflite_process.frame_ready, tracked_objects_queue, 
-            camera_processes[name]['fps'], camera_processes[name]['skipped_fps']))
+            camera_processes[name]['fps'], camera_processes[name]['skipped_fps'], camera_processes[name]['detection_fps']))
         camera_process.daemon = True
         camera_processes[name]['process'] = camera_process
 
@@ -143,12 +158,12 @@ def main():
         camera_process['process'].start()
         print(f"Camera_process started for {name}: {camera_process['process'].pid}")
     
-    camera_watchdog = CameraWatchdog(camera_processes, CONFIG['cameras'], tflite_process, tracked_objects_queue)
-    camera_watchdog.start()
-    
     object_processor = TrackedObjectProcessor(CONFIG['cameras'], client, MQTT_TOPIC_PREFIX, tracked_objects_queue)
     object_processor.start()
     
+    camera_watchdog = CameraWatchdog(camera_processes, CONFIG['cameras'], tflite_process, tracked_objects_queue, object_processor)
+    camera_watchdog.start()
+
     # create a flask app that encodes frames a mjpeg on demand
     app = Flask(__name__)
     log = logging.getLogger('werkzeug')
@@ -161,19 +176,23 @@ def main():
 
     @app.route('/debug/stats')
     def stats():
-        stats = {
-            'coral': {
-                'fps': tflite_process.fps.value,
-                'inference_speed': round(tflite_process.avg_inference_speed.value*1000, 2)
-            }
-        }
+        stats = {}
+
+        total_detection_fps = 0
 
         for name, camera_stats in camera_processes.items():
+            total_detection_fps += camera_stats['detection_fps'].value
             stats[name] = {
                 'fps': camera_stats['fps'].value,
-                'skipped_fps': camera_stats['skipped_fps'].value
+                'skipped_fps': camera_stats['skipped_fps'].value,
+                'detection_fps': camera_stats['detection_fps'].value
             }
         
+        stats['coral'] = {
+            'fps': total_detection_fps,
+            'inference_speed': round(tflite_process.avg_inference_speed.value*1000, 2)
+        }
+
         return jsonify(stats)
 
     @app.route('/<camera_name>/<label>/best.jpg')

frigate/edgetpu.py

@@ -78,16 +78,13 @@ class EdgeTPUProcess():
         self.detect_lock = mp.Lock()
         self.detect_ready = mp.Event()
         self.frame_ready = mp.Event()
-        self.fps = mp.Value('d', 0.0)
         self.avg_inference_speed = mp.Value('d', 0.01)
 
-        def run_detector(detect_ready, frame_ready, fps, avg_speed):
+        def run_detector(detect_ready, frame_ready, avg_speed):
             print(f"Starting detection process: {os.getpid()}")
             object_detector = ObjectDetector()
             input_frame = sa.attach("frame")
             detections = sa.attach("detections")
-            fps_tracker = EventsPerSecond()
-            fps_tracker.start()
 
             while True:
                 # wait until a frame is ready
@@ -98,12 +95,10 @@ class EdgeTPUProcess():
                 detections[:] = object_detector.detect_raw(input_frame)
                 # signal that the process is ready to detect
                 detect_ready.set()
-                fps_tracker.update()
-                fps.value = fps_tracker.eps()
                 duration = datetime.datetime.now().timestamp()-start
                 avg_speed.value = (avg_speed.value*9 + duration)/10
 
-        self.detect_process = mp.Process(target=run_detector, args=(self.detect_ready, self.frame_ready, self.fps, self.avg_inference_speed))
+        self.detect_process = mp.Process(target=run_detector, args=(self.detect_ready, self.frame_ready, self.avg_inference_speed))
         self.detect_process.daemon = True
         self.detect_process.start()
 
@@ -114,6 +109,8 @@ class RemoteObjectDetector():
         self.input_frame = sa.attach("frame")
         self.detections = sa.attach("detections")
 
+        self.fps = EventsPerSecond()
+
         self.detect_lock = detect_lock
         self.detect_ready = detect_ready
         self.frame_ready = frame_ready
@@ -135,4 +132,5 @@ class RemoteObjectDetector():
                     float(d[1]),
                     (d[2], d[3], d[4], d[5])
                 ))
+        self.fps.update()
         return detections

frigate/object_processing.py

@@ -33,7 +33,8 @@ class TrackedObjectProcessor(threading.Thread):
         self.camera_data = defaultdict(lambda: {
             'best_objects': {},
             'object_status': defaultdict(lambda: defaultdict(lambda: 'OFF')),
-            'tracked_objects': {}
+            'tracked_objects': {},
+            'current_frame_time': datetime.datetime.now().timestamp()
         })
         
     def get_best(self, camera, label):
@@ -45,6 +46,9 @@ class TrackedObjectProcessor(threading.Thread):
     def get_current_frame(self, camera):
         return self.camera_data[camera]['current_frame']
     
+    def get_current_frame_time(self, camera):
+        return self.camera_data[camera]['current_frame_time']
+
     def run(self):
         while True:
             camera, frame_time, tracked_objects = self.tracked_objects_queue.get()
@@ -86,6 +90,7 @@ class TrackedObjectProcessor(threading.Thread):
             # Set the current frame as ready
             ###
             self.camera_data[camera]['current_frame'] = current_frame
+            self.camera_data[camera]['current_frame_time'] = frame_time
             
             ###
             # Maintain the highest scoring recent object and frame for each label

frigate/video.py

@@ -99,7 +99,7 @@ def create_tensor_input(frame, region):
     # Expand dimensions since the model expects images to have shape: [1, 300, 300, 3]
     return np.expand_dims(cropped_frame, axis=0)
 
-def track_camera(name, config, ffmpeg_global_config, global_objects_config, detect_lock, detect_ready, frame_ready, detected_objects_queue, fps, skipped_fps):
+def track_camera(name, config, ffmpeg_global_config, global_objects_config, detect_lock, detect_ready, frame_ready, detected_objects_queue, fps, skipped_fps, detection_fps):
     print(f"Starting process for {name}: {os.getpid()}")
 
     # Merge the ffmpeg config with the global config
@@ -168,6 +168,7 @@ def track_camera(name, config, ffmpeg_global_config, global_objects_config, dete
     skipped_fps_tracker = EventsPerSecond()
     fps_tracker.start()
     skipped_fps_tracker.start()
+    object_detector.fps.start()
     while True:
         frame_bytes = ffmpeg_process.stdout.read(frame_size)
 
@@ -181,6 +182,7 @@ def track_camera(name, config, ffmpeg_global_config, global_objects_config, dete
 
         fps_tracker.update()
         fps.value = fps_tracker.eps()
+        detection_fps.value = object_detector.fps.eps()
 
         frame_time = datetime.datetime.now().timestamp()
         
@@ -193,6 +195,7 @@ def track_camera(name, config, ffmpeg_global_config, global_objects_config, dete
         motion_boxes = motion_detector.detect(frame)
 
         # skip object detection if we are below the min_fps
+        # TODO: its about more than just the FPS. also look at avg wait or min wait
         if frame_num > 100 and fps.value < expected_fps-1:
             skipped_fps_tracker.update()
             skipped_fps.value = skipped_fps_tracker.eps()