From 6f6d202c9946622784fd4b826588e505cc655cf1 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Fri, 21 Feb 2020 20:44:53 -0600 Subject: [PATCH] improve watchdog and coral fps tracking --- detect_objects.py | 45 +++++++++++++++++++++++++----------- frigate/edgetpu.py | 12 ++++------ frigate/object_processing.py | 7 +++++- frigate/video.py | 5 +++- 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/detect_objects.py b/detect_objects.py index 997d4e045..c61240fc0 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -1,5 +1,6 @@ import cv2 import time +import datetime import queue import yaml import threading @@ -53,12 +54,13 @@ WEB_PORT = CONFIG.get('web_port', 5000) DEBUG = (CONFIG.get('debug', '0') == '1') class CameraWatchdog(threading.Thread): - def __init__(self, camera_processes, config, tflite_process, tracked_objects_queue): + def __init__(self, camera_processes, config, tflite_process, tracked_objects_queue, object_processor): threading.Thread.__init__(self) self.camera_processes = camera_processes self.config = config self.tflite_process = tflite_process self.tracked_objects_queue = tracked_objects_queue + self.object_processor = object_processor def run(self): time.sleep(10) @@ -68,6 +70,17 @@ class CameraWatchdog(threading.Thread): for name, camera_process in self.camera_processes.items(): process = camera_process['process'] + if (datetime.datetime.now().timestamp() - self.object_processor.get_current_frame_time(name)) > 30: + print(f"Last frame for {name} is more than 30 seconds old...") + if process.is_alive(): + process.terminate() + try: + print("Waiting for process to exit gracefully...") + process.wait(timeout=30) + except sp.TimeoutExpired: + print("Process didnt exit. Force killing...") + process.kill() + process.wait() if not process.is_alive(): print(f"Process for {name} is not alive. Starting again...") camera_process['fps'].value = float(self.config[name]['fps']) @@ -131,11 +144,13 @@ def main(): for name, config in CONFIG['cameras'].items(): camera_processes[name] = { 'fps': mp.Value('d', float(config['fps'])), - 'skipped_fps': mp.Value('d', 0.0) + 'skipped_fps': mp.Value('d', 0.0), + 'detection_fps': mp.Value('d', 0.0), + 'last_frame': datetime.datetime.now().timestamp() } camera_process = mp.Process(target=track_camera, args=(name, config, FFMPEG_DEFAULT_CONFIG, GLOBAL_OBJECT_CONFIG, tflite_process.detect_lock, tflite_process.detect_ready, tflite_process.frame_ready, tracked_objects_queue, - camera_processes[name]['fps'], camera_processes[name]['skipped_fps'])) + camera_processes[name]['fps'], camera_processes[name]['skipped_fps'], camera_processes[name]['detection_fps'])) camera_process.daemon = True camera_processes[name]['process'] = camera_process @@ -143,11 +158,11 @@ def main(): camera_process['process'].start() print(f"Camera_process started for {name}: {camera_process['process'].pid}") - camera_watchdog = CameraWatchdog(camera_processes, CONFIG['cameras'], tflite_process, tracked_objects_queue) - camera_watchdog.start() - object_processor = TrackedObjectProcessor(CONFIG['cameras'], client, MQTT_TOPIC_PREFIX, tracked_objects_queue) object_processor.start() + + camera_watchdog = CameraWatchdog(camera_processes, CONFIG['cameras'], tflite_process, tracked_objects_queue, object_processor) + camera_watchdog.start() # create a flask app that encodes frames a mjpeg on demand app = Flask(__name__) @@ -161,18 +176,22 @@ def main(): @app.route('/debug/stats') def stats(): - stats = { - 'coral': { - 'fps': tflite_process.fps.value, - 'inference_speed': round(tflite_process.avg_inference_speed.value*1000, 2) - } - } + stats = {} + + total_detection_fps = 0 for name, camera_stats in camera_processes.items(): + total_detection_fps += camera_stats['detection_fps'].value stats[name] = { 'fps': camera_stats['fps'].value, - 'skipped_fps': camera_stats['skipped_fps'].value + 'skipped_fps': camera_stats['skipped_fps'].value, + 'detection_fps': camera_stats['detection_fps'].value } + + stats['coral'] = { + 'fps': total_detection_fps, + 'inference_speed': round(tflite_process.avg_inference_speed.value*1000, 2) + } return jsonify(stats) diff --git a/frigate/edgetpu.py b/frigate/edgetpu.py index 3ba4b3f0e..5471f7b51 100644 --- a/frigate/edgetpu.py +++ b/frigate/edgetpu.py @@ -78,16 +78,13 @@ class EdgeTPUProcess(): self.detect_lock = mp.Lock() self.detect_ready = mp.Event() self.frame_ready = mp.Event() - self.fps = mp.Value('d', 0.0) self.avg_inference_speed = mp.Value('d', 0.01) - def run_detector(detect_ready, frame_ready, fps, avg_speed): + def run_detector(detect_ready, frame_ready, avg_speed): print(f"Starting detection process: {os.getpid()}") object_detector = ObjectDetector() input_frame = sa.attach("frame") detections = sa.attach("detections") - fps_tracker = EventsPerSecond() - fps_tracker.start() while True: # wait until a frame is ready @@ -98,12 +95,10 @@ class EdgeTPUProcess(): detections[:] = object_detector.detect_raw(input_frame) # signal that the process is ready to detect detect_ready.set() - fps_tracker.update() - fps.value = fps_tracker.eps() duration = datetime.datetime.now().timestamp()-start avg_speed.value = (avg_speed.value*9 + duration)/10 - self.detect_process = mp.Process(target=run_detector, args=(self.detect_ready, self.frame_ready, self.fps, self.avg_inference_speed)) + self.detect_process = mp.Process(target=run_detector, args=(self.detect_ready, self.frame_ready, self.avg_inference_speed)) self.detect_process.daemon = True self.detect_process.start() @@ -114,6 +109,8 @@ class RemoteObjectDetector(): self.input_frame = sa.attach("frame") self.detections = sa.attach("detections") + self.fps = EventsPerSecond() + self.detect_lock = detect_lock self.detect_ready = detect_ready self.frame_ready = frame_ready @@ -135,4 +132,5 @@ class RemoteObjectDetector(): float(d[1]), (d[2], d[3], d[4], d[5]) )) + self.fps.update() return detections \ No newline at end of file diff --git a/frigate/object_processing.py b/frigate/object_processing.py index f2b0986ac..11e6c2562 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -33,7 +33,8 @@ class TrackedObjectProcessor(threading.Thread): self.camera_data = defaultdict(lambda: { 'best_objects': {}, 'object_status': defaultdict(lambda: defaultdict(lambda: 'OFF')), - 'tracked_objects': {} + 'tracked_objects': {}, + 'current_frame_time': datetime.datetime.now().timestamp() }) def get_best(self, camera, label): @@ -44,6 +45,9 @@ class TrackedObjectProcessor(threading.Thread): def get_current_frame(self, camera): return self.camera_data[camera]['current_frame'] + + def get_current_frame_time(self, camera): + return self.camera_data[camera]['current_frame_time'] def run(self): while True: @@ -86,6 +90,7 @@ class TrackedObjectProcessor(threading.Thread): # Set the current frame as ready ### self.camera_data[camera]['current_frame'] = current_frame + self.camera_data[camera]['current_frame_time'] = frame_time ### # Maintain the highest scoring recent object and frame for each label diff --git a/frigate/video.py b/frigate/video.py index 13890d928..40241f009 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -99,7 +99,7 @@ def create_tensor_input(frame, region): # Expand dimensions since the model expects images to have shape: [1, 300, 300, 3] return np.expand_dims(cropped_frame, axis=0) -def track_camera(name, config, ffmpeg_global_config, global_objects_config, detect_lock, detect_ready, frame_ready, detected_objects_queue, fps, skipped_fps): +def track_camera(name, config, ffmpeg_global_config, global_objects_config, detect_lock, detect_ready, frame_ready, detected_objects_queue, fps, skipped_fps, detection_fps): print(f"Starting process for {name}: {os.getpid()}") # Merge the ffmpeg config with the global config @@ -168,6 +168,7 @@ def track_camera(name, config, ffmpeg_global_config, global_objects_config, dete skipped_fps_tracker = EventsPerSecond() fps_tracker.start() skipped_fps_tracker.start() + object_detector.fps.start() while True: frame_bytes = ffmpeg_process.stdout.read(frame_size) @@ -181,6 +182,7 @@ def track_camera(name, config, ffmpeg_global_config, global_objects_config, dete fps_tracker.update() fps.value = fps_tracker.eps() + detection_fps.value = object_detector.fps.eps() frame_time = datetime.datetime.now().timestamp() @@ -193,6 +195,7 @@ def track_camera(name, config, ffmpeg_global_config, global_objects_config, dete motion_boxes = motion_detector.detect(frame) # skip object detection if we are below the min_fps + # TODO: its about more than just the FPS. also look at avg wait or min wait if frame_num > 100 and fps.value < expected_fps-1: skipped_fps_tracker.update() skipped_fps.value = skipped_fps_tracker.eps()