From 760e1ffe1dd5a2d16aebb57215a50f25335b6ed4 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sun, 19 Apr 2020 10:07:27 -0500 Subject: [PATCH] skip frames in the capture thread instead --- config/config.example.yml | 7 ------ detect_objects.py | 30 ++++++++++++++------------ frigate/video.py | 45 ++++++++++++++++++--------------------- 3 files changed, 37 insertions(+), 45 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index ca9844fec..0cac3afba 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -110,13 +110,6 @@ cameras: ################ take_frame: 1 - ################ - # The expected framerate for the camera. Frigate will try and ensure it maintains this framerate - # by dropping frames as necessary. Setting this lower than the actual framerate will allow frigate - # to process every frame at the expense of realtime processing. - ################ - fps: 5 - ################ # Configuration for the snapshots in the debug view and mqtt ################ diff --git a/detect_objects.py b/detect_objects.py index 4611191d5..6d77452b6 100644 --- a/detect_objects.py +++ b/detect_objects.py @@ -105,14 +105,13 @@ class CameraWatchdog(threading.Thread): process = camera_process['process'] if not process.is_alive(): print(f"Track process for {name} is not alive. Starting again...") - camera_process['fps'].value = float(self.config[name]['fps']) - camera_process['skipped_fps'].value = 0.0 + camera_process['process_fps'].value = 0.0 camera_process['detection_fps'].value = 0.0 camera_process['read_start'].value = 0.0 process = mp.Process(target=track_camera, args=(name, self.config[name], GLOBAL_OBJECT_CONFIG, camera_process['frame_queue'], camera_process['frame_shape'], self.tflite_process.detection_queue, self.tracked_objects_queue, - camera_process['fps'], camera_process['skipped_fps'], camera_process['detection_fps'], - camera_process['read_start'])) + camera_process['process_fps'], camera_process['detection_fps'], + camera_process['read_start'], camera_process['detection_frame'])) process.daemon = True camera_process['process'] = process process.start() @@ -123,7 +122,7 @@ class CameraWatchdog(threading.Thread): frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2] ffmpeg_process = start_or_restart_ffmpeg(camera_process['ffmpeg_cmd'], frame_size) camera_capture = CameraCapture(name, ffmpeg_process, frame_shape, camera_process['frame_queue'], - camera_process['take_frame'], camera_process['camera_fps']) + camera_process['take_frame'], camera_process['camera_fps'], camera_process['detection_frame']) camera_capture.start() camera_process['ffmpeg_process'] = ffmpeg_process camera_process['capture_thread'] = camera_capture @@ -193,20 +192,21 @@ def main(): frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2] take_frame = config.get('take_frame', 1) + detection_frame = mp.Value('d', 0.0) + ffmpeg_process = start_or_restart_ffmpeg(ffmpeg_cmd, frame_size) frame_queue = mp.SimpleQueue() camera_fps = EventsPerSecond() camera_fps.start() - camera_capture = CameraCapture(name, ffmpeg_process, frame_shape, frame_queue, take_frame, camera_fps) + camera_capture = CameraCapture(name, ffmpeg_process, frame_shape, frame_queue, take_frame, camera_fps, detection_frame) camera_capture.start() camera_processes[name] = { 'camera_fps': camera_fps, 'take_frame': take_frame, - 'fps': mp.Value('d', float(config['fps'])), - 'skipped_fps': mp.Value('d', 0.0), + 'process_fps': mp.Value('d', 0.0), 'detection_fps': mp.Value('d', 0.0), - 'detection_frame': mp.Value('d', 0.0), + 'detection_frame': detection_frame, 'read_start': mp.Value('d', 0.0), 'ffmpeg_process': ffmpeg_process, 'ffmpeg_cmd': ffmpeg_cmd, @@ -216,8 +216,8 @@ def main(): } camera_process = mp.Process(target=track_camera, args=(name, config, GLOBAL_OBJECT_CONFIG, frame_queue, frame_shape, - tflite_process.detection_queue, tracked_objects_queue, camera_processes[name]['fps'], - camera_processes[name]['skipped_fps'], camera_processes[name]['detection_fps'], + tflite_process.detection_queue, tracked_objects_queue, camera_processes[name]['process_fps'], + camera_processes[name]['detection_fps'], camera_processes[name]['read_start'], camera_processes[name]['detection_frame'])) camera_process.daemon = True camera_processes[name]['process'] = camera_process @@ -267,15 +267,17 @@ def main(): for name, camera_stats in camera_processes.items(): total_detection_fps += camera_stats['detection_fps'].value + capture_thread = camera_stats['capture_thread'] stats[name] = { - 'fps': round(camera_stats['fps'].value, 2), - 'skipped_fps': round(camera_stats['skipped_fps'].value, 2), + 'camera_fps': round(capture_thread.fps.eps(), 2), + 'process_fps': round(camera_stats['process_fps'].value, 2), + 'skipped_fps': round(capture_thread.skipped_fps.eps(), 2), 'detection_fps': round(camera_stats['detection_fps'].value, 2), 'read_start': camera_stats['read_start'].value, 'pid': camera_stats['process'].pid, 'ffmpeg_pid': camera_stats['ffmpeg_process'].pid, 'frame_info': { - 'read': camera_stats['capture_thread'].current_frame, + 'read': capture_thread.current_frame, 'detect': camera_stats['detection_frame'].value, 'process': object_processor.camera_data[name]['current_frame_time'] } diff --git a/frigate/video.py b/frigate/video.py index 95cb5bb75..a9b4e7198 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -115,7 +115,7 @@ def start_or_restart_ffmpeg(ffmpeg_cmd, frame_size, ffmpeg_process=None): return process class CameraCapture(threading.Thread): - def __init__(self, name, ffmpeg_process, frame_shape, frame_queue, take_frame, fps): + def __init__(self, name, ffmpeg_process, frame_shape, frame_queue, take_frame, fps, detection_frame): threading.Thread.__init__(self) self.name = name self.frame_shape = frame_shape @@ -123,12 +123,16 @@ class CameraCapture(threading.Thread): self.frame_queue = frame_queue self.take_frame = take_frame self.fps = fps + self.skipped_fps = EventsPerSecond() self.plasma_client = PlasmaManager() self.ffmpeg_process = ffmpeg_process self.current_frame = 0 + self.last_frame = 0 + self.detection_frame = detection_frame def run(self): frame_num = 0 + self.skipped_fps.start() while True: if self.ffmpeg_process.poll() != None: print(f"{self.name}: ffmpeg process is not running. exiting capture thread...") @@ -141,8 +145,16 @@ class CameraCapture(threading.Thread): print(f"{self.name}: ffmpeg didnt return a frame. something is wrong.") continue + self.fps.update() + frame_num += 1 if (frame_num % self.take_frame) != 0: + self.skipped_fps.update() + continue + + # if the detection process is more than 1 second behind, skip this frame + if self.detection_frame.value > 0.0 and (self.last_frame - self.detection_frame.value) > 1: + self.skipped_fps.update() continue # put the frame in the plasma store @@ -153,13 +165,14 @@ class CameraCapture(threading.Thread): ) # add to the queue self.frame_queue.put(self.current_frame) + self.last_frame = self.current_frame - self.fps.update() - -def track_camera(name, config, global_objects_config, frame_queue, frame_shape, detection_queue, detected_objects_queue, fps, skipped_fps, detection_fps, read_start, detection_frame): +def track_camera(name, config, global_objects_config, frame_queue, frame_shape, detection_queue, detected_objects_queue, fps, detection_fps, read_start, detection_frame): print(f"Starting process for {name}: {os.getpid()}") listen() + detection_frame.value = 0.0 + # Merge the tracked object config with the global config camera_objects_config = config.get('objects', {}) # combine tracked objects lists @@ -172,8 +185,6 @@ def track_camera(name, config, global_objects_config, frame_queue, frame_shape, for obj in objects_with_config: object_filters[obj] = {**global_object_filters.get(obj, {}), **camera_object_filters.get(obj, {})} - expected_fps = config['fps'] - frame = np.zeros(frame_shape, np.uint8) # load in the mask for object detection @@ -192,12 +203,9 @@ def track_camera(name, config, global_objects_config, frame_queue, frame_shape, object_tracker = ObjectTracker(10) plasma_client = PlasmaManager() - frame_num = 0 avg_wait = 0.0 fps_tracker = EventsPerSecond() - skipped_fps_tracker = EventsPerSecond() fps_tracker.start() - skipped_fps_tracker.start() object_detector.fps.start() while True: read_start.value = datetime.datetime.now().timestamp() @@ -206,31 +214,20 @@ def track_camera(name, config, global_objects_config, frame_queue, frame_shape, read_start.value = 0.0 avg_wait = (avg_wait*99+duration)/100 detection_frame.value = frame_time - - fps_tracker.update() - fps.value = fps_tracker.eps() - detection_fps.value = object_detector.fps.eps() # Get frame from plasma store frame = plasma_client.get(f"{name}{frame_time}") if frame is plasma.ObjectNotAvailable: - skipped_fps_tracker.update() - skipped_fps.value = skipped_fps_tracker.eps() continue + + fps_tracker.update() + fps.value = fps_tracker.eps() + detection_fps.value = object_detector.fps.eps() # look for motion motion_boxes = motion_detector.detect(frame) - # skip object detection if we are below the min_fps and wait time is less than half the average - if frame_num > 100 and fps.value < expected_fps-1 and duration < 0.5*avg_wait: - skipped_fps_tracker.update() - skipped_fps.value = skipped_fps_tracker.eps() - plasma_client.delete(f"{name}{frame_time}") - continue - - skipped_fps.value = skipped_fps_tracker.eps() - tracked_objects = object_tracker.tracked_objects.values() # merge areas of motion that intersect with a known tracked object into a single area to look at