add watchdog

frigate/__main__.py

@@ -15,6 +15,7 @@ from frigate.models import Event
 from frigate.mqtt import create_mqtt_client
 from frigate.object_processing import TrackedObjectProcessor
 from frigate.video import get_frame_shape, track_camera, get_ffmpeg_input, capture_camera
+from frigate.watchdog import FrigateWatchdog
 
 class FrigateApp():
     def __init__(self):
@@ -170,7 +171,8 @@ class FrigateApp():
         self.event_processor.start()
 
     def start_watchdog(self):
-        pass
+        self.frigate_watchdog = FrigateWatchdog(self.detectors, self.stop_event)
+        self.frigate_watchdog.start()
 
     def start(self):
         self.init_config()
@@ -193,6 +195,7 @@ class FrigateApp():
 
         self.detected_frames_processor.join()
         self.event_processor.join()
+        self.frigate_watchdog.join()
 
         for detector in self.detectors.values():
             detector.stop()

frigate/watchdog.py

@@ -0,0 +1,32 @@
+import datetime
+import time
+import threading
+
+class FrigateWatchdog(threading.Thread):
+    def __init__(self, detectors, stop_event):
+        threading.Thread.__init__(self)
+        self.detectors = detectors
+        self.stop_event = stop_event
+
+    def run(self):
+        time.sleep(10)
+        while True:
+            # wait a bit before checking
+            time.sleep(10)
+
+            if self.stop_event.is_set():
+                print(f"Exiting watchdog...")
+                break
+
+            now = datetime.datetime.now().timestamp()
+
+            # check the detection processes
+            for detector in self.detectors.values():
+                detection_start = detector.detection_start.value
+                if (detection_start > 0.0 and 
+                    now - detection_start > 10):
+                    print("Detection appears to be stuck. Restarting detection process")
+                    detector.start_or_restart()
+                elif not detector.detect_process.is_alive():
+                    print("Detection appears to have stopped. Restarting detection process")
+                    detector.start_or_restart()