From 15d989255cac61adc8db68fb392f2e13525085ac Mon Sep 17 00:00:00 2001
From: Blake Blackshear <blakeb@blakeshome.com>
Date: Sun, 1 Nov 2020 15:45:04 -0600
Subject: [PATCH] add watchdog

---
 frigate/__main__.py |  5 ++++-
 frigate/watchdog.py | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 frigate/watchdog.py

diff --git a/frigate/__main__.py b/frigate/__main__.py
index 1757582a2..ee2c8ef52 100644
--- a/frigate/__main__.py
+++ b/frigate/__main__.py
@@ -15,6 +15,7 @@ from frigate.models import Event
 from frigate.mqtt import create_mqtt_client
 from frigate.object_processing import TrackedObjectProcessor
 from frigate.video import get_frame_shape, track_camera, get_ffmpeg_input, capture_camera
+from frigate.watchdog import FrigateWatchdog
 
 class FrigateApp():
     def __init__(self):
@@ -170,7 +171,8 @@ class FrigateApp():
         self.event_processor.start()
 
     def start_watchdog(self):
-        pass
+        self.frigate_watchdog = FrigateWatchdog(self.detectors, self.stop_event)
+        self.frigate_watchdog.start()
 
     def start(self):
         self.init_config()
@@ -193,6 +195,7 @@ class FrigateApp():
 
         self.detected_frames_processor.join()
         self.event_processor.join()
+        self.frigate_watchdog.join()
 
         for detector in self.detectors.values():
             detector.stop()
diff --git a/frigate/watchdog.py b/frigate/watchdog.py
new file mode 100644
index 000000000..1b54eab64
--- /dev/null
+++ b/frigate/watchdog.py
@@ -0,0 +1,32 @@
+import datetime
+import time
+import threading
+
+class FrigateWatchdog(threading.Thread):
+    def __init__(self, detectors, stop_event):
+        threading.Thread.__init__(self)
+        self.detectors = detectors
+        self.stop_event = stop_event
+
+    def run(self):
+        time.sleep(10)
+        while True:
+            # wait a bit before checking
+            time.sleep(10)
+
+            if self.stop_event.is_set():
+                print(f"Exiting watchdog...")
+                break
+
+            now = datetime.datetime.now().timestamp()
+
+            # check the detection processes
+            for detector in self.detectors.values():
+                detection_start = detector.detection_start.value
+                if (detection_start > 0.0 and 
+                    now - detection_start > 10):
+                    print("Detection appears to be stuck. Restarting detection process")
+                    detector.start_or_restart()
+                elif not detector.detect_process.is_alive():
+                    print("Detection appears to have stopped. Restarting detection process")
+                    detector.start_or_restart()