diff --git a/frigate/__main__.py b/frigate/__main__.py index 1757582a2..ee2c8ef52 100644 --- a/frigate/__main__.py +++ b/frigate/__main__.py @@ -15,6 +15,7 @@ from frigate.models import Event from frigate.mqtt import create_mqtt_client from frigate.object_processing import TrackedObjectProcessor from frigate.video import get_frame_shape, track_camera, get_ffmpeg_input, capture_camera +from frigate.watchdog import FrigateWatchdog class FrigateApp(): def __init__(self): @@ -170,7 +171,8 @@ class FrigateApp(): self.event_processor.start() def start_watchdog(self): - pass + self.frigate_watchdog = FrigateWatchdog(self.detectors, self.stop_event) + self.frigate_watchdog.start() def start(self): self.init_config() @@ -193,6 +195,7 @@ class FrigateApp(): self.detected_frames_processor.join() self.event_processor.join() + self.frigate_watchdog.join() for detector in self.detectors.values(): detector.stop() diff --git a/frigate/watchdog.py b/frigate/watchdog.py new file mode 100644 index 000000000..1b54eab64 --- /dev/null +++ b/frigate/watchdog.py @@ -0,0 +1,32 @@ +import datetime +import time +import threading + +class FrigateWatchdog(threading.Thread): + def __init__(self, detectors, stop_event): + threading.Thread.__init__(self) + self.detectors = detectors + self.stop_event = stop_event + + def run(self): + time.sleep(10) + while True: + # wait a bit before checking + time.sleep(10) + + if self.stop_event.is_set(): + print(f"Exiting watchdog...") + break + + now = datetime.datetime.now().timestamp() + + # check the detection processes + for detector in self.detectors.values(): + detection_start = detector.detection_start.value + if (detection_start > 0.0 and + now - detection_start > 10): + print("Detection appears to be stuck. Restarting detection process") + detector.start_or_restart() + elif not detector.detect_process.is_alive(): + print("Detection appears to have stopped. Restarting detection process") + detector.start_or_restart()