Camera Health Status (#19709)

* Send status of camera streams to mqtt

* Update docs

* Formatting

* Fix frontend querying fps

docs/docs/integrations/mqtt.md

@@ -238,6 +238,14 @@ Topic with current state of notifications. Published values are `ON` and `OFF`.
 
 ## Frigate Camera Topics
 
+### `frigate/<camera_name>/<role>/status`
+
+Publishes the current health status of each role that is enabled (`audio`, `detect`, `record`). Possible values are:
+
+- `online`: Stream is running and being processed
+- `offline`: Stream is offline and is being restarted
+- `disabled`: Camera is currently disabled
+
 ### `frigate/<camera_name>/<object_name>`
 
 Publishes the count of objects for the camera for use as a sensor in Home Assistant.

frigate/events/audio.py

@@ -356,6 +356,7 @@ class AudioEventMaintainer(threading.Thread):
             self.chunk_size,
             self.audio_listener,
         )
+        self.requestor.send_data(f"{self.camera_config.name}/status/audio", "online")
 
     def read_audio(self) -> None:
         def log_and_restart() -> None:
@@ -371,6 +372,9 @@ class AudioEventMaintainer(threading.Thread):
 
             if not chunk:
                 if self.audio_listener.poll() is not None:
+                    self.requestor.send_data(
+                        f"{self.camera_config.name}/status/audio", "offline"
+                    )
                     self.logger.error("ffmpeg process is not running, restarting...")
                     log_and_restart()
                     return
@@ -396,6 +400,9 @@ class AudioEventMaintainer(threading.Thread):
                     )
                     self.start_or_restart_ffmpeg()
                 else:
+                    self.requestor.send_data(
+                        f"{self.camera_config.name}/status/audio", "disabled"
+                    )
                     self.logger.debug(
                         f"Disabling audio detections for {self.camera_config.name}, ending events"
                     )

frigate/video.py

@@ -71,7 +71,7 @@ def stop_ffmpeg(ffmpeg_process: sp.Popen[Any], logger: logging.Logger):
 
 def start_or_restart_ffmpeg(
     ffmpeg_cmd, logger, logpipe: LogPipe, frame_size=None, ffmpeg_process=None
-):
+) -> sp.Popen[Any]:
     if ffmpeg_process is not None:
         stop_ffmpeg(ffmpeg_process, logger)
 
@@ -96,7 +96,7 @@ def start_or_restart_ffmpeg(
 
 
 def capture_frames(
-    ffmpeg_process,
+    ffmpeg_process: sp.Popen[Any],
     config: CameraConfig,
     shm_frame_count: int,
     frame_index: int,
@@ -107,7 +107,7 @@ def capture_frames(
     skipped_fps: Value,
     current_frame: Value,
     stop_event: MpEvent,
-):
+) -> None:
     frame_size = frame_shape[0] * frame_shape[1]
     frame_rate = EventsPerSecond()
     frame_rate.start()
@@ -196,6 +196,7 @@ class CameraWatchdog(threading.Thread):
         self.config_subscriber = CameraConfigUpdateSubscriber(
             None, {config.name: config}, [CameraConfigUpdateEnum.enabled]
         )
+        self.requestor = InterProcessRequestor()
         self.was_enabled = self.config.enabled
 
     def _update_enabled_state(self) -> bool:
@@ -245,6 +246,14 @@ class CameraWatchdog(threading.Thread):
                 else:
                     self.logger.debug(f"Disabling camera {self.config.name}")
                     self.stop_all_ffmpeg()
+
+                    # update camera status
+                    self.requestor.send_data(
+                        f"{self.config.name}/status/detect", "disabled"
+                    )
+                    self.requestor.send_data(
+                        f"{self.config.name}/status/record", "disabled"
+                    )
                 self.was_enabled = enabled
                 continue
 
@@ -254,6 +263,7 @@ class CameraWatchdog(threading.Thread):
             now = datetime.datetime.now().timestamp()
 
             if not self.capture_thread.is_alive():
+                self.requestor.send_data(f"{self.config.name}/status/detect", "offline")
                 self.camera_fps.value = 0
                 self.logger.error(
                     f"Ffmpeg process crashed unexpectedly for {self.config.name}."
@@ -263,6 +273,9 @@ class CameraWatchdog(threading.Thread):
                 self.fps_overflow_count += 1
 
                 if self.fps_overflow_count == 3:
+                    self.requestor.send_data(
+                        f"{self.config.name}/status/detect", "offline"
+                    )
                     self.fps_overflow_count = 0
                     self.camera_fps.value = 0
                     self.logger.info(
@@ -270,6 +283,7 @@ class CameraWatchdog(threading.Thread):
                     )
                     self.reset_capture_thread(drain_output=False)
             elif now - self.capture_thread.current_frame.value > 20:
+                self.requestor.send_data(f"{self.config.name}/status/detect", "offline")
                 self.camera_fps.value = 0
                 self.logger.info(
                     f"No frames received from {self.config.name} in 20 seconds. Exiting ffmpeg..."
@@ -277,6 +291,7 @@ class CameraWatchdog(threading.Thread):
                 self.reset_capture_thread()
             else:
                 # process is running normally
+                self.requestor.send_data(f"{self.config.name}/status/detect", "online")
                 self.fps_overflow_count = 0
 
             for p in self.ffmpeg_other_processes:
@@ -302,13 +317,27 @@ class CameraWatchdog(threading.Thread):
                             p["logpipe"],
                             ffmpeg_process=p["process"],
                         )
+
+                        for role in p["roles"]:
+                            self.requestor.send_data(
+                                f"{self.config.name}/status/{role}", "offline"
+                            )
+
                         continue
                     else:
+                        self.requestor.send_data(
+                            f"{self.config.name}/status/record", "online"
+                        )
                         p["latest_segment_time"] = latest_segment_time
 
                 if poll is None:
                     continue
 
+                for role in p["roles"]:
+                    self.requestor.send_data(
+                        f"{self.config.name}/status/{role}", "offline"
+                    )
+
                 p["logpipe"].dump()
                 p["process"] = start_or_restart_ffmpeg(
                     p["cmd"], self.logger, p["logpipe"], ffmpeg_process=p["process"]

web/src/views/system/CameraMetrics.tsx

@@ -32,7 +32,12 @@ export default function CameraMetrics({
   // stats
 
   const { data: initialStats } = useSWR<FrigateStats[]>(
-    ["stats/history", { keys: "cpu_usages,cameras,detection_fps,service" }],
+    [
+      "stats/history",
+      {
+        keys: "cpu_usages,cameras,camera_fps,detection_fps,skipped_fps,service",
+      },
+    ],
     {
       revalidateOnFocus: false,
     },