Camera connection quality indicator (#21297)

* add camera connection quality metrics and indicator

* formatting

* move stall calcs to watchdog

* clean up

* change watchdog to 1s and separately track time for ffmpeg retry_interval

* implement status caching to reduce message volume

frigate/camera/__init__.py

@@ -19,6 +19,8 @@ class CameraMetrics:
     process_pid: Synchronized
     capture_process_pid: Synchronized
     ffmpeg_pid: Synchronized
+    reconnects_last_hour: Synchronized
+    stalls_last_hour: Synchronized
 
     def __init__(self, manager: SyncManager):
         self.camera_fps = manager.Value("d", 0)
@@ -35,6 +37,8 @@ class CameraMetrics:
         self.process_pid = manager.Value("i", 0)
         self.capture_process_pid = manager.Value("i", 0)
         self.ffmpeg_pid = manager.Value("i", 0)
+        self.reconnects_last_hour = manager.Value("i", 0)
+        self.stalls_last_hour = manager.Value("i", 0)
 
 
 class PTZMetrics:

frigate/stats/util.py

@@ -279,6 +279,32 @@ def stats_snapshot(
             if camera_stats.capture_process_pid.value
             else None
         )
+        # Calculate connection quality based on current state
+        # This is computed at stats-collection time so offline cameras
+        # correctly show as unusable rather than excellent
+        expected_fps = config.cameras[name].detect.fps
+        current_fps = camera_stats.camera_fps.value
+        reconnects = camera_stats.reconnects_last_hour.value
+        stalls = camera_stats.stalls_last_hour.value
+
+        if current_fps < 0.1:
+            quality_str = "unusable"
+        elif reconnects == 0 and current_fps >= 0.9 * expected_fps and stalls < 5:
+            quality_str = "excellent"
+        elif reconnects <= 2 and current_fps >= 0.6 * expected_fps:
+            quality_str = "fair"
+        elif reconnects > 10 or current_fps < 1.0 or stalls > 100:
+            quality_str = "unusable"
+        else:
+            quality_str = "poor"
+
+        connection_quality = {
+            "connection_quality": quality_str,
+            "expected_fps": expected_fps,
+            "reconnects_last_hour": reconnects,
+            "stalls_last_hour": stalls,
+        }
+
         stats["cameras"][name] = {
             "camera_fps": round(camera_stats.camera_fps.value, 2),
             "process_fps": round(camera_stats.process_fps.value, 2),
@@ -290,6 +316,7 @@ def stats_snapshot(
             "ffmpeg_pid": ffmpeg_pid,
             "audio_rms": round(camera_stats.audio_rms.value, 4),
             "audio_dBFS": round(camera_stats.audio_dBFS.value, 4),
+            **connection_quality,
         }
 
     stats["detectors"] = {}

frigate/video.py

@@ -3,6 +3,7 @@ import queue
 import subprocess as sp
 import threading
 import time
+from collections import deque
 from datetime import datetime, timedelta, timezone
 from multiprocessing import Queue, Value
 from multiprocessing.synchronize import Event as MpEvent
@@ -115,6 +116,7 @@ def capture_frames(
     frame_rate.start()
     skipped_eps = EventsPerSecond()
     skipped_eps.start()
+
     config_subscriber = CameraConfigUpdateSubscriber(
         None, {config.name: config}, [CameraConfigUpdateEnum.enabled]
     )
@@ -179,6 +181,9 @@ class CameraWatchdog(threading.Thread):
         camera_fps,
         skipped_fps,
         ffmpeg_pid,
+        stalls,
+        reconnects,
+        detection_frame,
         stop_event,
     ):
         threading.Thread.__init__(self)
@@ -199,6 +204,10 @@ class CameraWatchdog(threading.Thread):
         self.frame_index = 0
         self.stop_event = stop_event
         self.sleeptime = self.config.ffmpeg.retry_interval
+        self.reconnect_timestamps = deque()
+        self.stalls = stalls
+        self.reconnects = reconnects
+        self.detection_frame = detection_frame
 
         self.config_subscriber = CameraConfigUpdateSubscriber(
             None,
@@ -213,6 +222,35 @@ class CameraWatchdog(threading.Thread):
         self.latest_invalid_segment_time: float = 0
         self.latest_cache_segment_time: float = 0
 
+        # Stall tracking (based on last processed frame)
+        self._stall_timestamps: deque[float] = deque()
+        self._stall_active: bool = False
+
+        # Status caching to reduce message volume
+        self._last_detect_status: str | None = None
+        self._last_record_status: str | None = None
+        self._last_status_update_time: float = 0.0
+
+    def _send_detect_status(self, status: str, now: float) -> None:
+        """Send detect status only if changed or retry_interval has elapsed."""
+        if (
+            status != self._last_detect_status
+            or (now - self._last_status_update_time) >= self.sleeptime
+        ):
+            self.requestor.send_data(f"{self.config.name}/status/detect", status)
+            self._last_detect_status = status
+            self._last_status_update_time = now
+
+    def _send_record_status(self, status: str, now: float) -> None:
+        """Send record status only if changed or retry_interval has elapsed."""
+        if (
+            status != self._last_record_status
+            or (now - self._last_status_update_time) >= self.sleeptime
+        ):
+            self.requestor.send_data(f"{self.config.name}/status/record", status)
+            self._last_record_status = status
+            self._last_status_update_time = now
+
     def _update_enabled_state(self) -> bool:
         """Fetch the latest config and update enabled state."""
         self.config_subscriber.check_for_updates()
@@ -239,6 +277,14 @@ class CameraWatchdog(threading.Thread):
                 else:
                     self.ffmpeg_detect_process.wait()
 
+        # Update reconnects
+        now = datetime.now().timestamp()
+        self.reconnect_timestamps.append(now)
+        while self.reconnect_timestamps and self.reconnect_timestamps[0] < now - 3600:
+            self.reconnect_timestamps.popleft()
+        if self.reconnects:
+            self.reconnects.value = len(self.reconnect_timestamps)
+
         # Wait for old capture thread to fully exit before starting a new one
         if self.capture_thread is not None and self.capture_thread.is_alive():
             self.logger.info("Waiting for capture thread to exit...")
@@ -261,7 +307,10 @@ class CameraWatchdog(threading.Thread):
             self.start_all_ffmpeg()
 
         time.sleep(self.sleeptime)
-        while not self.stop_event.wait(self.sleeptime):
+        last_restart_time = datetime.now().timestamp()
+
+        # 1 second watchdog loop
+        while not self.stop_event.wait(1):
             enabled = self._update_enabled_state()
             if enabled != self.was_enabled:
                 if enabled:
@@ -277,12 +326,9 @@ class CameraWatchdog(threading.Thread):
                     self.stop_all_ffmpeg()
 
                     # update camera status
-                    self.requestor.send_data(
-                        f"{self.config.name}/status/detect", "disabled"
-                    )
-                    self.requestor.send_data(
-                        f"{self.config.name}/status/record", "disabled"
-                    )
+                    now = datetime.now().timestamp()
+                    self._send_detect_status("disabled", now)
+                    self._send_record_status("disabled", now)
                 self.was_enabled = enabled
                 continue
 
@@ -321,36 +367,44 @@ class CameraWatchdog(threading.Thread):
 
             now = datetime.now().timestamp()
 
+            # Check if enough time has passed to allow ffmpeg restart (backoff pacing)
+            time_since_last_restart = now - last_restart_time
+            can_restart = time_since_last_restart >= self.sleeptime
+
             if not self.capture_thread.is_alive():
-                self.requestor.send_data(f"{self.config.name}/status/detect", "offline")
+                self._send_detect_status("offline", now)
                 self.camera_fps.value = 0
                 self.logger.error(
                     f"Ffmpeg process crashed unexpectedly for {self.config.name}."
                 )
-                self.reset_capture_thread(terminate=False)
+                if can_restart:
+                    self.reset_capture_thread(terminate=False)
+                    last_restart_time = now
             elif self.camera_fps.value >= (self.config.detect.fps + 10):
                 self.fps_overflow_count += 1
 
                 if self.fps_overflow_count == 3:
-                    self.requestor.send_data(
-                        f"{self.config.name}/status/detect", "offline"
-                    )
+                    self._send_detect_status("offline", now)
                     self.fps_overflow_count = 0
                     self.camera_fps.value = 0
                     self.logger.info(
                         f"{self.config.name} exceeded fps limit. Exiting ffmpeg..."
                     )
-                    self.reset_capture_thread(drain_output=False)
+                    if can_restart:
+                        self.reset_capture_thread(drain_output=False)
+                        last_restart_time = now
             elif now - self.capture_thread.current_frame.value > 20:
-                self.requestor.send_data(f"{self.config.name}/status/detect", "offline")
+                self._send_detect_status("offline", now)
                 self.camera_fps.value = 0
                 self.logger.info(
                     f"No frames received from {self.config.name} in 20 seconds. Exiting ffmpeg..."
                 )
-                self.reset_capture_thread()
+                if can_restart:
+                    self.reset_capture_thread()
+                    last_restart_time = now
             else:
                 # process is running normally
-                self.requestor.send_data(f"{self.config.name}/status/detect", "online")
+                self._send_detect_status("online", now)
                 self.fps_overflow_count = 0
 
             for p in self.ffmpeg_other_processes:
@@ -421,9 +475,7 @@ class CameraWatchdog(threading.Thread):
 
                         continue
                     else:
-                        self.requestor.send_data(
-                            f"{self.config.name}/status/record", "online"
-                        )
+                        self._send_record_status("online", now)
                         p["latest_segment_time"] = self.latest_cache_segment_time
 
                 if poll is None:
@@ -439,6 +491,34 @@ class CameraWatchdog(threading.Thread):
                     p["cmd"], self.logger, p["logpipe"], ffmpeg_process=p["process"]
                 )
 
+            # Update stall metrics based on last processed frame timestamp
+            now = datetime.now().timestamp()
+            processed_ts = (
+                float(self.detection_frame.value) if self.detection_frame else 0.0
+            )
+            if processed_ts > 0:
+                delta = now - processed_ts
+                observed_fps = (
+                    self.camera_fps.value
+                    if self.camera_fps.value > 0
+                    else self.config.detect.fps
+                )
+                interval = 1.0 / max(observed_fps, 0.1)
+                stall_threshold = max(2.0 * interval, 2.0)
+
+                if delta > stall_threshold:
+                    if not self._stall_active:
+                        self._stall_timestamps.append(now)
+                        self._stall_active = True
+                else:
+                    self._stall_active = False
+
+                while self._stall_timestamps and self._stall_timestamps[0] < now - 3600:
+                    self._stall_timestamps.popleft()
+
+                if self.stalls:
+                    self.stalls.value = len(self._stall_timestamps)
+
         self.stop_all_ffmpeg()
         self.logpipe.close()
         self.config_subscriber.stop()
@@ -576,6 +656,9 @@ class CameraCapture(FrigateProcess):
             self.camera_metrics.camera_fps,
             self.camera_metrics.skipped_fps,
             self.camera_metrics.ffmpeg_pid,
+            self.camera_metrics.stalls_last_hour,
+            self.camera_metrics.reconnects_last_hour,
+            self.camera_metrics.detection_frame,
             self.stop_event,
         )
         camera_watchdog.start()

web/public/locales/en/views/system.json

@@ -151,6 +151,17 @@
       "cameraDetectionsPerSecond": "{{camName}} detections per second",
       "cameraSkippedDetectionsPerSecond": "{{camName}} skipped detections per second"
     },
+    "connectionQuality": {
+      "title": "Connection Quality",
+      "excellent": "Excellent",
+      "fair": "Fair",
+      "poor": "Poor",
+      "unusable": "Unusable",
+      "fps": "FPS",
+      "expectedFps": "Expected FPS",
+      "reconnectsLastHour": "Reconnects (last hour)",
+      "stallsLastHour": "Stalls (last hour)"
+    },
     "toast": {
       "success": {
         "copyToClipboard": "Copied probe data to clipboard."

web/src/components/camera/ConnectionQualityIndicator.tsx

@@ -0,0 +1,76 @@
+import { useTranslation } from "react-i18next";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
+import { cn } from "@/lib/utils";
+
+type ConnectionQualityIndicatorProps = {
+  quality: "excellent" | "fair" | "poor" | "unusable";
+  expectedFps: number;
+  reconnects: number;
+  stalls: number;
+};
+
+export function ConnectionQualityIndicator({
+  quality,
+  expectedFps,
+  reconnects,
+  stalls,
+}: ConnectionQualityIndicatorProps) {
+  const { t } = useTranslation(["views/system"]);
+
+  const getColorClass = (quality: string): string => {
+    switch (quality) {
+      case "excellent":
+        return "bg-success";
+      case "fair":
+        return "bg-yellow-500";
+      case "poor":
+        return "bg-orange-500";
+      case "unusable":
+        return "bg-destructive";
+      default:
+        return "bg-gray-500";
+    }
+  };
+
+  const qualityLabel = t(`cameras.connectionQuality.${quality}`);
+
+  return (
+    <Tooltip>
+      <TooltipTrigger asChild>
+        <div
+          className={cn(
+            "inline-block size-3 cursor-pointer rounded-full",
+            getColorClass(quality),
+          )}
+        />
+      </TooltipTrigger>
+      <TooltipContent className="max-w-xs">
+        <div className="space-y-2">
+          <div className="font-semibold">
+            {t("cameras.connectionQuality.title")}
+          </div>
+          <div className="text-sm">
+            <div className="capitalize">{qualityLabel}</div>
+            <div className="mt-2 space-y-1 text-xs">
+              <div>
+                {t("cameras.connectionQuality.expectedFps")}:{" "}
+                {expectedFps.toFixed(1)} {t("cameras.connectionQuality.fps")}
+              </div>
+              <div>
+                {t("cameras.connectionQuality.reconnectsLastHour")}:{" "}
+                {reconnects}
+              </div>
+              <div>
+                {t("cameras.connectionQuality.stallsLastHour")}: {stalls}
+              </div>
+            </div>
+          </div>
+        </div>
+      </TooltipContent>
+    </Tooltip>
+  );
+}

web/src/types/stats.ts

@@ -24,6 +24,10 @@ export type CameraStats = {
   pid: number;
   process_fps: number;
   skipped_fps: number;
+  connection_quality: "excellent" | "fair" | "poor" | "unusable";
+  expected_fps: number;
+  reconnects_last_hour: number;
+  stalls_last_hour: number;
 };
 
 export type CpuStats = {

web/src/views/system/CameraMetrics.tsx

@@ -1,6 +1,7 @@
 import { useFrigateStats } from "@/api/ws";
 import { CameraLineGraph } from "@/components/graph/LineGraph";
 import CameraInfoDialog from "@/components/overlay/CameraInfoDialog";
+import { ConnectionQualityIndicator } from "@/components/camera/ConnectionQualityIndicator";
 import { Skeleton } from "@/components/ui/skeleton";
 import { FrigateConfig } from "@/types/frigateConfig";
 import { FrigateStats } from "@/types/stats";
@@ -282,8 +283,37 @@ export default function CameraMetrics({
                   )}
                   <div className="flex w-full flex-col gap-3">
                     <div className="flex flex-row items-center justify-between">
-                      <div className="text-sm font-medium text-muted-foreground smart-capitalize">
-                        <CameraNameLabel camera={camera} />
+                      <div className="flex items-center gap-2">
+                        <div className="text-sm font-medium text-muted-foreground smart-capitalize">
+                          <CameraNameLabel camera={camera} />
+                        </div>
+                        {statsHistory.length > 0 &&
+                          statsHistory[statsHistory.length - 1]?.cameras[
+                            camera.name
+                          ] && (
+                            <ConnectionQualityIndicator
+                              quality={
+                                statsHistory[statsHistory.length - 1]?.cameras[
+                                  camera.name
+                                ]?.connection_quality
+                              }
+                              expectedFps={
+                                statsHistory[statsHistory.length - 1]?.cameras[
+                                  camera.name
+                                ]?.expected_fps || 0
+                              }
+                              reconnects={
+                                statsHistory[statsHistory.length - 1]?.cameras[
+                                  camera.name
+                                ]?.reconnects_last_hour || 0
+                              }
+                              stalls={
+                                statsHistory[statsHistory.length - 1]?.cameras[
+                                  camera.name
+                                ]?.stalls_last_hour || 0
+                              }
+                            />
+                          )}
                       </div>
                       <Tooltip>
                         <TooltipTrigger>