From 7cc16161b3dff0c15e7bf1e2200fc7b26bbdd803 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 15 Dec 2025 15:02:03 -0600 Subject: [PATCH] Camera connection quality indicator (#21297) * add camera connection quality metrics and indicator * formatting * move stall calcs to watchdog * clean up * change watchdog to 1s and separately track time for ffmpeg retry_interval * implement status caching to reduce message volume --- frigate/camera/__init__.py | 4 + frigate/stats/util.py | 27 ++++ frigate/video.py | 121 +++++++++++++++--- web/public/locales/en/views/system.json | 11 ++ .../camera/ConnectionQualityIndicator.tsx | 76 +++++++++++ web/src/types/stats.ts | 4 + web/src/views/system/CameraMetrics.tsx | 34 ++++- 7 files changed, 256 insertions(+), 21 deletions(-) create mode 100644 web/src/components/camera/ConnectionQualityIndicator.tsx diff --git a/frigate/camera/__init__.py b/frigate/camera/__init__.py index 77b1fd424..0461c98cb 100644 --- a/frigate/camera/__init__.py +++ b/frigate/camera/__init__.py @@ -19,6 +19,8 @@ class CameraMetrics: process_pid: Synchronized capture_process_pid: Synchronized ffmpeg_pid: Synchronized + reconnects_last_hour: Synchronized + stalls_last_hour: Synchronized def __init__(self, manager: SyncManager): self.camera_fps = manager.Value("d", 0) @@ -35,6 +37,8 @@ class CameraMetrics: self.process_pid = manager.Value("i", 0) self.capture_process_pid = manager.Value("i", 0) self.ffmpeg_pid = manager.Value("i", 0) + self.reconnects_last_hour = manager.Value("i", 0) + self.stalls_last_hour = manager.Value("i", 0) class PTZMetrics: diff --git a/frigate/stats/util.py b/frigate/stats/util.py index 17b45d1d4..c83b5dac3 100644 --- a/frigate/stats/util.py +++ b/frigate/stats/util.py @@ -279,6 +279,32 @@ def stats_snapshot( if camera_stats.capture_process_pid.value else None ) + # Calculate connection quality based on current state + # This is computed at stats-collection time so offline cameras + # correctly show as unusable rather than excellent + expected_fps = config.cameras[name].detect.fps + current_fps = camera_stats.camera_fps.value + reconnects = camera_stats.reconnects_last_hour.value + stalls = camera_stats.stalls_last_hour.value + + if current_fps < 0.1: + quality_str = "unusable" + elif reconnects == 0 and current_fps >= 0.9 * expected_fps and stalls < 5: + quality_str = "excellent" + elif reconnects <= 2 and current_fps >= 0.6 * expected_fps: + quality_str = "fair" + elif reconnects > 10 or current_fps < 1.0 or stalls > 100: + quality_str = "unusable" + else: + quality_str = "poor" + + connection_quality = { + "connection_quality": quality_str, + "expected_fps": expected_fps, + "reconnects_last_hour": reconnects, + "stalls_last_hour": stalls, + } + stats["cameras"][name] = { "camera_fps": round(camera_stats.camera_fps.value, 2), "process_fps": round(camera_stats.process_fps.value, 2), @@ -290,6 +316,7 @@ def stats_snapshot( "ffmpeg_pid": ffmpeg_pid, "audio_rms": round(camera_stats.audio_rms.value, 4), "audio_dBFS": round(camera_stats.audio_dBFS.value, 4), + **connection_quality, } stats["detectors"] = {} diff --git a/frigate/video.py b/frigate/video.py index a139c25f1..24c5ca462 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -3,6 +3,7 @@ import queue import subprocess as sp import threading import time +from collections import deque from datetime import datetime, timedelta, timezone from multiprocessing import Queue, Value from multiprocessing.synchronize import Event as MpEvent @@ -115,6 +116,7 @@ def capture_frames( frame_rate.start() skipped_eps = EventsPerSecond() skipped_eps.start() + config_subscriber = CameraConfigUpdateSubscriber( None, {config.name: config}, [CameraConfigUpdateEnum.enabled] ) @@ -179,6 +181,9 @@ class CameraWatchdog(threading.Thread): camera_fps, skipped_fps, ffmpeg_pid, + stalls, + reconnects, + detection_frame, stop_event, ): threading.Thread.__init__(self) @@ -199,6 +204,10 @@ class CameraWatchdog(threading.Thread): self.frame_index = 0 self.stop_event = stop_event self.sleeptime = self.config.ffmpeg.retry_interval + self.reconnect_timestamps = deque() + self.stalls = stalls + self.reconnects = reconnects + self.detection_frame = detection_frame self.config_subscriber = CameraConfigUpdateSubscriber( None, @@ -213,6 +222,35 @@ class CameraWatchdog(threading.Thread): self.latest_invalid_segment_time: float = 0 self.latest_cache_segment_time: float = 0 + # Stall tracking (based on last processed frame) + self._stall_timestamps: deque[float] = deque() + self._stall_active: bool = False + + # Status caching to reduce message volume + self._last_detect_status: str | None = None + self._last_record_status: str | None = None + self._last_status_update_time: float = 0.0 + + def _send_detect_status(self, status: str, now: float) -> None: + """Send detect status only if changed or retry_interval has elapsed.""" + if ( + status != self._last_detect_status + or (now - self._last_status_update_time) >= self.sleeptime + ): + self.requestor.send_data(f"{self.config.name}/status/detect", status) + self._last_detect_status = status + self._last_status_update_time = now + + def _send_record_status(self, status: str, now: float) -> None: + """Send record status only if changed or retry_interval has elapsed.""" + if ( + status != self._last_record_status + or (now - self._last_status_update_time) >= self.sleeptime + ): + self.requestor.send_data(f"{self.config.name}/status/record", status) + self._last_record_status = status + self._last_status_update_time = now + def _update_enabled_state(self) -> bool: """Fetch the latest config and update enabled state.""" self.config_subscriber.check_for_updates() @@ -239,6 +277,14 @@ class CameraWatchdog(threading.Thread): else: self.ffmpeg_detect_process.wait() + # Update reconnects + now = datetime.now().timestamp() + self.reconnect_timestamps.append(now) + while self.reconnect_timestamps and self.reconnect_timestamps[0] < now - 3600: + self.reconnect_timestamps.popleft() + if self.reconnects: + self.reconnects.value = len(self.reconnect_timestamps) + # Wait for old capture thread to fully exit before starting a new one if self.capture_thread is not None and self.capture_thread.is_alive(): self.logger.info("Waiting for capture thread to exit...") @@ -261,7 +307,10 @@ class CameraWatchdog(threading.Thread): self.start_all_ffmpeg() time.sleep(self.sleeptime) - while not self.stop_event.wait(self.sleeptime): + last_restart_time = datetime.now().timestamp() + + # 1 second watchdog loop + while not self.stop_event.wait(1): enabled = self._update_enabled_state() if enabled != self.was_enabled: if enabled: @@ -277,12 +326,9 @@ class CameraWatchdog(threading.Thread): self.stop_all_ffmpeg() # update camera status - self.requestor.send_data( - f"{self.config.name}/status/detect", "disabled" - ) - self.requestor.send_data( - f"{self.config.name}/status/record", "disabled" - ) + now = datetime.now().timestamp() + self._send_detect_status("disabled", now) + self._send_record_status("disabled", now) self.was_enabled = enabled continue @@ -321,36 +367,44 @@ class CameraWatchdog(threading.Thread): now = datetime.now().timestamp() + # Check if enough time has passed to allow ffmpeg restart (backoff pacing) + time_since_last_restart = now - last_restart_time + can_restart = time_since_last_restart >= self.sleeptime + if not self.capture_thread.is_alive(): - self.requestor.send_data(f"{self.config.name}/status/detect", "offline") + self._send_detect_status("offline", now) self.camera_fps.value = 0 self.logger.error( f"Ffmpeg process crashed unexpectedly for {self.config.name}." ) - self.reset_capture_thread(terminate=False) + if can_restart: + self.reset_capture_thread(terminate=False) + last_restart_time = now elif self.camera_fps.value >= (self.config.detect.fps + 10): self.fps_overflow_count += 1 if self.fps_overflow_count == 3: - self.requestor.send_data( - f"{self.config.name}/status/detect", "offline" - ) + self._send_detect_status("offline", now) self.fps_overflow_count = 0 self.camera_fps.value = 0 self.logger.info( f"{self.config.name} exceeded fps limit. Exiting ffmpeg..." ) - self.reset_capture_thread(drain_output=False) + if can_restart: + self.reset_capture_thread(drain_output=False) + last_restart_time = now elif now - self.capture_thread.current_frame.value > 20: - self.requestor.send_data(f"{self.config.name}/status/detect", "offline") + self._send_detect_status("offline", now) self.camera_fps.value = 0 self.logger.info( f"No frames received from {self.config.name} in 20 seconds. Exiting ffmpeg..." ) - self.reset_capture_thread() + if can_restart: + self.reset_capture_thread() + last_restart_time = now else: # process is running normally - self.requestor.send_data(f"{self.config.name}/status/detect", "online") + self._send_detect_status("online", now) self.fps_overflow_count = 0 for p in self.ffmpeg_other_processes: @@ -421,9 +475,7 @@ class CameraWatchdog(threading.Thread): continue else: - self.requestor.send_data( - f"{self.config.name}/status/record", "online" - ) + self._send_record_status("online", now) p["latest_segment_time"] = self.latest_cache_segment_time if poll is None: @@ -439,6 +491,34 @@ class CameraWatchdog(threading.Thread): p["cmd"], self.logger, p["logpipe"], ffmpeg_process=p["process"] ) + # Update stall metrics based on last processed frame timestamp + now = datetime.now().timestamp() + processed_ts = ( + float(self.detection_frame.value) if self.detection_frame else 0.0 + ) + if processed_ts > 0: + delta = now - processed_ts + observed_fps = ( + self.camera_fps.value + if self.camera_fps.value > 0 + else self.config.detect.fps + ) + interval = 1.0 / max(observed_fps, 0.1) + stall_threshold = max(2.0 * interval, 2.0) + + if delta > stall_threshold: + if not self._stall_active: + self._stall_timestamps.append(now) + self._stall_active = True + else: + self._stall_active = False + + while self._stall_timestamps and self._stall_timestamps[0] < now - 3600: + self._stall_timestamps.popleft() + + if self.stalls: + self.stalls.value = len(self._stall_timestamps) + self.stop_all_ffmpeg() self.logpipe.close() self.config_subscriber.stop() @@ -576,6 +656,9 @@ class CameraCapture(FrigateProcess): self.camera_metrics.camera_fps, self.camera_metrics.skipped_fps, self.camera_metrics.ffmpeg_pid, + self.camera_metrics.stalls_last_hour, + self.camera_metrics.reconnects_last_hour, + self.camera_metrics.detection_frame, self.stop_event, ) camera_watchdog.start() diff --git a/web/public/locales/en/views/system.json b/web/public/locales/en/views/system.json index 73c6d65b5..1ba46153e 100644 --- a/web/public/locales/en/views/system.json +++ b/web/public/locales/en/views/system.json @@ -151,6 +151,17 @@ "cameraDetectionsPerSecond": "{{camName}} detections per second", "cameraSkippedDetectionsPerSecond": "{{camName}} skipped detections per second" }, + "connectionQuality": { + "title": "Connection Quality", + "excellent": "Excellent", + "fair": "Fair", + "poor": "Poor", + "unusable": "Unusable", + "fps": "FPS", + "expectedFps": "Expected FPS", + "reconnectsLastHour": "Reconnects (last hour)", + "stallsLastHour": "Stalls (last hour)" + }, "toast": { "success": { "copyToClipboard": "Copied probe data to clipboard." diff --git a/web/src/components/camera/ConnectionQualityIndicator.tsx b/web/src/components/camera/ConnectionQualityIndicator.tsx new file mode 100644 index 000000000..3ea3c4f19 --- /dev/null +++ b/web/src/components/camera/ConnectionQualityIndicator.tsx @@ -0,0 +1,76 @@ +import { useTranslation } from "react-i18next"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { cn } from "@/lib/utils"; + +type ConnectionQualityIndicatorProps = { + quality: "excellent" | "fair" | "poor" | "unusable"; + expectedFps: number; + reconnects: number; + stalls: number; +}; + +export function ConnectionQualityIndicator({ + quality, + expectedFps, + reconnects, + stalls, +}: ConnectionQualityIndicatorProps) { + const { t } = useTranslation(["views/system"]); + + const getColorClass = (quality: string): string => { + switch (quality) { + case "excellent": + return "bg-success"; + case "fair": + return "bg-yellow-500"; + case "poor": + return "bg-orange-500"; + case "unusable": + return "bg-destructive"; + default: + return "bg-gray-500"; + } + }; + + const qualityLabel = t(`cameras.connectionQuality.${quality}`); + + return ( + + +
+ + +
+
+ {t("cameras.connectionQuality.title")} +
+
+
{qualityLabel}
+
+
+ {t("cameras.connectionQuality.expectedFps")}:{" "} + {expectedFps.toFixed(1)} {t("cameras.connectionQuality.fps")} +
+
+ {t("cameras.connectionQuality.reconnectsLastHour")}:{" "} + {reconnects} +
+
+ {t("cameras.connectionQuality.stallsLastHour")}: {stalls} +
+
+
+
+ + + ); +} diff --git a/web/src/types/stats.ts b/web/src/types/stats.ts index c98ebe80f..5432f3154 100644 --- a/web/src/types/stats.ts +++ b/web/src/types/stats.ts @@ -24,6 +24,10 @@ export type CameraStats = { pid: number; process_fps: number; skipped_fps: number; + connection_quality: "excellent" | "fair" | "poor" | "unusable"; + expected_fps: number; + reconnects_last_hour: number; + stalls_last_hour: number; }; export type CpuStats = { diff --git a/web/src/views/system/CameraMetrics.tsx b/web/src/views/system/CameraMetrics.tsx index 6e24ef5d0..b6c5be4fa 100644 --- a/web/src/views/system/CameraMetrics.tsx +++ b/web/src/views/system/CameraMetrics.tsx @@ -1,6 +1,7 @@ import { useFrigateStats } from "@/api/ws"; import { CameraLineGraph } from "@/components/graph/LineGraph"; import CameraInfoDialog from "@/components/overlay/CameraInfoDialog"; +import { ConnectionQualityIndicator } from "@/components/camera/ConnectionQualityIndicator"; import { Skeleton } from "@/components/ui/skeleton"; import { FrigateConfig } from "@/types/frigateConfig"; import { FrigateStats } from "@/types/stats"; @@ -282,8 +283,37 @@ export default function CameraMetrics({ )}
-
- +
+
+ +
+ {statsHistory.length > 0 && + statsHistory[statsHistory.length - 1]?.cameras[ + camera.name + ] && ( + + )}