From 0096a6d7783907dc42e5534a8847fcd6b3e10022 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 3 Apr 2024 21:22:11 -0600 Subject: [PATCH] Implement general page of system graphs (#10815) * Reorganize stats and show graphs in system metrics * Break apart all cpu / mem graphs * Auto update stats * Show camera graphs * Get system graphs working for inference time * Update stats every 10 seconds, keeping the last 10 minutes * Use types for thresholds * Use keys api * Break system metrics into different pages * Add dialog for viewing and copying vainfo * remove unused for now * Formatting * Make tooltip match theme * Make betters color in light mode * Include gpu * Make scaling consistent * Fix name * address feedback --- frigate/api/app.py | 7 +- frigate/stats/emitter.py | 39 +- web/src/components/graph/SystemGraph.tsx | 126 ++++ web/src/components/navigation/Bottombar.tsx | 5 +- web/src/components/overlay/VainfoDialog.tsx | 57 ++ web/src/hooks/use-stats.ts | 13 +- web/src/pages/Logs.tsx | 2 +- web/src/pages/System.tsx | 599 +++++++++++++++++++- web/src/types/graph.ts | 40 ++ web/src/types/stats.ts | 6 + web/vite.config.ts | 12 +- 11 files changed, 884 insertions(+), 22 deletions(-) create mode 100644 web/src/components/graph/SystemGraph.tsx create mode 100644 web/src/components/overlay/VainfoDialog.tsx diff --git a/frigate/api/app.py b/frigate/api/app.py index 3900aac05..d307e9384 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -127,7 +127,12 @@ def stats(): @bp.route("/stats/history") def stats_history(): - return jsonify(current_app.stats_emitter.get_stats_history()) + keys = request.args.get("keys", default=None) + + if keys: + keys = keys.split(",") + + return jsonify(current_app.stats_emitter.get_stats_history(keys)) @bp.route("/config") diff --git a/frigate/stats/emitter.py b/frigate/stats/emitter.py index 048436514..2c29548e9 100644 --- a/frigate/stats/emitter.py +++ b/frigate/stats/emitter.py @@ -1,10 +1,12 @@ """Emit stats to listeners.""" +import itertools import json import logging import threading import time from multiprocessing.synchronize import Event as MpEvent +from typing import Optional from frigate.comms.inter_process import InterProcessRequestor from frigate.config import FrigateConfig @@ -14,6 +16,9 @@ from frigate.types import StatsTrackingTypes logger = logging.getLogger(__name__) +MAX_STATS_POINTS = 120 + + class StatsEmitter(threading.Thread): def __init__( self, @@ -43,19 +48,43 @@ class StatsEmitter(threading.Thread): self.stats_history.append(stats) return stats - def get_stats_history(self) -> list[dict[str, any]]: + def get_stats_history( + self, keys: Optional[list[str]] = None + ) -> list[dict[str, any]]: """Get stats history.""" - return self.stats_history + if not keys: + return self.stats_history + + selected_stats: list[dict[str, any]] = [] + + for s in self.stats_history: + selected = {} + + for k in keys: + selected[k] = s.get(k) + + selected_stats.append(selected) + + return selected_stats def run(self) -> None: time.sleep(10) - while not self.stop_event.wait(self.config.mqtt.stats_interval): + for counter in itertools.cycle( + range(int(self.config.mqtt.stats_interval / 10)) + ): + if self.stop_event.wait(10): + break + logger.debug("Starting stats collection") stats = stats_snapshot( self.config, self.stats_tracking, self.hwaccel_errors ) self.stats_history.append(stats) - self.stats_history = self.stats_history[-10:] - self.requestor.send_data("stats", json.dumps(stats)) + self.stats_history = self.stats_history[-MAX_STATS_POINTS:] + + if counter == 0: + self.requestor.send_data("stats", json.dumps(stats)) + logger.debug("Finished stats collection") + logger.info("Exiting stats emitter...") diff --git a/web/src/components/graph/SystemGraph.tsx b/web/src/components/graph/SystemGraph.tsx new file mode 100644 index 000000000..ec750dceb --- /dev/null +++ b/web/src/components/graph/SystemGraph.tsx @@ -0,0 +1,126 @@ +import { useTheme } from "@/context/theme-provider"; +import { FrigateConfig } from "@/types/frigateConfig"; +import { Threshold } from "@/types/graph"; +import { useCallback, useEffect, useMemo } from "react"; +import Chart from "react-apexcharts"; +import useSWR from "swr"; + +type SystemGraphProps = { + graphId: string; + name: string; + unit: string; + threshold: Threshold; + updateTimes: number[]; + data: ApexAxisChartSeries; +}; +export default function SystemGraph({ + graphId, + name, + unit, + threshold, + updateTimes, + data, +}: SystemGraphProps) { + const { data: config } = useSWR("config", { + revalidateOnFocus: false, + }); + + const lastValue = useMemo( + // @ts-expect-error y is valid + () => data[0].data[data[0].data.length - 1]?.y ?? 0, + [data], + ); + + const { theme, systemTheme } = useTheme(); + + const formatTime = useCallback( + (val: unknown) => { + const date = new Date(updateTimes[Math.round(val as number)] * 1000); + return date.toLocaleTimeString([], { + hour12: config?.ui.time_format != "24hour", + hour: "2-digit", + minute: "2-digit", + }); + }, + [config, updateTimes], + ); + + const options = useMemo(() => { + return { + chart: { + id: graphId, + selection: { + enabled: false, + }, + toolbar: { + show: false, + }, + zoom: { + enabled: false, + }, + }, + colors: [ + ({ value }: { value: number }) => { + if (value >= threshold.error) { + return "#FA5252"; + } else if (value >= threshold.warning) { + return "#FF9966"; + } else { + return (systemTheme || theme) == "dark" ? "#404040" : "#E5E5E5"; + } + }, + ], + grid: { + show: false, + }, + legend: { + show: false, + }, + dataLabels: { + enabled: false, + }, + plotOptions: { + bar: { + distributed: true, + }, + }, + tooltip: { + theme: systemTheme || theme, + }, + xaxis: { + tickAmount: 6, + labels: { + formatter: formatTime, + }, + axisBorder: { + show: false, + }, + axisTicks: { + show: false, + }, + }, + yaxis: { + show: false, + min: 0, + max: threshold.warning + 10, + }, + }; + }, [graphId, threshold, systemTheme, theme, formatTime]); + + useEffect(() => { + ApexCharts.exec(graphId, "updateOptions", options, true, true); + }, [graphId, options]); + + return ( +
+
+
{name}
+
+ {lastValue} + {unit} +
+
+ +
+ ); +} diff --git a/web/src/components/navigation/Bottombar.tsx b/web/src/components/navigation/Bottombar.tsx index bee4a0616..6334b04b0 100644 --- a/web/src/components/navigation/Bottombar.tsx +++ b/web/src/components/navigation/Bottombar.tsx @@ -57,7 +57,10 @@ function StatusAlertNav() {
{potentialProblems.map((prob) => ( -
+
{prob.text}
diff --git a/web/src/components/overlay/VainfoDialog.tsx b/web/src/components/overlay/VainfoDialog.tsx new file mode 100644 index 000000000..51cd78f33 --- /dev/null +++ b/web/src/components/overlay/VainfoDialog.tsx @@ -0,0 +1,57 @@ +import useSWR from "swr"; +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from "../ui/dialog"; +import ActivityIndicator from "../indicators/activity-indicator"; +import { Vainfo } from "@/types/stats"; +import { Button } from "../ui/button"; +import copy from "copy-to-clipboard"; + +type VainfoDialogProps = { + showVainfo: boolean; + setShowVainfo: (show: boolean) => void; +}; +export default function VainfoDialog({ + showVainfo, + setShowVainfo, +}: VainfoDialogProps) { + const { data: vainfo } = useSWR(showVainfo ? "vainfo" : null); + + const onCopyVainfo = async () => { + copy(JSON.stringify(vainfo).replace(/[\\\s]+/gi, "")); + setShowVainfo(false); + }; + + return ( + + + + Vainfo Output + + {vainfo ? ( +
+
Return Code: {vainfo.return_code}
+
+
Process {vainfo.return_code == 0 ? "Output" : "Error"}:
+
+
{vainfo.return_code == 0 ? vainfo.stdout : vainfo.stderr}
+
+ ) : ( + + )} + + + + + + + ); +} diff --git a/web/src/hooks/use-stats.ts b/web/src/hooks/use-stats.ts index 18346538d..3a5c6e9ab 100644 --- a/web/src/hooks/use-stats.ts +++ b/web/src/hooks/use-stats.ts @@ -1,4 +1,9 @@ import { FrigateConfig } from "@/types/frigateConfig"; +import { + CameraDetectThreshold, + CameraFfmpegThreshold, + InferenceThreshold, +} from "@/types/graph"; import { FrigateStats, PotentialProblem } from "@/types/stats"; import { useMemo } from "react"; import useSWR from "swr"; @@ -15,12 +20,12 @@ export default function useStats(stats: FrigateStats | undefined) { // check detectors for high inference speeds Object.entries(stats["detectors"]).forEach(([key, det]) => { - if (det["inference_speed"] > 100) { + if (det["inference_speed"] > InferenceThreshold.error) { problems.push({ text: `${key} is very slow (${det["inference_speed"]} ms)`, color: "text-danger", }); - } else if (det["inference_speed"] > 50) { + } else if (det["inference_speed"] > InferenceThreshold.warning) { problems.push({ text: `${key} is slow (${det["inference_speed"]} ms)`, color: "text-orange-400", @@ -51,14 +56,14 @@ export default function useStats(stats: FrigateStats | undefined) { stats["cpu_usages"][cam["pid"]]?.cpu_average, ); - if (!isNaN(ffmpegAvg) && ffmpegAvg >= 20.0) { + if (!isNaN(ffmpegAvg) && ffmpegAvg >= CameraFfmpegThreshold.error) { problems.push({ text: `${name.replaceAll("_", " ")} has high FFMPEG CPU usage (${ffmpegAvg}%)`, color: "text-danger", }); } - if (!isNaN(detectAvg) && detectAvg >= 40.0) { + if (!isNaN(detectAvg) && detectAvg >= CameraDetectThreshold.error) { problems.push({ text: `${name.replaceAll("_", " ")} has high detect CPU usage (${detectAvg}%)`, color: "text-danger", diff --git a/web/src/pages/Logs.tsx b/web/src/pages/Logs.tsx index 3983a38d2..9f75c3cd0 100644 --- a/web/src/pages/Logs.tsx +++ b/web/src/pages/Logs.tsx @@ -283,7 +283,7 @@ function Logs() {
("general"); + const [lastUpdated, setLastUpdated] = useState(Date.now() / 1000); + + // stats collection + + const { data: statsSnapshot } = useSWR("stats", { + revalidateOnFocus: false, + }); + return ( - <> - System - +
+
+ { + if (value) { + setPage(value); + } + }} // don't allow the severity to be unselected + > + {Object.values(metrics).map((item) => ( + +
{item}
+ + ))} + + +
+ {lastUpdated && ( +
+ Last refreshed: +
+ )} +
+
+
+
System
+ {statsSnapshot && ( +
+ {statsSnapshot.service.version} +
+ )} +
+ {page == "general" && ( + + )} +
); } export default System; + +/** + * const cameraCpuSeries = useMemo(() => { + if (!statsHistory || statsHistory.length == 0) { + return {}; + } + + const series: { + [cam: string]: { + [key: string]: { name: string; data: { x: object; y: string }[] }; + }; + } = {}; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats) { + return; + } + + const statTime = new Date(stats.service.last_updated * 1000); + + Object.entries(stats.cameras).forEach(([key, camStats]) => { + if (!config?.cameras[key].enabled) { + return; + } + + if (!(key in series)) { + const camName = key.replaceAll("_", " "); + series[key] = {}; + series[key]["ffmpeg"] = { name: `${camName} ffmpeg`, data: [] }; + series[key]["capture"] = { name: `${camName} capture`, data: [] }; + series[key]["detect"] = { name: `${camName} detect`, data: [] }; + } + + series[key]["ffmpeg"].data.push({ + x: statsIdx, + y: stats.cpu_usages[camStats.ffmpeg_pid.toString()]?.cpu ?? 0.0, + }); + series[key]["capture"].data.push({ + x: statsIdx, + y: stats.cpu_usages[camStats.capture_pid?.toString()]?.cpu ?? 0, + }); + series[key]["detect"].data.push({ + x: statsIdx, + y: stats.cpu_usages[camStats.pid.toString()].cpu, + }); + }); + }); + return series; + }, [statsHistory]); + const cameraFpsSeries = useMemo(() => { + if (!statsHistory) { + return {}; + } + + const series: { + [cam: string]: { + [key: string]: { name: string; data: { x: object; y: number }[] }; + }; + } = {}; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats) { + return; + } + + const statTime = new Date(stats.service.last_updated * 1000); + + Object.entries(stats.cameras).forEach(([key, camStats]) => { + if (!(key in series)) { + const camName = key.replaceAll("_", " "); + series[key] = {}; + series[key]["det"] = { name: `${camName} detections`, data: [] }; + series[key]["skip"] = { + name: `${camName} skipped detections`, + data: [], + }; + } + + series[key]["det"].data.push({ + x: statsIdx, + y: camStats.detection_fps, + }); + series[key]["skip"].data.push({ + x: statsIdx, + y: camStats.skipped_fps, + }); + }); + }); + return series; + }, [statsHistory]); + * + *
+ Cameras +
+ {config && + Object.values(config.cameras).map((camera) => { + if (camera.enabled) { + return ( +
+ + +
+ ); + } + + return null; + })} +
+ */ + +type GeneralMetricsProps = { + lastUpdated: number; + setLastUpdated: (last: number) => void; +}; +function GeneralMetrics({ lastUpdated, setLastUpdated }: GeneralMetricsProps) { + // extra info + + const [showVainfo, setShowVainfo] = useState(false); + + // stats + + const { data: initialStats } = useSWR( + [ + "stats/history", + { keys: "cpu_usages,detectors,gpu_usages,processes,service" }, + ], + { + revalidateOnFocus: false, + }, + ); + + const [statsHistory, setStatsHistory] = useState([]); + const { payload: updatedStats } = useFrigateStats(); + + useEffect(() => { + if (initialStats == undefined || initialStats.length == 0) { + return; + } + + if (statsHistory.length == 0) { + setStatsHistory(initialStats); + return; + } + + if (!updatedStats) { + return; + } + + if (updatedStats.service.last_updated > lastUpdated) { + setStatsHistory([...statsHistory, updatedStats]); + setLastUpdated(Date.now() / 1000); + } + }, [initialStats, updatedStats, statsHistory, lastUpdated, setLastUpdated]); + + // timestamps + + const updateTimes = useMemo( + () => statsHistory.map((stats) => stats.service.last_updated), + [statsHistory], + ); + + // detectors stats + + const detInferenceTimeSeries = useMemo(() => { + if (!statsHistory) { + return []; + } + + const series: { + [key: string]: { name: string; data: { x: number; y: number }[] }; + } = {}; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats) { + return; + } + + Object.entries(stats.detectors).forEach(([key, stats]) => { + if (!(key in series)) { + series[key] = { name: key, data: [] }; + } + + series[key].data.push({ x: statsIdx, y: stats.inference_speed }); + }); + }); + return Object.values(series); + }, [statsHistory]); + + const detCpuSeries = useMemo(() => { + if (!statsHistory) { + return []; + } + + const series: { + [key: string]: { name: string; data: { x: number; y: string }[] }; + } = {}; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats) { + return; + } + + Object.entries(stats.detectors).forEach(([key, detStats]) => { + if (!(key in series)) { + series[key] = { name: key, data: [] }; + } + + series[key].data.push({ + x: statsIdx, + y: stats.cpu_usages[detStats.pid.toString()].cpu, + }); + }); + }); + return Object.values(series); + }, [statsHistory]); + + const detMemSeries = useMemo(() => { + if (!statsHistory) { + return []; + } + + const series: { + [key: string]: { name: string; data: { x: number; y: string }[] }; + } = {}; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats) { + return; + } + + Object.entries(stats.detectors).forEach(([key, detStats]) => { + if (!(key in series)) { + series[key] = { name: key, data: [] }; + } + + series[key].data.push({ + x: statsIdx, + y: stats.cpu_usages[detStats.pid.toString()].mem, + }); + }); + }); + return Object.values(series); + }, [statsHistory]); + + // gpu stats + + const gpuSeries = useMemo(() => { + if (!statsHistory) { + return []; + } + + const series: { + [key: string]: { name: string; data: { x: number; y: string }[] }; + } = {}; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats) { + return; + } + + Object.entries(stats.gpu_usages || []).forEach(([key, stats]) => { + if (!(key in series)) { + series[key] = { name: key, data: [] }; + } + + series[key].data.push({ x: statsIdx, y: stats.gpu }); + }); + }); + return Object.keys(series).length > 0 ? Object.values(series) : []; + }, [statsHistory]); + + const gpuMemSeries = useMemo(() => { + if (!statsHistory) { + return []; + } + + const series: { + [key: string]: { name: string; data: { x: number; y: string }[] }; + } = {}; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats) { + return; + } + + Object.entries(stats.gpu_usages || {}).forEach(([key, stats]) => { + if (!(key in series)) { + series[key] = { name: key, data: [] }; + } + + series[key].data.push({ x: statsIdx, y: stats.mem }); + }); + }); + return Object.values(series); + }, [statsHistory]); + + // other processes stats + + const otherProcessCpuSeries = useMemo(() => { + if (!statsHistory) { + return []; + } + + const series: { + [key: string]: { name: string; data: { x: number; y: string }[] }; + } = {}; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats) { + return; + } + + Object.entries(stats.processes).forEach(([key, procStats]) => { + if (procStats.pid.toString() in stats.cpu_usages) { + if (!(key in series)) { + series[key] = { name: key, data: [] }; + } + + series[key].data.push({ + x: statsIdx, + y: stats.cpu_usages[procStats.pid.toString()].cpu, + }); + } + }); + }); + return Object.keys(series).length > 0 ? Object.values(series) : []; + }, [statsHistory]); + + const otherProcessMemSeries = useMemo(() => { + if (!statsHistory) { + return []; + } + + const series: { + [key: string]: { name: string; data: { x: number; y: string }[] }; + } = {}; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats) { + return; + } + + Object.entries(stats.processes).forEach(([key, procStats]) => { + if (procStats.pid.toString() in stats.cpu_usages) { + if (!(key in series)) { + series[key] = { name: key, data: [] }; + } + + series[key].data.push({ + x: statsIdx, + y: stats.cpu_usages[procStats.pid.toString()].mem, + }); + } + }); + }); + return Object.values(series); + }, [statsHistory]); + + if (statsHistory.length == 0) { + return; + } + + return ( + <> + + +
+
+ Detectors +
+
+
+
Detector Inference Speed
+ {detInferenceTimeSeries.map((series) => ( + + ))} +
+
+
Detector CPU Usage
+ {detCpuSeries.map((series) => ( + + ))} +
+
+
Detector Memory Usage
+ {detMemSeries.map((series) => ( + + ))} +
+
+ + {statsHistory.length > 0 && statsHistory[0].gpu_usages && ( + <> +
+
+ GPUs +
+ {Object.keys(statsHistory[0].gpu_usages).filter( + (key) => + key == "amd-vaapi" || + key == "intel-vaapi" || + key == "intel-qsv", + ).length > 0 && ( + + )} +
+
+
+
GPU Usage
+ {gpuSeries.map((series) => ( + + ))} +
+
+
GPU Memory
+ {gpuMemSeries.map((series) => ( + + ))} +
+
+ + )} + +
+ Other Processes +
+
+
+
Process CPU Usage
+ {otherProcessCpuSeries.map((series) => ( + + ))} +
+
+
Process Memory Usage
+ {otherProcessMemSeries.map((series) => ( + + ))} +
+
+
+ + ); +} diff --git a/web/src/types/graph.ts b/web/src/types/graph.ts index de154480f..4963142e8 100644 --- a/web/src/types/graph.ts +++ b/web/src/types/graph.ts @@ -7,3 +7,43 @@ export type GraphData = { name?: string; data: GraphDataPoint[]; }; + +export type Threshold = { + warning: number; + error: number; +}; + +export const InferenceThreshold = { + warning: 50, + error: 100, +} as Threshold; + +export const DetectorCpuThreshold = { + warning: 25, + error: 50, +} as Threshold; + +export const DetectorMemThreshold = { + warning: 20, + error: 50, +} as Threshold; + +export const GPUUsageThreshold = { + warning: 75, + error: 95, +} as Threshold; + +export const GPUMemThreshold = { + warning: 75, + error: 95, +} as Threshold; + +export const CameraFfmpegThreshold = { + warning: 20, + error: 20, +} as Threshold; + +export const CameraDetectThreshold = { + warning: 20, + error: 40, +} as Threshold; diff --git a/web/src/types/stats.ts b/web/src/types/stats.ts index 831e2e639..3b293f76a 100644 --- a/web/src/types/stats.ts +++ b/web/src/types/stats.ts @@ -63,3 +63,9 @@ export type PotentialProblem = { text: string; color: string; }; + +export type Vainfo = { + return_code: number; + stdout: string; + stderr: string; +}; diff --git a/web/vite.config.ts b/web/vite.config.ts index cc3ead707..5afefa331 100644 --- a/web/vite.config.ts +++ b/web/vite.config.ts @@ -12,24 +12,24 @@ export default defineConfig({ server: { proxy: { "/api": { - target: "http://192.168.50.106:5000", + target: "http://localhost:5000", ws: true, }, "/vod": { - target: "http://192.168.50.106:5000", + target: "http://localhost:5000", }, "/clips": { - target: "http://192.168.50.106:5000", + target: "http://localhost:5000", }, "/exports": { - target: "http://192.168.50.106:5000", + target: "http://localhost:5000", }, "/ws": { - target: "ws://192.168.50.106:5000", + target: "ws://localhost:5000", ws: true, }, "/live": { - target: "ws://192.168.50.106:5000", + target: "ws://localhost:5000", changeOrigin: true, ws: true, },