Revamped debug UI and add camera / process info, ffprobe copying (#4349)

* Move each camera to a separate card and show per process info

* Install top

* Add support for cpu usage stats

* Use cpu usage stats in debug

* Increase number of runs to ensure good results

* Add ffprobe endpoint

* Get ffprobe for multiple inputs

* Copy ffprobe in output

* Add fps to camera metrics

* Fix lint errors

* Update stats config

* Add ffmpeg pid

* Use grid display so more cameras can take less vertical space

* Fix hanging characters

* Only show the current detector

* Fix bad if statement

* Return full output of ffprobe process

* Return full output of ffprobe process

* Don't specify rtsp_transport

* Make ffprobe button show dialog with output and option to copy

* Adjust ffprobe api to take paths directly

* Add docs for ffprobe api

docker/Dockerfile

@@ -63,6 +63,7 @@ RUN apt-get -qq update \
     apt-transport-https \
     gnupg \
     wget \
+    procps \
     unzip tzdata libxml2 xz-utils \
     python3-pip \
     # add raspberry pi repo

docs/docs/integrations/api.md

@@ -264,3 +264,11 @@ Get recording segment details for the given timestamp range.
 | -------- | ---- | ------------------------------------- |
 | `after`  | int  | Unix timestamp for beginning of range |
 | `before` | int  | Unix timestamp for end of range       |
+
+### `GET /api/ffprobe`
+
+Get ffprobe output for camera feed paths.
+
+| param   | Type   | Description                        |
+| ------- | ------ | ---------------------------------- |
+| `paths` | string | `,` separated list of camera paths |

frigate/http.py

@@ -1,8 +1,8 @@
 import base64
-from collections import OrderedDict
 from datetime import datetime, timedelta
 import copy
 import logging
+import json
 import os
 import subprocess as sp
 import time
@@ -28,9 +28,9 @@ from playhouse.shortcuts import model_to_dict
 
 from frigate.const import CLIPS_DIR
 from frigate.models import Event, Recordings
-from frigate.object_processing import TrackedObject, TrackedObjectProcessor
+from frigate.object_processing import TrackedObject
 from frigate.stats import stats_snapshot
-from frigate.util import clean_camera_user_pass
+from frigate.util import clean_camera_user_pass, ffprobe_stream
 from frigate.version import VERSION
 
 logger = logging.getLogger(__name__)
@@ -957,3 +957,37 @@ def imagestream(detected_frames_processor, camera_name, fps, height, draw_option
             b"--frame\r\n"
             b"Content-Type: image/jpeg\r\n\r\n" + jpg.tobytes() + b"\r\n\r\n"
         )
+
+
+@bp.route("/ffprobe", methods=["GET"])
+def ffprobe():
+    path_param = request.args.get("paths", "")
+
+    if not path_param:
+        return jsonify(
+            {"success": False, "message": f"Path needs to be provided."}, "404"
+        )
+
+    if "," in clean_camera_user_pass(path_param):
+        paths = path_param.split(",")
+    else:
+        paths = [path_param]
+
+    # user has multiple streams
+    output = []
+
+    for path in paths:
+        ffprobe = ffprobe_stream(path)
+        output.append(
+            {
+                "return_code": ffprobe.returncode,
+                "stderr": json.loads(ffprobe.stderr.decode("unicode_escape").strip())
+                if ffprobe.stderr.decode()
+                else {},
+                "stdout": json.loads(ffprobe.stdout.decode("unicode_escape").strip())
+                if ffprobe.stdout.decode()
+                else {},
+            }
+        )
+
+    return jsonify(output)

frigate/stats.py

@@ -14,6 +14,7 @@ from frigate.config import FrigateConfig
 from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR
 from frigate.types import StatsTrackingTypes, CameraMetricsTypes
 from frigate.version import VERSION
+from frigate.util import get_cpu_stats
 from frigate.object_detection import ObjectDetectProcess
 
 logger = logging.getLogger(__name__)
@@ -90,6 +91,9 @@ def stats_snapshot(stats_tracking: StatsTrackingTypes) -> dict[str, Any]:
     for name, camera_stats in camera_metrics.items():
         total_detection_fps += camera_stats["detection_fps"].value
         pid = camera_stats["process"].pid if camera_stats["process"] else None
+        ffmpeg_pid = (
+            camera_stats["ffmpeg_pid"].value if camera_stats["ffmpeg_pid"] else None
+        )
         cpid = (
             camera_stats["capture_process"].pid
             if camera_stats["capture_process"]
@@ -102,6 +106,7 @@ def stats_snapshot(stats_tracking: StatsTrackingTypes) -> dict[str, Any]:
             "detection_fps": round(camera_stats["detection_fps"].value, 2),
             "pid": pid,
             "capture_pid": cpid,
+            "ffmpeg_pid": ffmpeg_pid,
         }
 
     stats["detectors"] = {}
@@ -114,6 +119,8 @@ def stats_snapshot(stats_tracking: StatsTrackingTypes) -> dict[str, Any]:
         }
     stats["detection_fps"] = round(total_detection_fps, 2)
 
+    stats["cpu_usages"] = get_cpu_stats()
+
     stats["service"] = {
         "uptime": (int(time.time()) - stats_tracking["started"]),
         "version": VERSION,

frigate/util.py

@@ -1,6 +1,8 @@
 import copy
 import datetime
 import logging
+import subprocess as sp
+import json
 import re
 import signal
 import traceback
@@ -679,6 +681,52 @@ def escape_special_characters(path: str) -> str:
         return path
 
 
+def get_cpu_stats() -> dict[str, dict]:
+    """Get cpu usages for each process id"""
+    usages = {}
+    # -n=2 runs to ensure extraneous values are not included
+    top_command = ["top", "-b", "-n", "2"]
+
+    p = sp.run(
+        top_command,
+        encoding="ascii",
+        capture_output=True,
+    )
+
+    if p.returncode != 0:
+        logger.error(p.stderr)
+        return usages
+    else:
+        lines = p.stdout.split("\n")
+
+        for line in lines:
+            stats = list(filter(lambda a: a != "", line.strip().split(" ")))
+            try:
+                usages[stats[0]] = {
+                    "cpu": stats[8],
+                    "mem": stats[9],
+                }
+            except:
+                continue
+
+        return usages
+
+
+def ffprobe_stream(path: str) -> sp.CompletedProcess:
+    """Run ffprobe on stream."""
+    ffprobe_cmd = [
+        "ffprobe",
+        "-print_format",
+        "json",
+        "-show_entries",
+        "stream=codec_long_name,width,height,bit_rate,duration,display_aspect_ratio,avg_frame_rate",
+        "-loglevel",
+        "quiet",
+        path,
+    ]
+    return sp.run(ffprobe_cmd, capture_output=True)
+
+
 class FrameManager(ABC):
     @abstractmethod
     def create(self, name, size) -> AnyStr:

web/__test__/handlers.js

@@ -39,9 +39,10 @@ export const handlers = [
     return res(
       ctx.status(200),
       ctx.json({
+        cpu_usages: { 74: {cpu: 6, mem: 6}, 64: { cpu: 5, mem: 5 }, 54: { cpu: 4, mem: 4 }, 71: { cpu: 3, mem: 3}, 60: {cpu: 2, mem: 2}, 72: {cpu: 1, mem: 1} },
         detection_fps: 0.0,
         detectors: { coral: { detection_start: 0.0, inference_speed: 8.94, pid: 52 } },
-        front: { camera_fps: 5.0, capture_pid: 64, detection_fps: 0.0, pid: 54, process_fps: 0.0, skipped_fps: 0.0 },
+        front: { camera_fps: 5.0, capture_pid: 64, detection_fps: 0.0, pid: 54, process_fps: 0.0, skipped_fps: 0.0, ffmpeg_pid: 72 },
         side: {
           camera_fps: 6.9,
           capture_pid: 71,
@@ -49,6 +50,7 @@ export const handlers = [
           pid: 60,
           process_fps: 0.0,
           skipped_fps: 0.0,
+          ffmpeg_pid: 74,
         },
         service: { uptime: 34812, version: '0.8.1-d376f6b' },
       })

web/src/routes/Debug.jsx

@@ -5,12 +5,15 @@ import Heading from '../components/Heading';
 import Link from '../components/Link';
 import { useMqtt } from '../api/mqtt';
 import useSWR from 'swr';
+import axios from 'axios';
 import { Table, Tbody, Thead, Tr, Th, Td } from '../components/Table';
-import { useCallback } from 'preact/hooks';
+import { useCallback, useState } from 'preact/hooks';
+import Dialog from '../components/Dialog';
 
 const emptyObject = Object.freeze({});
 
 export default function Debug() {
+  const [state, setState] = useState({ showFfprobe: false, ffprobe: '' });
   const { data: config } = useSWR('config');
 
   const {
@@ -18,12 +21,10 @@ export default function Debug() {
   } = useMqtt('stats');
   const { data: initialStats } = useSWR('stats');
 
-  const { detectors, service = {}, detection_fps: _, ...cameras } = stats || initialStats || emptyObject;
+  const { cpu_usages, detectors, service = {}, detection_fps: _, ...cameras } = stats || initialStats || emptyObject;
 
   const detectorNames = Object.keys(detectors || emptyObject);
-  const detectorDataKeys = Object.keys(detectors ? detectors[detectorNames[0]] : emptyObject);
   const cameraNames = Object.keys(cameras || emptyObject);
-  const cameraDataKeys = Object.keys(cameras[cameraNames[0]] || emptyObject);
 
   const handleCopyConfig = useCallback(() => {
     async function copy() {
@@ -32,11 +33,64 @@ export default function Debug() {
     copy();
   }, [config]);
 
+  const onHandleFfprobe = async (camera, e) => {
+    if (e) {
+      e.stopPropagation();
+    }
+
+    setState({ ...state, showFfprobe: true });
+    let paths = '';
+    config.cameras[camera].ffmpeg.inputs.forEach((input) => {
+      if (paths) {
+        paths += ',';
+        paths += input.path;
+      } else {
+        paths = input.path;
+      }
+    });
+    const response = await axios.get('ffprobe', {
+      params: {
+        paths,
+      },
+    });
+
+    if (response.status === 200) {
+      setState({ showFfprobe: true, ffprobe: JSON.stringify(response.data, null, 2) });
+    } else {
+      setState({ ...state, ffprobe: 'There was an error getting the ffprobe output.' });
+    }
+  };
+
+  const onCopyFfprobe = async () => {
+    await window.navigator.clipboard.writeText(JSON.stringify(state.ffprobe, null, 2));
+    setState({ ...state, ffprobe: '', showFfprobe: false });
+  };
+
   return (
     <div className="space-y-4 p-2 px-4">
       <Heading>
         Debug <span className="text-sm">{service.version}</span>
       </Heading>
+      {state.showFfprobe && (
+        <Dialog>
+          <div className="p-4">
+            <Heading size="lg">Ffprobe Output</Heading>
+            {state.ffprobe != '' ? <p className="mb-2">{state.ffprobe}</p> : <ActivityIndicator />}
+          </div>
+          <div className="p-2 flex justify-start flex-row-reverse space-x-2">
+            <Button className="ml-2" onClick={() => onCopyFfprobe()} type="text">
+              Copy
+            </Button>
+            <Button
+              className="ml-2"
+              onClick={() => setState({ ...state, ffprobe: '', showFfprobe: false })}
+              type="text"
+            >
+              Close
+            </Button>
+          </div>
+        </Dialog>
+      )}
 
       {!detectors ? (
         <div>
@@ -44,52 +98,81 @@ export default function Debug() {
         </div>
       ) : (
         <Fragment>
-          <div data-testid="detectors" className="min-w-0 overflow-auto">
+          <Heading size="lg">Detectors</Heading>
-            <Table className="w-full">
+          <div data-testid="detectors" className="grid grid-cols-1 3xl:grid-cols-3 md:grid-cols-2 gap-4">
-              <Thead>
+            {detectorNames.map((detector) => (
-                <Tr>
+              <div key={detector} className="dark:bg-gray-800 shadow-md hover:shadow-lg rounded-lg transition-shadow">
-                  <Th>detector</Th>
+                <div className="text-lg flex justify-between p-4">{detector}</div>
-                  {detectorDataKeys.map((name) => (
+                <div className="p-2">
-                    <Th key={name}>{name.replace('_', ' ')}</Th>
+                  <Table className="w-full">
-                  ))}
+                    <Thead>
-                </Tr>
+                      <Tr>
-              </Thead>
+                        <Th>P-ID</Th>
-              <Tbody>
+                        <Th>Detection Start</Th>
-                {detectorNames.map((detector, i) => (
+                        <Th>Inference Speed</Th>
-                  <Tr key={i} index={i}>
+                      </Tr>
-                    <Td>{detector}</Td>
+                    </Thead>
-                    {detectorDataKeys.map((name) => (
+                    <Tbody>
-                      <Td key={`${name}-${detector}`}>{detectors[detector][name]}</Td>
+                      <Tr>
-                    ))}
+                        <Td>{detectors[detector]['pid']}</Td>
-                  </Tr>
+                        <Td>{detectors[detector]['detection_start']}</Td>
-                ))}
+                        <Td>{detectors[detector]['inference_speed']}</Td>
-              </Tbody>
+                      </Tr>
-            </Table>
+                    </Tbody>
+                  </Table>
+                </div>
+              </div>
+            ))}
           </div>
 
-          <div data-testid="cameras" className="min-w-0 overflow-auto">
+          <Heading size="lg">Cameras</Heading>
-            <Table className="w-full">
+          <div data-testid="cameras" className="grid grid-cols-1 3xl:grid-cols-3 md:grid-cols-2 gap-4">
-              <Thead>
+            {cameraNames.map((camera) => (
-                <Tr>
+              <div key={camera} className="dark:bg-gray-800 shadow-md hover:shadow-lg rounded-lg transition-shadow">
-                  <Th>camera</Th>
+                <div className="text-lg flex justify-between p-4">
-                  {cameraDataKeys.map((name) => (
+                  <Link href={`/cameras/${camera}`}>{camera.replaceAll('_', ' ')}</Link>
-                    <Th key={name}>{name.replace('_', ' ')}</Th>
+                  <Button onClick={(e) => onHandleFfprobe(camera, e)}>ffprobe</Button>
-                  ))}
+                </div>
-                </Tr>
+                <div className="p-2">
-              </Thead>
+                  <Table className="w-full">
-              <Tbody>
+                    <Thead>
-                {cameraNames.map((camera, i) => (
+                      <Tr>
-                  <Tr key={i} index={i}>
+                        <Th>Process</Th>
-                    <Td>
+                        <Th>P-ID</Th>
-                      <Link href={`/cameras/${camera}`}>{camera.replaceAll('_', ' ')}</Link>
+                        <Th>fps</Th>
-                    </Td>
+                        <Th>Cpu %</Th>
-                    {cameraDataKeys.map((name) => (
+                        <Th>Memory %</Th>
-                      <Td key={`${name}-${camera}`}>{cameras[camera][name]}</Td>
+                      </Tr>
-                    ))}
+                    </Thead>
-                  </Tr>
+                    <Tbody>
-                ))}
+                      <Tr key="capture" index="0">
-              </Tbody>
+                        <Td>Capture</Td>
-            </Table>
+                        <Td>{cameras[camera]['capture_pid']}</Td>
+                        <Td>{cameras[camera]['process_fps']}</Td>
+                        <Td>{cpu_usages[cameras[camera]['capture_pid']]['cpu']}%</Td>
+                        <Td>{cpu_usages[cameras[camera]['capture_pid']]['mem']}%</Td>
+                      </Tr>
+                      <Tr key="detect" index="1">
+                        <Td>Detect</Td>
+                        <Td>{cameras[camera]['pid']}</Td>
+                        <Td>
+                          {cameras[camera]['detection_fps']} ({cameras[camera]['skipped_fps']} skipped)
+                        </Td>
+                        <Td>{cpu_usages[cameras[camera]['pid']]['cpu']}%</Td>
+                        <Td>{cpu_usages[cameras[camera]['pid']]['cpu']}%</Td>
+                      </Tr>
+                      <Tr key="ffmpeg" index="2">
+                        <Td>ffmpeg</Td>
+                        <Td>{cameras[camera]['ffmpeg_pid']}</Td>
+                        <Td>{cameras[camera]['camera_fps']}</Td>
+                        <Td>{cpu_usages[cameras[camera]['ffmpeg_pid']]['cpu']}%</Td>
+                        <Td>{cpu_usages[cameras[camera]['ffmpeg_pid']]['cpu']}%</Td>
+                      </Tr>
+                    </Tbody>
+                  </Table>
+                </div>
+              </div>
+            ))}
           </div>
 
           <p>Debug stats update automatically every {config.mqtt.stats_interval} seconds.</p>