Refactor Process Stats and Bugfixes (#6344)

* test refactor process stats

* Update util.py

* bugfix

* black formatting

* add missing processes field to StatsTrackingTypes class

* fix python checks and tests...

* use psutil for calcilate cpu utilization on get_cpu_stats

* black...black...black...

* add cpu average

* calculate statiscts for logger process

* add P-ID for other processes in System.jsx

* Apply suggestions from code review

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

* make page beautiful again :)

---------

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

frigate/app.py

@@ -8,6 +8,7 @@ import signal
 import sys
 from typing import Optional
 from types import FrameType
+import psutil
 
 import traceback
 from peewee_migrate import Router
@@ -58,6 +59,7 @@ class FrigateApp:
         self.plus_api = PlusApi()
         self.camera_metrics: dict[str, CameraMetricsTypes] = {}
         self.record_metrics: dict[str, RecordMetricsTypes] = {}
+        self.processes: dict[str, int] = {}
 
     def set_environment_vars(self) -> None:
         for key, value in self.config.environment_vars.items():
@@ -77,6 +79,7 @@ class FrigateApp:
         )
         self.log_process.daemon = True
         self.log_process.start()
+        self.processes["logger"] = self.log_process.pid or 0
         root_configurer(self.log_queue)
 
     def init_config(self) -> None:
@@ -171,6 +174,12 @@ class FrigateApp:
 
         migrate_db.close()
 
+    def init_go2rtc(self) -> None:
+        for proc in psutil.process_iter(["pid", "name"]):
+            if proc.info["name"] == "go2rtc":
+                logger.info(f"go2rtc process pid: {proc.info['pid']}")
+                self.processes["go2rtc"] = proc.info["pid"]
+
     def init_recording_manager(self) -> None:
         recording_process = mp.Process(
             target=manage_recordings,
@@ -180,6 +189,7 @@ class FrigateApp:
         recording_process.daemon = True
         self.recording_process = recording_process
         recording_process.start()
+        self.processes["recording"] = recording_process.pid or 0
         logger.info(f"Recording process started: {recording_process.pid}")
 
     def bind_database(self) -> None:
@@ -191,7 +201,7 @@ class FrigateApp:
 
     def init_stats(self) -> None:
         self.stats_tracking = stats_init(
-            self.config, self.camera_metrics, self.detectors
+            self.config, self.camera_metrics, self.detectors, self.processes
         )
 
     def init_web_server(self) -> None:
@@ -412,6 +422,7 @@ class FrigateApp:
             self.init_database()
             self.init_onvif()
             self.init_recording_manager()
+            self.init_go2rtc()
             self.bind_database()
             self.init_dispatcher()
         except Exception as e:

frigate/stats.py

@@ -46,6 +46,7 @@ def stats_init(
     config: FrigateConfig,
     camera_metrics: dict[str, CameraMetricsTypes],
     detectors: dict[str, ObjectDetectProcess],
+    processes: dict[str, int],
 ) -> StatsTrackingTypes:
     stats_tracking: StatsTrackingTypes = {
         "camera_metrics": camera_metrics,
@@ -53,6 +54,7 @@ def stats_init(
         "started": int(time.time()),
         "latest_frigate_version": get_latest_version(config),
         "last_updated": int(time.time()),
+        "processes": processes,
     }
     return stats_tracking
 
@@ -260,6 +262,12 @@ def stats_snapshot(
             "mount_type": get_fs_type(path),
         }
 
+    stats["processes"] = {}
+    for name, pid in stats_tracking["processes"].items():
+        stats["processes"][name] = {
+            "pid": pid,
+        }
+
     return stats
 
 

frigate/types.py

@@ -34,3 +34,4 @@ class StatsTrackingTypes(TypedDict):
     started: int
     latest_frigate_version: str
     last_updated: int
+    processes: dict[str, int]

frigate/util.py

@@ -799,36 +799,46 @@ def get_cpu_stats() -> dict[str, dict]:
     docker_memlimit = get_docker_memlimit_bytes() / 1024
     total_mem = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / 1024
 
-    for pid in os.listdir("/proc"):
+    for process in psutil.process_iter(["pid", "name", "cpu_percent"]):
-        if pid.isdigit():
+        pid = process.info["pid"]
-            try:
+        try:
-                with open(f"/proc/{pid}/stat", "r") as f:
+            cpu_percent = process.info["cpu_percent"]
-                    stats = f.readline().split()
-                utime = int(stats[13])
-                stime = int(stats[14])
-                cpu_usage = round((utime + stime) / os.sysconf("SC_CLK_TCK"))
 
-                with open(f"/proc/{pid}/statm", "r") as f:
+            with open(f"/proc/{pid}/stat", "r") as f:
-                    mem_stats = f.readline().split()
+                stats = f.readline().split()
-                mem_res = int(mem_stats[1]) * os.sysconf("SC_PAGE_SIZE") / 1024
+            utime = int(stats[13])
+            stime = int(stats[14])
+            starttime = int(stats[21])
 
-                if docker_memlimit > 0:
+            with open("/proc/uptime") as f:
-                    mem_pct = round((mem_res / docker_memlimit) * 100, 1)
+                system_uptime_sec = int(float(f.read().split()[0]))
-                else:
-                    mem_pct = round((mem_res / total_mem) * 100, 1)
 
-                idx = pid
+            clk_tck = os.sysconf(os.sysconf_names["SC_CLK_TCK"])
-                if stats[1] == "(go2rtc)":
-                    idx = "go2rtc"
-                if stats[1].startswith("(frigate.r"):
-                    idx = "recording"
 
-                usages[idx] = {
+            process_utime_sec = utime // clk_tck
-                    "cpu": str(round(cpu_usage, 2)),
+            process_stime_sec = stime // clk_tck
-                    "mem": f"{mem_pct}",
+            process_starttime_sec = starttime // clk_tck
-                }
+
-            except:
+            process_elapsed_sec = system_uptime_sec - process_starttime_sec
-                continue
+            process_usage_sec = process_utime_sec + process_stime_sec
+            cpu_average_usage = process_usage_sec * 100 // process_elapsed_sec
+
+            with open(f"/proc/{pid}/statm", "r") as f:
+                mem_stats = f.readline().split()
+            mem_res = int(mem_stats[1]) * os.sysconf("SC_PAGE_SIZE") / 1024
+
+            if docker_memlimit > 0:
+                mem_pct = round((mem_res / docker_memlimit) * 100, 1)
+            else:
+                mem_pct = round((mem_res / total_mem) * 100, 1)
+
+            usages[pid] = {
+                "cpu": str(cpu_percent),
+                "cpu_average": str(round(cpu_average_usage, 2)),
+                "mem": f"{mem_pct}",
+            }
+        except:
+            continue
 
     return usages
 

web/src/routes/System.jsx

@@ -29,12 +29,14 @@ export default function System() {
     detectors,
     service = {},
     detection_fps: _,
+    processes,
     ...cameras
   } = stats || initialStats || emptyObject;
 
   const detectorNames = Object.keys(detectors || emptyObject);
   const gpuNames = Object.keys(gpu_usages || emptyObject);
   const cameraNames = Object.keys(cameras || emptyObject);
+  const processesNames = Object.keys(processes || emptyObject);
 
   const onHandleFfprobe = async (camera, e) => {
     if (e) {
@@ -347,7 +349,7 @@ export default function System() {
 
           <Heading size="lg">Other Processes</Heading>
           <div data-testid="cameras" className="grid grid-cols-1 3xl:grid-cols-3 md:grid-cols-2 gap-4">
-            {['go2rtc', 'recording'].map((process) => (
+            {processesNames.map((process) => (
               <div key={process} className="dark:bg-gray-800 shadow-md hover:shadow-lg rounded-lg transition-shadow">
                 <div className="capitalize text-lg flex justify-between p-4">
                   <div className="text-lg flex justify-between">{process}</div>
@@ -356,14 +358,18 @@ export default function System() {
                   <Table className="w-full">
                     <Thead>
                       <Tr>
+                        <Th>P-ID</Th>
                         <Th>CPU %</Th>
+                        <Th>Avg CPU %</Th>
                         <Th>Memory %</Th>
                       </Tr>
                     </Thead>
                     <Tbody>
-                      <Tr key="ffmpeg" index="0">
+                      <Tr key="other" index="0">
-                        <Td>{cpu_usages[process]?.['cpu'] || '- '}%</Td>
+                        <Td>{processes[process]['pid'] || '- '}</Td>
-                        <Td>{cpu_usages[process]?.['mem'] || '- '}%</Td>
+                        <Td>{cpu_usages[processes[process]['pid']]?.['cpu'] || '- '}%</Td>
+                        <Td>{cpu_usages[processes[process]['pid']]?.['cpu_average'] || '- '}%</Td>
+                        <Td>{cpu_usages[processes[process]['pid']]?.['mem'] || '- '}%</Td>
                       </Tr>
                     </Tbody>
                   </Table>