Pull count of detection events by label into prometheus metrics (#20119)

* pull count of detection events by label into prometheus metrics

* format changes with ruff

* remove unneeded f-string

* fix imports format

---------

Co-authored-by: iesad <iesad>

frigate/api/app.py

@@ -11,7 +11,7 @@ from datetime import datetime, timedelta
 from functools import reduce
 from io import StringIO
 from pathlib import Path as FilePath
-from typing import Any, Optional
+from typing import Any, Dict, List, Optional
 
 import aiofiles
 import requests
@@ -21,7 +21,7 @@ from fastapi.encoders import jsonable_encoder
 from fastapi.params import Depends
 from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse
 from markupsafe import escape
-from peewee import SQL, operator
+from peewee import SQL, fn, operator
 from pydantic import ValidationError
 
 from frigate.api.auth import require_role
@@ -130,7 +130,14 @@ def metrics(request: Request):
     """Expose Prometheus metrics endpoint and update metrics with latest stats"""
     # Retrieve the latest statistics and update the Prometheus metrics
     stats = request.app.stats_emitter.get_latest_stats()
-    update_metrics(stats)
+    # query DB for count of events by camera, label
+    event_counts: List[Dict[str, Any]] = (
+        Event.select(Event.camera, Event.label, fn.Count())
+        .group_by(Event.camera, Event.label)
+        .dicts()
+    )
+
+    update_metrics(stats=stats, event_counts=event_counts)
     content, content_type = get_metrics()
     return Response(content=content, media_type=content_type)
 

frigate/stats/prometheus.py

@@ -1,5 +1,6 @@
 import logging
 import re
+from typing import Any, Dict, List
 
 from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
 from prometheus_client.core import (
@@ -450,51 +451,17 @@ class CustomCollector(object):
         yield storage_total
         yield storage_used
 
-        # count events
-        events = []
-
-        if len(events) > 0:
-            # events[0] is newest event, last element is oldest, don't need to sort
-
-            if not self.previous_event_id:
-                # ignore all previous events on startup, prometheus might have already counted them
-                self.previous_event_id = events[0]["id"]
-                self.previous_event_start_time = int(events[0]["start_time"])
-
-            for event in events:
-                # break if event already counted
-                if event["id"] == self.previous_event_id:
-                    break
-
-                # break if event starts before previous event
-                if event["start_time"] < self.previous_event_start_time:
-                    break
-
-                # store counted events in a dict
-                try:
-                    cam = self.all_events[event["camera"]]
-                    try:
-                        cam[event["label"]] += 1
-                    except KeyError:
-                        # create label dict if not exists
-                        cam.update({event["label"]: 1})
-                except KeyError:
-                    # create camera and label dict if not exists
-                    self.all_events.update({event["camera"]: {event["label"]: 1}})
-
-            # don't recount events next time
-            self.previous_event_id = events[0]["id"]
-            self.previous_event_start_time = int(events[0]["start_time"])
-
         camera_events = CounterMetricFamily(
             "frigate_camera_events",
             "Count of camera events since exporter started",
             labels=["camera", "label"],
         )
 
-        for camera, cam_dict in self.all_events.items():
+        if len(self.all_events) > 0:
-            for label, label_value in cam_dict.items():
+            for event_count in self.all_events:
-                camera_events.add_metric([camera, label], label_value)
+                camera_events.add_metric(
+                    [event_count["camera"], event_count["label"]], event_count["Count"]
+                )
 
         yield camera_events
 
@@ -503,7 +470,7 @@ collector = CustomCollector(None)
 REGISTRY.register(collector)
 
 
-def update_metrics(stats):
+def update_metrics(stats: Dict[str, Any], event_counts: List[Dict[str, Any]]):
     """Updates the Prometheus metrics with the given stats data."""
     try:
         # Store the complete stats for later use by collect()
@@ -512,6 +479,8 @@ def update_metrics(stats):
         # For backwards compatibility
         collector.process_stats = stats.copy()
 
+        collector.all_events = event_counts
+
         # No need to call collect() here - it will be called by get_metrics()
     except Exception as e:
         logging.error(f"Error updating metrics: {e}")

frigate/test/http_api/test_http_event.py

@@ -8,7 +8,9 @@ from playhouse.shortcuts import model_to_dict
 from frigate.api.auth import get_allowed_cameras_for_filter, get_current_user
 from frigate.comms.event_metadata_updater import EventMetadataPublisher
 from frigate.models import Event, Recordings, ReviewSegment, Timeline
+from frigate.stats.emitter import StatsEmitter
 from frigate.test.http_api.base_http_test import BaseTestHttp
+from frigate.test.test_storage import _insert_mock_event
 
 
 class TestHttpApp(BaseTestHttp):
@@ -293,3 +295,108 @@ class TestHttpApp(BaseTestHttp):
             sub_labels = client.get("/sub_labels").json()
             assert sub_labels
             assert sub_labels == [sub_label]
+
+    ####################################################################################################################
+    ###################################  GET /metrics Endpoint   #########################################################
+    ####################################################################################################################
+    def test_get_metrics(self):
+        """ensure correct prometheus metrics api response"""
+        with TestClient(self.app) as client:
+            ts_start = datetime.now().timestamp()
+            ts_end = ts_start + 30
+            _insert_mock_event(
+                id="abcde.random", start=ts_start, end=ts_end, retain=True
+            )
+            _insert_mock_event(
+                id="01234.random", start=ts_start, end=ts_end, retain=True
+            )
+            _insert_mock_event(
+                id="56789.random", start=ts_start, end=ts_end, retain=True
+            )
+            _insert_mock_event(
+                id="101112.random",
+                label="outside",
+                start=ts_start,
+                end=ts_end,
+                retain=True,
+            )
+            _insert_mock_event(
+                id="131415.random",
+                label="outside",
+                start=ts_start,
+                end=ts_end,
+                retain=True,
+            )
+            _insert_mock_event(
+                id="161718.random",
+                camera="porch",
+                start=ts_start,
+                end=ts_end,
+                retain=True,
+            )
+            _insert_mock_event(
+                id="192021.random",
+                camera="porch",
+                start=ts_start,
+                end=ts_end,
+                retain=True,
+            )
+            _insert_mock_event(
+                id="222324.random",
+                camera="porch",
+                label="inside",
+                start=ts_start,
+                end=ts_end,
+                retain=True,
+            )
+            _insert_mock_event(
+                id="252627.random",
+                camera="porch",
+                label="inside",
+                start=ts_start,
+                end=ts_end,
+                retain=True,
+            )
+            _insert_mock_event(
+                id="282930.random",
+                label="inside",
+                start=ts_start,
+                end=ts_end,
+                retain=True,
+            )
+            _insert_mock_event(
+                id="313233.random",
+                label="inside",
+                start=ts_start,
+                end=ts_end,
+                retain=True,
+            )
+
+            stats_emitter = Mock(spec=StatsEmitter)
+            stats_emitter.get_latest_stats.return_value = self.test_stats
+            self.app.stats_emitter = stats_emitter
+            event = client.get("/metrics")
+
+        assert "# TYPE frigate_detection_total_fps gauge" in event.text
+        assert "frigate_detection_total_fps 13.7" in event.text
+        assert (
+            "# HELP frigate_camera_events_total Count of camera events since exporter started"
+            in event.text
+        )
+        assert "# TYPE frigate_camera_events_total counter" in event.text
+        assert (
+            'frigate_camera_events_total{camera="front_door",label="Mock"} 3.0'
+            in event.text
+        )
+        assert (
+            'frigate_camera_events_total{camera="front_door",label="inside"} 2.0'
+            in event.text
+        )
+        assert (
+            'frigate_camera_events_total{camera="front_door",label="outside"} 2.0'
+            in event.text
+        )
+        assert (
+            'frigate_camera_events_total{camera="porch",label="Mock"} 2.0' in event.text
+        )
+        assert 'frigate_camera_events_total{camera="porch",label="inside"} 2.0'

frigate/test/test_storage.py

@@ -261,12 +261,19 @@ class TestHttp(unittest.TestCase):
         assert Recordings.get(Recordings.id == rec_k3_id)
 
 
-def _insert_mock_event(id: str, start: int, end: int, retain: bool) -> Event:
+def _insert_mock_event(
+    id: str,
+    start: int,
+    end: int,
+    retain: bool,
+    camera: str = "front_door",
+    label: str = "Mock",
+) -> Event:
     """Inserts a basic event model with a given id."""
     return Event.insert(
         id=id,
-        label="Mock",
+        label=label,
-        camera="front_door",
+        camera=camera,
         start_time=start,
         end_time=end,
         top_score=100,