From dc96940eb96cd37c97bc5cff221d560c4ca1d3f1 Mon Sep 17 00:00:00 2001 From: iesad <123698118+iesad@users.noreply.github.com> Date: Fri, 19 Sep 2025 06:27:20 -0600 Subject: [PATCH] Pull count of detection events by label into prometheus metrics (#20119) * pull count of detection events by label into prometheus metrics * format changes with ruff * remove unneeded f-string * fix imports format --------- Co-authored-by: iesad --- frigate/api/app.py | 13 ++- frigate/stats/prometheus.py | 49 ++--------- frigate/test/http_api/test_http_event.py | 107 +++++++++++++++++++++++ frigate/test/test_storage.py | 13 ++- 4 files changed, 136 insertions(+), 46 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index d9e573d29..5c2c132fd 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -11,7 +11,7 @@ from datetime import datetime, timedelta from functools import reduce from io import StringIO from pathlib import Path as FilePath -from typing import Any, Optional +from typing import Any, Dict, List, Optional import aiofiles import requests @@ -21,7 +21,7 @@ from fastapi.encoders import jsonable_encoder from fastapi.params import Depends from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse from markupsafe import escape -from peewee import SQL, operator +from peewee import SQL, fn, operator from pydantic import ValidationError from frigate.api.auth import require_role @@ -130,7 +130,14 @@ def metrics(request: Request): """Expose Prometheus metrics endpoint and update metrics with latest stats""" # Retrieve the latest statistics and update the Prometheus metrics stats = request.app.stats_emitter.get_latest_stats() - update_metrics(stats) + # query DB for count of events by camera, label + event_counts: List[Dict[str, Any]] = ( + Event.select(Event.camera, Event.label, fn.Count()) + .group_by(Event.camera, Event.label) + .dicts() + ) + + update_metrics(stats=stats, event_counts=event_counts) content, content_type = get_metrics() return Response(content=content, media_type=content_type) diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py index bc545f21d..67d8d03d8 100644 --- a/frigate/stats/prometheus.py +++ b/frigate/stats/prometheus.py @@ -1,5 +1,6 @@ import logging import re +from typing import Any, Dict, List from prometheus_client import CONTENT_TYPE_LATEST, generate_latest from prometheus_client.core import ( @@ -450,51 +451,17 @@ class CustomCollector(object): yield storage_total yield storage_used - # count events - events = [] - - if len(events) > 0: - # events[0] is newest event, last element is oldest, don't need to sort - - if not self.previous_event_id: - # ignore all previous events on startup, prometheus might have already counted them - self.previous_event_id = events[0]["id"] - self.previous_event_start_time = int(events[0]["start_time"]) - - for event in events: - # break if event already counted - if event["id"] == self.previous_event_id: - break - - # break if event starts before previous event - if event["start_time"] < self.previous_event_start_time: - break - - # store counted events in a dict - try: - cam = self.all_events[event["camera"]] - try: - cam[event["label"]] += 1 - except KeyError: - # create label dict if not exists - cam.update({event["label"]: 1}) - except KeyError: - # create camera and label dict if not exists - self.all_events.update({event["camera"]: {event["label"]: 1}}) - - # don't recount events next time - self.previous_event_id = events[0]["id"] - self.previous_event_start_time = int(events[0]["start_time"]) - camera_events = CounterMetricFamily( "frigate_camera_events", "Count of camera events since exporter started", labels=["camera", "label"], ) - for camera, cam_dict in self.all_events.items(): - for label, label_value in cam_dict.items(): - camera_events.add_metric([camera, label], label_value) + if len(self.all_events) > 0: + for event_count in self.all_events: + camera_events.add_metric( + [event_count["camera"], event_count["label"]], event_count["Count"] + ) yield camera_events @@ -503,7 +470,7 @@ collector = CustomCollector(None) REGISTRY.register(collector) -def update_metrics(stats): +def update_metrics(stats: Dict[str, Any], event_counts: List[Dict[str, Any]]): """Updates the Prometheus metrics with the given stats data.""" try: # Store the complete stats for later use by collect() @@ -512,6 +479,8 @@ def update_metrics(stats): # For backwards compatibility collector.process_stats = stats.copy() + collector.all_events = event_counts + # No need to call collect() here - it will be called by get_metrics() except Exception as e: logging.error(f"Error updating metrics: {e}") diff --git a/frigate/test/http_api/test_http_event.py b/frigate/test/http_api/test_http_event.py index 4ac4f458d..2ef00aa05 100644 --- a/frigate/test/http_api/test_http_event.py +++ b/frigate/test/http_api/test_http_event.py @@ -8,7 +8,9 @@ from playhouse.shortcuts import model_to_dict from frigate.api.auth import get_allowed_cameras_for_filter, get_current_user from frigate.comms.event_metadata_updater import EventMetadataPublisher from frigate.models import Event, Recordings, ReviewSegment, Timeline +from frigate.stats.emitter import StatsEmitter from frigate.test.http_api.base_http_test import BaseTestHttp +from frigate.test.test_storage import _insert_mock_event class TestHttpApp(BaseTestHttp): @@ -293,3 +295,108 @@ class TestHttpApp(BaseTestHttp): sub_labels = client.get("/sub_labels").json() assert sub_labels assert sub_labels == [sub_label] + + #################################################################################################################### + ################################### GET /metrics Endpoint ######################################################### + #################################################################################################################### + def test_get_metrics(self): + """ensure correct prometheus metrics api response""" + with TestClient(self.app) as client: + ts_start = datetime.now().timestamp() + ts_end = ts_start + 30 + _insert_mock_event( + id="abcde.random", start=ts_start, end=ts_end, retain=True + ) + _insert_mock_event( + id="01234.random", start=ts_start, end=ts_end, retain=True + ) + _insert_mock_event( + id="56789.random", start=ts_start, end=ts_end, retain=True + ) + _insert_mock_event( + id="101112.random", + label="outside", + start=ts_start, + end=ts_end, + retain=True, + ) + _insert_mock_event( + id="131415.random", + label="outside", + start=ts_start, + end=ts_end, + retain=True, + ) + _insert_mock_event( + id="161718.random", + camera="porch", + start=ts_start, + end=ts_end, + retain=True, + ) + _insert_mock_event( + id="192021.random", + camera="porch", + start=ts_start, + end=ts_end, + retain=True, + ) + _insert_mock_event( + id="222324.random", + camera="porch", + label="inside", + start=ts_start, + end=ts_end, + retain=True, + ) + _insert_mock_event( + id="252627.random", + camera="porch", + label="inside", + start=ts_start, + end=ts_end, + retain=True, + ) + _insert_mock_event( + id="282930.random", + label="inside", + start=ts_start, + end=ts_end, + retain=True, + ) + _insert_mock_event( + id="313233.random", + label="inside", + start=ts_start, + end=ts_end, + retain=True, + ) + + stats_emitter = Mock(spec=StatsEmitter) + stats_emitter.get_latest_stats.return_value = self.test_stats + self.app.stats_emitter = stats_emitter + event = client.get("/metrics") + + assert "# TYPE frigate_detection_total_fps gauge" in event.text + assert "frigate_detection_total_fps 13.7" in event.text + assert ( + "# HELP frigate_camera_events_total Count of camera events since exporter started" + in event.text + ) + assert "# TYPE frigate_camera_events_total counter" in event.text + assert ( + 'frigate_camera_events_total{camera="front_door",label="Mock"} 3.0' + in event.text + ) + assert ( + 'frigate_camera_events_total{camera="front_door",label="inside"} 2.0' + in event.text + ) + assert ( + 'frigate_camera_events_total{camera="front_door",label="outside"} 2.0' + in event.text + ) + assert ( + 'frigate_camera_events_total{camera="porch",label="Mock"} 2.0' in event.text + ) + assert 'frigate_camera_events_total{camera="porch",label="inside"} 2.0' diff --git a/frigate/test/test_storage.py b/frigate/test/test_storage.py index d36960f47..4ae5715ca 100644 --- a/frigate/test/test_storage.py +++ b/frigate/test/test_storage.py @@ -261,12 +261,19 @@ class TestHttp(unittest.TestCase): assert Recordings.get(Recordings.id == rec_k3_id) -def _insert_mock_event(id: str, start: int, end: int, retain: bool) -> Event: +def _insert_mock_event( + id: str, + start: int, + end: int, + retain: bool, + camera: str = "front_door", + label: str = "Mock", +) -> Event: """Inserts a basic event model with a given id.""" return Event.insert( id=id, - label="Mock", - camera="front_door", + label=label, + camera=camera, start_time=start, end_time=end, top_score=100,