From 8409100623dc776c586ff3ce0834b00e0037e634 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 6 Jun 2025 10:29:44 -0600 Subject: [PATCH] Classification Model Metrics (#18595) * Add speed and rate metrics for custom classification models * Use metrics for classification models * Use keys * Cast to list --- frigate/app.py | 3 +- .../real_time/custom_classification.py | 28 ++++++++++++++++++- frigate/data_processing/types.py | 11 +++++++- frigate/stats/util.py | 8 ++++++ 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/frigate/app.py b/frigate/app.py index 65dc19472..04cdb2920 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -89,11 +89,12 @@ class FrigateApp: self.log_queue: Queue = mp.Queue() self.camera_metrics: dict[str, CameraMetrics] = {} self.embeddings_metrics: DataProcessorMetrics | None = ( - DataProcessorMetrics() + DataProcessorMetrics(list(config.classification.custom.keys())) if ( config.semantic_search.enabled or config.lpr.enabled or config.face_recognition.enabled + or len(config.classification.custom) > 0 ) else None ) diff --git a/frigate/data_processing/real_time/custom_classification.py b/frigate/data_processing/real_time/custom_classification.py index df4baf70b..a718956e2 100644 --- a/frigate/data_processing/real_time/custom_classification.py +++ b/frigate/data_processing/real_time/custom_classification.py @@ -19,7 +19,7 @@ from frigate.config import FrigateConfig from frigate.config.classification import CustomClassificationConfig from frigate.const import CLIPS_DIR, MODEL_CACHE_DIR, UPDATE_MODEL_STATE from frigate.types import ModelStatusTypesEnum -from frigate.util.builtin import load_labels +from frigate.util.builtin import EventsPerSecond, InferenceSpeed, load_labels from frigate.util.classification import train_classification_model from frigate.util.object import box_overlaps, calculate_region @@ -51,6 +51,10 @@ class CustomStateClassificationProcessor(RealTimeProcessorApi): self.tensor_input_details: dict[str, Any] = None self.tensor_output_details: dict[str, Any] = None self.labelmap: dict[int, str] = {} + self.classifications_per_second = EventsPerSecond() + self.inference_speed = InferenceSpeed( + self.metrics.classification_speeds[self.model_config.name] + ) self.last_run = datetime.datetime.now().timestamp() self.__build_detector() @@ -66,6 +70,7 @@ class CustomStateClassificationProcessor(RealTimeProcessorApi): os.path.join(self.model_dir, "labelmap.txt"), prefill=0, ) + self.classifications_per_second.start() def __retrain_model(self) -> None: train_classification_model(self.model_config.name) @@ -79,7 +84,14 @@ class CustomStateClassificationProcessor(RealTimeProcessorApi): ) logger.info(f"Successfully loaded updated model for {self.model_config.name}") + def __update_metrics(self, duration: float) -> None: + self.classifications_per_second.update() + self.inference_speed.update(duration) + def process_frame(self, frame_data: dict[str, Any], frame: np.ndarray): + self.metrics.classification_cps[ + self.model_config.name + ].value = self.classifications_per_second.eps() camera = frame_data.get("camera") if camera not in self.model_config.state_config.cameras: @@ -143,6 +155,7 @@ class CustomStateClassificationProcessor(RealTimeProcessorApi): probs = res / res.sum(axis=0) best_id = np.argmax(probs) score = round(probs[best_id], 2) + self.__update_metrics(datetime.datetime.now().timestamp() - now) write_classification_attempt( self.train_dir, @@ -200,6 +213,10 @@ class CustomObjectClassificationProcessor(RealTimeProcessorApi): self.tensor_output_details: dict[str, Any] = None self.detected_objects: dict[str, float] = {} self.labelmap: dict[int, str] = {} + self.classifications_per_second = EventsPerSecond() + self.inference_speed = InferenceSpeed( + self.metrics.classification_speeds[self.model_config.name] + ) self.__build_detector() def __build_detector(self) -> None: @@ -227,7 +244,15 @@ class CustomObjectClassificationProcessor(RealTimeProcessorApi): ) logger.info(f"Successfully loaded updated model for {self.model_config.name}") + def __update_metrics(self, duration: float) -> None: + self.classifications_per_second.update() + self.inference_speed.update(duration) + def process_frame(self, obj_data, frame): + self.metrics.classification_cps[ + self.model_config.name + ].value = self.classifications_per_second.eps() + if obj_data["label"] not in self.model_config.object_config.objects: return @@ -261,6 +286,7 @@ class CustomObjectClassificationProcessor(RealTimeProcessorApi): best_id = np.argmax(probs) score = round(probs[best_id], 2) previous_score = self.detected_objects.get(obj_data["id"], 0.0) + self.__update_metrics(datetime.datetime.now().timestamp() - now) write_classification_attempt( self.train_dir, diff --git a/frigate/data_processing/types.py b/frigate/data_processing/types.py index 5d083b32e..783b0798e 100644 --- a/frigate/data_processing/types.py +++ b/frigate/data_processing/types.py @@ -20,8 +20,10 @@ class DataProcessorMetrics: alpr_pps: Synchronized yolov9_lpr_speed: Synchronized yolov9_lpr_pps: Synchronized + classification_speeds: dict[str, Synchronized] + classification_cps: dict[str, Synchronized] - def __init__(self): + def __init__(self, custom_classification_models: list[str]): self.image_embeddings_speed = mp.Value("d", 0.0) self.image_embeddings_eps = mp.Value("d", 0.0) self.text_embeddings_speed = mp.Value("d", 0.0) @@ -33,6 +35,13 @@ class DataProcessorMetrics: self.yolov9_lpr_speed = mp.Value("d", 0.0) self.yolov9_lpr_pps = mp.Value("d", 0.0) + if custom_classification_models: + self.classification_speeds = {} + self.classification_cps = {} + for key in custom_classification_models: + self.classification_speeds[key] = mp.Value("d", 0.0) + self.classification_cps[key] = mp.Value("d", 0.0) + class DataProcessorModelRunner: def __init__(self, requestor, device: str = "CPU", model_size: str = "large"): diff --git a/frigate/stats/util.py b/frigate/stats/util.py index 72f76c07e..5078269eb 100644 --- a/frigate/stats/util.py +++ b/frigate/stats/util.py @@ -352,6 +352,14 @@ def stats_snapshot( embeddings_metrics.yolov9_lpr_pps.value, 2 ) + for key in embeddings_metrics.classification_speeds.keys(): + stats["embeddings"][f"{key}_classification_speed"] = round( + embeddings_metrics.classification_speeds[key].value * 1000, 2 + ) + stats["embeddings"][f"{key}_classification"] = round( + embeddings_metrics.classification_cps[key].value, 2 + ) + get_processing_stats(config, stats, hwaccel_errors) stats["service"] = {