import logging import numpy as np from pydantic import Field from typing_extensions import Literal from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import ( BaseDetectorConfig, ModelTypeEnum, ) from frigate.util.model import ( get_ort_providers, post_process_dfine, post_process_rfdetr, post_process_yolo, post_process_yolox, ) logger = logging.getLogger(__name__) DETECTOR_KEY = "onnx" class ONNXDetectorConfig(BaseDetectorConfig): type: Literal[DETECTOR_KEY] device: str = Field(default="AUTO", title="Device Type") class ONNXDetector(DetectionApi): type_key = DETECTOR_KEY def __init__(self, detector_config: ONNXDetectorConfig): try: import onnxruntime as ort logger.info("ONNX: loaded onnxruntime module") except ModuleNotFoundError: logger.error( "ONNX: module loading failed, need 'pip install onnxruntime'?!?" ) raise path = detector_config.model.path logger.info(f"ONNX: loading {detector_config.model.path}") providers, options = get_ort_providers( detector_config.device == "CPU", detector_config.device ) self.model = ort.InferenceSession( path, providers=providers, provider_options=options ) self.h = detector_config.model.height self.w = detector_config.model.width self.onnx_model_type = detector_config.model.model_type self.onnx_model_px = detector_config.model.input_pixel_format self.onnx_model_shape = detector_config.model.input_tensor path = detector_config.model.path if self.onnx_model_type == ModelTypeEnum.yolox: grids = [] expanded_strides = [] # decode and orient predictions strides = [8, 16, 32] hsizes = [self.h // stride for stride in strides] wsizes = [self.w // stride for stride in strides] for hsize, wsize, stride in zip(hsizes, wsizes, strides): xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize)) grid = np.stack((xv, yv), 2).reshape(1, -1, 2) grids.append(grid) shape = grid.shape[:2] expanded_strides.append(np.full((*shape, 1), stride)) self.grids = np.concatenate(grids, 1) self.expanded_strides = np.concatenate(expanded_strides, 1) logger.info(f"ONNX: {path} loaded") def detect_raw(self, tensor_input: np.ndarray): if self.onnx_model_type == ModelTypeEnum.dfine: tensor_output = self.model.run( None, { "images": tensor_input, "orig_target_sizes": np.array([[self.h, self.w]], dtype=np.int64), }, ) return post_process_dfine(tensor_output, self.w, self.h) model_input_name = self.model.get_inputs()[0].name tensor_output = self.model.run(None, {model_input_name: tensor_input}) if self.onnx_model_type == ModelTypeEnum.rfdetr: return post_process_rfdetr(tensor_output) elif self.onnx_model_type == ModelTypeEnum.yolonas: predictions = tensor_output[0] detections = np.zeros((20, 6), np.float32) for i, prediction in enumerate(predictions): if i == 20: break (_, x_min, y_min, x_max, y_max, confidence, class_id) = prediction # when running in GPU mode, empty predictions in the output have class_id of -1 if class_id < 0: break detections[i] = [ class_id, confidence, y_min / self.h, x_min / self.w, y_max / self.h, x_max / self.w, ] return detections elif self.onnx_model_type == ModelTypeEnum.yologeneric: return post_process_yolo(tensor_output, self.w, self.h) elif self.onnx_model_type == ModelTypeEnum.yolox: return post_process_yolox( tensor_output[0], self.w, self.h, self.grids, self.expanded_strides ) else: raise Exception( f"{self.onnx_model_type} is currently not supported for onnx. See the docs for more info on supported models." )