From ab7b12da54ea499b5eaaff320479e831e074752c Mon Sep 17 00:00:00 2001 From: Jimmy Date: Fri, 6 Jun 2025 14:41:04 -0500 Subject: [PATCH] Add Mesa Teflon as a TFLite detector (#18310) * Refactor common functions for tflite detector implementations * Add detector using mesa teflon delegate Non-EdgeTPU TFLite can use the standard .tflite format * Add mesa-teflon-delegate from bookworm-backports to arm64 images --- docker/main/install_deps.sh | 8 +++ frigate/config/config.py | 4 +- frigate/detectors/detector_utils.py | 74 +++++++++++++++++++++++++ frigate/detectors/plugins/cpu_tfl.py | 36 ++---------- frigate/detectors/plugins/teflon_tfl.py | 38 +++++++++++++ 5 files changed, 128 insertions(+), 32 deletions(-) create mode 100644 frigate/detectors/detector_utils.py create mode 100644 frigate/detectors/plugins/teflon_tfl.py diff --git a/docker/main/install_deps.sh b/docker/main/install_deps.sh index aed11dff4..bd9f363e9 100755 --- a/docker/main/install_deps.sh +++ b/docker/main/install_deps.sh @@ -31,6 +31,14 @@ unset DEBIAN_FRONTEND yes | dpkg -i /tmp/libedgetpu1-max.deb && export DEBIAN_FRONTEND=noninteractive rm /tmp/libedgetpu1-max.deb +# install mesa-teflon-delegate from bookworm-backports +# Only available for arm64 at the moment +if [[ "${TARGETARCH}" == "arm64" ]]; then + echo "deb http://deb.debian.org/debian bookworm-backports main" | tee /etc/apt/sources.list.d/bookworm-backports.list + apt-get -qq update + apt-get -qq install --no-install-recommends --no-install-suggests -y mesa-teflon-delegate/bookworm-backports +fi + # ffmpeg -> amd64 if [[ "${TARGETARCH}" == "amd64" ]]; then mkdir -p /usr/lib/ffmpeg/5.0 diff --git a/frigate/config/config.py b/frigate/config/config.py index 49e57f3cf..62c931c96 100644 --- a/frigate/config/config.py +++ b/frigate/config/config.py @@ -487,7 +487,9 @@ class FrigateConfig(FrigateBaseModel): model_config["path"] = detector_config.model_path if "path" not in model_config: - if detector_config.type == "cpu": + if detector_config.type == "cpu" or detector_config.type.endswith( + "_tfl" + ): model_config["path"] = "/cpu_model.tflite" elif detector_config.type == "edgetpu": model_config["path"] = "/edgetpu_model.tflite" diff --git a/frigate/detectors/detector_utils.py b/frigate/detectors/detector_utils.py new file mode 100644 index 000000000..d732de871 --- /dev/null +++ b/frigate/detectors/detector_utils.py @@ -0,0 +1,74 @@ +import logging +import os + +import numpy as np + +try: + from tflite_runtime.interpreter import Interpreter, load_delegate +except ModuleNotFoundError: + from tensorflow.lite.python.interpreter import Interpreter, load_delegate + + +logger = logging.getLogger(__name__) + + +def tflite_init(self, interpreter): + self.interpreter = interpreter + + self.interpreter.allocate_tensors() + + self.tensor_input_details = self.interpreter.get_input_details() + self.tensor_output_details = self.interpreter.get_output_details() + + +def tflite_detect_raw(self, tensor_input): + self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input) + self.interpreter.invoke() + + boxes = self.interpreter.tensor(self.tensor_output_details[0]["index"])()[0] + class_ids = self.interpreter.tensor(self.tensor_output_details[1]["index"])()[0] + scores = self.interpreter.tensor(self.tensor_output_details[2]["index"])()[0] + count = int(self.interpreter.tensor(self.tensor_output_details[3]["index"])()[0]) + + detections = np.zeros((20, 6), np.float32) + + for i in range(count): + if scores[i] < 0.4 or i == 20: + break + detections[i] = [ + class_ids[i], + float(scores[i]), + boxes[i][0], + boxes[i][1], + boxes[i][2], + boxes[i][3], + ] + + return detections + + +def tflite_load_delegate_interpreter( + delegate_library: str, detector_config, device_config +): + try: + logger.info("Attempting to load NPU") + tf_delegate = load_delegate(delegate_library, device_config) + logger.info("NPU found") + interpreter = Interpreter( + model_path=detector_config.model.path, + experimental_delegates=[tf_delegate], + ) + return interpreter + except ValueError: + _, ext = os.path.splitext(detector_config.model.path) + + if ext and ext != ".tflite": + logger.error( + "Incorrect model used with NPU. Only .tflite models can be used with a TFLite delegate." + ) + else: + logger.error( + "No NPU was detected. If you do not have a TFLite device yet, you must configure CPU detectors." + ) + + raise diff --git a/frigate/detectors/plugins/cpu_tfl.py b/frigate/detectors/plugins/cpu_tfl.py index 8a54363e1..fc8db0f4b 100644 --- a/frigate/detectors/plugins/cpu_tfl.py +++ b/frigate/detectors/plugins/cpu_tfl.py @@ -1,12 +1,13 @@ import logging -import numpy as np from pydantic import Field from typing_extensions import Literal from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig +from ..detector_utils import tflite_detect_raw, tflite_init + try: from tflite_runtime.interpreter import Interpreter except ModuleNotFoundError: @@ -27,39 +28,12 @@ class CpuTfl(DetectionApi): type_key = DETECTOR_KEY def __init__(self, detector_config: CpuDetectorConfig): - self.interpreter = Interpreter( + interpreter = Interpreter( model_path=detector_config.model.path, num_threads=detector_config.num_threads or 3, ) - self.interpreter.allocate_tensors() - - self.tensor_input_details = self.interpreter.get_input_details() - self.tensor_output_details = self.interpreter.get_output_details() + tflite_init(self, interpreter) def detect_raw(self, tensor_input): - self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input) - self.interpreter.invoke() - - boxes = self.interpreter.tensor(self.tensor_output_details[0]["index"])()[0] - class_ids = self.interpreter.tensor(self.tensor_output_details[1]["index"])()[0] - scores = self.interpreter.tensor(self.tensor_output_details[2]["index"])()[0] - count = int( - self.interpreter.tensor(self.tensor_output_details[3]["index"])()[0] - ) - - detections = np.zeros((20, 6), np.float32) - - for i in range(count): - if scores[i] < 0.4 or i == 20: - break - detections[i] = [ - class_ids[i], - float(scores[i]), - boxes[i][0], - boxes[i][1], - boxes[i][2], - boxes[i][3], - ] - - return detections + return tflite_detect_raw(self, tensor_input) diff --git a/frigate/detectors/plugins/teflon_tfl.py b/frigate/detectors/plugins/teflon_tfl.py new file mode 100644 index 000000000..7e29d6630 --- /dev/null +++ b/frigate/detectors/plugins/teflon_tfl.py @@ -0,0 +1,38 @@ +import logging + +from typing_extensions import Literal + +from frigate.detectors.detection_api import DetectionApi +from frigate.detectors.detector_config import BaseDetectorConfig + +from ..detector_utils import ( + tflite_detect_raw, + tflite_init, + tflite_load_delegate_interpreter, +) + +logger = logging.getLogger(__name__) + +# Use _tfl suffix to default tflite model +DETECTOR_KEY = "teflon_tfl" + + +class TeflonDetectorConfig(BaseDetectorConfig): + type: Literal[DETECTOR_KEY] + + +class TeflonTfl(DetectionApi): + type_key = DETECTOR_KEY + + def __init__(self, detector_config: TeflonDetectorConfig): + # Location in Debian's mesa-teflon-delegate + delegate_library = "/usr/lib/teflon/libteflon.so" + device_config = {} + + interpreter = tflite_load_delegate_interpreter( + delegate_library, detector_config, device_config + ) + tflite_init(self, interpreter) + + def detect_raw(self, tensor_input): + return tflite_detect_raw(self, tensor_input)