From 212784b68e4f708ea2391aab9db7b49d945fdbd5 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 23 Apr 2025 11:22:23 -0600 Subject: [PATCH] implement RKNN downloads for yolov9 and yolox models (#17875) * Add other rockchip download models * Specify newer release version * Specify newer release version * Update docs for rknn downloads * Update hardware docs --- docs/docs/configuration/object_detectors.md | 33 ++++++++++++--------- docs/docs/frigate/hardware.md | 2 +- frigate/detectors/plugins/rknn.py | 10 +++++-- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index fa6132dc8..ed783e093 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -844,14 +844,14 @@ detectors: # required The inference time was determined on a rk3588 with 3 NPU cores. -| Model | Size in mb | Inference time in ms | -| ------------------- | ---------- | -------------------- | -| deci-fp16-yolonas_s | 24 | 25 | -| deci-fp16-yolonas_m | 62 | 35 | -| deci-fp16-yolonas_l | 81 | 45 | -| yolov9_tiny | 8 | 35 | -| yolox_nano | 3 | 16 | -| yolox_tiny | 6 | 20 | +| Model | Size in mb | Inference time in ms | +| --------------------- | ---------- | -------------------- | +| deci-fp16-yolonas_s | 24 | 25 | +| deci-fp16-yolonas_m | 62 | 35 | +| deci-fp16-yolonas_l | 81 | 45 | +| frigate-fp16-yolov9-t | 6 | 35 | +| rock-i8-yolox_nano | 3 | 14 | +| rock-i8_yolox_tiny | 6 | 18 | - All models are automatically downloaded and stored in the folder `config/model_cache/rknn_cache`. After upgrading Frigate, you should remove older models to free up space. - You can also provide your own `.rknn` model. You should not save your own models in the `rknn_cache` folder, store them directly in the `model_cache` folder or another subfolder. To convert a model to `.rknn` format see the `rknn-toolkit2` (requires a x86 machine). Note, that there is only post-processing for the supported models. @@ -887,10 +887,13 @@ The pre-trained YOLO-NAS weights from DeciAI are subject to their license and ca model: # required # name of model (will be automatically downloaded) or path to your own .rknn model file # possible values are: - # - yolov9-t - # - yolov9-s + # - frigate-fp16-yolov9-t + # - frigate-fp16-yolov9-s + # - frigate-fp16-yolov9-m + # - frigate-fp16-yolov9-c + # - frigate-fp16-yolov9-e # your yolo_model.rknn - path: /config/model_cache/rknn_cache/yolov9-t.rknn + path: frigate-fp16-yolov9-t model_type: yolo-generic width: 320 height: 320 @@ -905,10 +908,12 @@ model: # required model: # required # name of model (will be automatically downloaded) or path to your own .rknn model file # possible values are: - # - yolox_nano - # - yolox_tiny + # - rock-i8-yolox_nano + # - rock-i8-yolox_tiny + # - rock-fp16-yolox_nano + # - rock-fp16-yolox_tiny # your yolox_model.rknn - path: yolox_tiny + path: rock-i8-yolox_nano model_type: yolox width: 416 height: 416 diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index f3612e327..fbd2d4057 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -168,7 +168,7 @@ Frigate supports hardware video processing on all Rockchip boards. However, hard | Name | YOLOv9 Inference Time | YOLO-NAS Inference Time | YOLOx Inference Time | | --------------- | --------------------- | --------------------------- | ------------------------- | -| rk3588 3 cores | ~ 35 ms | small: ~ 20 ms med: ~ 30 ms | nano: 18 ms tiny: 20 ms | +| rk3588 3 cores | tiny: ~ 35 ms | small: ~ 20 ms med: ~ 30 ms | nano: 14 ms tiny: 18 ms | | rk3566 1 core | | small: ~ 96 ms | | diff --git a/frigate/detectors/plugins/rknn.py b/frigate/detectors/plugins/rknn.py index 7794172c1..1fdd8b9c0 100644 --- a/frigate/detectors/plugins/rknn.py +++ b/frigate/detectors/plugins/rknn.py @@ -19,7 +19,11 @@ DETECTOR_KEY = "rknn" supported_socs = ["rk3562", "rk3566", "rk3568", "rk3576", "rk3588"] -supported_models = {ModelTypeEnum.yolonas: "^deci-fp16-yolonas_[sml]$"} +supported_models = { + ModelTypeEnum.yologeneric: "^frigate-fp16-yolov9-[cemst]$", + ModelTypeEnum.yolonas: "^deci-fp16-yolonas_[sml]$", + ModelTypeEnum.yolox: "^rock-(fp16|i8)-yolox_(nano|tiny)$", +} model_cache_dir = os.path.join(MODEL_CACHE_DIR, "rknn_cache/") @@ -115,7 +119,7 @@ class Rknn(DetectionApi): model_props["model_type"] = model_type if model_matched: - model_props["filename"] = model_path + f"-{soc}-v2.3.0-1.rknn" + model_props["filename"] = model_path + f"-{soc}-v2.3.2-1.rknn" model_props["path"] = model_cache_dir + model_props["filename"] @@ -136,7 +140,7 @@ class Rknn(DetectionApi): os.mkdir(model_cache_dir) urllib.request.urlretrieve( - f"https://github.com/MarcA711/rknn-models/releases/download/v2.3.0/{filename}", + f"https://github.com/MarcA711/rknn-models/releases/download/v2.3.2/{filename}", model_cache_dir + filename, )