diff --git a/docker/rockchip/COCO/coco_subset_20.txt b/docker/rockchip/COCO/coco_subset_20.txt new file mode 100644 index 000000000..aa372fe7a --- /dev/null +++ b/docker/rockchip/COCO/coco_subset_20.txt @@ -0,0 +1,20 @@ +./subset/000000005001.jpg +./subset/000000038829.jpg +./subset/000000052891.jpg +./subset/000000075612.jpg +./subset/000000098261.jpg +./subset/000000181542.jpg +./subset/000000215245.jpg +./subset/000000277005.jpg +./subset/000000288685.jpg +./subset/000000301421.jpg +./subset/000000334371.jpg +./subset/000000348481.jpg +./subset/000000373353.jpg +./subset/000000397681.jpg +./subset/000000414673.jpg +./subset/000000419312.jpg +./subset/000000465822.jpg +./subset/000000475732.jpg +./subset/000000559707.jpg +./subset/000000574315.jpg \ No newline at end of file diff --git a/docker/rockchip/COCO/subset/000000005001.jpg b/docker/rockchip/COCO/subset/000000005001.jpg new file mode 100644 index 000000000..a7d4437ec Binary files /dev/null and b/docker/rockchip/COCO/subset/000000005001.jpg differ diff --git a/docker/rockchip/COCO/subset/000000038829.jpg b/docker/rockchip/COCO/subset/000000038829.jpg new file mode 100644 index 000000000..f275500e8 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000038829.jpg differ diff --git a/docker/rockchip/COCO/subset/000000052891.jpg b/docker/rockchip/COCO/subset/000000052891.jpg new file mode 100644 index 000000000..57344ef00 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000052891.jpg differ diff --git a/docker/rockchip/COCO/subset/000000075612.jpg b/docker/rockchip/COCO/subset/000000075612.jpg new file mode 100644 index 000000000..16555e4b6 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000075612.jpg differ diff --git a/docker/rockchip/COCO/subset/000000098261.jpg b/docker/rockchip/COCO/subset/000000098261.jpg new file mode 100644 index 000000000..57412f7f3 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000098261.jpg differ diff --git a/docker/rockchip/COCO/subset/000000181542.jpg b/docker/rockchip/COCO/subset/000000181542.jpg new file mode 100644 index 000000000..e3676d39d Binary files /dev/null and b/docker/rockchip/COCO/subset/000000181542.jpg differ diff --git a/docker/rockchip/COCO/subset/000000215245.jpg b/docker/rockchip/COCO/subset/000000215245.jpg new file mode 100644 index 000000000..624e4f1d0 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000215245.jpg differ diff --git a/docker/rockchip/COCO/subset/000000277005.jpg b/docker/rockchip/COCO/subset/000000277005.jpg new file mode 100644 index 000000000..629cb6e61 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000277005.jpg differ diff --git a/docker/rockchip/COCO/subset/000000288685.jpg b/docker/rockchip/COCO/subset/000000288685.jpg new file mode 100644 index 000000000..4dc759dad Binary files /dev/null and b/docker/rockchip/COCO/subset/000000288685.jpg differ diff --git a/docker/rockchip/COCO/subset/000000301421.jpg b/docker/rockchip/COCO/subset/000000301421.jpg new file mode 100644 index 000000000..2cbfa4e65 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000301421.jpg differ diff --git a/docker/rockchip/COCO/subset/000000334371.jpg b/docker/rockchip/COCO/subset/000000334371.jpg new file mode 100644 index 000000000..b47ac6d2c Binary files /dev/null and b/docker/rockchip/COCO/subset/000000334371.jpg differ diff --git a/docker/rockchip/COCO/subset/000000348481.jpg b/docker/rockchip/COCO/subset/000000348481.jpg new file mode 100644 index 000000000..a2cb75cc0 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000348481.jpg differ diff --git a/docker/rockchip/COCO/subset/000000373353.jpg b/docker/rockchip/COCO/subset/000000373353.jpg new file mode 100644 index 000000000..c09251120 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000373353.jpg differ diff --git a/docker/rockchip/COCO/subset/000000397681.jpg b/docker/rockchip/COCO/subset/000000397681.jpg new file mode 100644 index 000000000..5b9425914 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000397681.jpg differ diff --git a/docker/rockchip/COCO/subset/000000414673.jpg b/docker/rockchip/COCO/subset/000000414673.jpg new file mode 100644 index 000000000..587c370a1 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000414673.jpg differ diff --git a/docker/rockchip/COCO/subset/000000419312.jpg b/docker/rockchip/COCO/subset/000000419312.jpg new file mode 100644 index 000000000..274ea879a Binary files /dev/null and b/docker/rockchip/COCO/subset/000000419312.jpg differ diff --git a/docker/rockchip/COCO/subset/000000465822.jpg b/docker/rockchip/COCO/subset/000000465822.jpg new file mode 100644 index 000000000..3510d113a Binary files /dev/null and b/docker/rockchip/COCO/subset/000000465822.jpg differ diff --git a/docker/rockchip/COCO/subset/000000475732.jpg b/docker/rockchip/COCO/subset/000000475732.jpg new file mode 100644 index 000000000..51d96851b Binary files /dev/null and b/docker/rockchip/COCO/subset/000000475732.jpg differ diff --git a/docker/rockchip/COCO/subset/000000559707.jpg b/docker/rockchip/COCO/subset/000000559707.jpg new file mode 100644 index 000000000..4811ef1c2 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000559707.jpg differ diff --git a/docker/rockchip/COCO/subset/000000574315.jpg b/docker/rockchip/COCO/subset/000000574315.jpg new file mode 100644 index 000000000..ad06b6dba Binary files /dev/null and b/docker/rockchip/COCO/subset/000000574315.jpg differ diff --git a/docker/rockchip/Dockerfile b/docker/rockchip/Dockerfile index 48d7d30e6..e9c9602a8 100644 --- a/docker/rockchip/Dockerfile +++ b/docker/rockchip/Dockerfile @@ -7,19 +7,23 @@ FROM wheels as rk-wheels COPY docker/main/requirements-wheels.txt /requirements-wheels.txt COPY docker/rockchip/requirements-wheels-rk.txt /requirements-wheels-rk.txt RUN sed -i "/https:\/\//d" /requirements-wheels.txt +RUN sed -i "/onnxruntime/d" /requirements-wheels.txt RUN python3 -m pip config set global.break-system-packages true RUN pip3 wheel --wheel-dir=/rk-wheels -c /requirements-wheels.txt -r /requirements-wheels-rk.txt +RUN rm -rf /rk-wheels/opencv_python-* FROM deps AS rk-frigate ARG TARGETARCH RUN --mount=type=bind,from=rk-wheels,source=/rk-wheels,target=/deps/rk-wheels \ - pip3 install -U /deps/rk-wheels/*.whl --break-system-packages + pip3 install --no-deps -U /deps/rk-wheels/*.whl --break-system-packages WORKDIR /opt/frigate/ COPY --from=rootfs / / +COPY docker/rockchip/COCO /COCO +COPY docker/rockchip/conv2rknn.py /opt/conv2rknn.py -ADD https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.0.0/librknnrt.so /usr/lib/ +ADD https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.0/librknnrt.so /usr/lib/ RUN rm -rf /usr/lib/btbn-ffmpeg/bin/ffmpeg RUN rm -rf /usr/lib/btbn-ffmpeg/bin/ffprobe diff --git a/docker/rockchip/conv2rknn.py b/docker/rockchip/conv2rknn.py new file mode 100644 index 000000000..4f4a315e1 --- /dev/null +++ b/docker/rockchip/conv2rknn.py @@ -0,0 +1,82 @@ +import os + +import rknn +import yaml +from rknn.api import RKNN + +try: + with open(rknn.__path__[0] + "/VERSION") as file: + tk_version = file.read().strip() +except FileNotFoundError: + pass + +try: + with open("/config/conv2rknn.yaml", "r") as config_file: + configuration = yaml.safe_load(config_file) +except FileNotFoundError: + raise Exception("Please place a config.yaml file in /config/conv2rknn.yaml") + +if configuration["config"] != None: + rknn_config = configuration["config"] +else: + rknn_config = {} + +if not os.path.isdir("/config/model_cache/rknn_cache/onnx"): + raise Exception( + "Place the onnx models you want to convert to rknn format in /config/model_cache/rknn_cache/onnx" + ) + +if "soc" not in configuration: + try: + with open("/proc/device-tree/compatible") as file: + soc = file.read().split(",")[-1].strip("\x00") + except FileNotFoundError: + raise Exception("Make sure to run docker in privileged mode.") + + configuration["soc"] = [ + soc, + ] + +if "quantization" not in configuration: + configuration["quantization"] = False + +if "output_name" not in configuration: + configuration["output_name"] = "{{input_basename}}" + +for input_filename in os.listdir("/config/model_cache/rknn_cache/onnx"): + for soc in configuration["soc"]: + quant = "i8" if configuration["quantization"] else "fp16" + + input_path = "/config/model_cache/rknn_cache/onnx/" + input_filename + input_basename = input_filename[: input_filename.rfind(".")] + + output_filename = ( + configuration["output_name"].format( + quant=quant, + input_basename=input_basename, + soc=soc, + tk_version=tk_version, + ) + + ".rknn" + ) + output_path = "/config/model_cache/rknn_cache/" + output_filename + + rknn_config["target_platform"] = soc + + rknn = RKNN(verbose=True) + rknn.config(**rknn_config) + + if rknn.load_onnx(model=input_path) != 0: + raise Exception("Error loading model.") + + if ( + rknn.build( + do_quantization=configuration["quantization"], + dataset="/COCO/coco_subset_20.txt", + ) + != 0 + ): + raise Exception("Error building model.") + + if rknn.export_rknn(output_path) != 0: + raise Exception("Error exporting rknn model.") diff --git a/docker/rockchip/requirements-wheels-rk.txt b/docker/rockchip/requirements-wheels-rk.txt index 71a9f58d1..8d5b5efe0 100644 --- a/docker/rockchip/requirements-wheels-rk.txt +++ b/docker/rockchip/requirements-wheels-rk.txt @@ -1 +1,2 @@ -rknn-toolkit-lite2 @ https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.0.0/rknn_toolkit_lite2-2.0.0b0-cp311-cp311-linux_aarch64.whl \ No newline at end of file +rknn-toolkit2 == 2.3.0 +rknn-toolkit-lite2 == 2.3.0 \ No newline at end of file diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 3e03daab3..8b12c7e54 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -550,7 +550,7 @@ Hardware accelerated object detection is supported on the following SoCs: - RK3576 - RK3588 -This implementation uses the [Rockchip's RKNN-Toolkit2](https://github.com/airockchip/rknn-toolkit2/), version v2.0.0.beta0. Currently, only [Yolo-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) is supported as object detection model. +This implementation uses the [Rockchip's RKNN-Toolkit2](https://github.com/airockchip/rknn-toolkit2/), version v2.3.0. Currently, only [Yolo-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) is supported as object detection model. ### Prerequisites @@ -623,7 +623,41 @@ $ cat /sys/kernel/debug/rknpu/load ::: - All models are automatically downloaded and stored in the folder `config/model_cache/rknn_cache`. After upgrading Frigate, you should remove older models to free up space. -- You can also provide your own `.rknn` model. You should not save your own models in the `rknn_cache` folder, store them directly in the `model_cache` folder or another subfolder. To convert a model to `.rknn` format see the `rknn-toolkit2` (requires a x86 machine). Note, that there is only post-processing for the supported models. +- You can also provide your own `.rknn` model. You should not save your own models in the `rknn_cache` folder, store them directly in the `model_cache` folder or another subfolder. To convert a model to `.rknn` format see the `rknn-toolkit2`. Note, that there is only post-processing for the supported models. + +### Converting your own onnx model to rknn format + +To convert a onnx model to the rknn format using the [rknn-toolkit2](https://github.com/airockchip/rknn-toolkit2/) you have to: + +- Place one ore more models in onnx format in the directory `config/model_cache/rknn_cache/onnx` on your docker host (this might require `sudo` privileges). +- Save the configuration file under `config/conv2rknn.yaml` (see below for details). +- Run `docker exec python3 /opt/conv2rknn.py`. If the conversion was successful, the rknn models will be placed in `config/model_cache/rknn_cache`. + +This is an example configuration file that you need to adjust to your specific onnx model: + +```yaml +soc: ["rk3562","rk3566", "rk3568", "rk3576", "rk3588"] +quantization: false + +output_name: "{input_basename}" + +config: + mean_values: [[0, 0, 0]] + std_values: [[255, 255, 255]] + quant_img_rgb2bgr: true +``` + +Explanation of the paramters: + +- `soc`: A list of all SoCs you want to build the rknn model for. If you don't specify this parameter, the script tries to find out your SoC and builds the rknn model for this one. +- `quantization`: true: 8 bit integer (i8) quantization, false: 16 bit float (fp16). Default: false. +- `output_name`: The output name of the model. The following variables are available: + - `quant`: "i8" or "fp16" depending on the config + - `input_basename`: the basename of the input model (e.g. "my_model" if the input model is calles "my_model.onnx") + - `soc`: the SoC this model was build for (e.g. "rk3588") + - `tk_version`: Version of `rknn-toolkit2` (e.g. "2.3.0") + - **example**: Specifying `output_name = "frigate-{quant}-{input_basename}-{soc}-v{tk_version}"` could result in a model called `frigate-i8-my_model-rk3588-v2.3.0.rknn`. +- `config`: Configuration passed to `rknn-toolkit2` for model conversion. For an explanation of all available parameters have a look at section "2.2. Model configuration" of [this manual](https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.0/03_Rockchip_RKNPU_API_Reference_RKNN_Toolkit2_V2.3.0_EN.pdf). ## Hailo-8l diff --git a/frigate/detectors/plugins/rknn.py b/frigate/detectors/plugins/rknn.py index df94d7b62..bfd7866e6 100644 --- a/frigate/detectors/plugins/rknn.py +++ b/frigate/detectors/plugins/rknn.py @@ -108,7 +108,7 @@ class Rknn(DetectionApi): model_props["model_type"] = model_type if model_matched: - model_props["filename"] = model_path + f"-{soc}-v2.0.0-1.rknn" + model_props["filename"] = model_path + f"-{soc}-v2.3.0-1.rknn" model_props["path"] = model_cache_dir + model_props["filename"] @@ -129,7 +129,7 @@ class Rknn(DetectionApi): os.mkdir(model_cache_dir) urllib.request.urlretrieve( - f"https://github.com/MarcA711/rknn-models/releases/download/v2.0.0/{filename}", + f"https://github.com/MarcA711/rknn-models/releases/download/v2.3.0/{filename}", model_cache_dir + filename, )