Add hardware accelerated scaling when using ffmpeg hwaccel presets (#4804)

* Use hardware accelerated scaling when hwaccel preset is set

* Set output types

* Add tests for scale, fix bugs

* Need to copy specific scale too

frigate/config.py

@@ -27,7 +27,8 @@ from frigate.util import (
     load_labels,
 )
 from frigate.ffmpeg_presets import (
-    parse_preset_hardware_acceleration,
+    parse_preset_hardware_acceleration_decode,
+    parse_preset_hardware_acceleration_scale,
     parse_preset_input,
     parse_preset_output_record,
     parse_preset_output_rtmp,
@@ -626,18 +627,15 @@ class CameraConfig(FrigateBaseModel):
         ffmpeg_output_args = []
         if "detect" in ffmpeg_input.roles:
             detect_args = get_ffmpeg_arg_list(self.ffmpeg.output_args.detect)
-
+            scale_detect_args = parse_preset_hardware_acceleration_scale(
-            ffmpeg_output_args = (
+                ffmpeg_input.hwaccel_args or self.ffmpeg.hwaccel_args,
-                [
+                detect_args,
-                    "-r",
+                self.detect.fps,
-                    str(self.detect.fps),
+                self.detect.width,
-                    "-s",
+                self.detect.height,
-                    f"{self.detect.width}x{self.detect.height}",
-                ]
-                + detect_args
-                + ffmpeg_output_args
-                + ["pipe:"]
             )
+
+            ffmpeg_output_args = scale_detect_args + ffmpeg_output_args + ["pipe:"]
         if "rtmp" in ffmpeg_input.roles and self.rtmp.enabled:
             rtmp_args = get_ffmpeg_arg_list(
                 parse_preset_output_rtmp(self.ffmpeg.output_args.rtmp)
@@ -667,12 +665,14 @@ class CameraConfig(FrigateBaseModel):
             ffmpeg_input.global_args or self.ffmpeg.global_args
         )
         hwaccel_args = get_ffmpeg_arg_list(
-            ffmpeg_input.hwaccel_args
+            parse_preset_hardware_acceleration_decode(ffmpeg_input.hwaccel_args)
-            or parse_preset_hardware_acceleration(self.ffmpeg.hwaccel_args)
+            or ffmpeg_input.hwaccel_args
+            or parse_preset_hardware_acceleration_decode(self.ffmpeg.hwaccel_args)
             or self.ffmpeg.hwaccel_args
         )
         input_args = get_ffmpeg_arg_list(
-            ffmpeg_input.input_args
+            parse_preset_input(ffmpeg_input.input_args, self.detect.fps)
+            or ffmpeg_input.input_args
             or parse_preset_input(self.ffmpeg.input_args, self.detect.fps)
             or self.ffmpeg.input_args
         )

frigate/ffmpeg_presets.py

@@ -9,39 +9,153 @@ _user_agent_args = [
     f"FFmpeg Frigate/{VERSION}",
 ]
 
-PRESETS_HW_ACCEL = {
+PRESETS_HW_ACCEL_DECODE = {
     "preset-rpi-32-h264": ["-c:v", "h264_v4l2m2m"],
     "preset-rpi-64-h264": ["-c:v", "h264_v4l2m2m"],
     "preset-intel-vaapi": [
+        "-hwaccel_flags",
+        "allow_profile_mismatch",
         "-hwaccel",
         "vaapi",
         "-hwaccel_device",
         "/dev/dri/renderD128",
         "-hwaccel_output_format",
-        "yuv420p",
+        "vaapi",
+    ],
+    "preset-intel-qsv-h264": [
+        "-hwaccel",
+        "qsv",
+        "-qsv_device",
+        "/dev/dri/renderD128",
+        "-hwaccel_output_format",
+        "qsv",
+        "-c:v",
+        "h264_qsv",
+    ],
+    "preset-intel-qsv-h265": [
+        "-hwaccel",
+        "qsv",
+        "-qsv_device",
+        "/dev/dri/renderD128",
+        "-hwaccel_output_format",
+        "qsv",
+        "-c:v",
+        "hevc_qsv",
     ],
-    "preset-intel-qsv-h264": ["-c:v", "h264_qsv"],
-    "preset-intel-qsv-h265": ["-c:v", "hevc_qsv"],
     "preset-amd-vaapi": [
+        "-hwaccel_flags",
+        "allow_profile_mismatch",
         "-hwaccel",
         "vaapi",
         "-hwaccel_device",
         "/dev/dri/renderD128",
         "-hwaccel_output_format",
-        "yuv420p",
+        "vaapi",
+    ],
+    "preset-nvidia-h264": [
+        "-hwaccel",
+        "cuda",
+        "-hwaccel_output_format",
+        "cuda",
+        "-extra_hw_frames",
+        "2",
+        "-c:v",
+        "h264_cuvid",
+    ],
+    "preset-nvidia-h265": [
+        "-hwaccel",
+        "cuda",
+        "-hwaccel_output_format",
+        "cuda",
+        "-extra_hw_frames",
+        "2",
+        "-c:v",
+        "hevc_cuvid",
+    ],
+    "preset-nvidia-mjpeg": [
+        "-hwaccel",
+        "cuda",
+        "-hwaccel_output_format",
+        "cuda",
+        "-extra_hw_frames",
+        "2",
+        "-c:v",
+        "mjpeg_cuvid",
+    ],
+}
+
+PRESETS_HW_ACCEL_SCALE = {
+    "preset-intel-vaapi": [
+        "-vf",
+        "fps={},deinterlace_vaapi=rate=field:auto=1,scale_vaapi=w={}:h={},hwdownload,format=yuv420p",
+        "-f",
+        "rawvideo",
+    ],
+    "preset-intel-qsv-h264": [
+        "-vf",
+        "vpp_qsv=framerate={}:scale_mode=1:w={}:h={}:detail=50:denoise=100:deinterlace=2:format=nv12,hwdownload,format=nv12,format=yuv420p",
+        "-f",
+        "rawvideo",
+    ],
+    "preset-intel-qsv-h265": [
+        "-vf",
+        "vpp_qsv=framerate={}:scale_mode=1:w={}:h={}:detail=50:denoise=100:deinterlace=2:format=nv12,hwdownload,format=nv12,format=yuv420p",
+        "-f",
+        "rawvideo",
+    ],
+    "preset-amd-vaapi": [
+        "-vf",
+        "fps={},deinterlace_vaapi=rate=field:auto=1,scale_vaapi=w={}:h={},hwdownload,format=yuv420p",
+        "-f",
+        "rawvideo",
+    ],
+    "preset-nvidia-h264": [
+        "-vf",
+        "fps={},scale_cuda=w={}:h={}:format=nv12,hwdownload,format=nv12,format=yuv420p",
+        "-f",
+        "rawvideo",
+    ],
+    "preset-nvidia-h265": [
+        "-vf",
+        "fps={},scale_cuda=w={}:h={}:format=nv12,hwdownload,format=nv12,format=yuv420p",
+        "-f",
+        "rawvideo",
+    ],
+    "default": [
+        "-r",
+        "{}",
+        "-s",
+        "{}",
     ],
-    "preset-nvidia-h264": ["-c:v", "h264_cuvid"],
-    "preset-nvidia-h265": ["-c:v", "hevc_cuvid"],
-    "preset-nvidia-mjpeg": ["-c:v", "mjpeg_cuvid"],
 }
 
 
-def parse_preset_hardware_acceleration(arg: Any) -> list[str]:
+def parse_preset_hardware_acceleration_decode(arg: Any) -> list[str]:
     """Return the correct preset if in preset format otherwise return None."""
     if not isinstance(arg, str):
         return None
 
-    return PRESETS_HW_ACCEL.get(arg, None)
+    return PRESETS_HW_ACCEL_DECODE.get(arg, None)
+
+
+def parse_preset_hardware_acceleration_scale(
+    arg: Any,
+    detect_args: list[str],
+    fps: int,
+    width: int,
+    height: int,
+) -> list[str]:
+    """Return the correct scaling preset or default preset if none is set."""
+    if not isinstance(arg, str):
+        scale = PRESETS_HW_ACCEL_SCALE["default"].copy()
+        scale[1] = str(fps)
+        scale[3] = f"{width}x{height}"
+        scale.extend(detect_args)
+        return scale
+
+    scale = PRESETS_HW_ACCEL_SCALE.get(arg, PRESETS_HW_ACCEL_SCALE["default"]).copy()
+    scale[1] = scale[1].format(fps, width, height)
+    return scale
 
 
 PRESETS_INPUT = {
@@ -170,7 +284,9 @@ def parse_preset_input(arg: Any, detect_fps: int) -> list[str]:
         return None
 
     if arg == "preset-jpeg-generic":
-        return PRESETS_INPUT[arg].format(f"{detect_fps}")
+        input = PRESETS_INPUT[arg].copy()
+        input[1] = str(detect_fps)
+        return input
 
     return PRESETS_INPUT.get(arg, None)
 

frigate/test/test_ffmpeg_presets.py

@@ -66,6 +66,25 @@ class TestFfmpegPresets(unittest.TestCase):
             " ".join(frigate_config.cameras["back"].ffmpeg_cmds[0]["cmd"])
         )
 
+    def test_ffmpeg_hwaccel_scale_preset(self):
+        self.default_ffmpeg["cameras"]["back"]["ffmpeg"][
+            "hwaccel_args"
+        ] = "preset-nvidia-h264"
+        self.default_ffmpeg["cameras"]["back"]["detect"] = {
+            "height": 1920,
+            "width": 2560,
+            "fps": 10,
+        }
+        frigate_config = FrigateConfig(**self.default_ffmpeg)
+        frigate_config.cameras["back"].create_ffmpeg_cmds()
+        assert "preset-nvidia-h264" not in (
+            " ".join(frigate_config.cameras["back"].ffmpeg_cmds[0]["cmd"])
+        )
+        assert (
+            "fps=10,scale_cuda=w=2560:h=1920:format=nv12,hwdownload,format=nv12,format=yuv420p"
+            in (" ".join(frigate_config.cameras["back"].ffmpeg_cmds[0]["cmd"]))
+        )
+
     def test_default_ffmpeg_input_arg_preset(self):
         frigate_config = FrigateConfig(**self.default_ffmpeg)