Add initial support for Rockchip Media Process Platform (rkmpp) hardware acceleration (#2418)

* Add Rockchip Media Process Platform (rkmpp) acceleration

* remove fourcc stuff; it's exclusive to videotoolbox

* update changelog

---------

Co-authored-by: Jason Dove <1695733+jasongdove@users.noreply.github.com>

CHANGELOG.md

@@ -16,9 +16,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     - This means resetting the playout will reset the rerun history
   - Items will still be scheduled from the rerun collection if it is used before the first run collection
     - Otherwise, the rerun collection would be considered "empty" which prevents the playout build altogether
+- Add `Rkmpp` hardware acceleration by @peterdey
+  - This is supported using jellyfin-ffmpeg7 on devices like Orange Pi 5 Plus and NanoPi R6S
 
 ### Fixed
-- Fix green output when libplacebo tonemapping is used with NVIDIA acceleration
+- Fix green output when libplacebo tonemapping is used with NVIDIA acceleration and 10-bit output in FFmpeg Profile
 
 ## [25.6.0] - 2025-09-14
 ### Added

ErsatzTV.Application/FFmpegProfiles/Queries/GetSupportedHardwareAccelerationKindsHandler.cs

@@ -72,6 +72,11 @@ public class
         //     result.Add(HardwareAccelerationKind.V4l2m2m);
         // }
 
+        if (ffmpegCapabilities.HasHardwareAcceleration(HardwareAccelerationMode.Rkmpp))
+        {
+            result.Add(HardwareAccelerationKind.Rkmpp);
+        }
+
         return result;
     }
 

ErsatzTV.Core/Domain/HardwareAccelerationKind.cs

@@ -8,5 +8,6 @@ public enum HardwareAccelerationKind
     Vaapi = 3,
     VideoToolbox = 4,
     Amf = 5,
-    V4l2m2m = 6
+    V4l2m2m = 6,
+    Rkmpp = 7
 }

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -1129,6 +1129,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             HardwareAccelerationKind.VideoToolbox => HardwareAccelerationMode.VideoToolbox,
             HardwareAccelerationKind.Amf => HardwareAccelerationMode.Amf,
             HardwareAccelerationKind.V4l2m2m => HardwareAccelerationMode.V4l2m2m,
+            HardwareAccelerationKind.Rkmpp => HardwareAccelerationMode.Rkmpp,
             _ => HardwareAccelerationMode.None
         };
 }

ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs

@@ -51,6 +51,7 @@ public class FFmpegCapabilities : IFFmpegCapabilities
             HardwareAccelerationMode.VideoToolbox => FFmpegKnownHardwareAcceleration.VideoToolbox,
             HardwareAccelerationMode.OpenCL => FFmpegKnownHardwareAcceleration.OpenCL,
             HardwareAccelerationMode.Vulkan => FFmpegKnownHardwareAcceleration.Vulkan,
+            HardwareAccelerationMode.Rkmpp => FFmpegKnownHardwareAcceleration.Rkmpp,
             _ => Option<FFmpegKnownHardwareAcceleration>.None
         };
 

ErsatzTV.FFmpeg/Capabilities/FFmpegKnownEncoder.cs

@@ -16,6 +16,8 @@ public record FFmpegKnownEncoder
         "h264_v4l2m2m",
         "hevc_v4l2m2m",
         "h264_videotoolbox",
-        "hevc_videotoolbox"
+        "hevc_videotoolbox",
+        "h264_rkmpp",
+        "hevc_rkmpp"
     ];
 }

ErsatzTV.FFmpeg/Capabilities/FFmpegKnownHardwareAcceleration.cs

@@ -10,6 +10,7 @@ public record FFmpegKnownHardwareAcceleration
     public static readonly FFmpegKnownHardwareAcceleration OpenCL = new("opencl");
     public static readonly FFmpegKnownHardwareAcceleration Vulkan = new("vulkan");
     public static readonly FFmpegKnownHardwareAcceleration V4l2m2m = new("v4l2m2m");
+    public static readonly FFmpegKnownHardwareAcceleration Rkmpp = new("rkmpp");
 
     private FFmpegKnownHardwareAcceleration(string Name) => this.Name = Name;
 
@@ -24,6 +25,7 @@ public record FFmpegKnownHardwareAcceleration
         VideoToolbox.Name,
         OpenCL.Name,
         Vulkan.Name,
-        V4l2m2m.Name
+        V4l2m2m.Name,
+        Rkmpp.Name
     ];
 }

ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs

@@ -110,6 +110,7 @@ public class HardwareCapabilitiesFactory : IHardwareCapabilitiesFactory
             HardwareAccelerationMode.VideoToolbox => new VideoToolboxHardwareCapabilities(ffmpegCapabilities, _logger),
             HardwareAccelerationMode.Amf => new AmfHardwareCapabilities(),
             HardwareAccelerationMode.V4l2m2m => new V4l2m2mHardwareCapabilities(ffmpegCapabilities),
+            HardwareAccelerationMode.Rkmpp => new RkmppHardwareCapabilities(),
             _ => new DefaultHardwareCapabilities()
         };
     }

ErsatzTV.FFmpeg/Capabilities/RkmppHardwareCapabilities.cs

@@ -0,0 +1,34 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Capabilities;
+
+public class RkmppHardwareCapabilities : IHardwareCapabilities
+{
+    public FFmpegCapability CanDecode(
+        string videoFormat,
+        Option<string> videoProfile,
+        Option<IPixelFormat> maybePixelFormat,
+        bool isHdr) => FFmpegCapability.Hardware;
+
+    public FFmpegCapability CanEncode(
+        string videoFormat,
+        Option<string> videoProfile,
+        Option<IPixelFormat> maybePixelFormat)
+    {
+        int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
+
+        return (videoFormat, bitDepth) switch
+        {
+            // 10-bit hevc encoding is not yet supported by ffmpeg
+            (VideoFormat.Hevc, 10) => FFmpegCapability.Software,
+
+            // 10-bit h264 encoding is not support by any hardware
+            (VideoFormat.H264, 10) => FFmpegCapability.Software,
+
+            _ => FFmpegCapability.Hardware
+        };
+    }
+
+    public Option<RateControlMode> GetRateControlMode(string videoFormat, Option<IPixelFormat> maybePixelFormat) =>
+        Option<RateControlMode>.None;
+}

ErsatzTV.FFmpeg/Decoder/DecoderRkmpp.cs

@@ -0,0 +1,27 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Decoder;
+
+public class DecoderRkmpp : DecoderBase
+{
+    protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Software;
+
+    public override string Name => "implicit_rkmpp";
+    public override string[] InputOptions(InputFile inputFile) => Array.Empty<string>();
+    
+    /*
+    public override string[] InputOptions(InputFile inputFile) =>
+        new[] { "-hwaccel_output_format", "drm_prime" };
+    */
+
+    public override FrameState NextState(FrameState currentState)
+    {
+        FrameState nextState = base.NextState(currentState);
+
+        return currentState.PixelFormat.Match(
+            pixelFormat => pixelFormat.BitDepth == 8
+                ? nextState with { PixelFormat = new PixelFormatNv12(pixelFormat.Name) }
+                : nextState with { PixelFormat = new PixelFormatNv15(pixelFormat.Name) },
+            () => nextState);
+    }
+}

ErsatzTV.FFmpeg/Encoder/Rkmpp/EncoderH264Rkmpp.cs

@@ -0,0 +1,32 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Encoder.Rkmpp;
+
+public class EncoderH264Rkmpp(Option<string> maybeVideoProfile) : EncoderBase
+{
+    public override string Name => "h264_rkmpp";
+    public override StreamKind Kind => StreamKind.Video;
+
+    public override string[] OutputOptions
+    {
+        get
+        {
+            foreach (string videoProfile in maybeVideoProfile)
+            {
+                return
+                [
+                    "-c:v", Name,
+                    "-profile:v", videoProfile.ToLowerInvariant()
+                ];
+            }
+
+            return base.OutputOptions;
+        }
+    }
+
+    public override FrameState NextState(FrameState currentState) => currentState with
+    {
+        VideoFormat = VideoFormat.H264,
+        FrameDataLocation = FrameDataLocation.Hardware
+    };
+}

ErsatzTV.FFmpeg/Encoder/Rkmpp/EncoderHevcRkmpp.cs

@@ -0,0 +1,26 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Encoder.Rkmpp;
+
+public class EncoderHevcRkmpp : EncoderBase
+{
+    private readonly int _desiredBitDepth;
+
+    public EncoderHevcRkmpp(int desiredBitDepth) => _desiredBitDepth = desiredBitDepth;
+
+    public override string Name => "hevc_rkmpp";
+    public override StreamKind Kind => StreamKind.Video;
+
+    public override string[] OutputOptions => base.OutputOptions.Concat(
+        new[]
+        {
+            "-profile:v",
+            _desiredBitDepth == 10 ? "main10" : "main"
+        }).ToArray();
+
+    public override FrameState NextState(FrameState currentState) => currentState with
+    {
+        VideoFormat = VideoFormat.Hevc,
+        FrameDataLocation = FrameDataLocation.Hardware
+    };
+}

ErsatzTV.FFmpeg/Filter/SubtitleHardwareUploadFilter.cs

@@ -31,6 +31,9 @@ public class SubtitleHardwareUploadFilter : BaseFilter
             // leave v4l2m2m in software since we use a software overlay filter
             HardwareAccelerationMode.V4l2m2m => string.Empty,
 
+            // leave rkmpp in software since we use a software overlay filter
+            HardwareAccelerationMode.Rkmpp => string.Empty,
+
             _ => "hwupload"
         };
 

ErsatzTV.FFmpeg/Format/FFmpegFormat.cs

@@ -8,5 +8,6 @@ public class FFmpegFormat
     public const string YUVA420P = "yuva420p";
     public const string P010LE = "p010le";
     public const string NV12 = "nv12";
+    public const string NV15 = "nv15";
     public const string VAAPI = "vaapi";
 }

ErsatzTV.FFmpeg/Format/PixelFormatNv15.cs

@@ -0,0 +1,11 @@
+namespace ErsatzTV.FFmpeg.Format;
+
+public class PixelFormatNv15 : IPixelFormat
+{
+    public PixelFormatNv15(string name) => Name = name;
+
+    public string Name { get; }
+
+    public string FFmpegName => "nv15";
+    public int BitDepth => 8;
+}

ErsatzTV.FFmpeg/GlobalOption/HardwareAcceleration/RkmppHardwareAccelerationOption.cs

@@ -0,0 +1,11 @@
+namespace ErsatzTV.FFmpeg.GlobalOption.HardwareAcceleration;
+
+public class RkmppHardwareAccelerationOption : GlobalOption
+{
+    public override string[] GlobalOptions => new[] { "-hwaccel", "rkmpp" };
+
+    public override FrameState NextState(FrameState currentState) => currentState with
+    {
+        FrameDataLocation = FrameDataLocation.Hardware
+    };
+}

ErsatzTV.FFmpeg/HardwareAccelerationMode.cs

@@ -10,5 +10,6 @@ public enum HardwareAccelerationMode
     Amf = 5,
     OpenCL = 6,
     Vulkan = 7,
-    V4l2m2m = 8
+    V4l2m2m = 8,
+    Rkmpp = 9
 }

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderFactory.cs

@@ -87,7 +87,22 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory
                 fontsFolder,
                 _logger),
 
-            // force software pipeline when content is HDR (and not VAAPI or NVENC or QSV)
+            HardwareAccelerationMode.Rkmpp when capabilities is not NoHardwareCapabilities => new
+                RkmppPipelineBuilder(
+                    ffmpegCapabilities,
+                    capabilities,
+                    hardwareAccelerationMode,
+                    videoInputFile,
+                    audioInputFile,
+                    watermarkInputFile,
+                    subtitleInputFile,
+                    concatInputFile,
+                    graphicsEngineInput,
+                    reportsFolder,
+                    fontsFolder,
+                    _logger),
+
+            // force software pipeline when content is HDR (and not VAAPI or NVENC or QSV or Rkmpp)
             _ when isHdrContent => new SoftwarePipelineBuilder(
                 ffmpegCapabilities,
                 HardwareAccelerationMode.None,

ErsatzTV.FFmpeg/Pipeline/RkmppPipelineBuilder.cs

@@ -0,0 +1,160 @@
+using ErsatzTV.FFmpeg.Capabilities;
+using ErsatzTV.FFmpeg.Decoder;
+using ErsatzTV.FFmpeg.Encoder;
+using ErsatzTV.FFmpeg.Encoder.Rkmpp;
+using ErsatzTV.FFmpeg.Filter;
+using ErsatzTV.FFmpeg.Format;
+using ErsatzTV.FFmpeg.GlobalOption.HardwareAcceleration;
+using ErsatzTV.FFmpeg.OutputFormat;
+using ErsatzTV.FFmpeg.OutputOption;
+using Microsoft.Extensions.Logging;
+
+namespace ErsatzTV.FFmpeg.Pipeline;
+
+public class RkmppPipelineBuilder : SoftwarePipelineBuilder
+{
+    private readonly IHardwareCapabilities _hardwareCapabilities;
+    private readonly ILogger _logger;
+
+    public RkmppPipelineBuilder(
+        IFFmpegCapabilities ffmpegCapabilities,
+        IHardwareCapabilities hardwareCapabilities,
+        HardwareAccelerationMode hardwareAccelerationMode,
+        Option<VideoInputFile> videoInputFile,
+        Option<AudioInputFile> audioInputFile,
+        Option<WatermarkInputFile> watermarkInputFile,
+        Option<SubtitleInputFile> subtitleInputFile,
+        Option<ConcatInputFile> concatInputFile,
+        Option<GraphicsEngineInput> graphicsEngineInput,
+        string reportsFolder,
+        string fontsFolder,
+        ILogger logger) : base(
+        ffmpegCapabilities,
+        hardwareAccelerationMode,
+        videoInputFile,
+        audioInputFile,
+        watermarkInputFile,
+        subtitleInputFile,
+        concatInputFile,
+        graphicsEngineInput,
+        reportsFolder,
+        fontsFolder,
+        logger)
+    {
+        _hardwareCapabilities = hardwareCapabilities;
+        _logger = logger;
+        _logger.LogDebug("Using RkmppPipelineBuilder");
+    }
+
+    protected override FFmpegState SetAccelState(
+        VideoStream videoStream,
+        FFmpegState ffmpegState,
+        FrameState desiredState,
+        PipelineContext context,
+        ICollection<IPipelineStep> pipelineSteps)
+    {
+        FFmpegCapability decodeCapability = _hardwareCapabilities.CanDecode(
+            videoStream.Codec,
+            videoStream.Profile,
+            videoStream.PixelFormat,
+            videoStream.ColorParams.IsHdr);
+        FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
+            desiredState.VideoFormat,
+            desiredState.VideoProfile,
+            desiredState.PixelFormat);
+
+        // use software encoding (rawvideo) when piping to parent hls segmenter
+        if (ffmpegState.OutputFormat is OutputFormatKind.Nut)
+        {
+            encodeCapability = FFmpegCapability.Software;
+            _logger.LogDebug("Using software encoder");
+        }
+
+        if (decodeCapability is FFmpegCapability.Hardware)
+        {
+            pipelineSteps.Add(new RkmppHardwareAccelerationOption());
+            _logger.LogDebug("Using RkmppHardwareAccelerationOption decoder");
+        }
+
+        // disable hw accel if decoder/encoder isn't supported
+        return ffmpegState with
+        {
+            DecoderHardwareAccelerationMode = decodeCapability == FFmpegCapability.Hardware
+                ? HardwareAccelerationMode.Rkmpp
+                : HardwareAccelerationMode.None,
+            EncoderHardwareAccelerationMode = encodeCapability == FFmpegCapability.Hardware
+                ? HardwareAccelerationMode.Rkmpp
+                : HardwareAccelerationMode.None
+        };
+    }
+
+    protected override Option<IDecoder> SetDecoder(
+        VideoInputFile videoInputFile,
+        VideoStream videoStream,
+        FFmpegState ffmpegState,
+        PipelineContext context)
+    {
+        Option<IDecoder> maybeDecoder = (ffmpegState.DecoderHardwareAccelerationMode, videoStream.Codec) switch
+        {
+            (HardwareAccelerationMode.Rkmpp, _) => new DecoderRkmpp(),
+
+            _ => GetSoftwareDecoder(videoStream)
+        };
+
+        foreach (IDecoder decoder in maybeDecoder)
+        {
+            videoInputFile.AddOption(decoder);
+            return Some(decoder);
+        }
+
+        return None;
+    }
+
+    protected override Option<IEncoder> GetEncoder(
+        FFmpegState ffmpegState,
+        FrameState currentState,
+        FrameState desiredState) =>
+        (ffmpegState.EncoderHardwareAccelerationMode, desiredState.VideoFormat) switch
+        {
+            (HardwareAccelerationMode.Rkmpp, VideoFormat.Hevc) =>
+                new EncoderHevcRkmpp(desiredState.BitDepth),
+            (HardwareAccelerationMode.Rkmpp, VideoFormat.H264) =>
+                new EncoderH264Rkmpp(desiredState.VideoProfile),
+
+            _ => GetSoftwareEncoder(ffmpegState, currentState, desiredState)
+        };
+
+    protected override List<IPipelineFilterStep> SetPixelFormat(
+        VideoStream videoStream,
+        Option<IPixelFormat> desiredPixelFormat,
+        FrameState currentState,
+        ICollection<IPipelineStep> pipelineSteps)
+    {
+        var result = new List<IPipelineFilterStep>();
+
+        foreach (IPixelFormat pixelFormat in desiredPixelFormat)
+        {
+            if (!videoStream.ColorParams.IsBt709)
+            {
+                // _logger.LogDebug("Adding colorspace filter");
+                var colorspace = new ColorspaceFilter(currentState, videoStream, pixelFormat);
+                currentState = colorspace.NextState(currentState);
+                result.Add(colorspace);
+            }
+
+            if (currentState.PixelFormat.Map(f => f.FFmpegName) != pixelFormat.FFmpegName)
+            {
+                _logger.LogDebug(
+                    "Format {A} doesn't equal {B}",
+                    currentState.PixelFormat.Map(f => f.FFmpegName),
+                    pixelFormat.FFmpegName);
+
+                //result.Add(new PixelFormatFilter(pixelFormat));
+            }
+
+            pipelineSteps.Add(new PixelFormatOutputOption(pixelFormat));
+        }
+
+        return result;
+    }
+}

ErsatzTV.Infrastructure/Health/Checks/HardwareAccelerationHealthCheck.cs

@@ -129,6 +129,9 @@ public class HardwareAccelerationHealthCheck : BaseHealthCheck, IHardwareAcceler
                 case "videotoolbox":
                     result.Add(HardwareAccelerationKind.VideoToolbox);
                     break;
+                case "rkmpp":
+                    result.Add(HardwareAccelerationKind.Rkmpp);
+                    break;
             }
         }
 

ErsatzTV/Pages/FFmpegEditor.razor

@@ -77,7 +77,7 @@
             </div>
             <MudSelect @bind-Value="_model.VideoProfile"
                        For="@(() => _model.VideoProfile)"
-                       Disabled="@(_model.VideoFormat != FFmpegProfileVideoFormat.H264 || _model.HardwareAcceleration != HardwareAccelerationKind.Nvenc && _model.HardwareAcceleration != HardwareAccelerationKind.Qsv && _model.HardwareAcceleration != HardwareAccelerationKind.VideoToolbox && _model.HardwareAcceleration != HardwareAccelerationKind.None)"
+                       Disabled="@(_model.VideoFormat != FFmpegProfileVideoFormat.H264 || _model.HardwareAcceleration != HardwareAccelerationKind.Nvenc && _model.HardwareAcceleration != HardwareAccelerationKind.Qsv && _model.HardwareAcceleration != HardwareAccelerationKind.VideoToolbox && _model.HardwareAcceleration != HardwareAccelerationKind.Rkmpp && _model.HardwareAcceleration != HardwareAccelerationKind.None)"
                        Clearable="true">
                 <MudSelectItem Value="@VideoProfile.Main">main</MudSelectItem>
                 <MudSelectItem Value="@VideoProfile.High">high</MudSelectItem>
@@ -431,6 +431,7 @@
             HardwareAccelerationKind.Vaapi => HardwareAccelerationMode.Vaapi,
             HardwareAccelerationKind.VideoToolbox => HardwareAccelerationMode.VideoToolbox,
             HardwareAccelerationKind.V4l2m2m => HardwareAccelerationMode.V4l2m2m,
+            HardwareAccelerationKind.Rkmpp => HardwareAccelerationMode.Rkmpp,
             _ => HardwareAccelerationMode.None
         };
 

ErsatzTV/Validators/FFmpegProfileEditViewModelValidator.cs

@@ -45,6 +45,12 @@ public class FFmpegProfileEditViewModelValidator : AbstractValidator<FFmpegProfi
         FFmpegProfileVideoFormat.H264,
         FFmpegProfileVideoFormat.Hevc
     ];
+
+    private static readonly List<FFmpegProfileVideoFormat> RkmppFormats =
+    [
+        FFmpegProfileVideoFormat.H264,
+        FFmpegProfileVideoFormat.Hevc
+    ];
     public FFmpegProfileEditViewModelValidator()
     {
         RuleFor(x => x.Name).NotEmpty();
@@ -106,6 +112,14 @@ public class FFmpegProfileEditViewModelValidator : AbstractValidator<FFmpegProfi
                     .WithMessage("V4L2 M2M supports formats (h264, hevc)");
             });
 
+        When(
+            x => x.HardwareAcceleration == HardwareAccelerationKind.Rkmpp,
+            () =>
+            {
+                RuleFor(x => x.VideoFormat).Must(c => RkmppFormats.Contains(c))
+                    .WithMessage("Rkmpp supports formats (h264, hevc)");
+            });
+
         When(
             x => x.VideoFormat == FFmpegProfileVideoFormat.Mpeg2Video,
             () => RuleFor(x => x.BitDepth)