use libplacebo to tonemap with nvidia/vulkan (#2029)

CHANGELOG.md

@@ -23,7 +23,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     - `Strict` will add nearly 24h (23:58) of unscheduled time so that it can start exactly at 6:00 AM the next day
     - `Flexible` will NOT add unscheduled time, and will schedule its item at 6:02 AM (which may also affect the scheduling of later items)
 - Add basic HDR transcoding support
-  - VAAPI may use hardware-accelerated tone mapping
+  - VAAPI may use hardware-accelerated tone mapping (when opencl accel is also available)
+  - NVIDIA may use hardware-accelerated tone mapping (when vulkan accel and libplacebo filter are also available)
   - In all other cases, HDR content will use a software pipeline
 
 ### Changed

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -398,7 +398,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             ptsOffset,
             playbackSettings.ThreadCount,
             qsvExtraHardwareFrames,
-            videoVersion is BackgroundImageMediaVersion { IsSongWithProgress: true });
+            videoVersion is BackgroundImageMediaVersion { IsSongWithProgress: true },
+            IsHdrTonemap: false);
 
         _logger.LogDebug("FFmpeg desired state {FrameState}", desiredState);
 
@@ -552,7 +553,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             ptsOffset,
             Option<int>.None,
             qsvExtraHardwareFrames,
-            IsSongWithProgress: false);
+            IsSongWithProgress: false,
+            IsHdrTonemap: false);
 
         var ffmpegSubtitleStream = new ErsatzTV.FFmpeg.MediaStream(0, "ass", StreamKind.Video);
 
@@ -746,7 +748,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             0,
             playbackSettings.ThreadCount,
             Optional(channel.FFmpegProfile.QsvExtraHardwareFrames),
-            IsSongWithProgress: false);
+            IsSongWithProgress: false,
+            IsHdrTonemap: false);
 
         _logger.LogDebug("FFmpeg desired state {FrameState}", desiredState);
 

ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs

@@ -93,6 +93,7 @@ public class PipelineBuilderBaseTests
             0,
             Option<int>.None,
             Option<int>.None,
+            false,
             false);
 
         var builder = new SoftwarePipelineBuilder(
@@ -187,6 +188,7 @@ public class PipelineBuilderBaseTests
             0,
             Option<int>.None,
             Option<int>.None,
+            false,
             false);
 
         var builder = new SoftwarePipelineBuilder(
@@ -337,6 +339,7 @@ public class PipelineBuilderBaseTests
             0,
             Option<int>.None,
             Option<int>.None,
+            false,
             false);
 
         var builder = new SoftwarePipelineBuilder(
@@ -425,6 +428,7 @@ public class PipelineBuilderBaseTests
             0,
             Option<int>.None,
             Option<int>.None,
+            false,
             false);
 
         var builder = new SoftwarePipelineBuilder(

ErsatzTV.FFmpeg/Capabilities/AmfHardwareCapabilities.cs

@@ -7,7 +7,8 @@ public class AmfHardwareCapabilities : IHardwareCapabilities
     public FFmpegCapability CanDecode(
         string videoFormat,
         Option<string> videoProfile,
-        Option<IPixelFormat> maybePixelFormat) => FFmpegCapability.Software;
+        Option<IPixelFormat> maybePixelFormat,
+        bool isHdr) => FFmpegCapability.Software;
 
     public FFmpegCapability CanEncode(
         string videoFormat,

ErsatzTV.FFmpeg/Capabilities/DefaultHardwareCapabilities.cs

@@ -7,7 +7,8 @@ public class DefaultHardwareCapabilities : IHardwareCapabilities
     public FFmpegCapability CanDecode(
         string videoFormat,
         Option<string> videoProfile,
-        Option<IPixelFormat> maybePixelFormat)
+        Option<IPixelFormat> maybePixelFormat,
+        bool isHdr)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 

ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs

@@ -41,6 +41,7 @@ public class FFmpegCapabilities : IFFmpegCapabilities
             HardwareAccelerationMode.Vaapi => FFmpegKnownHardwareAcceleration.Vaapi,
             HardwareAccelerationMode.VideoToolbox => FFmpegKnownHardwareAcceleration.VideoToolbox,
             HardwareAccelerationMode.OpenCL =>  FFmpegKnownHardwareAcceleration.OpenCL,
+            HardwareAccelerationMode.Vulkan => FFmpegKnownHardwareAcceleration.Vulkan,
             _ => Option<FFmpegKnownHardwareAcceleration>.None
         };
 

ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs

@@ -4,6 +4,7 @@ public record FFmpegKnownFilter
 {
     public static readonly FFmpegKnownFilter ScaleNpp = new("scale_npp");
     public static readonly FFmpegKnownFilter TonemapOpenCL = new("tonemap_opencl");
+    public static readonly FFmpegKnownFilter Libplacebo = new("libplacebo");
 
     private FFmpegKnownFilter(string Name) => this.Name = Name;
 

ErsatzTV.FFmpeg/Capabilities/FFmpegKnownHardwareAcceleration.cs

@@ -8,19 +8,20 @@ public record FFmpegKnownHardwareAcceleration
     public static readonly FFmpegKnownHardwareAcceleration Vaapi = new("vaapi");
     public static readonly FFmpegKnownHardwareAcceleration VideoToolbox = new("videotoolbox");
     public static readonly FFmpegKnownHardwareAcceleration OpenCL = new("opencl");
+    public static readonly FFmpegKnownHardwareAcceleration Vulkan = new("vulkan");
 
     private FFmpegKnownHardwareAcceleration(string Name) => this.Name = Name;
 
     public string Name { get; }
 
     public static IList<string> AllAccels =>
-        new[]
-        {
-            Amf.Name,
-            Cuda.Name,
-            Qsv.Name,
-            Vaapi.Name,
-            VideoToolbox.Name,
-            OpenCL.Name
-        };
+    [
+        Amf.Name,
+        Cuda.Name,
+        Qsv.Name,
+        Vaapi.Name,
+        VideoToolbox.Name,
+        OpenCL.Name,
+        Vulkan.Name
+    ];
 }

ErsatzTV.FFmpeg/Capabilities/IHardwareCapabilities.cs

@@ -7,7 +7,8 @@ public interface IHardwareCapabilities
     public FFmpegCapability CanDecode(
         string videoFormat,
         Option<string> videoProfile,
-        Option<IPixelFormat> maybePixelFormat);
+        Option<IPixelFormat> maybePixelFormat,
+        bool isHdr);
 
     public FFmpegCapability CanEncode(
         string videoFormat,

ErsatzTV.FFmpeg/Capabilities/NoHardwareCapabilities.cs

@@ -7,7 +7,8 @@ public class NoHardwareCapabilities : IHardwareCapabilities
     public FFmpegCapability CanDecode(
         string videoFormat,
         Option<string> videoProfile,
-        Option<IPixelFormat> maybePixelFormat) =>
+        Option<IPixelFormat> maybePixelFormat,
+        bool isHdr) =>
         FFmpegCapability.Software;
 
     public FFmpegCapability CanEncode(

ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs

@@ -29,8 +29,11 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities
     public FFmpegCapability CanDecode(
         string videoFormat,
         Option<string> videoProfile,
-        Option<IPixelFormat> maybePixelFormat)
+        Option<IPixelFormat> maybePixelFormat,
+        bool isHdr)
     {
+        // we use vulkan for hdr, so only support h264, hevc and av1 when isHdr == true
+
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 
         bool isHardware = videoFormat switch
@@ -39,17 +42,17 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities
             VideoFormat.Hevc => _architecture == 52 && _maxwellGm206.Contains(_model) || _architecture >= 60,
 
             // pascal is required to decode vp9 10-bit
-            VideoFormat.Vp9 when bitDepth == 10 => _architecture >= 60,
+            VideoFormat.Vp9 when bitDepth == 10 => !isHdr && _architecture >= 60,
 
             // some second gen maxwell can decode vp9, otherwise pascal is required
-            VideoFormat.Vp9 => _architecture == 52 && _maxwellGm206.Contains(_model) || _architecture >= 60,
+            VideoFormat.Vp9 => !isHdr && _architecture == 52 && _maxwellGm206.Contains(_model) || _architecture >= 60,
 
             // no hardware decoding of 10-bit h264
             VideoFormat.H264 => bitDepth < 10,
 
-            VideoFormat.Mpeg2Video => true,
+            VideoFormat.Mpeg2Video => !isHdr,
 
-            VideoFormat.Vc1 => true,
+            VideoFormat.Vc1 => !isHdr,
 
             // too many issues with odd mpeg4 content, so use software
             VideoFormat.Mpeg4 => false,

ErsatzTV.FFmpeg/Capabilities/VaapiHardwareCapabilities.cs

@@ -18,7 +18,8 @@ public class VaapiHardwareCapabilities : IHardwareCapabilities
     public FFmpegCapability CanDecode(
         string videoFormat,
         Option<string> videoProfile,
-        Option<IPixelFormat> maybePixelFormat)
+        Option<IPixelFormat> maybePixelFormat,
+        bool isHdr)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 

ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderImplicitVulkan.cs

@@ -0,0 +1,13 @@
+namespace ErsatzTV.FFmpeg.Decoder.Cuvid;
+
+public class DecoderImplicitVulkan : DecoderBase
+{
+    protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Hardware;
+    public override string Name => string.Empty;
+
+    public override string[] InputOptions(InputFile inputFile) =>
+    [
+        "-hwaccel_output_format",
+        "vulkan"
+    ];
+}

ErsatzTV.FFmpeg/FFmpegState.cs

@@ -22,7 +22,8 @@ public record FFmpegState(
     long PtsOffset,
     Option<int> ThreadCount,
     Option<int> MaybeQsvExtraHardwareFrames,
-    bool IsSongWithProgress)
+    bool IsSongWithProgress,
+    bool IsHdrTonemap)
 {
     public int QsvExtraHardwareFrames => MaybeQsvExtraHardwareFrames.IfNone(64);
 
@@ -47,5 +48,6 @@ public record FFmpegState(
             0,
             Option<int>.None,
             Option<int>.None,
+            false,
             false);
 }

ErsatzTV.FFmpeg/Filter/Cuda/TonemapCudaFilter.cs

@@ -0,0 +1,16 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Filter.Cuda;
+
+public class TonemapCudaFilter(IPixelFormat desiredPixelFormat) : BaseFilter
+{
+    public override string Filter =>
+        $"libplacebo=tonemapping=auto:colorspace=bt709:color_primaries=bt709:color_trc=bt709:format={desiredPixelFormat.FFmpegName},hwupload_cuda";
+
+    public override FrameState NextState(FrameState currentState) =>
+        currentState with
+        {
+            PixelFormat = Some(desiredPixelFormat),
+            FrameDataLocation = FrameDataLocation.Hardware
+        };
+}

ErsatzTV.FFmpeg/GlobalOption/HardwareAcceleration/CudaHardwareAccelerationOption.cs

@@ -1,6 +1,17 @@
 namespace ErsatzTV.FFmpeg.GlobalOption.HardwareAcceleration;
 
-public class CudaHardwareAccelerationOption : GlobalOption
+public class CudaHardwareAccelerationOption(bool isVulkanHdr) : GlobalOption
 {
-    public override string[] GlobalOptions => new[] { "-hwaccel", "cuda" };
+    public override string[] GlobalOptions
+    {
+        get
+        {
+            if (isVulkanHdr)
+            {
+                return ["-init_hw_device", "cuda=nv", "-init_hw_device", "vulkan=vk@nv", "-hwaccel", "vulkan"];
+            }
+
+            return ["-hwaccel", "cuda"];
+        }
+    }
 }

ErsatzTV.FFmpeg/HardwareAccelerationMode.cs

@@ -8,5 +8,6 @@ public enum HardwareAccelerationMode
     Vaapi = 3,
     VideoToolbox = 4,
     Amf = 5,
-    OpenCL = 6
+    OpenCL = 6,
+    Vulkan = 7
 }

ErsatzTV.FFmpeg/Pipeline/AmfPipelineBuilder.cs

@@ -51,7 +51,8 @@ public class AmfPipelineBuilder : SoftwarePipelineBuilder
         FFmpegCapability decodeCapability = _hardwareCapabilities.CanDecode(
             videoStream.Codec,
             videoStream.Profile,
-            videoStream.PixelFormat);
+            videoStream.PixelFormat,
+            videoStream.ColorParams.IsHdr);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,

ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs

@@ -59,7 +59,8 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
         FFmpegCapability decodeCapability = _hardwareCapabilities.CanDecode(
             videoStream.Codec,
             videoStream.Profile,
-            videoStream.PixelFormat);
+            videoStream.PixelFormat,
+            videoStream.ColorParams.IsHdr);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,
@@ -77,9 +78,13 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
             decodeCapability = FFmpegCapability.Software;
         }
 
+        bool isHdrTonemap = decodeCapability == FFmpegCapability.Hardware
+                            && _ffmpegCapabilities.HasHardwareAcceleration(HardwareAccelerationMode.Vulkan)
+                            && videoStream.ColorParams.IsHdr;
+
         if (decodeCapability == FFmpegCapability.Hardware || encodeCapability == FFmpegCapability.Hardware)
         {
-            pipelineSteps.Add(new CudaHardwareAccelerationOption());
+            pipelineSteps.Add(new CudaHardwareAccelerationOption(isHdrTonemap));
         }
 
         // disable hw accel if decoder/encoder isn't supported
@@ -90,7 +95,9 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
                 : HardwareAccelerationMode.None,
             EncoderHardwareAccelerationMode = encodeCapability == FFmpegCapability.Hardware
                 ? HardwareAccelerationMode.Nvenc
-                : HardwareAccelerationMode.None
+                : HardwareAccelerationMode.None,
+
+            IsHdrTonemap = isHdrTonemap
         };
     }
 
@@ -100,6 +107,14 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
         FFmpegState ffmpegState,
         PipelineContext context)
     {
+        // use implicit vulkan decoder with HDR tonemap
+        if (ffmpegState.IsHdrTonemap)
+        {
+            IDecoder decoder = new DecoderImplicitVulkan();
+            videoInputFile.AddOption(decoder);
+            return Some(decoder);
+        }
+
         Option<IDecoder> maybeDecoder = (ffmpegState.DecoderHardwareAccelerationMode, videoStream.Codec) switch
         {
             (HardwareAccelerationMode.Nvenc, VideoFormat.Hevc) => new DecoderHevcCuvid(HardwareAccelerationMode.Nvenc),
@@ -166,6 +181,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
         //     desiredState = desiredState with { PixelFormat = Some(pixelFormat) };
         // }
 
+        currentState = SetTonemap(videoInputFile, videoStream, ffmpegState, desiredState, currentState);
         currentState = SetDeinterlace(videoInputFile, context, currentState);
         currentState = SetScale(videoInputFile, videoStream, context, ffmpegState, desiredState, currentState);
         currentState = SetPad(videoInputFile, videoStream, desiredState, currentState);
@@ -710,4 +726,35 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
 
         return currentState;
     }
+
+    private static FrameState SetTonemap(
+        VideoInputFile videoInputFile,
+        VideoStream videoStream,
+        FFmpegState ffmpegState,
+        FrameState desiredState,
+        FrameState currentState)
+    {
+        if (videoStream.ColorParams.IsHdr)
+        {
+            foreach (IPixelFormat pixelFormat in desiredState.PixelFormat)
+            {
+                if (ffmpegState.IsHdrTonemap)
+                {
+                    var filter = new TonemapCudaFilter(pixelFormat);
+                    currentState = filter.NextState(currentState);
+                    videoStream.ResetColorParams(ColorParams.Default);
+                    videoInputFile.FilterSteps.Add(filter);
+                }
+                else
+                {
+                    var filter = new TonemapFilter(currentState, pixelFormat);
+                    currentState = filter.NextState(currentState);
+                    videoStream.ResetColorParams(ColorParams.Default);
+                    videoInputFile.FilterSteps.Add(filter);
+                }
+            }
+        }
+
+        return currentState;
+    }
 }

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderFactory.cs

@@ -57,10 +57,10 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory
                 fontsFolder,
                 _logger),
 
-            // force software pipeline when content is HDR (and not VAAPI)
-            _ when isHdrContent => new SoftwarePipelineBuilder(
+            HardwareAccelerationMode.Nvenc when capabilities is not NoHardwareCapabilities => new NvidiaPipelineBuilder(
                 ffmpegCapabilities,
-                HardwareAccelerationMode.None,
+                capabilities,
+                hardwareAccelerationMode,
                 videoInputFile,
                 audioInputFile,
                 watermarkInputFile,
@@ -70,10 +70,10 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory
                 fontsFolder,
                 _logger),
 
-            HardwareAccelerationMode.Nvenc when capabilities is not NoHardwareCapabilities => new NvidiaPipelineBuilder(
+            // force software pipeline when content is HDR (and not VAAPI or NVENC)
+            _ when isHdrContent => new SoftwarePipelineBuilder(
                 ffmpegCapabilities,
-                capabilities,
-                hardwareAccelerationMode,
+                HardwareAccelerationMode.None,
                 videoInputFile,
                 audioInputFile,
                 watermarkInputFile,

ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs

@@ -61,7 +61,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
         FFmpegCapability decodeCapability = _hardwareCapabilities.CanDecode(
             videoStream.Codec,
             videoStream.Profile,
-            videoStream.PixelFormat);
+            videoStream.PixelFormat,
+            videoStream.ColorParams.IsHdr);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -65,7 +65,8 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
         FFmpegCapability decodeCapability = _hardwareCapabilities.CanDecode(
             videoStream.Codec,
             videoStream.Profile,
-            videoStream.PixelFormat);
+            videoStream.PixelFormat,
+            videoStream.ColorParams.IsHdr);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,

ErsatzTV.FFmpeg/Pipeline/VideoToolboxPipelineBuilder.cs

@@ -52,7 +52,8 @@ public class VideoToolboxPipelineBuilder : SoftwarePipelineBuilder
         FFmpegCapability decodeCapability = _hardwareCapabilities.CanDecode(
             videoStream.Codec,
             videoStream.Profile,
-            videoStream.PixelFormat);
+            videoStream.PixelFormat,
+            videoStream.ColorParams.IsHdr);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,