use hardware-accelerated tonemapping with vaapi (#2028)

* add tonemap_vaapi filter

* let vaapi pipeline handle hdr content

* use tonemap_opencl with vaapi

* update changelog

CHANGELOG.md

@@ -23,7 +23,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     - `Strict` will add nearly 24h (23:58) of unscheduled time so that it can start exactly at 6:00 AM the next day
     - `Flexible` will NOT add unscheduled time, and will schedule its item at 6:02 AM (which may also affect the scheduling of later items)
 - Add basic HDR transcoding support
-  - For this initial implementation, HDR content will *always* use a software pipeline
+  - VAAPI may use hardware-accelerated tone mapping
+  - In all other cases, HDR content will use a software pipeline
 
 ### Changed
 - Start to make UI minimally responsive (functional on smaller screens)

ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs

@@ -40,6 +40,7 @@ public class FFmpegCapabilities : IFFmpegCapabilities
             HardwareAccelerationMode.Qsv => FFmpegKnownHardwareAcceleration.Qsv,
             HardwareAccelerationMode.Vaapi => FFmpegKnownHardwareAcceleration.Vaapi,
             HardwareAccelerationMode.VideoToolbox => FFmpegKnownHardwareAcceleration.VideoToolbox,
+            HardwareAccelerationMode.OpenCL =>  FFmpegKnownHardwareAcceleration.OpenCL,
             _ => Option<FFmpegKnownHardwareAcceleration>.None
         };
 

ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs

@@ -3,6 +3,7 @@ namespace ErsatzTV.FFmpeg.Capabilities;
 public record FFmpegKnownFilter
 {
     public static readonly FFmpegKnownFilter ScaleNpp = new("scale_npp");
+    public static readonly FFmpegKnownFilter TonemapOpenCL = new("tonemap_opencl");
 
     private FFmpegKnownFilter(string Name) => this.Name = Name;
 
@@ -11,6 +12,7 @@ public record FFmpegKnownFilter
     public static IList<string> AllFilters =>
         new[]
         {
-            ScaleNpp.Name
+            ScaleNpp.Name,
+            TonemapOpenCL.Name,
         };
 }

ErsatzTV.FFmpeg/Capabilities/FFmpegKnownHardwareAcceleration.cs

@@ -7,6 +7,7 @@ public record FFmpegKnownHardwareAcceleration
     public static readonly FFmpegKnownHardwareAcceleration Qsv = new("qsv");
     public static readonly FFmpegKnownHardwareAcceleration Vaapi = new("vaapi");
     public static readonly FFmpegKnownHardwareAcceleration VideoToolbox = new("videotoolbox");
+    public static readonly FFmpegKnownHardwareAcceleration OpenCL = new("opencl");
 
     private FFmpegKnownHardwareAcceleration(string Name) => this.Name = Name;
 
@@ -19,6 +20,7 @@ public record FFmpegKnownHardwareAcceleration
             Cuda.Name,
             Qsv.Name,
             Vaapi.Name,
-            VideoToolbox.Name
+            VideoToolbox.Name,
+            OpenCL.Name
         };
 }

ErsatzTV.FFmpeg/Filter/Vaapi/TonemapVaapiFilter.cs

@@ -0,0 +1,12 @@
+namespace ErsatzTV.FFmpeg.Filter.Vaapi;
+
+public class TonemapVaapiFilter : BaseFilter
+{
+    public override string Filter => "hwupload=derive_device=vaapi,hwmap=derive_device=opencl,tonemap_opencl,hwmap=derive_device=vaapi:reverse=1";
+
+    public override FrameState NextState(FrameState currentState) =>
+        currentState with
+        {
+            FrameDataLocation = FrameDataLocation.Hardware
+        };
+}

ErsatzTV.FFmpeg/HardwareAccelerationMode.cs

@@ -7,5 +7,6 @@ public enum HardwareAccelerationMode
     Nvenc = 2,
     Vaapi = 3,
     VideoToolbox = 4,
-    Amf = 5
+    Amf = 5,
+    OpenCL = 6
 }

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderFactory.cs

@@ -44,10 +44,10 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory
 
         return hardwareAccelerationMode switch
         {
-            // force software pipeline when content is HDR
+            HardwareAccelerationMode.Vaapi when capabilities is not NoHardwareCapabilities => new VaapiPipelineBuilder(
-            _ when isHdrContent => new SoftwarePipelineBuilder(
                 ffmpegCapabilities,
-                HardwareAccelerationMode.None,
+                capabilities,
+                hardwareAccelerationMode,
                 videoInputFile,
                 audioInputFile,
                 watermarkInputFile,
@@ -57,10 +57,10 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory
                 fontsFolder,
                 _logger),
 
-            HardwareAccelerationMode.Nvenc when capabilities is not NoHardwareCapabilities => new NvidiaPipelineBuilder(
+            // force software pipeline when content is HDR (and not VAAPI)
+            _ when isHdrContent => new SoftwarePipelineBuilder(
                 ffmpegCapabilities,
-                capabilities,
+                HardwareAccelerationMode.None,
-                hardwareAccelerationMode,
                 videoInputFile,
                 audioInputFile,
                 watermarkInputFile,
@@ -69,7 +69,8 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory
                 reportsFolder,
                 fontsFolder,
                 _logger),
-            HardwareAccelerationMode.Vaapi when capabilities is not NoHardwareCapabilities => new VaapiPipelineBuilder(
+
+            HardwareAccelerationMode.Nvenc when capabilities is not NoHardwareCapabilities => new NvidiaPipelineBuilder(
                 ffmpegCapabilities,
                 capabilities,
                 hardwareAccelerationMode,
@@ -81,6 +82,7 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory
                 reportsFolder,
                 fontsFolder,
                 _logger),
+
             HardwareAccelerationMode.Qsv when capabilities is not NoHardwareCapabilities => new QsvPipelineBuilder(
                 ffmpegCapabilities,
                 capabilities,
@@ -93,6 +95,7 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory
                 reportsFolder,
                 fontsFolder,
                 _logger),
+
             HardwareAccelerationMode.VideoToolbox when capabilities is not NoHardwareCapabilities => new
                 VideoToolboxPipelineBuilder(
                     ffmpegCapabilities,
@@ -106,6 +109,7 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory
                     reportsFolder,
                     fontsFolder,
                     _logger),
+
             HardwareAccelerationMode.Amf when capabilities is not NoHardwareCapabilities => new AmfPipelineBuilder(
                 ffmpegCapabilities,
                 capabilities,
@@ -118,6 +122,7 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory
                 reportsFolder,
                 fontsFolder,
                 _logger),
+
             _ => new SoftwarePipelineBuilder(
                 ffmpegCapabilities,
                 HardwareAccelerationMode.None,

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -17,6 +17,7 @@ namespace ErsatzTV.FFmpeg.Pipeline;
 
 public class VaapiPipelineBuilder : SoftwarePipelineBuilder
 {
+    private readonly IFFmpegCapabilities _ffmpegCapabilities;
     private readonly IHardwareCapabilities _hardwareCapabilities;
     private readonly ILogger _logger;
 
@@ -43,6 +44,7 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
         fontsFolder,
         logger)
     {
+        _ffmpegCapabilities = ffmpegCapabilities;
         _hardwareCapabilities = hardwareCapabilities;
         _logger = logger;
     }
@@ -168,6 +170,8 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
 
         // _logger.LogDebug("After decode: {PixelFormat}", currentState.PixelFormat);
 
+        currentState = SetTonemap(videoInputFile, videoStream, ffmpegState, desiredState, currentState);
+
         currentState = SetDeinterlace(videoInputFile, context, ffmpegState, currentState);
         // _logger.LogDebug("After deinterlace: {PixelFormat}", currentState.PixelFormat);
 
@@ -619,4 +623,35 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
 
         return currentState;
     }
+
+    private FrameState SetTonemap(
+        VideoInputFile videoInputFile,
+        VideoStream videoStream,
+        FFmpegState ffmpegState,
+        FrameState desiredState,
+        FrameState currentState)
+    {
+        if (videoStream.ColorParams.IsHdr)
+        {
+            foreach (IPixelFormat pixelFormat in desiredState.PixelFormat)
+            {
+                if (ffmpegState.DecoderHardwareAccelerationMode == HardwareAccelerationMode.Vaapi && _ffmpegCapabilities.HasFilter(FFmpegKnownFilter.TonemapOpenCL))
+                {
+                    var filter = new TonemapVaapiFilter();
+                    currentState = filter.NextState(currentState);
+                    videoStream.ResetColorParams(ColorParams.Default);
+                    videoInputFile.FilterSteps.Add(filter);
+                }
+                else
+                {
+                    var filter = new TonemapFilter(currentState, pixelFormat);
+                    currentState = filter.NextState(currentState);
+                    videoStream.ResetColorParams(ColorParams.Default);
+                    videoInputFile.FilterSteps.Add(filter);
+                }
+            }
+        }
+
+        return currentState;
+    }
 }

ErsatzTV.sln.DotSettings

@@ -1,5 +1,6 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
 	<s:Boolean x:Key="/Default/CodeInspection/CodeAnnotations/NamespacesWithAnnotations/=ErsatzTV_002EAnnotations/@EntryIndexedValue">True</s:Boolean>
+	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=CL/@EntryIndexedValue">CL</s:String>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=DTO/@EntryIndexedValue">DTO</s:String>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=EPG/@EntryIndexedValue">EPG</s:String>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=FF/@EntryIndexedValue">FF</s:String>