fix vaapi watermarks with new transcoder (#657)

ErsatzTV.FFmpeg/Filter/AvailableOverlayFilters.cs

@@ -8,13 +8,13 @@ public static class AvailableOverlayFilters
 {
     public static IPipelineFilterStep ForAcceleration(
         HardwareAccelerationMode accelMode,
+        FrameState currentState,
         WatermarkState watermarkState,
         FrameSize resolution) =>
         accelMode switch
         {
-            HardwareAccelerationMode.Nvenc => new OverlayCudaFilter(watermarkState, resolution),
+            HardwareAccelerationMode.Nvenc => new OverlayCudaFilter(currentState, watermarkState, resolution),
-            HardwareAccelerationMode.Qsv => new OverlayQsvFilter(watermarkState, resolution),
+            HardwareAccelerationMode.Qsv => new OverlayQsvFilter(currentState, watermarkState, resolution),
-            // HardwareAccelerationMode.Vaapi => new DeinterlaceVaapiFilter(),
+            _ => new OverlayFilter(currentState, watermarkState, resolution)
-            _ => new OverlayFilter(watermarkState, resolution)
         };
 }

ErsatzTV.FFmpeg/Filter/ComplexFilter.cs

@@ -5,6 +5,7 @@ namespace ErsatzTV.FFmpeg.Filter;
 
 public class ComplexFilter : IPipelineStep
 {
+    private readonly FrameState _currentState;
     private readonly FFmpegState _ffmpegState;
     private readonly Option<VideoInputFile> _maybeVideoInputFile;
     private readonly Option<AudioInputFile> _maybeAudioInputFile;
@@ -12,12 +13,14 @@ public class ComplexFilter : IPipelineStep
     private readonly FrameSize _resolution;
 
     public ComplexFilter(
+        FrameState currentState,
         FFmpegState ffmpegState,
         Option<VideoInputFile> maybeVideoInputFile,
         Option<AudioInputFile> maybeAudioInputFile,
         Option<WatermarkInputFile> maybeWatermarkInputFile,
         FrameSize resolution)
     {
+        _currentState = currentState;
         _ffmpegState = ffmpegState;
         _maybeVideoInputFile = maybeVideoInputFile;
         _maybeAudioInputFile = maybeAudioInputFile;
@@ -117,6 +120,7 @@ public class ComplexFilter : IPipelineStep
 
                 IPipelineFilterStep overlayFilter = AvailableOverlayFilters.ForAcceleration(
                     _ffmpegState.HardwareAccelerationMode,
+                    _currentState,
                     watermarkInputFile.DesiredState,
                     _resolution);
 
@@ -125,8 +129,20 @@ public class ComplexFilter : IPipelineStep
                     string tempVideoLabel = string.IsNullOrWhiteSpace(videoFilterComplex)
                         ? $"[{videoLabel}]"
                         : videoLabel;
+
+                    // vaapi uses software overlay and needs to upload
+                    string uploadFilter = string.Empty;
+                    if (_ffmpegState.HardwareAccelerationMode == HardwareAccelerationMode.Vaapi)
+                    {
+                        uploadFilter = new HardwareUploadFilter(_ffmpegState).Filter;
+                    }
+
+                    if (!string.IsNullOrWhiteSpace(uploadFilter))
+                    {
+                        uploadFilter = "," + uploadFilter;
+                    }
                     
-                    overlayFilterComplex = $"{tempVideoLabel}{watermarkLabel}{overlayFilter.Filter}[vf]";
+                    overlayFilterComplex = $"{tempVideoLabel}{watermarkLabel}{overlayFilter.Filter}{uploadFilter}[vf]";
                     
                     // change the mapped label
                     videoLabel = "[vf]";

ErsatzTV.FFmpeg/Filter/Cuda/OverlayCudaFilter.cs

@@ -4,7 +4,10 @@ namespace ErsatzTV.FFmpeg.Filter.Cuda;
 
 public class OverlayCudaFilter : OverlayFilter
 {
-    public OverlayCudaFilter(WatermarkState watermarkState, FrameSize resolution) : base(watermarkState, resolution)
+    public OverlayCudaFilter(FrameState currentState, WatermarkState watermarkState, FrameSize resolution) : base(
+        currentState,
+        watermarkState,
+        resolution)
     {
     }
 

ErsatzTV.FFmpeg/Filter/HardwareDownloadFilter.cs

@@ -0,0 +1,37 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Filter;
+
+public class HardwareDownloadFilter : BaseFilter
+{
+    private readonly FrameState _currentState;
+
+    public HardwareDownloadFilter(FrameState currentState)
+    {
+        _currentState = currentState;
+    }
+
+    public override FrameState NextState(FrameState currentState) =>
+        currentState with { FrameDataLocation = FrameDataLocation.Software };
+
+    public override string Filter
+    {
+        get
+        {
+            string hwdownload = string.Empty;
+            if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)
+            {
+                hwdownload = "hwdownload";
+                foreach (IPixelFormat pixelFormat in _currentState.PixelFormat)
+                {
+                    if (!string.IsNullOrWhiteSpace(pixelFormat.FFmpegName))
+                    {
+                        hwdownload = $"hwdownload,format={pixelFormat.FFmpegName}";
+                    }
+                }
+            }
+
+            return hwdownload;
+        }
+    }
+}

ErsatzTV.FFmpeg/Filter/HardwareUploadFilter.cs

@@ -16,6 +16,7 @@ public class HardwareUploadFilter : BaseFilter
         HardwareAccelerationMode.None => string.Empty,
         HardwareAccelerationMode.Nvenc => "hwupload_cuda",
         HardwareAccelerationMode.Qsv => "hwupload=extra_hw_frames=64",
+        HardwareAccelerationMode.Vaapi => "format=nv12|vaapi,hwupload",
         _ => "hwupload"
     };
 }

ErsatzTV.FFmpeg/Filter/OverlayFilter.cs

@@ -1,22 +1,44 @@
-using ErsatzTV.FFmpeg.State;
+using ErsatzTV.FFmpeg.Format;
+using ErsatzTV.FFmpeg.State;
 
 namespace ErsatzTV.FFmpeg.Filter;
 
 public class OverlayFilter : BaseFilter
 {
+    private readonly FrameState _currentState;
     private readonly WatermarkState _watermarkState;
     private readonly FrameSize _resolution;
 
-    public OverlayFilter(WatermarkState watermarkState, FrameSize resolution)
+    public OverlayFilter(FrameState currentState, WatermarkState watermarkState, FrameSize resolution)
     {
+        _currentState = currentState;
         _watermarkState = watermarkState;
         _resolution = resolution;
     }
 
     public override FrameState NextState(FrameState currentState) => currentState;
 
-    public override string Filter => $"overlay={Position}";
+    public override string Filter
-    
+    {
+        get
+        {
+            string hwdownload = string.Empty;
+            if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)
+            {
+                hwdownload = "hwdownload,";
+                foreach (IPixelFormat pixelFormat in _currentState.PixelFormat)
+                {
+                    if (pixelFormat.FFmpegName == FFmpegFormat.NV12)
+                    {
+                        hwdownload = "hwdownload,format=nv12,";
+                    }
+                }
+            }
+
+            return $"{hwdownload}overlay={Position}";
+        }
+    }
+
     protected string Position
     {
         get

ErsatzTV.FFmpeg/Filter/Qsv/OverlayQsvFilter.cs

@@ -4,7 +4,10 @@ namespace ErsatzTV.FFmpeg.Filter.Qsv;
 
 public class OverlayQsvFilter : OverlayFilter
 {
-    public OverlayQsvFilter(WatermarkState watermarkState, FrameSize resolution) : base(watermarkState, resolution)
+    public OverlayQsvFilter(FrameState currentState, WatermarkState watermarkState, FrameSize resolution) : base(
+        currentState,
+        watermarkState,
+        resolution)
     {
     }
 

ErsatzTV.FFmpeg/Filter/WatermarkHardwareUploadFilter.cs

@@ -0,0 +1,28 @@
+namespace ErsatzTV.FFmpeg.Filter;
+
+public class WatermarkHardwareUploadFilter : BaseFilter
+{
+    private readonly FrameState _currentState;
+    private readonly FFmpegState _ffmpegState;
+
+    public WatermarkHardwareUploadFilter(FrameState currentState, FFmpegState ffmpegState)
+    {
+        _currentState = currentState;
+        _ffmpegState = ffmpegState;
+    }
+
+    public override FrameState NextState(FrameState currentState) => currentState;
+
+    public override string Filter => _ffmpegState.HardwareAccelerationMode switch
+    {
+        HardwareAccelerationMode.None => string.Empty,
+        HardwareAccelerationMode.Nvenc => "hwupload_cuda",
+        HardwareAccelerationMode.Qsv => "hwupload=extra_hw_frames=64",
+        
+        // leave vaapi in software since we don't (yet) use overlay_vaapi
+        HardwareAccelerationMode.Vaapi when _currentState.FrameDataLocation == FrameDataLocation.Software =>
+            string.Empty,
+        
+        _ => "hwupload"
+    };
+}

ErsatzTV.FFmpeg/PipelineBuilder.cs

@@ -443,6 +443,14 @@ public class PipelineBuilder
 
             foreach (WatermarkInputFile watermarkInputFile in _watermarkInputFile)
             {
+                // vaapi uses a software overlay, so we need to ensure the background is already in software
+                if (ffmpegState.HardwareAccelerationMode == HardwareAccelerationMode.Vaapi)
+                {
+                    var downloadFilter = new HardwareDownloadFilter(currentState);
+                    currentState = downloadFilter.NextState(currentState);
+                    _videoInputFile.Iter(f => f.FilterSteps.Add(downloadFilter));
+                }
+
                 watermarkInputFile.FilterSteps.Add(
                     new WatermarkPixelFormatFilter(ffmpegState, watermarkInputFile.DesiredState));
 
@@ -475,7 +483,7 @@ public class PipelineBuilder
                     watermarkInputFile.FilterSteps.AddRange(fadePoints.Map(fp => new WatermarkFadeFilter(fp)));
                 }
 
-                watermarkInputFile.FilterSteps.Add(new HardwareUploadFilter(ffmpegState));
+                watermarkInputFile.FilterSteps.Add(new WatermarkHardwareUploadFilter(currentState, ffmpegState));
             }
 
             if (ffmpegState.DoNotMapMetadata)
@@ -524,6 +532,7 @@ public class PipelineBuilder
             }
 
             var complexFilter = new ComplexFilter(
+                currentState,
                 ffmpegState,
                 _videoInputFile,
                 _audioInputFile,