vaapi tonemap fixes (#2125)

CHANGELOG.md

@@ -96,6 +96,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Update Plex movie/other video plot ("summary") during library deep scan
 - Fix compatibility with ffmpeg 7.2+ when using NVIDIA accel and 10-bit source content
 - Fix some NVIDIA edge cases when media servers don't provide video bit depth information
+- Fix VAAPI tonemap failure
+- Fix green bars after VAAPI tonemap
 
 ## [25.2.0] - 2025-06-24
 ### Added

ErsatzTV.FFmpeg/Filter/Vaapi/TonemapVaapiFilter.cs

@@ -1,13 +1,16 @@
+using ErsatzTV.FFmpeg.Format;
+
 namespace ErsatzTV.FFmpeg.Filter.Vaapi;
 
 public class TonemapVaapiFilter(FFmpegState ffmpegState) : BaseFilter
 {
     public override string Filter =>
-        $"hwupload=derive_device=vaapi,hwmap=derive_device=opencl,tonemap_opencl=tonemap={ffmpegState.TonemapAlgorithm},hwmap=derive_device=vaapi:reverse=1";
+        $"hwupload=derive_device=vaapi,hwmap=derive_device=opencl,tonemap_opencl=tonemap={ffmpegState.TonemapAlgorithm},hwmap=derive_device=vaapi:reverse=1,scale_vaapi=format=p010le";
 
     public override FrameState NextState(FrameState currentState) =>
         currentState with
         {
-            FrameDataLocation = FrameDataLocation.Hardware
+            FrameDataLocation = FrameDataLocation.Hardware,
+            PixelFormat = new PixelFormatVaapi(PixelFormat.YUV420P10LE)
         };
 }

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -177,9 +177,10 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
         currentState = SetScale(videoInputFile, videoStream, context, ffmpegState, desiredState, currentState);
         // _logger.LogDebug("After scale: {PixelFormat}", currentState.PixelFormat);
 
+        bool isHdrTonemap = videoStream.ColorParams.IsHdr;
         currentState = SetTonemap(videoInputFile, videoStream, ffmpegState, desiredState, currentState);
 
-        currentState = SetPad(videoInputFile, desiredState, currentState);
+        currentState = SetPad(videoInputFile, desiredState, currentState, isHdrTonemap);
         // _logger.LogDebug("After pad: {PixelFormat}", currentState.PixelFormat);
 
         currentState = SetCrop(videoInputFile, desiredState, currentState);
@@ -532,13 +533,24 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
     private static FrameState SetPad(
         VideoInputFile videoInputFile,
         FrameState desiredState,
-        FrameState currentState)
+        FrameState currentState,
+        bool isHdrTonemap)
     {
         if (desiredState.CroppedSize.IsNone && currentState.PaddedSize != desiredState.PaddedSize)
         {
-            var padStep = new PadVaapiFilter(currentState, desiredState.PaddedSize);
-            currentState = padStep.NextState(currentState);
-            videoInputFile.FilterSteps.Add(padStep);
+            // pad_vaapi seems to pad with green when input is HDR, so use software pad here
+            if (isHdrTonemap)
+            {
+                var padStep = new PadFilter(currentState, desiredState.PaddedSize);
+                currentState = padStep.NextState(currentState);
+                videoInputFile.FilterSteps.Add(padStep);
+            }
+            else
+            {
+                var padStep = new PadVaapiFilter(currentState, desiredState.PaddedSize);
+                currentState = padStep.NextState(currentState);
+                videoInputFile.FilterSteps.Add(padStep);
+            }
         }
 
         return currentState;