Merge cdef25ffc2 into 15db3c4ca0

ErsatzTV.FFmpeg.Tests/Capabilities/FFmpegFilterOptionParserTests.cs

@@ -0,0 +1,99 @@
+using System.Collections.Generic;
+using ErsatzTV.FFmpeg.Capabilities;
+using NUnit.Framework;
+using Shouldly;
+
+namespace ErsatzTV.FFmpeg.Tests.Capabilities;
+
+[TestFixture]
+public class FFmpegFilterOptionParserTests
+{
+    // stock ffmpeg vpp_qsv, without the pad_w/pad_h/pad_x/pad_y/pad_color options
+    private const string WithoutPadOptions = @"Filter vpp_qsv
+  Quick Sync Video VPP.
+    Inputs:
+       #0: default (video)
+    Outputs:
+       #0: default (video)
+vpp_qsv AVOptions:
+   deinterlace       <int>        ..FV....... deinterlace mode (from 0 to 2) (default 0)
+     bob             1            ..FV....... Get one frame for each field
+     advanced        2            ..FV....... Get one frame for each frame
+   denoise           <int>        ..FV....... denoise level [0, 100] (from 0 to 100) (default 0)
+   framerate         <rational>   ..FV....... output framerate (from 0 to DBL_MAX) (default 0/1)
+   w                 <string>     ..FV....... Output video width
+   h                 <string>     ..FV....... Output video height
+   format            <string>     ..FV....... Output pixel format
+   async_depth       <int>        ..FV....... Internal parallelization depth (from 0 to INT_MAX) (default 4)
+   scale_mode        <int>        ..FV....... scaling mode (from 0 to 2) (default 0)
+";
+
+    // patched ffmpeg vpp_qsv, carrying the pad_w/pad_h/pad_x/pad_y/pad_color options
+    private const string WithPadOptions = @"Filter vpp_qsv
+  Quick Sync Video VPP.
+    Inputs:
+       #0: default (video)
+    Outputs:
+       #0: default (video)
+vpp_qsv AVOptions:
+   deinterlace       <int>        ..FV....... deinterlace mode (from 0 to 2) (default 0)
+     bob             1            ..FV....... Get one frame for each field
+     advanced        2            ..FV....... Get one frame for each frame
+   denoise           <int>        ..FV....... denoise level [0, 100] (from 0 to 100) (default 0)
+   framerate         <rational>   ..FV....... output framerate (from 0 to DBL_MAX) (default 0/1)
+   w                 <string>     ..FV....... Output video width
+   h                 <string>     ..FV....... Output video height
+   format            <string>     ..FV....... Output pixel format
+   async_depth       <int>        ..FV....... Internal parallelization depth (from 0 to INT_MAX) (default 4)
+   scale_mode        <int>        ..FV....... scaling mode (from 0 to 2) (default 0)
+   pad_w             <int>        ..FV....... Padded width (from 0 to 8192) (default 0)
+   pad_h             <int>        ..FV....... Padded height (from 0 to 8192) (default 0)
+   pad_x             <int>        ..FV....... Horizontal position (from -1 to 8192) (default -1)
+   pad_y             <int>        ..FV....... Vertical position (from -1 to 8192) (default -1)
+   pad_color         <color>      ..FV....... Padding color (default ""black"")
+";
+
+    [Test]
+    public void Should_Parse_Option_Names()
+    {
+        IReadOnlySet<string> options = FFmpegFilterOptionParser.Parse(WithoutPadOptions);
+
+        options.ShouldContain("deinterlace");
+        options.ShouldContain("denoise");
+        options.ShouldContain("framerate");
+        options.ShouldContain("w");
+        options.ShouldContain("h");
+        options.ShouldContain("format");
+    }
+
+    [Test]
+    public void Should_Not_Parse_Enum_Constants()
+    {
+        IReadOnlySet<string> options = FFmpegFilterOptionParser.Parse(WithoutPadOptions);
+
+        options.ShouldNotContain("bob");
+        options.ShouldNotContain("advanced");
+    }
+
+    [Test]
+    public void Should_Detect_Pad_Options_When_Present()
+    {
+        IReadOnlySet<string> options = FFmpegFilterOptionParser.Parse(WithPadOptions);
+
+        options.ShouldContain("pad_w");
+        options.ShouldContain("pad_h");
+        options.ShouldContain("pad_x");
+        options.ShouldContain("pad_y");
+        options.ShouldContain("pad_color");
+    }
+
+    [Test]
+    public void Should_Not_Detect_Pad_Options_When_Absent()
+    {
+        IReadOnlySet<string> options = FFmpegFilterOptionParser.Parse(WithoutPadOptions);
+
+        options.ShouldNotContain("pad_w");
+        options.ShouldNotContain("pad_h");
+        options.ShouldNotContain("pad_color");
+    }
+}

ErsatzTV.FFmpeg.Tests/Filter/Qsv/PadQsvFilterTests.cs

@@ -0,0 +1,72 @@
+using ErsatzTV.FFmpeg;
+using ErsatzTV.FFmpeg.Filter.Qsv;
+using ErsatzTV.FFmpeg.Format;
+using ErsatzTV.FFmpeg.State;
+using LanguageExt;
+using NUnit.Framework;
+using Shouldly;
+
+namespace ErsatzTV.FFmpeg.Tests.Filter.Qsv;
+
+[TestFixture]
+public class PadQsvFilterTests
+{
+    private static FrameState FrameStateAt(FrameDataLocation location, Option<IPixelFormat> pixelFormat) => new(
+        false,
+        false,
+        string.Empty,
+        Option<string>.None,
+        Option<string>.None,
+        true,
+        pixelFormat,
+        new FrameSize(1920, 1080),
+        new FrameSize(1920, 1080),
+        Option<FrameSize>.None,
+        FFmpegFilterMode.HardwareIfPossible,
+        false,
+        Option<FrameRate>.None,
+        Option<int>.None,
+        Option<int>.None,
+        Option<int>.None,
+        false,
+        false,
+        location);
+
+    [Test]
+    public void Should_Emit_Hardware_Pad_When_Frame_In_Hardware()
+    {
+        var filter = new PadQsvFilter(
+            FrameStateAt(FrameDataLocation.Hardware, Option<IPixelFormat>.None),
+            new FrameSize(1920, 1080),
+            64);
+
+        filter.Filter.ShouldBe("vpp_qsv=pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black");
+    }
+
+    [Test]
+    public void Should_Upload_Before_Pad_When_Frame_In_Software()
+    {
+        var filter = new PadQsvFilter(
+            FrameStateAt(FrameDataLocation.Software, Option<IPixelFormat>.Some(new PixelFormatNv12("yuv420p"))),
+            new FrameSize(1920, 1080),
+            64);
+
+        filter.Filter.ShouldBe(
+            "format=nv12,hwupload=extra_hw_frames=64,vpp_qsv=pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black");
+    }
+
+    [Test]
+    public void Should_Report_Padded_Size_And_Hardware_Location()
+    {
+        var filter = new PadQsvFilter(
+            FrameStateAt(FrameDataLocation.Software, Option<IPixelFormat>.None),
+            new FrameSize(1920, 1080),
+            64);
+
+        FrameState nextState = filter.NextState(
+            FrameStateAt(FrameDataLocation.Software, Option<IPixelFormat>.None));
+
+        nextState.PaddedSize.ShouldBe(new FrameSize(1920, 1080));
+        nextState.FrameDataLocation.ShouldBe(FrameDataLocation.Hardware);
+    }
+}

ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs

@@ -0,0 +1,201 @@
+using System;
+using System.Collections.Generic;
+using ErsatzTV.FFmpeg;
+using ErsatzTV.FFmpeg.Capabilities;
+using ErsatzTV.FFmpeg.Format;
+using ErsatzTV.FFmpeg.InputOption;
+using ErsatzTV.FFmpeg.OutputFormat;
+using ErsatzTV.FFmpeg.Pipeline;
+using ErsatzTV.FFmpeg.Preset;
+using ErsatzTV.FFmpeg.State;
+using LanguageExt;
+using Microsoft.Extensions.Logging;
+using NSubstitute;
+using NUnit.Framework;
+using Shouldly;
+using static LanguageExt.Prelude;
+
+namespace ErsatzTV.FFmpeg.Tests.Pipeline;
+
+[TestFixture]
+public class QsvPipelineBuilderTests
+{
+    private readonly ILogger _logger = Substitute.For<ILogger>();
+
+    [Test]
+    public void Scale_And_Pad_Fold_Into_Single_Hardware_Vpp_Qsv()
+    {
+        string command = BuildPadCommand(vppQsvSupportsPad: true);
+
+        // scale and pad fold into a single vpp_qsv instance that pads on the GPU
+        command.ShouldContain("vpp_qsv=w=1440:h=1080:pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black");
+
+        // exactly one vpp_qsv instance (two chained instances drop the final frame at EOF)
+        command.Split("vpp_qsv").Length.ShouldBe(2);
+
+        // no software pad
+        command.ShouldNotContain("pad=1920:1080");
+    }
+
+    [Test]
+    public void Frames_Stay_In_Hardware_Through_Setpts_Into_The_Qsv_Encoder()
+    {
+        string command = BuildPadCommand(vppQsvSupportsPad: true);
+
+        // setpts/fps run on the hardware frame and hevc_qsv ingests it directly: no download anywhere
+        command.ShouldNotContain("hwdownload");
+        command.ShouldContain("pad_color=black,setsar=1,setpts=PTS-STARTPTS,fps=24");
+        command.ShouldContain("hevc_qsv");
+    }
+
+    [Test]
+    public void Download_Is_Retained_When_A_Colorspace_Conversion_Follows()
+    {
+        string command = BuildPadCommand(vppQsvSupportsPad: true, colorsAreBt709: true);
+
+        // a downstream software colorspace filter still needs system memory, so the download stays
+        // exactly where it is today: right after the pad, before setpts, and byte-identical to the
+        // unfolded path (format=nv12, not a bare hwdownload)
+        command.ShouldContain("pad_color=black,setsar=1,hwdownload,format=nv12,setpts=PTS-STARTPTS");
+        command.ShouldContain("colorspace=");
+    }
+
+    [Test]
+    public void Pad_Falls_Back_To_Software_When_Pad_Option_Unavailable()
+    {
+        string command = BuildPadCommand(vppQsvSupportsPad: false);
+
+        // software fallback: download to system memory, pad, then continue
+        command.ShouldNotContain("vpp_qsv=pad_w");
+        command.ShouldContain("hwdownload,format=nv12,pad=1920:1080:-1:-1:color=black");
+    }
+
+    private string BuildPadCommand(bool vppQsvSupportsPad, bool colorsAreBt709 = false)
+    {
+        var videoInputFile = new VideoInputFile(
+            "/tmp/whatever.mkv",
+            new List<VideoStream>
+            {
+                new(
+                    0,
+                    VideoFormat.H264,
+                    VideoProfile.Main,
+                    new PixelFormatYuv420P(),
+                    ColorParams.Default,
+                    new FrameSize(640, 480),
+                    "1:1",
+                    "4:3",
+                    FrameRate.DefaultFrameRate,
+                    false,
+                    ScanKind.Progressive)
+            });
+
+        var desiredState = new FrameState(
+            true,
+            false,
+            VideoFormat.Hevc,
+            VideoProfile.Main,
+            VideoPreset.Unset,
+            false,
+            new PixelFormatNv12("yuv420p"),
+            new FrameSize(1440, 1080),
+            new FrameSize(1920, 1080),
+            Option<FrameSize>.None,
+            FFmpegFilterMode.HardwareIfPossible,
+            false,
+            Option<FrameRate>.None,
+            2000,
+            4000,
+            90_000,
+            colorsAreBt709,
+            false);
+
+        var ffmpegState = new FFmpegState(
+            false,
+            HardwareAccelerationMode.Qsv,
+            HardwareAccelerationMode.Qsv,
+            Option<string>.None,
+            Option<string>.None,
+            TimeSpan.FromSeconds(1),
+            Option<TimeSpan>.None,
+            false,
+            Option<string>.None,
+            Option<string>.None,
+            Option<string>.None,
+            Option<string>.None,
+            Option<string>.None,
+            OutputFormatKind.MpegTs,
+            Option<string>.None,
+            Option<string>.None,
+            Option<string>.None,
+            Option<string>.None,
+            TimeSpan.Zero,
+            Option<int>.None,
+            Option<int>.None,
+            false,
+            false,
+            "clip",
+            false);
+
+        var hardwareCapabilities = Substitute.For<IHardwareCapabilities>();
+        hardwareCapabilities.CanDecode(
+                Arg.Any<string>(),
+                Arg.Any<Option<string>>(),
+                Arg.Any<Option<IPixelFormat>>(),
+                Arg.Any<ColorParams>())
+            .Returns(FFmpegCapability.Hardware);
+        hardwareCapabilities.CanEncode(
+                Arg.Any<string>(),
+                Arg.Any<Option<string>>(),
+                Arg.Any<Option<IPixelFormat>>())
+            .Returns(FFmpegCapability.Hardware);
+        hardwareCapabilities.GetRateControlMode(Arg.Any<string>(), Arg.Any<Option<IPixelFormat>>())
+            .Returns(Option<RateControlMode>.None);
+
+        var vppQsvOptions = new System.Collections.Generic.HashSet<string> { "w", "h", "format", "deinterlace" };
+        if (vppQsvSupportsPad)
+        {
+            vppQsvOptions.UnionWith(["pad_w", "pad_h", "pad_x", "pad_y", "pad_color"]);
+        }
+
+        var ffmpegCapabilities = new FFmpegCapabilities(
+            string.Empty,
+            new System.Collections.Generic.HashSet<string> { FFmpegKnownHardwareAcceleration.Qsv.Name },
+            new System.Collections.Generic.HashSet<string>(),
+            new System.Collections.Generic.HashSet<string>(),
+            new System.Collections.Generic.HashSet<string>(),
+            new System.Collections.Generic.HashSet<string>(),
+            new System.Collections.Generic.HashSet<string>(),
+            new System.Collections.Generic.Dictionary<string, IReadOnlySet<string>>
+            {
+                [FFmpegKnownFilter.VppQsv.Name] = vppQsvOptions
+            });
+
+        var builder = new QsvPipelineBuilder(
+            ffmpegCapabilities,
+            hardwareCapabilities,
+            HardwareAccelerationMode.Qsv,
+            videoInputFile,
+            Option<AudioInputFile>.None,
+            Option<WatermarkInputFile>.None,
+            Option<SubtitleInputFile>.None,
+            None,
+            Option<GraphicsEngineInput>.None,
+            "",
+            "",
+            _logger);
+
+        FFmpegPipeline result = builder.Build(ffmpegState, desiredState);
+
+        IList<string> arguments = CommandGenerator.GenerateArguments(
+            videoInputFile,
+            Option<AudioInputFile>.None,
+            Option<WatermarkInputFile>.None,
+            Option<ConcatInputFile>.None,
+            Option<GraphicsEngineInput>.None,
+            result.PipelineSteps,
+            result.IsIntelVaapiOrQsv);
+
+        return string.Join(" ", arguments);
+    }
+}

ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs

@@ -562,5 +562,6 @@ public class PipelineBuilderBaseTests
         new System.Collections.Generic.HashSet<string>(),
         new System.Collections.Generic.HashSet<string>(),
         new System.Collections.Generic.HashSet<string>(),
-        new System.Collections.Generic.HashSet<string>());
+        new System.Collections.Generic.HashSet<string>(),
+        new System.Collections.Generic.Dictionary<string, IReadOnlySet<string>>());
 }

ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs

@@ -10,7 +10,8 @@ public class FFmpegCapabilities(
     IReadOnlySet<string> ffmpegFilters,
     IReadOnlySet<string> ffmpegEncoders,
     IReadOnlySet<string> ffmpegOptions,
-    IReadOnlySet<string> ffmpegDemuxFormats)
+    IReadOnlySet<string> ffmpegDemuxFormats,
+    IReadOnlyDictionary<string, IReadOnlySet<string>> ffmpegFilterOptions)
     : IFFmpegCapabilities
 {
     public string Version => ffmpegVersion;
@@ -59,6 +60,9 @@ public class FFmpegCapabilities(
 
     public bool HasFilter(FFmpegKnownFilter filter) => ffmpegFilters.Contains(filter.Name);
 
+    public bool HasFilterOption(FFmpegKnownFilter filter, string optionName) =>
+        ffmpegFilterOptions.TryGetValue(filter.Name, out IReadOnlySet<string>? options) && options.Contains(optionName);
+
     public bool HasOption(FFmpegKnownOption ffmpegOption) => ffmpegOptions.Contains(ffmpegOption.Name);
 
     public bool HasDemuxFormat(FFmpegKnownFormat format) => ffmpegDemuxFormats.Contains(format.Name);

ErsatzTV.FFmpeg/Capabilities/FFmpegFilterOptionParser.cs

@@ -0,0 +1,21 @@
+using System.Collections.Immutable;
+using System.Text.RegularExpressions;
+
+namespace ErsatzTV.FFmpeg.Capabilities;
+
+public static partial class FFmpegFilterOptionParser
+{
+    public static IReadOnlySet<string> Parse(string filterHelpOutput) =>
+        filterHelpOutput.Split("\n").Map(s => s.Trim())
+            .Bind(l => ParseLine(l))
+            .ToImmutableHashSet();
+
+    private static Option<string> ParseLine(string input)
+    {
+        Match match = FilterOptionRegex().Match(input);
+        return match.Success ? match.Groups[1].Value : Option<string>.None;
+    }
+
+    [GeneratedRegex(@"^([\w-]+)\s+<")]
+    private static partial Regex FilterOptionRegex();
+}

ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs

@@ -24,6 +24,7 @@ public record FFmpegKnownFilter
     public static readonly FFmpegKnownFilter Scale = new("scale");
     public static readonly FFmpegKnownFilter Subtitles = new("subtitles");
     public static readonly FFmpegKnownFilter Tonemap = new("tonemap");
+    public static readonly FFmpegKnownFilter VppQsv = new("vpp_qsv");
     public static readonly FFmpegKnownFilter Yadif = new("yadif");
     public static readonly FFmpegKnownFilter ZScale = new("zscale");
 

ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs

@@ -36,6 +36,9 @@ public partial class HardwareCapabilitiesFactory(
     private static readonly CompositeFormat QsvCacheKeyFormat = CompositeFormat.Parse("ffmpeg.hardware.qsv.{0}");
     private static readonly CompositeFormat FFmpegCapabilitiesCacheKeyFormat = CompositeFormat.Parse("ffmpeg.{0}");
 
+    private static readonly CompositeFormat FFmpegFilterOptionsCacheKeyFormat =
+        CompositeFormat.Parse("ffmpeg.filter.{0}");
+
     private static readonly string[] QsvArguments =
     {
         "-f", "lavfi",
@@ -54,6 +57,8 @@ public partial class HardwareCapabilitiesFactory(
         memoryCache.Remove(string.Format(CultureInfo.InvariantCulture, FFmpegCapabilitiesCacheKeyFormat, "encoders"));
         memoryCache.Remove(string.Format(CultureInfo.InvariantCulture, FFmpegCapabilitiesCacheKeyFormat, "options"));
         memoryCache.Remove(string.Format(CultureInfo.InvariantCulture, FFmpegCapabilitiesCacheKeyFormat, "formats"));
+        memoryCache.Remove(
+            string.Format(CultureInfo.InvariantCulture, FFmpegFilterOptionsCacheKeyFormat, FFmpegKnownFilter.VppQsv.Name));
     }
 
     public async Task<IFFmpegCapabilities> GetFFmpegCapabilities(string ffmpegPath, CancellationToken cancellationToken)
@@ -87,6 +92,13 @@ public partial class HardwareCapabilitiesFactory(
         IReadOnlySet<string> ffmpegDemuxFormats = await GetFFmpegFormats(ffmpegPath, "D", cancellationToken)
             .Map(set => set.Intersect(FFmpegKnownFormat.AllFormats).ToImmutableHashSet());
 
+        IReadOnlySet<string> vppQsvOptions =
+            await GetFFmpegFilterOptions(ffmpegPath, FFmpegKnownFilter.VppQsv.Name, cancellationToken);
+        var ffmpegFilterOptions = new Dictionary<string, IReadOnlySet<string>>
+        {
+            [FFmpegKnownFilter.VppQsv.Name] = vppQsvOptions
+        };
+
         return new FFmpegCapabilities(
             ffmpegVersion,
             ffmpegHardwareAccelerations,
@@ -94,7 +106,8 @@ public partial class HardwareCapabilitiesFactory(
             ffmpegFilters,
             ffmpegEncoders,
             ffmpegOptions,
-            ffmpegDemuxFormats);
+            ffmpegDemuxFormats,
+            ffmpegFilterOptions);
     }
 
     public async Task<IHardwareCapabilities> GetHardwareCapabilities(
@@ -438,6 +451,36 @@ public partial class HardwareCapabilitiesFactory(
         return capabilitiesResult;
     }
 
+    private async Task<IReadOnlySet<string>> GetFFmpegFilterOptions(
+        string ffmpegPath,
+        string filterName,
+        CancellationToken cancellationToken)
+    {
+        var cacheKey = string.Format(CultureInfo.InvariantCulture, FFmpegFilterOptionsCacheKeyFormat, filterName);
+        if (memoryCache.TryGetValue(cacheKey, out IReadOnlySet<string>? cachedCapabilities) &&
+            cachedCapabilities is not null)
+        {
+            return cachedCapabilities;
+        }
+
+        string[] arguments = ["-hide_banner", "-h", $"filter={filterName}"];
+
+        BufferedCommandResult result = await Cli.Wrap(ffmpegPath)
+            .WithArguments(arguments)
+            .WithValidation(CommandResultValidation.None)
+            .ExecuteBufferedAsync(Encoding.UTF8, cancellationToken);
+
+        string output = string.IsNullOrWhiteSpace(result.StandardOutput)
+            ? result.StandardError
+            : result.StandardOutput;
+
+        IReadOnlySet<string> capabilitiesResult = FFmpegFilterOptionParser.Parse(output);
+
+        memoryCache.Set(cacheKey, capabilitiesResult);
+
+        return capabilitiesResult;
+    }
+
     private async Task<IReadOnlySet<string>> GetFFmpegFormats(
         string ffmpegPath,
         string muxDemux,

ErsatzTV.FFmpeg/Capabilities/IFFmpegCapabilities.cs

@@ -9,6 +9,7 @@ public interface IFFmpegCapabilities
     bool HasDecoder(FFmpegKnownDecoder decoder);
     bool HasEncoder(FFmpegKnownEncoder encoder);
     bool HasFilter(FFmpegKnownFilter filter);
+    bool HasFilterOption(FFmpegKnownFilter filter, string optionName);
     bool HasOption(FFmpegKnownOption ffmpegOption);
     bool HasDemuxFormat(FFmpegKnownFormat format);
     Option<IDecoder> SoftwareDecoderForVideoFormat(string videoFormat);

ErsatzTV.FFmpeg/Filter/Qsv/PadQsvFilter.cs

@@ -0,0 +1,40 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Filter.Qsv;
+
+public class PadQsvFilter : BaseFilter
+{
+    private readonly FrameState _currentState;
+    private readonly FrameSize _paddedSize;
+    private readonly int _extraHardwareFrames;
+
+    public PadQsvFilter(FrameState currentState, FrameSize paddedSize, int extraHardwareFrames)
+    {
+        _currentState = currentState;
+        _paddedSize = paddedSize;
+        _extraHardwareFrames = extraHardwareFrames;
+    }
+
+    public override string Filter
+    {
+        get
+        {
+            var pad =
+                $"vpp_qsv=pad_w={_paddedSize.Width}:pad_h={_paddedSize.Height}:pad_x=-1:pad_y=-1:pad_color=black";
+
+            if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)
+            {
+                return pad;
+            }
+
+            string initialPixelFormat = _currentState.PixelFormat.Match(pf => pf.FFmpegName, FFmpegFormat.NV12);
+            return $"format={initialPixelFormat},hwupload=extra_hw_frames={_extraHardwareFrames},{pad}";
+        }
+    }
+
+    public override FrameState NextState(FrameState currentState) => currentState with
+    {
+        PaddedSize = _paddedSize,
+        FrameDataLocation = FrameDataLocation.Hardware
+    };
+}

ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs

@@ -0,0 +1,121 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Filter.Qsv;
+
+public class ScaleAndPadQsvFilter : BaseFilter
+{
+    private readonly FrameState _currentState;
+    private readonly int _extraHardwareFrames;
+    private readonly bool _isAnamorphicEdgeCase;
+    private readonly FrameSize _paddedSize;
+    private readonly string _sampleAspectRatio;
+    private readonly FrameSize _scaledSize;
+
+    public ScaleAndPadQsvFilter(
+        FrameState currentState,
+        FrameSize scaledSize,
+        FrameSize paddedSize,
+        int extraHardwareFrames,
+        bool isAnamorphicEdgeCase,
+        string sampleAspectRatio)
+    {
+        _currentState = currentState;
+        _scaledSize = scaledSize;
+        _paddedSize = paddedSize;
+        _extraHardwareFrames = extraHardwareFrames;
+        _isAnamorphicEdgeCase = isAnamorphicEdgeCase;
+        _sampleAspectRatio = sampleAspectRatio;
+    }
+
+    public override string Filter
+    {
+        get
+        {
+            // fold scale and pad into a single vpp_qsv instance; chaining two vpp_qsv instances
+            // drops the final frame at EOF
+
+            string padOptions =
+                $"pad_w={_paddedSize.Width}:pad_h={_paddedSize.Height}:pad_x=-1:pad_y=-1:pad_color=black";
+
+            string scale;
+
+            if (_currentState.ScaledSize == _scaledSize)
+            {
+                string format = string.Empty;
+                foreach (IPixelFormat pixelFormat in _currentState.PixelFormat)
+                {
+                    format = $"format={pixelFormat.FFmpegName}:";
+                }
+
+                scale = $"vpp_qsv={format}{padOptions}";
+            }
+            else
+            {
+                string format = string.Empty;
+                foreach (IPixelFormat pixelFormat in _currentState.PixelFormat)
+                {
+                    format = $":format={pixelFormat.FFmpegName}";
+                }
+
+                string squareScale = string.Empty;
+                string setsar = string.Empty;
+                string sar = _sampleAspectRatio.Replace(':', '/');
+                if (_isAnamorphicEdgeCase)
+                {
+                    squareScale = $"vpp_qsv=w=iw:h={sar}*ih{format},setsar=1,";
+                }
+                else if (_currentState.IsAnamorphic)
+                {
+                    squareScale = $"vpp_qsv=w=iw*{sar}:h=ih{format},setsar=1,";
+                }
+                else
+                {
+                    setsar = ",setsar=1";
+                }
+
+                scale =
+                    $"{squareScale}vpp_qsv=w={_scaledSize.Width}:h={_scaledSize.Height}{format}:{padOptions}{setsar}";
+            }
+
+            if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)
+            {
+                return scale;
+            }
+
+            string initialPixelFormat = _currentState.PixelFormat.Match(pf => pf.FFmpegName, FFmpegFormat.NV12);
+            return $"format={initialPixelFormat},hwupload=extra_hw_frames={_extraHardwareFrames},{scale}";
+        }
+    }
+
+    public override FrameState NextState(FrameState currentState)
+    {
+        FrameState result = currentState with
+        {
+            ScaledSize = _scaledSize,
+            PaddedSize = _paddedSize,
+            FrameDataLocation = FrameDataLocation.Hardware,
+            IsAnamorphic = false // this filter always outputs square pixels
+        };
+
+        if (_currentState.PixelFormat.IsNone &&
+            _currentState.FrameDataLocation == FrameDataLocation.Software &&
+            currentState.PixelFormat.Map(pf => pf is not PixelFormatNv12).IfNone(false))
+        {
+            // wrap in nv12
+            result = result with
+            {
+                PixelFormat = currentState.PixelFormat
+                    .Map(pf => (IPixelFormat)new PixelFormatNv12(pf.Name))
+            };
+        }
+        else
+        {
+            foreach (IPixelFormat pixelFormat in _currentState.PixelFormat)
+            {
+                result = result with { PixelFormat = Some(pixelFormat) };
+            }
+        }
+
+        return result;
+    }
+}

ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs

@@ -30,6 +30,12 @@ public class ScaleQsvFilter : BaseFilter
         _sampleAspectRatio = sampleAspectRatio;
     }
 
+    public FrameState CurrentState => _currentState;
+    public FrameSize ScaledSize => _scaledSize;
+    public int ExtraHardwareFrames => _extraHardwareFrames;
+    public bool IsAnamorphicEdgeCase => _isAnamorphicEdgeCase;
+    public string SampleAspectRatio => _sampleAspectRatio;
+
     public override string Filter
     {
         get

ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs

@@ -18,6 +18,7 @@ namespace ErsatzTV.FFmpeg.Pipeline;
 
 public class QsvPipelineBuilder : SoftwarePipelineBuilder
 {
+    private readonly IFFmpegCapabilities _ffmpegCapabilities;
     private readonly IHardwareCapabilities _hardwareCapabilities;
     private readonly ILogger _logger;
 
@@ -46,6 +47,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
         fontsFolder,
         logger)
     {
+        _ffmpegCapabilities = ffmpegCapabilities;
         _hardwareCapabilities = hardwareCapabilities;
         _logger = logger;
     }
@@ -194,15 +196,26 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
         // _logger.LogDebug("After deinterlace: {PixelFormat}", currentState.PixelFormat);
         currentState = SetScale(videoInputFile, videoStream, context, ffmpegState, desiredState, currentState);
         // _logger.LogDebug("After scale: {PixelFormat}", currentState.PixelFormat);
+        bool isHdrTonemap = videoStream.ColorParams.IsHdr;
         currentState = SetTonemap(videoInputFile, videoStream, ffmpegState, desiredState, currentState);
-        currentState = SetPad(videoInputFile, videoStream, desiredState, currentState);
+        currentState = SetPad(videoInputFile, videoStream, ffmpegState, desiredState, currentState, isHdrTonemap);
         // _logger.LogDebug("After pad: {PixelFormat}", currentState.PixelFormat);
         currentState = SetCrop(videoInputFile, desiredState, currentState);
         SetStillImageLoop(videoInputFile, videoStream, ffmpegState, desiredState, pipelineSteps);
 
-        // need to download for any sort of overlay (and always for setpts)
-        if (currentState.FrameDataLocation == FrameDataLocation.Hardware) //&&
-            //(context.HasSubtitleOverlay || context.HasWatermark || context.HasGraphicsEngine))
+        // setpts/fps operate on timestamps only and accept hardware frames, and the qsv encoder
+        // ingests hardware frames directly, so keep the frame on the GPU through both when nothing
+        // downstream needs it in system memory. any software-requiring step (overlays, subtitle
+        // burn-in, a colorspace conversion, or a software encoder) still forces the download.
+        bool canKeepHardwareFrames =
+            ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Qsv
+            && !context.HasWatermark
+            && !context.HasSubtitleOverlay
+            && !context.HasSubtitleText
+            && !context.HasGraphicsEngine
+            && !WillDownloadForColorspace(videoInputFile, videoStream, desiredState);
+
+        if (currentState.FrameDataLocation == FrameDataLocation.Hardware && !canKeepHardwareFrames)
         {
             var hardwareDownload = new HardwareDownloadFilter(currentState);
             currentState = hardwareDownload.NextState(currentState);
@@ -311,7 +324,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
 
             bool usesVppQsv =
                 videoInputFile.FilterSteps.Any(f =>
-                    f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter);
+                    f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter
+                        or ScaleAndPadQsvFilter);
 
             // if we have no filters, check whether we need to convert pixel format
             // since qsv doesn't seem to like doing that at the encoder
@@ -554,7 +568,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
                     }
 
                     // only scale if scaling or padding was used for main video stream
-                    if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleQsvFilter or PadFilter))
+                    if (videoInputFile.FilterSteps.Any(s =>
+                            s is ScaleFilter or ScaleQsvFilter or PadFilter or ScaleAndPadQsvFilter))
                     {
                         var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize);
                         subtitle.FilterSteps.Add(scaleFilter);
@@ -598,22 +613,78 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
         }
     }
 
-    private static FrameState SetPad(
+    private FrameState SetPad(
         VideoInputFile videoInputFile,
         VideoStream videoStream,
+        FFmpegState ffmpegState,
         FrameState desiredState,
-        FrameState currentState)
+        FrameState currentState,
+        bool isHdrTonemap)
     {
         if (desiredState.CroppedSize.IsNone && currentState.PaddedSize != desiredState.PaddedSize)
         {
-            var padStep = new PadFilter(currentState, desiredState.PaddedSize);
-            currentState = padStep.NextState(currentState);
-            videoInputFile.FilterSteps.Add(padStep);
+            if (desiredState.PadMode is FFmpegFilterMode.Software
+                || isHdrTonemap
+                || !_ffmpegCapabilities.HasFilterOption(FFmpegKnownFilter.VppQsv, "pad_w"))
+            {
+                var padStep = new PadFilter(currentState, desiredState.PaddedSize);
+                currentState = padStep.NextState(currentState);
+                videoInputFile.FilterSteps.Add(padStep);
+            }
+            else
+            {
+                // fold an existing hardware scale and this pad into a single vpp_qsv instance;
+                // two chained vpp_qsv instances drop the final frame at EOF
+                Option<ScaleQsvFilter> maybeScale = videoInputFile.FilterSteps.OfType<ScaleQsvFilter>().HeadOrNone();
+                if (maybeScale.IsSome)
+                {
+                    foreach (ScaleQsvFilter scaleStep in maybeScale)
+                    {
+                        videoInputFile.FilterSteps.Remove(scaleStep);
+
+                        var scalePadStep = new ScaleAndPadQsvFilter(
+                            scaleStep.CurrentState,
+                            scaleStep.ScaledSize,
+                            desiredState.PaddedSize,
+                            scaleStep.ExtraHardwareFrames,
+                            scaleStep.IsAnamorphicEdgeCase,
+                            scaleStep.SampleAspectRatio);
+
+                        // advance the live pipeline state (which carries the pixel format the
+                        // unfolded ScaleQsvFilter produced) so any later download keeps format=X
+                        currentState = scalePadStep.NextState(currentState);
+                        videoInputFile.FilterSteps.Add(scalePadStep);
+                    }
+                }
+                else
+                {
+                    var padStep = new PadQsvFilter(
+                        currentState,
+                        desiredState.PaddedSize,
+                        ffmpegState.QsvExtraHardwareFrames);
+                    currentState = padStep.NextState(currentState);
+                    videoInputFile.FilterSteps.Add(padStep);
+                }
+            }
         }
 
         return currentState;
     }
 
+    private static bool WillDownloadForColorspace(
+        VideoInputFile videoInputFile,
+        VideoStream videoStream,
+        FrameState desiredState)
+    {
+        // mirrors the colorspace emission in SetPixelFormat: when it runs on a hardware frame it
+        // prepends its own hwdownload, so the frame would leave the GPU regardless
+        bool usesVppQsv = videoInputFile.FilterSteps.Any(f =>
+            f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter
+                or ScaleAndPadQsvFilter);
+
+        return desiredState.ColorsAreBt709 && (!videoStream.ColorParams.IsBt709 || usesVppQsv);
+    }
+
     private static FrameState SetScale(
         VideoInputFile videoInputFile,
         VideoStream videoStream,