From 30280aef0b2a28ad7678e6212ea41f4d403814ca Mon Sep 17 00:00:00 2001 From: McKenzie Pepper Date: Thu, 30 Jul 2026 11:28:51 -0400 Subject: [PATCH 1/2] feat: add QSV hardware padding via vpp_qsv pad options with capability gate The QSV pipeline previously always padded in software, forcing a hwdownload/pad/hwupload round-trip. ffmpeg's vpp_qsv filter is gaining pad_w/pad_h/pad_x/pad_y/pad_color options, so when the ffmpeg binary exposes them we can pad on the GPU instead, mirroring the existing pad_vaapi hardware path. A new capability probe runs `ffmpeg -h filter=vpp_qsv` and parses the available AVOption names, surfaced as IFFmpegCapabilities.HasFilterOption. QsvPipelineBuilder.SetPad now emits PadQsvFilter (vpp_qsv=pad_w=..:pad_h=.. :pad_x=-1:pad_y=-1:pad_color=black) when vpp_qsv advertises pad_w, and keeps the software PadFilter fallback for HDR tonemap, forced software pad mode, and stock ffmpeg builds without the pad options. The gate is what makes this safe to upstream, since stock ffmpeg has no vpp_qsv pad options yet. Co-Authored-By: Claude Fable 5 --- .../FFmpegFilterOptionParserTests.cs | 99 ++++++++++ .../Filter/Qsv/PadQsvFilterTests.cs | 72 ++++++++ .../Pipeline/QsvPipelineBuilderTests.cs | 174 ++++++++++++++++++ .../PipelineBuilderBaseTests.cs | 3 +- .../Capabilities/FFmpegCapabilities.cs | 6 +- .../Capabilities/FFmpegFilterOptionParser.cs | 21 +++ .../Capabilities/FFmpegKnownFilter.cs | 1 + .../HardwareCapabilitiesFactory.cs | 45 ++++- .../Capabilities/IFFmpegCapabilities.cs | 1 + ErsatzTV.FFmpeg/Filter/Qsv/PadQsvFilter.cs | 40 ++++ .../Pipeline/QsvPipelineBuilder.cs | 33 +++- 11 files changed, 485 insertions(+), 10 deletions(-) create mode 100644 ErsatzTV.FFmpeg.Tests/Capabilities/FFmpegFilterOptionParserTests.cs create mode 100644 ErsatzTV.FFmpeg.Tests/Filter/Qsv/PadQsvFilterTests.cs create mode 100644 ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs create mode 100644 ErsatzTV.FFmpeg/Capabilities/FFmpegFilterOptionParser.cs create mode 100644 ErsatzTV.FFmpeg/Filter/Qsv/PadQsvFilter.cs diff --git a/ErsatzTV.FFmpeg.Tests/Capabilities/FFmpegFilterOptionParserTests.cs b/ErsatzTV.FFmpeg.Tests/Capabilities/FFmpegFilterOptionParserTests.cs new file mode 100644 index 000000000..7a4e08eaf --- /dev/null +++ b/ErsatzTV.FFmpeg.Tests/Capabilities/FFmpegFilterOptionParserTests.cs @@ -0,0 +1,99 @@ +using System.Collections.Generic; +using ErsatzTV.FFmpeg.Capabilities; +using NUnit.Framework; +using Shouldly; + +namespace ErsatzTV.FFmpeg.Tests.Capabilities; + +[TestFixture] +public class FFmpegFilterOptionParserTests +{ + // stock ffmpeg vpp_qsv, without the pad_w/pad_h/pad_x/pad_y/pad_color options + private const string WithoutPadOptions = @"Filter vpp_qsv + Quick Sync Video VPP. + Inputs: + #0: default (video) + Outputs: + #0: default (video) +vpp_qsv AVOptions: + deinterlace ..FV....... deinterlace mode (from 0 to 2) (default 0) + bob 1 ..FV....... Get one frame for each field + advanced 2 ..FV....... Get one frame for each frame + denoise ..FV....... denoise level [0, 100] (from 0 to 100) (default 0) + framerate ..FV....... output framerate (from 0 to DBL_MAX) (default 0/1) + w ..FV....... Output video width + h ..FV....... Output video height + format ..FV....... Output pixel format + async_depth ..FV....... Internal parallelization depth (from 0 to INT_MAX) (default 4) + scale_mode ..FV....... scaling mode (from 0 to 2) (default 0) +"; + + // patched ffmpeg vpp_qsv, carrying the pad_w/pad_h/pad_x/pad_y/pad_color options + private const string WithPadOptions = @"Filter vpp_qsv + Quick Sync Video VPP. + Inputs: + #0: default (video) + Outputs: + #0: default (video) +vpp_qsv AVOptions: + deinterlace ..FV....... deinterlace mode (from 0 to 2) (default 0) + bob 1 ..FV....... Get one frame for each field + advanced 2 ..FV....... Get one frame for each frame + denoise ..FV....... denoise level [0, 100] (from 0 to 100) (default 0) + framerate ..FV....... output framerate (from 0 to DBL_MAX) (default 0/1) + w ..FV....... Output video width + h ..FV....... Output video height + format ..FV....... Output pixel format + async_depth ..FV....... Internal parallelization depth (from 0 to INT_MAX) (default 4) + scale_mode ..FV....... scaling mode (from 0 to 2) (default 0) + pad_w ..FV....... Padded width (from 0 to 8192) (default 0) + pad_h ..FV....... Padded height (from 0 to 8192) (default 0) + pad_x ..FV....... Horizontal position (from -1 to 8192) (default -1) + pad_y ..FV....... Vertical position (from -1 to 8192) (default -1) + pad_color ..FV....... Padding color (default ""black"") +"; + + [Test] + public void Should_Parse_Option_Names() + { + IReadOnlySet options = FFmpegFilterOptionParser.Parse(WithoutPadOptions); + + options.ShouldContain("deinterlace"); + options.ShouldContain("denoise"); + options.ShouldContain("framerate"); + options.ShouldContain("w"); + options.ShouldContain("h"); + options.ShouldContain("format"); + } + + [Test] + public void Should_Not_Parse_Enum_Constants() + { + IReadOnlySet options = FFmpegFilterOptionParser.Parse(WithoutPadOptions); + + options.ShouldNotContain("bob"); + options.ShouldNotContain("advanced"); + } + + [Test] + public void Should_Detect_Pad_Options_When_Present() + { + IReadOnlySet options = FFmpegFilterOptionParser.Parse(WithPadOptions); + + options.ShouldContain("pad_w"); + options.ShouldContain("pad_h"); + options.ShouldContain("pad_x"); + options.ShouldContain("pad_y"); + options.ShouldContain("pad_color"); + } + + [Test] + public void Should_Not_Detect_Pad_Options_When_Absent() + { + IReadOnlySet options = FFmpegFilterOptionParser.Parse(WithoutPadOptions); + + options.ShouldNotContain("pad_w"); + options.ShouldNotContain("pad_h"); + options.ShouldNotContain("pad_color"); + } +} diff --git a/ErsatzTV.FFmpeg.Tests/Filter/Qsv/PadQsvFilterTests.cs b/ErsatzTV.FFmpeg.Tests/Filter/Qsv/PadQsvFilterTests.cs new file mode 100644 index 000000000..fad03b0f0 --- /dev/null +++ b/ErsatzTV.FFmpeg.Tests/Filter/Qsv/PadQsvFilterTests.cs @@ -0,0 +1,72 @@ +using ErsatzTV.FFmpeg; +using ErsatzTV.FFmpeg.Filter.Qsv; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.State; +using LanguageExt; +using NUnit.Framework; +using Shouldly; + +namespace ErsatzTV.FFmpeg.Tests.Filter.Qsv; + +[TestFixture] +public class PadQsvFilterTests +{ + private static FrameState FrameStateAt(FrameDataLocation location, Option pixelFormat) => new( + false, + false, + string.Empty, + Option.None, + Option.None, + true, + pixelFormat, + new FrameSize(1920, 1080), + new FrameSize(1920, 1080), + Option.None, + FFmpegFilterMode.HardwareIfPossible, + false, + Option.None, + Option.None, + Option.None, + Option.None, + false, + false, + location); + + [Test] + public void Should_Emit_Hardware_Pad_When_Frame_In_Hardware() + { + var filter = new PadQsvFilter( + FrameStateAt(FrameDataLocation.Hardware, Option.None), + new FrameSize(1920, 1080), + 64); + + filter.Filter.ShouldBe("vpp_qsv=pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + } + + [Test] + public void Should_Upload_Before_Pad_When_Frame_In_Software() + { + var filter = new PadQsvFilter( + FrameStateAt(FrameDataLocation.Software, Option.Some(new PixelFormatNv12("yuv420p"))), + new FrameSize(1920, 1080), + 64); + + filter.Filter.ShouldBe( + "format=nv12,hwupload=extra_hw_frames=64,vpp_qsv=pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + } + + [Test] + public void Should_Report_Padded_Size_And_Hardware_Location() + { + var filter = new PadQsvFilter( + FrameStateAt(FrameDataLocation.Software, Option.None), + new FrameSize(1920, 1080), + 64); + + FrameState nextState = filter.NextState( + FrameStateAt(FrameDataLocation.Software, Option.None)); + + nextState.PaddedSize.ShouldBe(new FrameSize(1920, 1080)); + nextState.FrameDataLocation.ShouldBe(FrameDataLocation.Hardware); + } +} diff --git a/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs b/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs new file mode 100644 index 000000000..21abfe446 --- /dev/null +++ b/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; +using ErsatzTV.FFmpeg; +using ErsatzTV.FFmpeg.Capabilities; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.InputOption; +using ErsatzTV.FFmpeg.OutputFormat; +using ErsatzTV.FFmpeg.Pipeline; +using ErsatzTV.FFmpeg.Preset; +using ErsatzTV.FFmpeg.State; +using LanguageExt; +using Microsoft.Extensions.Logging; +using NSubstitute; +using NUnit.Framework; +using Shouldly; +using static LanguageExt.Prelude; + +namespace ErsatzTV.FFmpeg.Tests.Pipeline; + +[TestFixture] +public class QsvPipelineBuilderTests +{ + private readonly ILogger _logger = Substitute.For(); + + [Test] + public void Pad_Uses_Hardware_Vpp_Qsv_When_Pad_Option_Available() + { + string command = BuildPadCommand(vppQsvSupportsPad: true); + + // hardware pad: vpp_qsv does the padding, with no software pad and no hwdownload round-trip around it + command.ShouldContain("vpp_qsv=pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + command.ShouldNotContain("pad=1920:1080"); + command.ShouldNotContain("hwdownload,format=nv12,pad"); + } + + [Test] + public void Pad_Falls_Back_To_Software_When_Pad_Option_Unavailable() + { + string command = BuildPadCommand(vppQsvSupportsPad: false); + + // software fallback: download to system memory, pad, then continue + command.ShouldNotContain("vpp_qsv=pad_w"); + command.ShouldContain("hwdownload,format=nv12,pad=1920:1080:-1:-1:color=black"); + } + + private string BuildPadCommand(bool vppQsvSupportsPad) + { + var videoInputFile = new VideoInputFile( + "/tmp/whatever.mkv", + new List + { + new( + 0, + VideoFormat.H264, + VideoProfile.Main, + new PixelFormatYuv420P(), + ColorParams.Default, + new FrameSize(1440, 1080), + "1:1", + "4:3", + FrameRate.DefaultFrameRate, + false, + ScanKind.Progressive) + }); + + var desiredState = new FrameState( + true, + false, + VideoFormat.Hevc, + VideoProfile.Main, + VideoPreset.Unset, + false, + new PixelFormatNv12("yuv420p"), + new FrameSize(1440, 1080), + new FrameSize(1920, 1080), + Option.None, + FFmpegFilterMode.HardwareIfPossible, + false, + Option.None, + 2000, + 4000, + 90_000, + false, + false); + + var ffmpegState = new FFmpegState( + false, + HardwareAccelerationMode.Qsv, + HardwareAccelerationMode.Qsv, + Option.None, + Option.None, + TimeSpan.FromSeconds(1), + Option.None, + false, + Option.None, + Option.None, + Option.None, + Option.None, + Option.None, + OutputFormatKind.MpegTs, + Option.None, + Option.None, + Option.None, + Option.None, + TimeSpan.Zero, + Option.None, + Option.None, + false, + false, + "clip", + false); + + var hardwareCapabilities = Substitute.For(); + hardwareCapabilities.CanDecode( + Arg.Any(), + Arg.Any>(), + Arg.Any>(), + Arg.Any()) + .Returns(FFmpegCapability.Hardware); + hardwareCapabilities.CanEncode( + Arg.Any(), + Arg.Any>(), + Arg.Any>()) + .Returns(FFmpegCapability.Hardware); + hardwareCapabilities.GetRateControlMode(Arg.Any(), Arg.Any>()) + .Returns(Option.None); + + var vppQsvOptions = new System.Collections.Generic.HashSet { "w", "h", "format", "deinterlace" }; + if (vppQsvSupportsPad) + { + vppQsvOptions.UnionWith(["pad_w", "pad_h", "pad_x", "pad_y", "pad_color"]); + } + + var ffmpegCapabilities = new FFmpegCapabilities( + string.Empty, + new System.Collections.Generic.HashSet { FFmpegKnownHardwareAcceleration.Qsv.Name }, + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.Dictionary> + { + [FFmpegKnownFilter.VppQsv.Name] = vppQsvOptions + }); + + var builder = new QsvPipelineBuilder( + ffmpegCapabilities, + hardwareCapabilities, + HardwareAccelerationMode.Qsv, + videoInputFile, + Option.None, + Option.None, + Option.None, + None, + Option.None, + "", + "", + _logger); + + FFmpegPipeline result = builder.Build(ffmpegState, desiredState); + + IList arguments = CommandGenerator.GenerateArguments( + videoInputFile, + Option.None, + Option.None, + Option.None, + Option.None, + result.PipelineSteps, + result.IsIntelVaapiOrQsv); + + return string.Join(" ", arguments); + } +} diff --git a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs index 64e485b4e..5c3c4cc91 100644 --- a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs +++ b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs @@ -562,5 +562,6 @@ public class PipelineBuilderBaseTests new System.Collections.Generic.HashSet(), new System.Collections.Generic.HashSet(), new System.Collections.Generic.HashSet(), - new System.Collections.Generic.HashSet()); + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.Dictionary>()); } diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs index 9fa18ee11..a008b58b1 100644 --- a/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs @@ -10,7 +10,8 @@ public class FFmpegCapabilities( IReadOnlySet ffmpegFilters, IReadOnlySet ffmpegEncoders, IReadOnlySet ffmpegOptions, - IReadOnlySet ffmpegDemuxFormats) + IReadOnlySet ffmpegDemuxFormats, + IReadOnlyDictionary> ffmpegFilterOptions) : IFFmpegCapabilities { public string Version => ffmpegVersion; @@ -59,6 +60,9 @@ public class FFmpegCapabilities( public bool HasFilter(FFmpegKnownFilter filter) => ffmpegFilters.Contains(filter.Name); + public bool HasFilterOption(FFmpegKnownFilter filter, string optionName) => + ffmpegFilterOptions.TryGetValue(filter.Name, out IReadOnlySet? options) && options.Contains(optionName); + public bool HasOption(FFmpegKnownOption ffmpegOption) => ffmpegOptions.Contains(ffmpegOption.Name); public bool HasDemuxFormat(FFmpegKnownFormat format) => ffmpegDemuxFormats.Contains(format.Name); diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegFilterOptionParser.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegFilterOptionParser.cs new file mode 100644 index 000000000..41209495e --- /dev/null +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegFilterOptionParser.cs @@ -0,0 +1,21 @@ +using System.Collections.Immutable; +using System.Text.RegularExpressions; + +namespace ErsatzTV.FFmpeg.Capabilities; + +public static partial class FFmpegFilterOptionParser +{ + public static IReadOnlySet Parse(string filterHelpOutput) => + filterHelpOutput.Split("\n").Map(s => s.Trim()) + .Bind(l => ParseLine(l)) + .ToImmutableHashSet(); + + private static Option ParseLine(string input) + { + Match match = FilterOptionRegex().Match(input); + return match.Success ? match.Groups[1].Value : Option.None; + } + + [GeneratedRegex(@"^([\w-]+)\s+<")] + private static partial Regex FilterOptionRegex(); +} diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs index 5ef26f5a2..2a10425e4 100644 --- a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs @@ -24,6 +24,7 @@ public record FFmpegKnownFilter public static readonly FFmpegKnownFilter Scale = new("scale"); public static readonly FFmpegKnownFilter Subtitles = new("subtitles"); public static readonly FFmpegKnownFilter Tonemap = new("tonemap"); + public static readonly FFmpegKnownFilter VppQsv = new("vpp_qsv"); public static readonly FFmpegKnownFilter Yadif = new("yadif"); public static readonly FFmpegKnownFilter ZScale = new("zscale"); diff --git a/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs b/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs index 1bb27b9ec..65d338db7 100644 --- a/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs +++ b/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs @@ -36,6 +36,9 @@ public partial class HardwareCapabilitiesFactory( private static readonly CompositeFormat QsvCacheKeyFormat = CompositeFormat.Parse("ffmpeg.hardware.qsv.{0}"); private static readonly CompositeFormat FFmpegCapabilitiesCacheKeyFormat = CompositeFormat.Parse("ffmpeg.{0}"); + private static readonly CompositeFormat FFmpegFilterOptionsCacheKeyFormat = + CompositeFormat.Parse("ffmpeg.filter.{0}"); + private static readonly string[] QsvArguments = { "-f", "lavfi", @@ -54,6 +57,8 @@ public partial class HardwareCapabilitiesFactory( memoryCache.Remove(string.Format(CultureInfo.InvariantCulture, FFmpegCapabilitiesCacheKeyFormat, "encoders")); memoryCache.Remove(string.Format(CultureInfo.InvariantCulture, FFmpegCapabilitiesCacheKeyFormat, "options")); memoryCache.Remove(string.Format(CultureInfo.InvariantCulture, FFmpegCapabilitiesCacheKeyFormat, "formats")); + memoryCache.Remove( + string.Format(CultureInfo.InvariantCulture, FFmpegFilterOptionsCacheKeyFormat, FFmpegKnownFilter.VppQsv.Name)); } public async Task GetFFmpegCapabilities(string ffmpegPath, CancellationToken cancellationToken) @@ -87,6 +92,13 @@ public partial class HardwareCapabilitiesFactory( IReadOnlySet ffmpegDemuxFormats = await GetFFmpegFormats(ffmpegPath, "D", cancellationToken) .Map(set => set.Intersect(FFmpegKnownFormat.AllFormats).ToImmutableHashSet()); + IReadOnlySet vppQsvOptions = + await GetFFmpegFilterOptions(ffmpegPath, FFmpegKnownFilter.VppQsv.Name, cancellationToken); + var ffmpegFilterOptions = new Dictionary> + { + [FFmpegKnownFilter.VppQsv.Name] = vppQsvOptions + }; + return new FFmpegCapabilities( ffmpegVersion, ffmpegHardwareAccelerations, @@ -94,7 +106,8 @@ public partial class HardwareCapabilitiesFactory( ffmpegFilters, ffmpegEncoders, ffmpegOptions, - ffmpegDemuxFormats); + ffmpegDemuxFormats, + ffmpegFilterOptions); } public async Task GetHardwareCapabilities( @@ -438,6 +451,36 @@ public partial class HardwareCapabilitiesFactory( return capabilitiesResult; } + private async Task> GetFFmpegFilterOptions( + string ffmpegPath, + string filterName, + CancellationToken cancellationToken) + { + var cacheKey = string.Format(CultureInfo.InvariantCulture, FFmpegFilterOptionsCacheKeyFormat, filterName); + if (memoryCache.TryGetValue(cacheKey, out IReadOnlySet? cachedCapabilities) && + cachedCapabilities is not null) + { + return cachedCapabilities; + } + + string[] arguments = ["-hide_banner", "-h", $"filter={filterName}"]; + + BufferedCommandResult result = await Cli.Wrap(ffmpegPath) + .WithArguments(arguments) + .WithValidation(CommandResultValidation.None) + .ExecuteBufferedAsync(Encoding.UTF8, cancellationToken); + + string output = string.IsNullOrWhiteSpace(result.StandardOutput) + ? result.StandardError + : result.StandardOutput; + + IReadOnlySet capabilitiesResult = FFmpegFilterOptionParser.Parse(output); + + memoryCache.Set(cacheKey, capabilitiesResult); + + return capabilitiesResult; + } + private async Task> GetFFmpegFormats( string ffmpegPath, string muxDemux, diff --git a/ErsatzTV.FFmpeg/Capabilities/IFFmpegCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/IFFmpegCapabilities.cs index 3dcfc5fec..6938f047d 100644 --- a/ErsatzTV.FFmpeg/Capabilities/IFFmpegCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/IFFmpegCapabilities.cs @@ -9,6 +9,7 @@ public interface IFFmpegCapabilities bool HasDecoder(FFmpegKnownDecoder decoder); bool HasEncoder(FFmpegKnownEncoder encoder); bool HasFilter(FFmpegKnownFilter filter); + bool HasFilterOption(FFmpegKnownFilter filter, string optionName); bool HasOption(FFmpegKnownOption ffmpegOption); bool HasDemuxFormat(FFmpegKnownFormat format); Option SoftwareDecoderForVideoFormat(string videoFormat); diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/PadQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/PadQsvFilter.cs new file mode 100644 index 000000000..08b5dcaf5 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Qsv/PadQsvFilter.cs @@ -0,0 +1,40 @@ +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter.Qsv; + +public class PadQsvFilter : BaseFilter +{ + private readonly FrameState _currentState; + private readonly FrameSize _paddedSize; + private readonly int _extraHardwareFrames; + + public PadQsvFilter(FrameState currentState, FrameSize paddedSize, int extraHardwareFrames) + { + _currentState = currentState; + _paddedSize = paddedSize; + _extraHardwareFrames = extraHardwareFrames; + } + + public override string Filter + { + get + { + var pad = + $"vpp_qsv=pad_w={_paddedSize.Width}:pad_h={_paddedSize.Height}:pad_x=-1:pad_y=-1:pad_color=black"; + + if (_currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + return pad; + } + + string initialPixelFormat = _currentState.PixelFormat.Match(pf => pf.FFmpegName, FFmpegFormat.NV12); + return $"format={initialPixelFormat},hwupload=extra_hw_frames={_extraHardwareFrames},{pad}"; + } + } + + public override FrameState NextState(FrameState currentState) => currentState with + { + PaddedSize = _paddedSize, + FrameDataLocation = FrameDataLocation.Hardware + }; +} diff --git a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs index 2e3970f81..d4ca4d767 100644 --- a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs @@ -18,6 +18,7 @@ namespace ErsatzTV.FFmpeg.Pipeline; public class QsvPipelineBuilder : SoftwarePipelineBuilder { + private readonly IFFmpegCapabilities _ffmpegCapabilities; private readonly IHardwareCapabilities _hardwareCapabilities; private readonly ILogger _logger; @@ -46,6 +47,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder fontsFolder, logger) { + _ffmpegCapabilities = ffmpegCapabilities; _hardwareCapabilities = hardwareCapabilities; _logger = logger; } @@ -194,8 +196,9 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder // _logger.LogDebug("After deinterlace: {PixelFormat}", currentState.PixelFormat); currentState = SetScale(videoInputFile, videoStream, context, ffmpegState, desiredState, currentState); // _logger.LogDebug("After scale: {PixelFormat}", currentState.PixelFormat); + bool isHdrTonemap = videoStream.ColorParams.IsHdr; currentState = SetTonemap(videoInputFile, videoStream, ffmpegState, desiredState, currentState); - currentState = SetPad(videoInputFile, videoStream, desiredState, currentState); + currentState = SetPad(videoInputFile, videoStream, ffmpegState, desiredState, currentState, isHdrTonemap); // _logger.LogDebug("After pad: {PixelFormat}", currentState.PixelFormat); currentState = SetCrop(videoInputFile, desiredState, currentState); SetStillImageLoop(videoInputFile, videoStream, ffmpegState, desiredState, pipelineSteps); @@ -311,7 +314,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder bool usesVppQsv = videoInputFile.FilterSteps.Any(f => - f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter); + f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter); // if we have no filters, check whether we need to convert pixel format // since qsv doesn't seem to like doing that at the encoder @@ -598,17 +601,33 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder } } - private static FrameState SetPad( + private FrameState SetPad( VideoInputFile videoInputFile, VideoStream videoStream, + FFmpegState ffmpegState, FrameState desiredState, - FrameState currentState) + FrameState currentState, + bool isHdrTonemap) { if (desiredState.CroppedSize.IsNone && currentState.PaddedSize != desiredState.PaddedSize) { - var padStep = new PadFilter(currentState, desiredState.PaddedSize); - currentState = padStep.NextState(currentState); - videoInputFile.FilterSteps.Add(padStep); + if (desiredState.PadMode is FFmpegFilterMode.Software + || isHdrTonemap + || !_ffmpegCapabilities.HasFilterOption(FFmpegKnownFilter.VppQsv, "pad_w")) + { + var padStep = new PadFilter(currentState, desiredState.PaddedSize); + currentState = padStep.NextState(currentState); + videoInputFile.FilterSteps.Add(padStep); + } + else + { + var padStep = new PadQsvFilter( + currentState, + desiredState.PaddedSize, + ffmpegState.QsvExtraHardwareFrames); + currentState = padStep.NextState(currentState); + videoInputFile.FilterSteps.Add(padStep); + } } return currentState; From cdef25ffc214559001530b97baa6fe8adf18f630 Mon Sep 17 00:00:00 2001 From: McKenzie Pepper Date: Thu, 30 Jul 2026 11:52:33 -0400 Subject: [PATCH 2/2] perf: keep QSV frames in hardware through setpts when no software filters are needed The QSV pipeline downloaded every hardware frame to system memory before setpts "always for setpts", then re-fed software frames to the qsv encoder. Combined with commit 30280ae's GPU pad this was a net loss on Intel B50 (224 fps / 6.5s CPU vs. 242 fps / 5.6s CPU for the old download+CPU-pad path) because the padded frame was downloaded only to be uploaded again by h264_qsv. setpts and fps operate on timestamps only and accept hardware frames, and the qsv encoder ingests hardware frames directly, so the download is now skipped whenever nothing downstream needs system memory: hardware qsv encoder, no watermark, no subtitle overlay or burn-in, no graphics engine, and no follow-on colorspace conversion (which prepends its own hwdownload). Every config that still needs software frames keeps the download in exactly the same place as before. On B50 the full-GPU chain measures 503 fps / 1.8s CPU, 2.1x the old download+CPU-pad baseline. Because two chained vpp_qsv instances drop the final frame at EOF (1439/1440, a pre-existing ffmpeg quirk reproducible with two plain scales on stock ffmpeg), a hardware scale followed by a hardware pad is now emitted as one vpp_qsv instance carrying both the scale and the pad_w/pad_h options. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0139ePZuiHJGo2g4PY2tQJ9Q --- .../Pipeline/QsvPipelineBuilderTests.cs | 41 +++++- .../Filter/Qsv/ScaleAndPadQsvFilter.cs | 121 ++++++++++++++++++ ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs | 6 + .../Pipeline/QsvPipelineBuilder.cs | 74 +++++++++-- 4 files changed, 224 insertions(+), 18 deletions(-) create mode 100644 ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs diff --git a/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs b/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs index 21abfe446..8f3b5b164 100644 --- a/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs +++ b/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs @@ -23,14 +23,41 @@ public class QsvPipelineBuilderTests private readonly ILogger _logger = Substitute.For(); [Test] - public void Pad_Uses_Hardware_Vpp_Qsv_When_Pad_Option_Available() + public void Scale_And_Pad_Fold_Into_Single_Hardware_Vpp_Qsv() { string command = BuildPadCommand(vppQsvSupportsPad: true); - // hardware pad: vpp_qsv does the padding, with no software pad and no hwdownload round-trip around it - command.ShouldContain("vpp_qsv=pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + // scale and pad fold into a single vpp_qsv instance that pads on the GPU + command.ShouldContain("vpp_qsv=w=1440:h=1080:pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + + // exactly one vpp_qsv instance (two chained instances drop the final frame at EOF) + command.Split("vpp_qsv").Length.ShouldBe(2); + + // no software pad command.ShouldNotContain("pad=1920:1080"); - command.ShouldNotContain("hwdownload,format=nv12,pad"); + } + + [Test] + public void Frames_Stay_In_Hardware_Through_Setpts_Into_The_Qsv_Encoder() + { + string command = BuildPadCommand(vppQsvSupportsPad: true); + + // setpts/fps run on the hardware frame and hevc_qsv ingests it directly: no download anywhere + command.ShouldNotContain("hwdownload"); + command.ShouldContain("pad_color=black,setsar=1,setpts=PTS-STARTPTS,fps=24"); + command.ShouldContain("hevc_qsv"); + } + + [Test] + public void Download_Is_Retained_When_A_Colorspace_Conversion_Follows() + { + string command = BuildPadCommand(vppQsvSupportsPad: true, colorsAreBt709: true); + + // a downstream software colorspace filter still needs system memory, so the download stays + // exactly where it is today: right after the pad, before setpts, and byte-identical to the + // unfolded path (format=nv12, not a bare hwdownload) + command.ShouldContain("pad_color=black,setsar=1,hwdownload,format=nv12,setpts=PTS-STARTPTS"); + command.ShouldContain("colorspace="); } [Test] @@ -43,7 +70,7 @@ public class QsvPipelineBuilderTests command.ShouldContain("hwdownload,format=nv12,pad=1920:1080:-1:-1:color=black"); } - private string BuildPadCommand(bool vppQsvSupportsPad) + private string BuildPadCommand(bool vppQsvSupportsPad, bool colorsAreBt709 = false) { var videoInputFile = new VideoInputFile( "/tmp/whatever.mkv", @@ -55,7 +82,7 @@ public class QsvPipelineBuilderTests VideoProfile.Main, new PixelFormatYuv420P(), ColorParams.Default, - new FrameSize(1440, 1080), + new FrameSize(640, 480), "1:1", "4:3", FrameRate.DefaultFrameRate, @@ -80,7 +107,7 @@ public class QsvPipelineBuilderTests 2000, 4000, 90_000, - false, + colorsAreBt709, false); var ffmpegState = new FFmpegState( diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs new file mode 100644 index 000000000..183801d07 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs @@ -0,0 +1,121 @@ +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter.Qsv; + +public class ScaleAndPadQsvFilter : BaseFilter +{ + private readonly FrameState _currentState; + private readonly int _extraHardwareFrames; + private readonly bool _isAnamorphicEdgeCase; + private readonly FrameSize _paddedSize; + private readonly string _sampleAspectRatio; + private readonly FrameSize _scaledSize; + + public ScaleAndPadQsvFilter( + FrameState currentState, + FrameSize scaledSize, + FrameSize paddedSize, + int extraHardwareFrames, + bool isAnamorphicEdgeCase, + string sampleAspectRatio) + { + _currentState = currentState; + _scaledSize = scaledSize; + _paddedSize = paddedSize; + _extraHardwareFrames = extraHardwareFrames; + _isAnamorphicEdgeCase = isAnamorphicEdgeCase; + _sampleAspectRatio = sampleAspectRatio; + } + + public override string Filter + { + get + { + // fold scale and pad into a single vpp_qsv instance; chaining two vpp_qsv instances + // drops the final frame at EOF + + string padOptions = + $"pad_w={_paddedSize.Width}:pad_h={_paddedSize.Height}:pad_x=-1:pad_y=-1:pad_color=black"; + + string scale; + + if (_currentState.ScaledSize == _scaledSize) + { + string format = string.Empty; + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + format = $"format={pixelFormat.FFmpegName}:"; + } + + scale = $"vpp_qsv={format}{padOptions}"; + } + else + { + string format = string.Empty; + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + format = $":format={pixelFormat.FFmpegName}"; + } + + string squareScale = string.Empty; + string setsar = string.Empty; + string sar = _sampleAspectRatio.Replace(':', '/'); + if (_isAnamorphicEdgeCase) + { + squareScale = $"vpp_qsv=w=iw:h={sar}*ih{format},setsar=1,"; + } + else if (_currentState.IsAnamorphic) + { + squareScale = $"vpp_qsv=w=iw*{sar}:h=ih{format},setsar=1,"; + } + else + { + setsar = ",setsar=1"; + } + + scale = + $"{squareScale}vpp_qsv=w={_scaledSize.Width}:h={_scaledSize.Height}{format}:{padOptions}{setsar}"; + } + + if (_currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + return scale; + } + + string initialPixelFormat = _currentState.PixelFormat.Match(pf => pf.FFmpegName, FFmpegFormat.NV12); + return $"format={initialPixelFormat},hwupload=extra_hw_frames={_extraHardwareFrames},{scale}"; + } + } + + public override FrameState NextState(FrameState currentState) + { + FrameState result = currentState with + { + ScaledSize = _scaledSize, + PaddedSize = _paddedSize, + FrameDataLocation = FrameDataLocation.Hardware, + IsAnamorphic = false // this filter always outputs square pixels + }; + + if (_currentState.PixelFormat.IsNone && + _currentState.FrameDataLocation == FrameDataLocation.Software && + currentState.PixelFormat.Map(pf => pf is not PixelFormatNv12).IfNone(false)) + { + // wrap in nv12 + result = result with + { + PixelFormat = currentState.PixelFormat + .Map(pf => (IPixelFormat)new PixelFormatNv12(pf.Name)) + }; + } + else + { + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + result = result with { PixelFormat = Some(pixelFormat) }; + } + } + + return result; + } +} diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs index f7f9a8860..d094c9a0a 100644 --- a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs @@ -30,6 +30,12 @@ public class ScaleQsvFilter : BaseFilter _sampleAspectRatio = sampleAspectRatio; } + public FrameState CurrentState => _currentState; + public FrameSize ScaledSize => _scaledSize; + public int ExtraHardwareFrames => _extraHardwareFrames; + public bool IsAnamorphicEdgeCase => _isAnamorphicEdgeCase; + public string SampleAspectRatio => _sampleAspectRatio; + public override string Filter { get diff --git a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs index d4ca4d767..902902478 100644 --- a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs @@ -203,9 +203,19 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder currentState = SetCrop(videoInputFile, desiredState, currentState); SetStillImageLoop(videoInputFile, videoStream, ffmpegState, desiredState, pipelineSteps); - // need to download for any sort of overlay (and always for setpts) - if (currentState.FrameDataLocation == FrameDataLocation.Hardware) //&& - //(context.HasSubtitleOverlay || context.HasWatermark || context.HasGraphicsEngine)) + // setpts/fps operate on timestamps only and accept hardware frames, and the qsv encoder + // ingests hardware frames directly, so keep the frame on the GPU through both when nothing + // downstream needs it in system memory. any software-requiring step (overlays, subtitle + // burn-in, a colorspace conversion, or a software encoder) still forces the download. + bool canKeepHardwareFrames = + ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Qsv + && !context.HasWatermark + && !context.HasSubtitleOverlay + && !context.HasSubtitleText + && !context.HasGraphicsEngine + && !WillDownloadForColorspace(videoInputFile, videoStream, desiredState); + + if (currentState.FrameDataLocation == FrameDataLocation.Hardware && !canKeepHardwareFrames) { var hardwareDownload = new HardwareDownloadFilter(currentState); currentState = hardwareDownload.NextState(currentState); @@ -314,7 +324,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder bool usesVppQsv = videoInputFile.FilterSteps.Any(f => - f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter); + f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter + or ScaleAndPadQsvFilter); // if we have no filters, check whether we need to convert pixel format // since qsv doesn't seem to like doing that at the encoder @@ -557,7 +568,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder } // only scale if scaling or padding was used for main video stream - if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleQsvFilter or PadFilter)) + if (videoInputFile.FilterSteps.Any(s => + s is ScaleFilter or ScaleQsvFilter or PadFilter or ScaleAndPadQsvFilter)) { var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize); subtitle.FilterSteps.Add(scaleFilter); @@ -621,18 +633,58 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder } else { - var padStep = new PadQsvFilter( - currentState, - desiredState.PaddedSize, - ffmpegState.QsvExtraHardwareFrames); - currentState = padStep.NextState(currentState); - videoInputFile.FilterSteps.Add(padStep); + // fold an existing hardware scale and this pad into a single vpp_qsv instance; + // two chained vpp_qsv instances drop the final frame at EOF + Option maybeScale = videoInputFile.FilterSteps.OfType().HeadOrNone(); + if (maybeScale.IsSome) + { + foreach (ScaleQsvFilter scaleStep in maybeScale) + { + videoInputFile.FilterSteps.Remove(scaleStep); + + var scalePadStep = new ScaleAndPadQsvFilter( + scaleStep.CurrentState, + scaleStep.ScaledSize, + desiredState.PaddedSize, + scaleStep.ExtraHardwareFrames, + scaleStep.IsAnamorphicEdgeCase, + scaleStep.SampleAspectRatio); + + // advance the live pipeline state (which carries the pixel format the + // unfolded ScaleQsvFilter produced) so any later download keeps format=X + currentState = scalePadStep.NextState(currentState); + videoInputFile.FilterSteps.Add(scalePadStep); + } + } + else + { + var padStep = new PadQsvFilter( + currentState, + desiredState.PaddedSize, + ffmpegState.QsvExtraHardwareFrames); + currentState = padStep.NextState(currentState); + videoInputFile.FilterSteps.Add(padStep); + } } } return currentState; } + private static bool WillDownloadForColorspace( + VideoInputFile videoInputFile, + VideoStream videoStream, + FrameState desiredState) + { + // mirrors the colorspace emission in SetPixelFormat: when it runs on a hardware frame it + // prepends its own hwdownload, so the frame would leave the GPU regardless + bool usesVppQsv = videoInputFile.FilterSteps.Any(f => + f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter + or ScaleAndPadQsvFilter); + + return desiredState.ColorsAreBt709 && (!videoStream.ColorParams.IsBt709 || usesVppQsv); + } + private static FrameState SetScale( VideoInputFile videoInputFile, VideoStream videoStream,