From cdef25ffc214559001530b97baa6fe8adf18f630 Mon Sep 17 00:00:00 2001 From: McKenzie Pepper Date: Thu, 30 Jul 2026 11:52:33 -0400 Subject: [PATCH] perf: keep QSV frames in hardware through setpts when no software filters are needed The QSV pipeline downloaded every hardware frame to system memory before setpts "always for setpts", then re-fed software frames to the qsv encoder. Combined with commit 30280ae's GPU pad this was a net loss on Intel B50 (224 fps / 6.5s CPU vs. 242 fps / 5.6s CPU for the old download+CPU-pad path) because the padded frame was downloaded only to be uploaded again by h264_qsv. setpts and fps operate on timestamps only and accept hardware frames, and the qsv encoder ingests hardware frames directly, so the download is now skipped whenever nothing downstream needs system memory: hardware qsv encoder, no watermark, no subtitle overlay or burn-in, no graphics engine, and no follow-on colorspace conversion (which prepends its own hwdownload). Every config that still needs software frames keeps the download in exactly the same place as before. On B50 the full-GPU chain measures 503 fps / 1.8s CPU, 2.1x the old download+CPU-pad baseline. Because two chained vpp_qsv instances drop the final frame at EOF (1439/1440, a pre-existing ffmpeg quirk reproducible with two plain scales on stock ffmpeg), a hardware scale followed by a hardware pad is now emitted as one vpp_qsv instance carrying both the scale and the pad_w/pad_h options. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0139ePZuiHJGo2g4PY2tQJ9Q --- .../Pipeline/QsvPipelineBuilderTests.cs | 41 +++++- .../Filter/Qsv/ScaleAndPadQsvFilter.cs | 121 ++++++++++++++++++ ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs | 6 + .../Pipeline/QsvPipelineBuilder.cs | 74 +++++++++-- 4 files changed, 224 insertions(+), 18 deletions(-) create mode 100644 ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs diff --git a/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs b/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs index 21abfe446..8f3b5b164 100644 --- a/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs +++ b/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs @@ -23,14 +23,41 @@ public class QsvPipelineBuilderTests private readonly ILogger _logger = Substitute.For(); [Test] - public void Pad_Uses_Hardware_Vpp_Qsv_When_Pad_Option_Available() + public void Scale_And_Pad_Fold_Into_Single_Hardware_Vpp_Qsv() { string command = BuildPadCommand(vppQsvSupportsPad: true); - // hardware pad: vpp_qsv does the padding, with no software pad and no hwdownload round-trip around it - command.ShouldContain("vpp_qsv=pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + // scale and pad fold into a single vpp_qsv instance that pads on the GPU + command.ShouldContain("vpp_qsv=w=1440:h=1080:pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + + // exactly one vpp_qsv instance (two chained instances drop the final frame at EOF) + command.Split("vpp_qsv").Length.ShouldBe(2); + + // no software pad command.ShouldNotContain("pad=1920:1080"); - command.ShouldNotContain("hwdownload,format=nv12,pad"); + } + + [Test] + public void Frames_Stay_In_Hardware_Through_Setpts_Into_The_Qsv_Encoder() + { + string command = BuildPadCommand(vppQsvSupportsPad: true); + + // setpts/fps run on the hardware frame and hevc_qsv ingests it directly: no download anywhere + command.ShouldNotContain("hwdownload"); + command.ShouldContain("pad_color=black,setsar=1,setpts=PTS-STARTPTS,fps=24"); + command.ShouldContain("hevc_qsv"); + } + + [Test] + public void Download_Is_Retained_When_A_Colorspace_Conversion_Follows() + { + string command = BuildPadCommand(vppQsvSupportsPad: true, colorsAreBt709: true); + + // a downstream software colorspace filter still needs system memory, so the download stays + // exactly where it is today: right after the pad, before setpts, and byte-identical to the + // unfolded path (format=nv12, not a bare hwdownload) + command.ShouldContain("pad_color=black,setsar=1,hwdownload,format=nv12,setpts=PTS-STARTPTS"); + command.ShouldContain("colorspace="); } [Test] @@ -43,7 +70,7 @@ public class QsvPipelineBuilderTests command.ShouldContain("hwdownload,format=nv12,pad=1920:1080:-1:-1:color=black"); } - private string BuildPadCommand(bool vppQsvSupportsPad) + private string BuildPadCommand(bool vppQsvSupportsPad, bool colorsAreBt709 = false) { var videoInputFile = new VideoInputFile( "/tmp/whatever.mkv", @@ -55,7 +82,7 @@ public class QsvPipelineBuilderTests VideoProfile.Main, new PixelFormatYuv420P(), ColorParams.Default, - new FrameSize(1440, 1080), + new FrameSize(640, 480), "1:1", "4:3", FrameRate.DefaultFrameRate, @@ -80,7 +107,7 @@ public class QsvPipelineBuilderTests 2000, 4000, 90_000, - false, + colorsAreBt709, false); var ffmpegState = new FFmpegState( diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs new file mode 100644 index 000000000..183801d07 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs @@ -0,0 +1,121 @@ +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter.Qsv; + +public class ScaleAndPadQsvFilter : BaseFilter +{ + private readonly FrameState _currentState; + private readonly int _extraHardwareFrames; + private readonly bool _isAnamorphicEdgeCase; + private readonly FrameSize _paddedSize; + private readonly string _sampleAspectRatio; + private readonly FrameSize _scaledSize; + + public ScaleAndPadQsvFilter( + FrameState currentState, + FrameSize scaledSize, + FrameSize paddedSize, + int extraHardwareFrames, + bool isAnamorphicEdgeCase, + string sampleAspectRatio) + { + _currentState = currentState; + _scaledSize = scaledSize; + _paddedSize = paddedSize; + _extraHardwareFrames = extraHardwareFrames; + _isAnamorphicEdgeCase = isAnamorphicEdgeCase; + _sampleAspectRatio = sampleAspectRatio; + } + + public override string Filter + { + get + { + // fold scale and pad into a single vpp_qsv instance; chaining two vpp_qsv instances + // drops the final frame at EOF + + string padOptions = + $"pad_w={_paddedSize.Width}:pad_h={_paddedSize.Height}:pad_x=-1:pad_y=-1:pad_color=black"; + + string scale; + + if (_currentState.ScaledSize == _scaledSize) + { + string format = string.Empty; + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + format = $"format={pixelFormat.FFmpegName}:"; + } + + scale = $"vpp_qsv={format}{padOptions}"; + } + else + { + string format = string.Empty; + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + format = $":format={pixelFormat.FFmpegName}"; + } + + string squareScale = string.Empty; + string setsar = string.Empty; + string sar = _sampleAspectRatio.Replace(':', '/'); + if (_isAnamorphicEdgeCase) + { + squareScale = $"vpp_qsv=w=iw:h={sar}*ih{format},setsar=1,"; + } + else if (_currentState.IsAnamorphic) + { + squareScale = $"vpp_qsv=w=iw*{sar}:h=ih{format},setsar=1,"; + } + else + { + setsar = ",setsar=1"; + } + + scale = + $"{squareScale}vpp_qsv=w={_scaledSize.Width}:h={_scaledSize.Height}{format}:{padOptions}{setsar}"; + } + + if (_currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + return scale; + } + + string initialPixelFormat = _currentState.PixelFormat.Match(pf => pf.FFmpegName, FFmpegFormat.NV12); + return $"format={initialPixelFormat},hwupload=extra_hw_frames={_extraHardwareFrames},{scale}"; + } + } + + public override FrameState NextState(FrameState currentState) + { + FrameState result = currentState with + { + ScaledSize = _scaledSize, + PaddedSize = _paddedSize, + FrameDataLocation = FrameDataLocation.Hardware, + IsAnamorphic = false // this filter always outputs square pixels + }; + + if (_currentState.PixelFormat.IsNone && + _currentState.FrameDataLocation == FrameDataLocation.Software && + currentState.PixelFormat.Map(pf => pf is not PixelFormatNv12).IfNone(false)) + { + // wrap in nv12 + result = result with + { + PixelFormat = currentState.PixelFormat + .Map(pf => (IPixelFormat)new PixelFormatNv12(pf.Name)) + }; + } + else + { + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + result = result with { PixelFormat = Some(pixelFormat) }; + } + } + + return result; + } +} diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs index f7f9a8860..d094c9a0a 100644 --- a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs @@ -30,6 +30,12 @@ public class ScaleQsvFilter : BaseFilter _sampleAspectRatio = sampleAspectRatio; } + public FrameState CurrentState => _currentState; + public FrameSize ScaledSize => _scaledSize; + public int ExtraHardwareFrames => _extraHardwareFrames; + public bool IsAnamorphicEdgeCase => _isAnamorphicEdgeCase; + public string SampleAspectRatio => _sampleAspectRatio; + public override string Filter { get diff --git a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs index d4ca4d767..902902478 100644 --- a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs @@ -203,9 +203,19 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder currentState = SetCrop(videoInputFile, desiredState, currentState); SetStillImageLoop(videoInputFile, videoStream, ffmpegState, desiredState, pipelineSteps); - // need to download for any sort of overlay (and always for setpts) - if (currentState.FrameDataLocation == FrameDataLocation.Hardware) //&& - //(context.HasSubtitleOverlay || context.HasWatermark || context.HasGraphicsEngine)) + // setpts/fps operate on timestamps only and accept hardware frames, and the qsv encoder + // ingests hardware frames directly, so keep the frame on the GPU through both when nothing + // downstream needs it in system memory. any software-requiring step (overlays, subtitle + // burn-in, a colorspace conversion, or a software encoder) still forces the download. + bool canKeepHardwareFrames = + ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Qsv + && !context.HasWatermark + && !context.HasSubtitleOverlay + && !context.HasSubtitleText + && !context.HasGraphicsEngine + && !WillDownloadForColorspace(videoInputFile, videoStream, desiredState); + + if (currentState.FrameDataLocation == FrameDataLocation.Hardware && !canKeepHardwareFrames) { var hardwareDownload = new HardwareDownloadFilter(currentState); currentState = hardwareDownload.NextState(currentState); @@ -314,7 +324,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder bool usesVppQsv = videoInputFile.FilterSteps.Any(f => - f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter); + f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter + or ScaleAndPadQsvFilter); // if we have no filters, check whether we need to convert pixel format // since qsv doesn't seem to like doing that at the encoder @@ -557,7 +568,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder } // only scale if scaling or padding was used for main video stream - if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleQsvFilter or PadFilter)) + if (videoInputFile.FilterSteps.Any(s => + s is ScaleFilter or ScaleQsvFilter or PadFilter or ScaleAndPadQsvFilter)) { var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize); subtitle.FilterSteps.Add(scaleFilter); @@ -621,18 +633,58 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder } else { - var padStep = new PadQsvFilter( - currentState, - desiredState.PaddedSize, - ffmpegState.QsvExtraHardwareFrames); - currentState = padStep.NextState(currentState); - videoInputFile.FilterSteps.Add(padStep); + // fold an existing hardware scale and this pad into a single vpp_qsv instance; + // two chained vpp_qsv instances drop the final frame at EOF + Option maybeScale = videoInputFile.FilterSteps.OfType().HeadOrNone(); + if (maybeScale.IsSome) + { + foreach (ScaleQsvFilter scaleStep in maybeScale) + { + videoInputFile.FilterSteps.Remove(scaleStep); + + var scalePadStep = new ScaleAndPadQsvFilter( + scaleStep.CurrentState, + scaleStep.ScaledSize, + desiredState.PaddedSize, + scaleStep.ExtraHardwareFrames, + scaleStep.IsAnamorphicEdgeCase, + scaleStep.SampleAspectRatio); + + // advance the live pipeline state (which carries the pixel format the + // unfolded ScaleQsvFilter produced) so any later download keeps format=X + currentState = scalePadStep.NextState(currentState); + videoInputFile.FilterSteps.Add(scalePadStep); + } + } + else + { + var padStep = new PadQsvFilter( + currentState, + desiredState.PaddedSize, + ffmpegState.QsvExtraHardwareFrames); + currentState = padStep.NextState(currentState); + videoInputFile.FilterSteps.Add(padStep); + } } } return currentState; } + private static bool WillDownloadForColorspace( + VideoInputFile videoInputFile, + VideoStream videoStream, + FrameState desiredState) + { + // mirrors the colorspace emission in SetPixelFormat: when it runs on a hardware frame it + // prepends its own hwdownload, so the frame would leave the GPU regardless + bool usesVppQsv = videoInputFile.FilterSteps.Any(f => + f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter + or ScaleAndPadQsvFilter); + + return desiredState.ColorsAreBt709 && (!videoStream.ColorParams.IsBt709 || usesVppQsv); + } + private static FrameState SetScale( VideoInputFile videoInputFile, VideoStream videoStream,