diff --git a/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs b/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs index 21abfe446..8f3b5b164 100644 --- a/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs +++ b/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs @@ -23,14 +23,41 @@ public class QsvPipelineBuilderTests private readonly ILogger _logger = Substitute.For(); [Test] - public void Pad_Uses_Hardware_Vpp_Qsv_When_Pad_Option_Available() + public void Scale_And_Pad_Fold_Into_Single_Hardware_Vpp_Qsv() { string command = BuildPadCommand(vppQsvSupportsPad: true); - // hardware pad: vpp_qsv does the padding, with no software pad and no hwdownload round-trip around it - command.ShouldContain("vpp_qsv=pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + // scale and pad fold into a single vpp_qsv instance that pads on the GPU + command.ShouldContain("vpp_qsv=w=1440:h=1080:pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + + // exactly one vpp_qsv instance (two chained instances drop the final frame at EOF) + command.Split("vpp_qsv").Length.ShouldBe(2); + + // no software pad command.ShouldNotContain("pad=1920:1080"); - command.ShouldNotContain("hwdownload,format=nv12,pad"); + } + + [Test] + public void Frames_Stay_In_Hardware_Through_Setpts_Into_The_Qsv_Encoder() + { + string command = BuildPadCommand(vppQsvSupportsPad: true); + + // setpts/fps run on the hardware frame and hevc_qsv ingests it directly: no download anywhere + command.ShouldNotContain("hwdownload"); + command.ShouldContain("pad_color=black,setsar=1,setpts=PTS-STARTPTS,fps=24"); + command.ShouldContain("hevc_qsv"); + } + + [Test] + public void Download_Is_Retained_When_A_Colorspace_Conversion_Follows() + { + string command = BuildPadCommand(vppQsvSupportsPad: true, colorsAreBt709: true); + + // a downstream software colorspace filter still needs system memory, so the download stays + // exactly where it is today: right after the pad, before setpts, and byte-identical to the + // unfolded path (format=nv12, not a bare hwdownload) + command.ShouldContain("pad_color=black,setsar=1,hwdownload,format=nv12,setpts=PTS-STARTPTS"); + command.ShouldContain("colorspace="); } [Test] @@ -43,7 +70,7 @@ public class QsvPipelineBuilderTests command.ShouldContain("hwdownload,format=nv12,pad=1920:1080:-1:-1:color=black"); } - private string BuildPadCommand(bool vppQsvSupportsPad) + private string BuildPadCommand(bool vppQsvSupportsPad, bool colorsAreBt709 = false) { var videoInputFile = new VideoInputFile( "/tmp/whatever.mkv", @@ -55,7 +82,7 @@ public class QsvPipelineBuilderTests VideoProfile.Main, new PixelFormatYuv420P(), ColorParams.Default, - new FrameSize(1440, 1080), + new FrameSize(640, 480), "1:1", "4:3", FrameRate.DefaultFrameRate, @@ -80,7 +107,7 @@ public class QsvPipelineBuilderTests 2000, 4000, 90_000, - false, + colorsAreBt709, false); var ffmpegState = new FFmpegState( diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs new file mode 100644 index 000000000..183801d07 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs @@ -0,0 +1,121 @@ +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter.Qsv; + +public class ScaleAndPadQsvFilter : BaseFilter +{ + private readonly FrameState _currentState; + private readonly int _extraHardwareFrames; + private readonly bool _isAnamorphicEdgeCase; + private readonly FrameSize _paddedSize; + private readonly string _sampleAspectRatio; + private readonly FrameSize _scaledSize; + + public ScaleAndPadQsvFilter( + FrameState currentState, + FrameSize scaledSize, + FrameSize paddedSize, + int extraHardwareFrames, + bool isAnamorphicEdgeCase, + string sampleAspectRatio) + { + _currentState = currentState; + _scaledSize = scaledSize; + _paddedSize = paddedSize; + _extraHardwareFrames = extraHardwareFrames; + _isAnamorphicEdgeCase = isAnamorphicEdgeCase; + _sampleAspectRatio = sampleAspectRatio; + } + + public override string Filter + { + get + { + // fold scale and pad into a single vpp_qsv instance; chaining two vpp_qsv instances + // drops the final frame at EOF + + string padOptions = + $"pad_w={_paddedSize.Width}:pad_h={_paddedSize.Height}:pad_x=-1:pad_y=-1:pad_color=black"; + + string scale; + + if (_currentState.ScaledSize == _scaledSize) + { + string format = string.Empty; + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + format = $"format={pixelFormat.FFmpegName}:"; + } + + scale = $"vpp_qsv={format}{padOptions}"; + } + else + { + string format = string.Empty; + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + format = $":format={pixelFormat.FFmpegName}"; + } + + string squareScale = string.Empty; + string setsar = string.Empty; + string sar = _sampleAspectRatio.Replace(':', '/'); + if (_isAnamorphicEdgeCase) + { + squareScale = $"vpp_qsv=w=iw:h={sar}*ih{format},setsar=1,"; + } + else if (_currentState.IsAnamorphic) + { + squareScale = $"vpp_qsv=w=iw*{sar}:h=ih{format},setsar=1,"; + } + else + { + setsar = ",setsar=1"; + } + + scale = + $"{squareScale}vpp_qsv=w={_scaledSize.Width}:h={_scaledSize.Height}{format}:{padOptions}{setsar}"; + } + + if (_currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + return scale; + } + + string initialPixelFormat = _currentState.PixelFormat.Match(pf => pf.FFmpegName, FFmpegFormat.NV12); + return $"format={initialPixelFormat},hwupload=extra_hw_frames={_extraHardwareFrames},{scale}"; + } + } + + public override FrameState NextState(FrameState currentState) + { + FrameState result = currentState with + { + ScaledSize = _scaledSize, + PaddedSize = _paddedSize, + FrameDataLocation = FrameDataLocation.Hardware, + IsAnamorphic = false // this filter always outputs square pixels + }; + + if (_currentState.PixelFormat.IsNone && + _currentState.FrameDataLocation == FrameDataLocation.Software && + currentState.PixelFormat.Map(pf => pf is not PixelFormatNv12).IfNone(false)) + { + // wrap in nv12 + result = result with + { + PixelFormat = currentState.PixelFormat + .Map(pf => (IPixelFormat)new PixelFormatNv12(pf.Name)) + }; + } + else + { + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + result = result with { PixelFormat = Some(pixelFormat) }; + } + } + + return result; + } +} diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs index f7f9a8860..d094c9a0a 100644 --- a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs @@ -30,6 +30,12 @@ public class ScaleQsvFilter : BaseFilter _sampleAspectRatio = sampleAspectRatio; } + public FrameState CurrentState => _currentState; + public FrameSize ScaledSize => _scaledSize; + public int ExtraHardwareFrames => _extraHardwareFrames; + public bool IsAnamorphicEdgeCase => _isAnamorphicEdgeCase; + public string SampleAspectRatio => _sampleAspectRatio; + public override string Filter { get diff --git a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs index d4ca4d767..902902478 100644 --- a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs @@ -203,9 +203,19 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder currentState = SetCrop(videoInputFile, desiredState, currentState); SetStillImageLoop(videoInputFile, videoStream, ffmpegState, desiredState, pipelineSteps); - // need to download for any sort of overlay (and always for setpts) - if (currentState.FrameDataLocation == FrameDataLocation.Hardware) //&& - //(context.HasSubtitleOverlay || context.HasWatermark || context.HasGraphicsEngine)) + // setpts/fps operate on timestamps only and accept hardware frames, and the qsv encoder + // ingests hardware frames directly, so keep the frame on the GPU through both when nothing + // downstream needs it in system memory. any software-requiring step (overlays, subtitle + // burn-in, a colorspace conversion, or a software encoder) still forces the download. + bool canKeepHardwareFrames = + ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Qsv + && !context.HasWatermark + && !context.HasSubtitleOverlay + && !context.HasSubtitleText + && !context.HasGraphicsEngine + && !WillDownloadForColorspace(videoInputFile, videoStream, desiredState); + + if (currentState.FrameDataLocation == FrameDataLocation.Hardware && !canKeepHardwareFrames) { var hardwareDownload = new HardwareDownloadFilter(currentState); currentState = hardwareDownload.NextState(currentState); @@ -314,7 +324,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder bool usesVppQsv = videoInputFile.FilterSteps.Any(f => - f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter); + f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter + or ScaleAndPadQsvFilter); // if we have no filters, check whether we need to convert pixel format // since qsv doesn't seem to like doing that at the encoder @@ -557,7 +568,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder } // only scale if scaling or padding was used for main video stream - if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleQsvFilter or PadFilter)) + if (videoInputFile.FilterSteps.Any(s => + s is ScaleFilter or ScaleQsvFilter or PadFilter or ScaleAndPadQsvFilter)) { var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize); subtitle.FilterSteps.Add(scaleFilter); @@ -621,18 +633,58 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder } else { - var padStep = new PadQsvFilter( - currentState, - desiredState.PaddedSize, - ffmpegState.QsvExtraHardwareFrames); - currentState = padStep.NextState(currentState); - videoInputFile.FilterSteps.Add(padStep); + // fold an existing hardware scale and this pad into a single vpp_qsv instance; + // two chained vpp_qsv instances drop the final frame at EOF + Option maybeScale = videoInputFile.FilterSteps.OfType().HeadOrNone(); + if (maybeScale.IsSome) + { + foreach (ScaleQsvFilter scaleStep in maybeScale) + { + videoInputFile.FilterSteps.Remove(scaleStep); + + var scalePadStep = new ScaleAndPadQsvFilter( + scaleStep.CurrentState, + scaleStep.ScaledSize, + desiredState.PaddedSize, + scaleStep.ExtraHardwareFrames, + scaleStep.IsAnamorphicEdgeCase, + scaleStep.SampleAspectRatio); + + // advance the live pipeline state (which carries the pixel format the + // unfolded ScaleQsvFilter produced) so any later download keeps format=X + currentState = scalePadStep.NextState(currentState); + videoInputFile.FilterSteps.Add(scalePadStep); + } + } + else + { + var padStep = new PadQsvFilter( + currentState, + desiredState.PaddedSize, + ffmpegState.QsvExtraHardwareFrames); + currentState = padStep.NextState(currentState); + videoInputFile.FilterSteps.Add(padStep); + } } } return currentState; } + private static bool WillDownloadForColorspace( + VideoInputFile videoInputFile, + VideoStream videoStream, + FrameState desiredState) + { + // mirrors the colorspace emission in SetPixelFormat: when it runs on a hardware frame it + // prepends its own hwdownload, so the frame would leave the GPU regardless + bool usesVppQsv = videoInputFile.FilterSteps.Any(f => + f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter + or ScaleAndPadQsvFilter); + + return desiredState.ColorsAreBt709 && (!videoStream.ColorParams.IsBt709 || usesVppQsv); + } + private static FrameState SetScale( VideoInputFile videoInputFile, VideoStream videoStream,