diff --git a/ErsatzTV.FFmpeg.Tests/Capabilities/FFmpegFilterOptionParserTests.cs b/ErsatzTV.FFmpeg.Tests/Capabilities/FFmpegFilterOptionParserTests.cs new file mode 100644 index 000000000..7a4e08eaf --- /dev/null +++ b/ErsatzTV.FFmpeg.Tests/Capabilities/FFmpegFilterOptionParserTests.cs @@ -0,0 +1,99 @@ +using System.Collections.Generic; +using ErsatzTV.FFmpeg.Capabilities; +using NUnit.Framework; +using Shouldly; + +namespace ErsatzTV.FFmpeg.Tests.Capabilities; + +[TestFixture] +public class FFmpegFilterOptionParserTests +{ + // stock ffmpeg vpp_qsv, without the pad_w/pad_h/pad_x/pad_y/pad_color options + private const string WithoutPadOptions = @"Filter vpp_qsv + Quick Sync Video VPP. + Inputs: + #0: default (video) + Outputs: + #0: default (video) +vpp_qsv AVOptions: + deinterlace ..FV....... deinterlace mode (from 0 to 2) (default 0) + bob 1 ..FV....... Get one frame for each field + advanced 2 ..FV....... Get one frame for each frame + denoise ..FV....... denoise level [0, 100] (from 0 to 100) (default 0) + framerate ..FV....... output framerate (from 0 to DBL_MAX) (default 0/1) + w ..FV....... Output video width + h ..FV....... Output video height + format ..FV....... Output pixel format + async_depth ..FV....... Internal parallelization depth (from 0 to INT_MAX) (default 4) + scale_mode ..FV....... scaling mode (from 0 to 2) (default 0) +"; + + // patched ffmpeg vpp_qsv, carrying the pad_w/pad_h/pad_x/pad_y/pad_color options + private const string WithPadOptions = @"Filter vpp_qsv + Quick Sync Video VPP. + Inputs: + #0: default (video) + Outputs: + #0: default (video) +vpp_qsv AVOptions: + deinterlace ..FV....... deinterlace mode (from 0 to 2) (default 0) + bob 1 ..FV....... Get one frame for each field + advanced 2 ..FV....... Get one frame for each frame + denoise ..FV....... denoise level [0, 100] (from 0 to 100) (default 0) + framerate ..FV....... output framerate (from 0 to DBL_MAX) (default 0/1) + w ..FV....... Output video width + h ..FV....... Output video height + format ..FV....... Output pixel format + async_depth ..FV....... Internal parallelization depth (from 0 to INT_MAX) (default 4) + scale_mode ..FV....... scaling mode (from 0 to 2) (default 0) + pad_w ..FV....... Padded width (from 0 to 8192) (default 0) + pad_h ..FV....... Padded height (from 0 to 8192) (default 0) + pad_x ..FV....... Horizontal position (from -1 to 8192) (default -1) + pad_y ..FV....... Vertical position (from -1 to 8192) (default -1) + pad_color ..FV....... Padding color (default ""black"") +"; + + [Test] + public void Should_Parse_Option_Names() + { + IReadOnlySet options = FFmpegFilterOptionParser.Parse(WithoutPadOptions); + + options.ShouldContain("deinterlace"); + options.ShouldContain("denoise"); + options.ShouldContain("framerate"); + options.ShouldContain("w"); + options.ShouldContain("h"); + options.ShouldContain("format"); + } + + [Test] + public void Should_Not_Parse_Enum_Constants() + { + IReadOnlySet options = FFmpegFilterOptionParser.Parse(WithoutPadOptions); + + options.ShouldNotContain("bob"); + options.ShouldNotContain("advanced"); + } + + [Test] + public void Should_Detect_Pad_Options_When_Present() + { + IReadOnlySet options = FFmpegFilterOptionParser.Parse(WithPadOptions); + + options.ShouldContain("pad_w"); + options.ShouldContain("pad_h"); + options.ShouldContain("pad_x"); + options.ShouldContain("pad_y"); + options.ShouldContain("pad_color"); + } + + [Test] + public void Should_Not_Detect_Pad_Options_When_Absent() + { + IReadOnlySet options = FFmpegFilterOptionParser.Parse(WithoutPadOptions); + + options.ShouldNotContain("pad_w"); + options.ShouldNotContain("pad_h"); + options.ShouldNotContain("pad_color"); + } +} diff --git a/ErsatzTV.FFmpeg.Tests/Filter/Qsv/PadQsvFilterTests.cs b/ErsatzTV.FFmpeg.Tests/Filter/Qsv/PadQsvFilterTests.cs new file mode 100644 index 000000000..fad03b0f0 --- /dev/null +++ b/ErsatzTV.FFmpeg.Tests/Filter/Qsv/PadQsvFilterTests.cs @@ -0,0 +1,72 @@ +using ErsatzTV.FFmpeg; +using ErsatzTV.FFmpeg.Filter.Qsv; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.State; +using LanguageExt; +using NUnit.Framework; +using Shouldly; + +namespace ErsatzTV.FFmpeg.Tests.Filter.Qsv; + +[TestFixture] +public class PadQsvFilterTests +{ + private static FrameState FrameStateAt(FrameDataLocation location, Option pixelFormat) => new( + false, + false, + string.Empty, + Option.None, + Option.None, + true, + pixelFormat, + new FrameSize(1920, 1080), + new FrameSize(1920, 1080), + Option.None, + FFmpegFilterMode.HardwareIfPossible, + false, + Option.None, + Option.None, + Option.None, + Option.None, + false, + false, + location); + + [Test] + public void Should_Emit_Hardware_Pad_When_Frame_In_Hardware() + { + var filter = new PadQsvFilter( + FrameStateAt(FrameDataLocation.Hardware, Option.None), + new FrameSize(1920, 1080), + 64); + + filter.Filter.ShouldBe("vpp_qsv=pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + } + + [Test] + public void Should_Upload_Before_Pad_When_Frame_In_Software() + { + var filter = new PadQsvFilter( + FrameStateAt(FrameDataLocation.Software, Option.Some(new PixelFormatNv12("yuv420p"))), + new FrameSize(1920, 1080), + 64); + + filter.Filter.ShouldBe( + "format=nv12,hwupload=extra_hw_frames=64,vpp_qsv=pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + } + + [Test] + public void Should_Report_Padded_Size_And_Hardware_Location() + { + var filter = new PadQsvFilter( + FrameStateAt(FrameDataLocation.Software, Option.None), + new FrameSize(1920, 1080), + 64); + + FrameState nextState = filter.NextState( + FrameStateAt(FrameDataLocation.Software, Option.None)); + + nextState.PaddedSize.ShouldBe(new FrameSize(1920, 1080)); + nextState.FrameDataLocation.ShouldBe(FrameDataLocation.Hardware); + } +} diff --git a/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs b/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs new file mode 100644 index 000000000..8f3b5b164 --- /dev/null +++ b/ErsatzTV.FFmpeg.Tests/Pipeline/QsvPipelineBuilderTests.cs @@ -0,0 +1,201 @@ +using System; +using System.Collections.Generic; +using ErsatzTV.FFmpeg; +using ErsatzTV.FFmpeg.Capabilities; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.InputOption; +using ErsatzTV.FFmpeg.OutputFormat; +using ErsatzTV.FFmpeg.Pipeline; +using ErsatzTV.FFmpeg.Preset; +using ErsatzTV.FFmpeg.State; +using LanguageExt; +using Microsoft.Extensions.Logging; +using NSubstitute; +using NUnit.Framework; +using Shouldly; +using static LanguageExt.Prelude; + +namespace ErsatzTV.FFmpeg.Tests.Pipeline; + +[TestFixture] +public class QsvPipelineBuilderTests +{ + private readonly ILogger _logger = Substitute.For(); + + [Test] + public void Scale_And_Pad_Fold_Into_Single_Hardware_Vpp_Qsv() + { + string command = BuildPadCommand(vppQsvSupportsPad: true); + + // scale and pad fold into a single vpp_qsv instance that pads on the GPU + command.ShouldContain("vpp_qsv=w=1440:h=1080:pad_w=1920:pad_h=1080:pad_x=-1:pad_y=-1:pad_color=black"); + + // exactly one vpp_qsv instance (two chained instances drop the final frame at EOF) + command.Split("vpp_qsv").Length.ShouldBe(2); + + // no software pad + command.ShouldNotContain("pad=1920:1080"); + } + + [Test] + public void Frames_Stay_In_Hardware_Through_Setpts_Into_The_Qsv_Encoder() + { + string command = BuildPadCommand(vppQsvSupportsPad: true); + + // setpts/fps run on the hardware frame and hevc_qsv ingests it directly: no download anywhere + command.ShouldNotContain("hwdownload"); + command.ShouldContain("pad_color=black,setsar=1,setpts=PTS-STARTPTS,fps=24"); + command.ShouldContain("hevc_qsv"); + } + + [Test] + public void Download_Is_Retained_When_A_Colorspace_Conversion_Follows() + { + string command = BuildPadCommand(vppQsvSupportsPad: true, colorsAreBt709: true); + + // a downstream software colorspace filter still needs system memory, so the download stays + // exactly where it is today: right after the pad, before setpts, and byte-identical to the + // unfolded path (format=nv12, not a bare hwdownload) + command.ShouldContain("pad_color=black,setsar=1,hwdownload,format=nv12,setpts=PTS-STARTPTS"); + command.ShouldContain("colorspace="); + } + + [Test] + public void Pad_Falls_Back_To_Software_When_Pad_Option_Unavailable() + { + string command = BuildPadCommand(vppQsvSupportsPad: false); + + // software fallback: download to system memory, pad, then continue + command.ShouldNotContain("vpp_qsv=pad_w"); + command.ShouldContain("hwdownload,format=nv12,pad=1920:1080:-1:-1:color=black"); + } + + private string BuildPadCommand(bool vppQsvSupportsPad, bool colorsAreBt709 = false) + { + var videoInputFile = new VideoInputFile( + "/tmp/whatever.mkv", + new List + { + new( + 0, + VideoFormat.H264, + VideoProfile.Main, + new PixelFormatYuv420P(), + ColorParams.Default, + new FrameSize(640, 480), + "1:1", + "4:3", + FrameRate.DefaultFrameRate, + false, + ScanKind.Progressive) + }); + + var desiredState = new FrameState( + true, + false, + VideoFormat.Hevc, + VideoProfile.Main, + VideoPreset.Unset, + false, + new PixelFormatNv12("yuv420p"), + new FrameSize(1440, 1080), + new FrameSize(1920, 1080), + Option.None, + FFmpegFilterMode.HardwareIfPossible, + false, + Option.None, + 2000, + 4000, + 90_000, + colorsAreBt709, + false); + + var ffmpegState = new FFmpegState( + false, + HardwareAccelerationMode.Qsv, + HardwareAccelerationMode.Qsv, + Option.None, + Option.None, + TimeSpan.FromSeconds(1), + Option.None, + false, + Option.None, + Option.None, + Option.None, + Option.None, + Option.None, + OutputFormatKind.MpegTs, + Option.None, + Option.None, + Option.None, + Option.None, + TimeSpan.Zero, + Option.None, + Option.None, + false, + false, + "clip", + false); + + var hardwareCapabilities = Substitute.For(); + hardwareCapabilities.CanDecode( + Arg.Any(), + Arg.Any>(), + Arg.Any>(), + Arg.Any()) + .Returns(FFmpegCapability.Hardware); + hardwareCapabilities.CanEncode( + Arg.Any(), + Arg.Any>(), + Arg.Any>()) + .Returns(FFmpegCapability.Hardware); + hardwareCapabilities.GetRateControlMode(Arg.Any(), Arg.Any>()) + .Returns(Option.None); + + var vppQsvOptions = new System.Collections.Generic.HashSet { "w", "h", "format", "deinterlace" }; + if (vppQsvSupportsPad) + { + vppQsvOptions.UnionWith(["pad_w", "pad_h", "pad_x", "pad_y", "pad_color"]); + } + + var ffmpegCapabilities = new FFmpegCapabilities( + string.Empty, + new System.Collections.Generic.HashSet { FFmpegKnownHardwareAcceleration.Qsv.Name }, + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.Dictionary> + { + [FFmpegKnownFilter.VppQsv.Name] = vppQsvOptions + }); + + var builder = new QsvPipelineBuilder( + ffmpegCapabilities, + hardwareCapabilities, + HardwareAccelerationMode.Qsv, + videoInputFile, + Option.None, + Option.None, + Option.None, + None, + Option.None, + "", + "", + _logger); + + FFmpegPipeline result = builder.Build(ffmpegState, desiredState); + + IList arguments = CommandGenerator.GenerateArguments( + videoInputFile, + Option.None, + Option.None, + Option.None, + Option.None, + result.PipelineSteps, + result.IsIntelVaapiOrQsv); + + return string.Join(" ", arguments); + } +} diff --git a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs index 64e485b4e..5c3c4cc91 100644 --- a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs +++ b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs @@ -562,5 +562,6 @@ public class PipelineBuilderBaseTests new System.Collections.Generic.HashSet(), new System.Collections.Generic.HashSet(), new System.Collections.Generic.HashSet(), - new System.Collections.Generic.HashSet()); + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.Dictionary>()); } diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs index 9fa18ee11..a008b58b1 100644 --- a/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs @@ -10,7 +10,8 @@ public class FFmpegCapabilities( IReadOnlySet ffmpegFilters, IReadOnlySet ffmpegEncoders, IReadOnlySet ffmpegOptions, - IReadOnlySet ffmpegDemuxFormats) + IReadOnlySet ffmpegDemuxFormats, + IReadOnlyDictionary> ffmpegFilterOptions) : IFFmpegCapabilities { public string Version => ffmpegVersion; @@ -59,6 +60,9 @@ public class FFmpegCapabilities( public bool HasFilter(FFmpegKnownFilter filter) => ffmpegFilters.Contains(filter.Name); + public bool HasFilterOption(FFmpegKnownFilter filter, string optionName) => + ffmpegFilterOptions.TryGetValue(filter.Name, out IReadOnlySet? options) && options.Contains(optionName); + public bool HasOption(FFmpegKnownOption ffmpegOption) => ffmpegOptions.Contains(ffmpegOption.Name); public bool HasDemuxFormat(FFmpegKnownFormat format) => ffmpegDemuxFormats.Contains(format.Name); diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegFilterOptionParser.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegFilterOptionParser.cs new file mode 100644 index 000000000..41209495e --- /dev/null +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegFilterOptionParser.cs @@ -0,0 +1,21 @@ +using System.Collections.Immutable; +using System.Text.RegularExpressions; + +namespace ErsatzTV.FFmpeg.Capabilities; + +public static partial class FFmpegFilterOptionParser +{ + public static IReadOnlySet Parse(string filterHelpOutput) => + filterHelpOutput.Split("\n").Map(s => s.Trim()) + .Bind(l => ParseLine(l)) + .ToImmutableHashSet(); + + private static Option ParseLine(string input) + { + Match match = FilterOptionRegex().Match(input); + return match.Success ? match.Groups[1].Value : Option.None; + } + + [GeneratedRegex(@"^([\w-]+)\s+<")] + private static partial Regex FilterOptionRegex(); +} diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs index 5ef26f5a2..2a10425e4 100644 --- a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs @@ -24,6 +24,7 @@ public record FFmpegKnownFilter public static readonly FFmpegKnownFilter Scale = new("scale"); public static readonly FFmpegKnownFilter Subtitles = new("subtitles"); public static readonly FFmpegKnownFilter Tonemap = new("tonemap"); + public static readonly FFmpegKnownFilter VppQsv = new("vpp_qsv"); public static readonly FFmpegKnownFilter Yadif = new("yadif"); public static readonly FFmpegKnownFilter ZScale = new("zscale"); diff --git a/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs b/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs index 1bb27b9ec..65d338db7 100644 --- a/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs +++ b/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs @@ -36,6 +36,9 @@ public partial class HardwareCapabilitiesFactory( private static readonly CompositeFormat QsvCacheKeyFormat = CompositeFormat.Parse("ffmpeg.hardware.qsv.{0}"); private static readonly CompositeFormat FFmpegCapabilitiesCacheKeyFormat = CompositeFormat.Parse("ffmpeg.{0}"); + private static readonly CompositeFormat FFmpegFilterOptionsCacheKeyFormat = + CompositeFormat.Parse("ffmpeg.filter.{0}"); + private static readonly string[] QsvArguments = { "-f", "lavfi", @@ -54,6 +57,8 @@ public partial class HardwareCapabilitiesFactory( memoryCache.Remove(string.Format(CultureInfo.InvariantCulture, FFmpegCapabilitiesCacheKeyFormat, "encoders")); memoryCache.Remove(string.Format(CultureInfo.InvariantCulture, FFmpegCapabilitiesCacheKeyFormat, "options")); memoryCache.Remove(string.Format(CultureInfo.InvariantCulture, FFmpegCapabilitiesCacheKeyFormat, "formats")); + memoryCache.Remove( + string.Format(CultureInfo.InvariantCulture, FFmpegFilterOptionsCacheKeyFormat, FFmpegKnownFilter.VppQsv.Name)); } public async Task GetFFmpegCapabilities(string ffmpegPath, CancellationToken cancellationToken) @@ -87,6 +92,13 @@ public partial class HardwareCapabilitiesFactory( IReadOnlySet ffmpegDemuxFormats = await GetFFmpegFormats(ffmpegPath, "D", cancellationToken) .Map(set => set.Intersect(FFmpegKnownFormat.AllFormats).ToImmutableHashSet()); + IReadOnlySet vppQsvOptions = + await GetFFmpegFilterOptions(ffmpegPath, FFmpegKnownFilter.VppQsv.Name, cancellationToken); + var ffmpegFilterOptions = new Dictionary> + { + [FFmpegKnownFilter.VppQsv.Name] = vppQsvOptions + }; + return new FFmpegCapabilities( ffmpegVersion, ffmpegHardwareAccelerations, @@ -94,7 +106,8 @@ public partial class HardwareCapabilitiesFactory( ffmpegFilters, ffmpegEncoders, ffmpegOptions, - ffmpegDemuxFormats); + ffmpegDemuxFormats, + ffmpegFilterOptions); } public async Task GetHardwareCapabilities( @@ -438,6 +451,36 @@ public partial class HardwareCapabilitiesFactory( return capabilitiesResult; } + private async Task> GetFFmpegFilterOptions( + string ffmpegPath, + string filterName, + CancellationToken cancellationToken) + { + var cacheKey = string.Format(CultureInfo.InvariantCulture, FFmpegFilterOptionsCacheKeyFormat, filterName); + if (memoryCache.TryGetValue(cacheKey, out IReadOnlySet? cachedCapabilities) && + cachedCapabilities is not null) + { + return cachedCapabilities; + } + + string[] arguments = ["-hide_banner", "-h", $"filter={filterName}"]; + + BufferedCommandResult result = await Cli.Wrap(ffmpegPath) + .WithArguments(arguments) + .WithValidation(CommandResultValidation.None) + .ExecuteBufferedAsync(Encoding.UTF8, cancellationToken); + + string output = string.IsNullOrWhiteSpace(result.StandardOutput) + ? result.StandardError + : result.StandardOutput; + + IReadOnlySet capabilitiesResult = FFmpegFilterOptionParser.Parse(output); + + memoryCache.Set(cacheKey, capabilitiesResult); + + return capabilitiesResult; + } + private async Task> GetFFmpegFormats( string ffmpegPath, string muxDemux, diff --git a/ErsatzTV.FFmpeg/Capabilities/IFFmpegCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/IFFmpegCapabilities.cs index 3dcfc5fec..6938f047d 100644 --- a/ErsatzTV.FFmpeg/Capabilities/IFFmpegCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/IFFmpegCapabilities.cs @@ -9,6 +9,7 @@ public interface IFFmpegCapabilities bool HasDecoder(FFmpegKnownDecoder decoder); bool HasEncoder(FFmpegKnownEncoder encoder); bool HasFilter(FFmpegKnownFilter filter); + bool HasFilterOption(FFmpegKnownFilter filter, string optionName); bool HasOption(FFmpegKnownOption ffmpegOption); bool HasDemuxFormat(FFmpegKnownFormat format); Option SoftwareDecoderForVideoFormat(string videoFormat); diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/PadQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/PadQsvFilter.cs new file mode 100644 index 000000000..08b5dcaf5 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Qsv/PadQsvFilter.cs @@ -0,0 +1,40 @@ +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter.Qsv; + +public class PadQsvFilter : BaseFilter +{ + private readonly FrameState _currentState; + private readonly FrameSize _paddedSize; + private readonly int _extraHardwareFrames; + + public PadQsvFilter(FrameState currentState, FrameSize paddedSize, int extraHardwareFrames) + { + _currentState = currentState; + _paddedSize = paddedSize; + _extraHardwareFrames = extraHardwareFrames; + } + + public override string Filter + { + get + { + var pad = + $"vpp_qsv=pad_w={_paddedSize.Width}:pad_h={_paddedSize.Height}:pad_x=-1:pad_y=-1:pad_color=black"; + + if (_currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + return pad; + } + + string initialPixelFormat = _currentState.PixelFormat.Match(pf => pf.FFmpegName, FFmpegFormat.NV12); + return $"format={initialPixelFormat},hwupload=extra_hw_frames={_extraHardwareFrames},{pad}"; + } + } + + public override FrameState NextState(FrameState currentState) => currentState with + { + PaddedSize = _paddedSize, + FrameDataLocation = FrameDataLocation.Hardware + }; +} diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs new file mode 100644 index 000000000..183801d07 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleAndPadQsvFilter.cs @@ -0,0 +1,121 @@ +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter.Qsv; + +public class ScaleAndPadQsvFilter : BaseFilter +{ + private readonly FrameState _currentState; + private readonly int _extraHardwareFrames; + private readonly bool _isAnamorphicEdgeCase; + private readonly FrameSize _paddedSize; + private readonly string _sampleAspectRatio; + private readonly FrameSize _scaledSize; + + public ScaleAndPadQsvFilter( + FrameState currentState, + FrameSize scaledSize, + FrameSize paddedSize, + int extraHardwareFrames, + bool isAnamorphicEdgeCase, + string sampleAspectRatio) + { + _currentState = currentState; + _scaledSize = scaledSize; + _paddedSize = paddedSize; + _extraHardwareFrames = extraHardwareFrames; + _isAnamorphicEdgeCase = isAnamorphicEdgeCase; + _sampleAspectRatio = sampleAspectRatio; + } + + public override string Filter + { + get + { + // fold scale and pad into a single vpp_qsv instance; chaining two vpp_qsv instances + // drops the final frame at EOF + + string padOptions = + $"pad_w={_paddedSize.Width}:pad_h={_paddedSize.Height}:pad_x=-1:pad_y=-1:pad_color=black"; + + string scale; + + if (_currentState.ScaledSize == _scaledSize) + { + string format = string.Empty; + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + format = $"format={pixelFormat.FFmpegName}:"; + } + + scale = $"vpp_qsv={format}{padOptions}"; + } + else + { + string format = string.Empty; + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + format = $":format={pixelFormat.FFmpegName}"; + } + + string squareScale = string.Empty; + string setsar = string.Empty; + string sar = _sampleAspectRatio.Replace(':', '/'); + if (_isAnamorphicEdgeCase) + { + squareScale = $"vpp_qsv=w=iw:h={sar}*ih{format},setsar=1,"; + } + else if (_currentState.IsAnamorphic) + { + squareScale = $"vpp_qsv=w=iw*{sar}:h=ih{format},setsar=1,"; + } + else + { + setsar = ",setsar=1"; + } + + scale = + $"{squareScale}vpp_qsv=w={_scaledSize.Width}:h={_scaledSize.Height}{format}:{padOptions}{setsar}"; + } + + if (_currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + return scale; + } + + string initialPixelFormat = _currentState.PixelFormat.Match(pf => pf.FFmpegName, FFmpegFormat.NV12); + return $"format={initialPixelFormat},hwupload=extra_hw_frames={_extraHardwareFrames},{scale}"; + } + } + + public override FrameState NextState(FrameState currentState) + { + FrameState result = currentState with + { + ScaledSize = _scaledSize, + PaddedSize = _paddedSize, + FrameDataLocation = FrameDataLocation.Hardware, + IsAnamorphic = false // this filter always outputs square pixels + }; + + if (_currentState.PixelFormat.IsNone && + _currentState.FrameDataLocation == FrameDataLocation.Software && + currentState.PixelFormat.Map(pf => pf is not PixelFormatNv12).IfNone(false)) + { + // wrap in nv12 + result = result with + { + PixelFormat = currentState.PixelFormat + .Map(pf => (IPixelFormat)new PixelFormatNv12(pf.Name)) + }; + } + else + { + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + result = result with { PixelFormat = Some(pixelFormat) }; + } + } + + return result; + } +} diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs index f7f9a8860..d094c9a0a 100644 --- a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs @@ -30,6 +30,12 @@ public class ScaleQsvFilter : BaseFilter _sampleAspectRatio = sampleAspectRatio; } + public FrameState CurrentState => _currentState; + public FrameSize ScaledSize => _scaledSize; + public int ExtraHardwareFrames => _extraHardwareFrames; + public bool IsAnamorphicEdgeCase => _isAnamorphicEdgeCase; + public string SampleAspectRatio => _sampleAspectRatio; + public override string Filter { get diff --git a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs index 2e3970f81..902902478 100644 --- a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs @@ -18,6 +18,7 @@ namespace ErsatzTV.FFmpeg.Pipeline; public class QsvPipelineBuilder : SoftwarePipelineBuilder { + private readonly IFFmpegCapabilities _ffmpegCapabilities; private readonly IHardwareCapabilities _hardwareCapabilities; private readonly ILogger _logger; @@ -46,6 +47,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder fontsFolder, logger) { + _ffmpegCapabilities = ffmpegCapabilities; _hardwareCapabilities = hardwareCapabilities; _logger = logger; } @@ -194,15 +196,26 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder // _logger.LogDebug("After deinterlace: {PixelFormat}", currentState.PixelFormat); currentState = SetScale(videoInputFile, videoStream, context, ffmpegState, desiredState, currentState); // _logger.LogDebug("After scale: {PixelFormat}", currentState.PixelFormat); + bool isHdrTonemap = videoStream.ColorParams.IsHdr; currentState = SetTonemap(videoInputFile, videoStream, ffmpegState, desiredState, currentState); - currentState = SetPad(videoInputFile, videoStream, desiredState, currentState); + currentState = SetPad(videoInputFile, videoStream, ffmpegState, desiredState, currentState, isHdrTonemap); // _logger.LogDebug("After pad: {PixelFormat}", currentState.PixelFormat); currentState = SetCrop(videoInputFile, desiredState, currentState); SetStillImageLoop(videoInputFile, videoStream, ffmpegState, desiredState, pipelineSteps); - // need to download for any sort of overlay (and always for setpts) - if (currentState.FrameDataLocation == FrameDataLocation.Hardware) //&& - //(context.HasSubtitleOverlay || context.HasWatermark || context.HasGraphicsEngine)) + // setpts/fps operate on timestamps only and accept hardware frames, and the qsv encoder + // ingests hardware frames directly, so keep the frame on the GPU through both when nothing + // downstream needs it in system memory. any software-requiring step (overlays, subtitle + // burn-in, a colorspace conversion, or a software encoder) still forces the download. + bool canKeepHardwareFrames = + ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Qsv + && !context.HasWatermark + && !context.HasSubtitleOverlay + && !context.HasSubtitleText + && !context.HasGraphicsEngine + && !WillDownloadForColorspace(videoInputFile, videoStream, desiredState); + + if (currentState.FrameDataLocation == FrameDataLocation.Hardware && !canKeepHardwareFrames) { var hardwareDownload = new HardwareDownloadFilter(currentState); currentState = hardwareDownload.NextState(currentState); @@ -311,7 +324,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder bool usesVppQsv = videoInputFile.FilterSteps.Any(f => - f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter); + f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter + or ScaleAndPadQsvFilter); // if we have no filters, check whether we need to convert pixel format // since qsv doesn't seem to like doing that at the encoder @@ -554,7 +568,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder } // only scale if scaling or padding was used for main video stream - if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleQsvFilter or PadFilter)) + if (videoInputFile.FilterSteps.Any(s => + s is ScaleFilter or ScaleQsvFilter or PadFilter or ScaleAndPadQsvFilter)) { var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize); subtitle.FilterSteps.Add(scaleFilter); @@ -598,22 +613,78 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder } } - private static FrameState SetPad( + private FrameState SetPad( VideoInputFile videoInputFile, VideoStream videoStream, + FFmpegState ffmpegState, FrameState desiredState, - FrameState currentState) + FrameState currentState, + bool isHdrTonemap) { if (desiredState.CroppedSize.IsNone && currentState.PaddedSize != desiredState.PaddedSize) { - var padStep = new PadFilter(currentState, desiredState.PaddedSize); - currentState = padStep.NextState(currentState); - videoInputFile.FilterSteps.Add(padStep); + if (desiredState.PadMode is FFmpegFilterMode.Software + || isHdrTonemap + || !_ffmpegCapabilities.HasFilterOption(FFmpegKnownFilter.VppQsv, "pad_w")) + { + var padStep = new PadFilter(currentState, desiredState.PaddedSize); + currentState = padStep.NextState(currentState); + videoInputFile.FilterSteps.Add(padStep); + } + else + { + // fold an existing hardware scale and this pad into a single vpp_qsv instance; + // two chained vpp_qsv instances drop the final frame at EOF + Option maybeScale = videoInputFile.FilterSteps.OfType().HeadOrNone(); + if (maybeScale.IsSome) + { + foreach (ScaleQsvFilter scaleStep in maybeScale) + { + videoInputFile.FilterSteps.Remove(scaleStep); + + var scalePadStep = new ScaleAndPadQsvFilter( + scaleStep.CurrentState, + scaleStep.ScaledSize, + desiredState.PaddedSize, + scaleStep.ExtraHardwareFrames, + scaleStep.IsAnamorphicEdgeCase, + scaleStep.SampleAspectRatio); + + // advance the live pipeline state (which carries the pixel format the + // unfolded ScaleQsvFilter produced) so any later download keeps format=X + currentState = scalePadStep.NextState(currentState); + videoInputFile.FilterSteps.Add(scalePadStep); + } + } + else + { + var padStep = new PadQsvFilter( + currentState, + desiredState.PaddedSize, + ffmpegState.QsvExtraHardwareFrames); + currentState = padStep.NextState(currentState); + videoInputFile.FilterSteps.Add(padStep); + } + } } return currentState; } + private static bool WillDownloadForColorspace( + VideoInputFile videoInputFile, + VideoStream videoStream, + FrameState desiredState) + { + // mirrors the colorspace emission in SetPixelFormat: when it runs on a hardware frame it + // prepends its own hwdownload, so the frame would leave the GPU regardless + bool usesVppQsv = videoInputFile.FilterSteps.Any(f => + f is QsvFormatFilter or ScaleQsvFilter or DeinterlaceQsvFilter or TonemapQsvFilter or PadQsvFilter + or ScaleAndPadQsvFilter); + + return desiredState.ColorsAreBt709 && (!videoStream.ColorParams.IsBt709 || usesVppQsv); + } + private static FrameState SetScale( VideoInputFile videoInputFile, VideoStream videoStream,