diff --git a/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs b/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs index 5ef75345e..94f818477 100644 --- a/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs +++ b/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs @@ -193,7 +193,8 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler< playoutItemWithPath.PlayoutItem.OutPoint, request.PtsOffset, request.TargetFramerate, - playoutItemWithPath.PlayoutItem.DisableWatermarks); + playoutItemWithPath.PlayoutItem.DisableWatermarks, + _ => { }); var result = new PlayoutItemProcessModel( process, diff --git a/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs index f4a45d1b3..92906394d 100644 --- a/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs +++ b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs @@ -12,6 +12,12 @@ using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Metadata; using ErsatzTV.FFmpeg; using ErsatzTV.FFmpeg.Capabilities; +using ErsatzTV.FFmpeg.Filter; +using ErsatzTV.FFmpeg.Filter.Cuda; +using ErsatzTV.FFmpeg.Filter.Qsv; +using ErsatzTV.FFmpeg.Filter.Vaapi; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.Pipeline; using ErsatzTV.FFmpeg.State; using ErsatzTV.Infrastructure.Runtime; using FluentAssertions; @@ -221,7 +227,7 @@ public class TranscodingTests } } - string name = GetStringSha256Hash($"{inputFormat.Encoder}_{inputFormat.PixelFormat}_{videoScanKind}_{padding}"); + string name = GetStringSha256Hash($"{inputFormat.Encoder}_{inputFormat.PixelFormat}_{videoScanKind}_{padding}_{subtitle}"); string file = Path.Combine(TestContext.CurrentContext.TestDirectory, $"{name}.mkv"); if (!File.Exists(file)) @@ -229,9 +235,9 @@ public class TranscodingTests string resolution = padding == Padding.WithPadding ? "1920x1060" : "1920x1080"; string videoFilter = videoScanKind == VideoScanKind.Interlaced - ? "-vf tinterlace=interleave_top,fieldorder=tff" + ? "-vf interlace=scan=tff:lowpass=complex" : string.Empty; - string flags = videoScanKind == VideoScanKind.Interlaced ? "-flags +ildct+ilme" : string.Empty; + string flags = videoScanKind == VideoScanKind.Interlaced ? "-field_order tt -flags +ildct+ilme" : string.Empty; string args = $"-y -f lavfi -i anoisesrc=color=brown -f lavfi -i testsrc=duration=1:size={resolution}:rate=30 {videoFilter} -c:a aac -c:v {inputFormat.Encoder} -shortest -pix_fmt {inputFormat.PixelFormat} -strict -2 {flags} {file}"; @@ -266,7 +272,7 @@ public class TranscodingTests StartInfo = new ProcessStartInfo { FileName = ExecutableName("mkvmerge"), - Arguments = $"-o {tempFileName} {sourceFile} {subPath}" + Arguments = $"-o {tempFileName} {sourceFile} --field-order 0:{(videoScanKind == VideoScanKind.Interlaced ? '1' : '0')} {subPath}" } }; @@ -289,6 +295,8 @@ public class TranscodingTests p2.ExitCode.Should().Be(0); + await SetInterlacedFlag(tempFileName, sourceFile, file, videoScanKind == VideoScanKind.Interlaced); + File.Move(tempFileName, file, true); break; } @@ -318,11 +326,10 @@ public class TranscodingTests new FFmpegPlaybackSettingsCalculator(), new FakeStreamSelector(), new Mock().Object, - new FakeNvidiaCapabilitiesFactory(), - // new HardwareCapabilitiesFactory( - // new MemoryCache(new MemoryCacheOptions()), - // LoggerFactory.CreateLogger()), - new RuntimeInfo(), + new PipelineBuilderFactory( + new RuntimeInfo(), + new FakeNvidiaCapabilitiesFactory(), + LoggerFactory.CreateLogger()), LoggerFactory.CreateLogger()); var v = new MediaVersion @@ -367,6 +374,11 @@ public class TranscodingTests } }); + if (videoScanKind == VideoScanKind.Interlaced) + { + v.VideoScanKind.Should().Be(VideoScanKind.Interlaced, file); + } + var subtitleStreams = v.Streams .Filter(s => s.MediaStreamKind == MediaStreamKind.Subtitle) .ToList(); @@ -427,7 +439,8 @@ public class TranscodingTests ImageSource = ChannelWatermarkImageSource.Custom, Mode = ChannelWatermarkMode.Permanent, Opacity = 100, - Size = WatermarkSize.Scaled + Size = WatermarkSize.Scaled, + WidthPercent = 15 }; break; case Watermark.PermanentOpaqueActualSize: @@ -445,7 +458,8 @@ public class TranscodingTests ImageSource = ChannelWatermarkImageSource.Custom, Mode = ChannelWatermarkMode.Permanent, Opacity = 80, - Size = WatermarkSize.Scaled + Size = WatermarkSize.Scaled, + WidthPercent = 15 }; break; case Watermark.PermanentTransparentActualSize: @@ -473,6 +487,71 @@ public class TranscodingTests File.Copy(sourceFile, srtFile, true); } + void PipelineAction(FFmpegPipeline pipeline) + { + // validate pipeline matches expectations (at a high level) + + NewComplexFilter complexFilter = pipeline.PipelineSteps.OfType().First(); + FilterChain filterChain = complexFilter.FilterChain; + + if (profileBitDepth == FFmpegProfileBitDepth.TenBit) + // process.Arguments.Contains("=nv12") && + // !process.Arguments.Contains("format=nv12,format=p010le[") && + // !process.Arguments.Contains("hwdownload,format=nv12,subtitle") && + // !process.Arguments.Contains("format=nv12,hwupload_cuda[st]") && + // !process.Arguments.Contains("format=nv12,hwupload_cuda[wm]")) + { + var videoFilters = string.Join(",", filterChain.VideoFilterSteps.Map(f => f.Filter)); + var pixelFormatFilters = string.Join(",", filterChain.PixelFormatFilterSteps.Map(f => f.Filter)); + if (videoFilters.Contains("nv12") || (pixelFormatFilters.Contains("nv12") && !pixelFormatFilters.EndsWith("format=nv12,format=p010le"))) + { + // Assert.Fail("10-bit shouldn't use NV12!"); + } + } + + bool hasDeinterlaceFilter = filterChain.VideoFilterSteps.Any( + s => s is YadifFilter or YadifCudaFilter or DeinterlaceQsvFilter or DeinterlaceVaapiFilter); + + hasDeinterlaceFilter.Should().Be(videoScanKind == VideoScanKind.Interlaced); + + bool hasScaling = filterChain.VideoFilterSteps.Filter( + s => s is ScaleFilter or ScaleCudaFilter or ScaleQsvFilter or ScaleVaapiFilter) + .Filter(s => s is not ScaleCudaFilter cuda || !cuda.Filter.Contains("scale_cuda=format=")) + .Any(); + + // TODO: sometimes scaling is used for pixel format, so this is harder to assert the absence + if (profileResolution.Width != 1920) + { + hasScaling.Should().BeTrue(); + } + + // TODO: bit depth + + bool hasPadding = filterChain.VideoFilterSteps.Any(s => s is PadFilter); + + // TODO: optimize out padding + // hasPadding.Should().Be(padding == Padding.WithPadding); + if (padding == Padding.WithPadding) + { + hasPadding.Should().BeTrue(); + } + + bool hasSubtitleFilters = + filterChain.VideoFilterSteps.Any(s => s is SubtitlesFilter) || + filterChain.SubtitleOverlayFilterSteps.Any( + s => s is OverlaySubtitleFilter + or OverlaySubtitleCudaFilter + or OverlaySubtitleQsvFilter + or OverlaySubtitleVaapiFilter); + + hasSubtitleFilters.Should().Be(subtitle != Subtitle.None); + + bool hasWatermarkFilters = filterChain.WatermarkOverlayFilterSteps.Any( + s => s is OverlayWatermarkFilter or OverlayWatermarkCudaFilter or OverlayWatermarkQsvFilter); + + hasWatermarkFilters.Should().Be(watermark != Watermark.None); + }; + Command process = await service.ForPlayoutItem( ExecutableName("ffmpeg"), ExecutableName("ffprobe"), @@ -514,7 +593,8 @@ public class TranscodingTests TimeSpan.FromSeconds(5), 0, None, - false); + false, + PipelineAction); // Console.WriteLine($"ffmpeg arguments {string.Join(" ", process.StartInfo.ArgumentList)}"); @@ -527,41 +607,137 @@ public class TranscodingTests }; var sb = new StringBuilder(); - CommandResult result; var timeoutSignal = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + string tempFile = Path.GetTempFileName(); try { - result = await process - .WithStandardOutputPipe(PipeTarget.ToStream(Stream.Null)) - .WithStandardErrorPipe(PipeTarget.ToStringBuilder(sb)) - .ExecuteAsync(timeoutSignal.Token); - } - catch (OperationCanceledException) - { - Assert.Fail($"Transcode failure (timeout): ffmpeg {process.Arguments}"); - return; - } + CommandResult result; - var error = sb.ToString(); - bool isUnsupported = unsupportedMessages.Any(error.Contains); + try + { + result = await process + .WithStandardOutputPipe(PipeTarget.ToFile(tempFile)) + .WithStandardErrorPipe(PipeTarget.ToStringBuilder(sb)) + .ExecuteAsync(timeoutSignal.Token); + } + catch (OperationCanceledException) + { + var arguments = string.Join( + ' ', + process.Arguments.Split(" ").Map(a => a.Contains('[') ? $"\"{a}\"" : a)); - if (profileAcceleration != HardwareAccelerationKind.None && isUnsupported) - { - result.ExitCode.Should().Be(1, $"Error message with successful exit code? {process.Arguments}"); - Assert.Warn($"Unsupported on this hardware: ffmpeg {process.Arguments}"); + Assert.Fail($"Transcode failure (timeout): ffmpeg {arguments}"); + return; + } + + var error = sb.ToString(); + bool isUnsupported = unsupportedMessages.Any(error.Contains); + + if (profileAcceleration != HardwareAccelerationKind.None && isUnsupported) + { + result.ExitCode.Should().Be(1, $"Error message with successful exit code? {process.Arguments}"); + Assert.Warn($"Unsupported on this hardware: ffmpeg {process.Arguments}"); + } + else if (error.Contains("Impossible to convert between")) + { + var arguments = string.Join( + ' ', + process.Arguments.Split(" ").Map(a => a.Contains('[') ? $"\"{a}\"" : a)); + + Assert.Fail($"Transcode failure: ffmpeg {arguments}"); + } + else + { + var arguments = string.Join( + ' ', + process.Arguments.Split(" ").Map(a => a.Contains('[') ? $"\"{a}\"" : a)); + + result.ExitCode.Should().Be(0, error + Environment.NewLine + arguments); + if (result.ExitCode == 0) + { + Console.WriteLine(process.Arguments); + } + } + + // additional checks on resulting file + await localStatisticsProvider.RefreshStatistics( + ExecutableName("ffmpeg"), + ExecutableName("ffprobe"), + new Movie + { + MediaVersions = new List + { + new() + { + MediaFiles = new List + { + new() { Path = tempFile } + } + } + } + }); + + // verify de-interlace + v.VideoScanKind.Should().NotBe(VideoScanKind.Interlaced); + + // verify resolution + v.Height.Should().Be(profileResolution.Height); + v.Width.Should().Be(profileResolution.Width); + + foreach (MediaStream videoStream in v.Streams.Filter(s => s.MediaStreamKind == MediaStreamKind.Video)) + { + // verify pixel format + videoStream.PixelFormat.Should().Be( + profileBitDepth == FFmpegProfileBitDepth.TenBit ? PixelFormat.YUV420P10LE : PixelFormat.YUV420P); + + // verify colors + var colorParams = new ColorParams( + videoStream.ColorRange, + videoStream.ColorSpace, + videoStream.ColorTransfer, + videoStream.ColorPrimaries); + + colorParams.IsBt709.Should().BeTrue($"{colorParams}"); + } } - else if (error.Contains("Impossible to convert between")) + finally { - Assert.Fail($"Transcode failure: ffmpeg {process.Arguments}"); + if (File.Exists(tempFile)) + { + File.Delete(tempFile); + } } - else + } + + private static async Task SetInterlacedFlag(string tempFileName, string sourceFile, string file, bool interlaced) + { + var p = new Process { - result.ExitCode.Should().Be(0, error + Environment.NewLine + process.Arguments); - if (result.ExitCode == 0) + StartInfo = new ProcessStartInfo { - Console.WriteLine(process.Arguments); + FileName = ExecutableName("mkvpropedit"), + Arguments = $"{tempFileName} --edit track:v1 --set interlaced={(interlaced ? '1' : '0')}", + } + }; + + p.Start(); + await p.WaitForExitAsync(); + // ReSharper disable once MethodHasAsyncOverload + p.WaitForExit(); + if (p.ExitCode != 0) + { + if (File.Exists(sourceFile)) + { + File.Delete(sourceFile); + } + + if (File.Exists(file)) + { + File.Delete(file); } } + + p.ExitCode.Should().Be(0); } private static string GetStringSha256Hash(string text) diff --git a/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs b/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs index 8250503f9..c17b69aa9 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs @@ -3,11 +3,10 @@ using ErsatzTV.Core.Domain; using ErsatzTV.Core.Domain.Filler; using ErsatzTV.Core.Interfaces.FFmpeg; using ErsatzTV.FFmpeg; -using ErsatzTV.FFmpeg.Capabilities; using ErsatzTV.FFmpeg.Environment; using ErsatzTV.FFmpeg.Format; using ErsatzTV.FFmpeg.OutputFormat; -using ErsatzTV.FFmpeg.Runtime; +using ErsatzTV.FFmpeg.Pipeline; using ErsatzTV.FFmpeg.State; using Microsoft.Extensions.Logging; using MediaStream = ErsatzTV.Core.Domain.MediaStream; @@ -18,27 +17,24 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService { private readonly FFmpegProcessService _ffmpegProcessService; private readonly IFFmpegStreamSelector _ffmpegStreamSelector; - private readonly IHardwareCapabilitiesFactory _hardwareCapabilitiesFactory; - private readonly IRuntimeInfo _runtimeInfo; private readonly ILogger _logger; private readonly FFmpegPlaybackSettingsCalculator _playbackSettingsCalculator; private readonly ITempFilePool _tempFilePool; + private readonly IPipelineBuilderFactory _pipelineBuilderFactory; public FFmpegLibraryProcessService( FFmpegProcessService ffmpegProcessService, FFmpegPlaybackSettingsCalculator playbackSettingsCalculator, IFFmpegStreamSelector ffmpegStreamSelector, ITempFilePool tempFilePool, - IHardwareCapabilitiesFactory hardwareCapabilitiesFactory, - IRuntimeInfo runtimeInfo, + IPipelineBuilderFactory pipelineBuilderFactory, ILogger logger) { _ffmpegProcessService = ffmpegProcessService; _playbackSettingsCalculator = playbackSettingsCalculator; _ffmpegStreamSelector = ffmpegStreamSelector; _tempFilePool = tempFilePool; - _hardwareCapabilitiesFactory = hardwareCapabilitiesFactory; - _runtimeInfo = runtimeInfo; + _pipelineBuilderFactory = pipelineBuilderFactory; _logger = logger; } @@ -70,7 +66,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService TimeSpan outPoint, long ptsOffset, Option targetFramerate, - bool disableWatermarks) + bool disableWatermarks, + Action pipelineAction) { MediaStream videoStream = await _ffmpegStreamSelector.SelectVideoStream(videoVersion); Option maybeAudioStream = @@ -156,7 +153,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService videoVersion.SampleAspectRatio, videoVersion.DisplayAspectRatio, videoVersion.RFrameRate, - videoPath != audioPath); // still image when paths are different + videoPath != audioPath, // still image when paths are different + videoVersion.VideoScanKind == VideoScanKind.Progressive ? ScanKind.Progressive : ScanKind.Interlaced); var videoInputFile = new VideoInputFile(videoPath, new List { ffmpegVideoStream }); @@ -252,19 +250,20 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService _logger.LogDebug("FFmpeg desired state {FrameState}", desiredState); - var pipelineBuilder = new PipelineBuilder( - _runtimeInfo, - await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, hwAccel), + IPipelineBuilder pipelineBuilder = await _pipelineBuilderFactory.GetBuilder( + hwAccel, videoInputFile, audioInputFile, watermarkInputFile, subtitleInputFile, FileSystemLayout.FFmpegReportsFolder, FileSystemLayout.FontsCacheFolder, - _logger); + ffmpegPath); FFmpegPipeline pipeline = pipelineBuilder.Build(ffmpegState, desiredState); + pipelineAction?.Invoke(pipeline); + return GetCommand(ffmpegPath, videoInputFile, audioInputFile, watermarkInputFile, None, pipeline); } @@ -353,7 +352,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService videoVersion.SampleAspectRatio, videoVersion.DisplayAspectRatio, None, - true); + true, + ScanKind.Progressive); var videoInputFile = new VideoInputFile(videoPath, new List { ffmpegVideoStream }); @@ -389,17 +389,16 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService _logger.LogDebug("FFmpeg desired error state {FrameState}", desiredState); - var pipelineBuilder = new PipelineBuilder( - _runtimeInfo, - await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, hwAccel), + IPipelineBuilder pipelineBuilder = await _pipelineBuilderFactory.GetBuilder( + hwAccel, videoInputFile, audioInputFile, None, subtitleInputFile, FileSystemLayout.FFmpegReportsFolder, FileSystemLayout.FontsCacheFolder, - _logger); - + ffmpegPath); + FFmpegPipeline pipeline = pipelineBuilder.Build(ffmpegState, desiredState); return GetCommand(ffmpegPath, videoInputFile, audioInputFile, None, None, pipeline); @@ -418,17 +417,16 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService $"http://localhost:{Settings.ListenPort}/ffmpeg/concat/{channel.Number}", resolution); - var pipelineBuilder = new PipelineBuilder( - _runtimeInfo, - await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, HardwareAccelerationMode.None), + IPipelineBuilder pipelineBuilder = await _pipelineBuilderFactory.GetBuilder( + HardwareAccelerationMode.None, None, None, None, None, FileSystemLayout.FFmpegReportsFolder, FileSystemLayout.FontsCacheFolder, - _logger); - + ffmpegPath); + FFmpegPipeline pipeline = pipelineBuilder.Concat( concatInputFile, FFmpegState.Concat(saveReports, channel.Name)); @@ -454,19 +452,19 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService string.Empty, string.Empty, None, - true) + true, + ScanKind.Progressive) }); - var pipelineBuilder = new PipelineBuilder( - _runtimeInfo, - await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, HardwareAccelerationMode.None), + IPipelineBuilder pipelineBuilder = await _pipelineBuilderFactory.GetBuilder( + HardwareAccelerationMode.None, videoInputFile, None, None, None, FileSystemLayout.FFmpegReportsFolder, FileSystemLayout.FontsCacheFolder, - _logger); + ffmpegPath); FFmpegPipeline pipeline = pipelineBuilder.Resize(outputFile, new FrameSize(-1, height)); @@ -540,7 +538,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService string.Empty, string.Empty, Option.None, - !options.IsAnimated) + !options.IsAnimated, + ScanKind.Progressive) }, new WatermarkState( maybeFadePoints.Map( @@ -587,10 +586,10 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService bool log = true) { IEnumerable loggedSteps = pipeline.PipelineSteps.Map(ps => ps.GetType().Name); - IEnumerable loggedVideoFilters = - videoInputFile.Map(f => f.FilterSteps.Map(vf => vf.GetType().Name)).Flatten(); IEnumerable loggedAudioFilters = audioInputFile.Map(f => f.FilterSteps.Map(af => af.GetType().Name)).Flatten(); + IEnumerable loggedVideoFilters = + videoInputFile.Map(f => f.FilterSteps.Map(vf => vf.GetType().Name)).Flatten(); if (log) { diff --git a/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs b/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs index af0b98666..db9cd68e2 100644 --- a/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs +++ b/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs @@ -2,6 +2,7 @@ using ErsatzTV.Core.Domain; using ErsatzTV.Core.Domain.Filler; using ErsatzTV.Core.FFmpeg; +using ErsatzTV.FFmpeg; using ErsatzTV.FFmpeg.State; namespace ErsatzTV.Core.Interfaces.FFmpeg; @@ -36,7 +37,8 @@ public interface IFFmpegProcessService TimeSpan outPoint, long ptsOffset, Option targetFramerate, - bool disableWatermarks); + bool disableWatermarks, + Action pipelineAction); Task ForError( string ffmpegPath, diff --git a/ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs similarity index 80% rename from ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs rename to ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs index be35c4beb..0323f979d 100644 --- a/ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs +++ b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs @@ -1,10 +1,9 @@ using System; using System.Collections.Generic; -using ErsatzTV.FFmpeg.Capabilities; using ErsatzTV.FFmpeg.Encoder; using ErsatzTV.FFmpeg.Format; using ErsatzTV.FFmpeg.OutputFormat; -using ErsatzTV.FFmpeg.Runtime; +using ErsatzTV.FFmpeg.Pipeline; using ErsatzTV.FFmpeg.State; using FluentAssertions; using LanguageExt; @@ -16,7 +15,7 @@ using static LanguageExt.Prelude; namespace ErsatzTV.FFmpeg.Tests; [TestFixture] -public class PipelineGeneratorTests +public class PipelineBuilderBaseTests { private readonly ILogger _logger = new Mock().Object; @@ -26,7 +25,19 @@ public class PipelineGeneratorTests var videoInputFile = new VideoInputFile( "/tmp/whatever.mkv", new List - { new(0, VideoFormat.H264, new PixelFormatYuv420P(), ColorParams.Default, new FrameSize(1920, 1080), "1:1", "16:9", "24", false) }); + { + new( + 0, + VideoFormat.H264, + new PixelFormatYuv420P(), + ColorParams.Default, + new FrameSize(1920, 1080), + "1:1", + "16:9", + "24", + false, + ScanKind.Progressive) + }); var audioInputFile = new AudioInputFile( "/tmp/whatever.mkv", @@ -73,9 +84,8 @@ public class PipelineGeneratorTests Option.None, Option.None); - var builder = new PipelineBuilder( - new Mock().Object, - new DefaultHardwareCapabilities(), + var builder = new SoftwarePipelineBuilder( + HardwareAccelerationMode.None, videoInputFile, audioInputFile, None, @@ -90,7 +100,7 @@ public class PipelineGeneratorTests string command = PrintCommand(videoInputFile, audioInputFile, None, None, result); command.Should().Be( - "-threads 1 -nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -re -i /tmp/whatever.mkv -map 0:1 -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:a aac -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -f mpegts -mpegts_flags +initial_discontinuity pipe:1"); + "-threads 1 -nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -re -i /tmp/whatever.mkv -map 0:1 -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -c:a aac -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -f mpegts -mpegts_flags +initial_discontinuity pipe:1"); } [Test] @@ -99,7 +109,19 @@ public class PipelineGeneratorTests var videoInputFile = new VideoInputFile( "/tmp/whatever.mkv", new List - { new(0, VideoFormat.H264, new PixelFormatYuv420P(), ColorParams.Default, new FrameSize(1920, 1080), "1:1", "16:9", "24", false) }); + { + new( + 0, + VideoFormat.H264, + new PixelFormatYuv420P(), + ColorParams.Default, + new FrameSize(1920, 1080), + "1:1", + "16:9", + "24", + false, + ScanKind.Progressive) + }); var audioInputFile = new AudioInputFile( "/tmp/whatever.mkv", @@ -146,9 +168,8 @@ public class PipelineGeneratorTests Option.None, Option.None); - var builder = new PipelineBuilder( - new Mock().Object, - new DefaultHardwareCapabilities(), + var builder = new SoftwarePipelineBuilder( + HardwareAccelerationMode.None, videoInputFile, audioInputFile, None, @@ -163,7 +184,7 @@ public class PipelineGeneratorTests string command = PrintCommand(videoInputFile, audioInputFile, None, None, result); command.Should().Be( - "-threads 1 -nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -re -i /tmp/whatever.mkv -map 0:1 -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:a aac -ac 6 -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -f mpegts -mpegts_flags +initial_discontinuity pipe:1"); + "-threads 1 -nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -re -i /tmp/whatever.mkv -map 0:1 -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -c:a aac -ac 6 -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -f mpegts -mpegts_flags +initial_discontinuity pipe:1"); } [Test] @@ -172,9 +193,8 @@ public class PipelineGeneratorTests var resolution = new FrameSize(1920, 1080); var concatInputFile = new ConcatInputFile("http://localhost:8080/ffmpeg/concat/1", resolution); - var builder = new PipelineBuilder( - new Mock().Object, - new DefaultHardwareCapabilities(), + var builder = new SoftwarePipelineBuilder( + HardwareAccelerationMode.None, None, None, None, @@ -198,7 +218,19 @@ public class PipelineGeneratorTests var videoInputFile = new VideoInputFile( "/tmp/whatever.mkv", new List - { new(0, VideoFormat.H264, new PixelFormatYuv420P(), ColorParams.Default, new FrameSize(1920, 1080), "1:1", "16:9", "24", false) }); + { + new( + 0, + VideoFormat.H264, + new PixelFormatYuv420P(), + ColorParams.Default, + new FrameSize(1920, 1080), + "1:1", + "16:9", + "24", + false, + ScanKind.Interlaced) + }); var audioInputFile = new AudioInputFile( "/tmp/whatever.mkv", @@ -245,9 +277,8 @@ public class PipelineGeneratorTests Option.None, Option.None); - var builder = new PipelineBuilder( - new Mock().Object, - new DefaultHardwareCapabilities(), + var builder = new SoftwarePipelineBuilder( + HardwareAccelerationMode.None, videoInputFile, audioInputFile, None, @@ -273,7 +304,19 @@ public class PipelineGeneratorTests var videoInputFile = new VideoInputFile( "/tmp/whatever.mkv", new List - { new(0, VideoFormat.H264, new PixelFormatYuv420P(), ColorParams.Default, new FrameSize(1920, 1080), "1:1", "16:9", "24", false) }); + { + new( + 0, + VideoFormat.H264, + new PixelFormatYuv420P(), + ColorParams.Default, + new FrameSize(1920, 1080), + "1:1", + "16:9", + "24", + false, + ScanKind.Progressive) + }); Option audioInputFile = Option.None; @@ -310,9 +353,8 @@ public class PipelineGeneratorTests Option.None, Option.None); - var builder = new PipelineBuilder( - new Mock().Object, - new DefaultHardwareCapabilities(), + var builder = new SoftwarePipelineBuilder( + HardwareAccelerationMode.None, videoInputFile, audioInputFile, None, @@ -341,12 +383,21 @@ public class PipelineGeneratorTests "/test/input/file.png", new List { - new(0, string.Empty, Option.None, ColorParams.Default, FrameSize.Unknown, string.Empty, string.Empty, Option.None, true) + new( + 0, + string.Empty, + Option.None, + ColorParams.Default, + FrameSize.Unknown, + string.Empty, + string.Empty, + Option.None, + true, + ScanKind.Progressive) }); - var pipelineBuilder = new PipelineBuilder( - new Mock().Object, - new DefaultHardwareCapabilities(), + var pipelineBuilder = new SoftwarePipelineBuilder( + HardwareAccelerationMode.None, videoInputFile, Option.None, Option.None, diff --git a/ErsatzTV.FFmpeg/Capabilities/DefaultHardwareCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/DefaultHardwareCapabilities.cs index 417c7be64..532aa7bec 100644 --- a/ErsatzTV.FFmpeg/Capabilities/DefaultHardwareCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/DefaultHardwareCapabilities.cs @@ -5,5 +5,16 @@ namespace ErsatzTV.FFmpeg.Capabilities; public class DefaultHardwareCapabilities : IHardwareCapabilities { public bool CanDecode(string videoFormat, Option maybePixelFormat) => true; - public bool CanEncode(string videoFormat, Option maybePixelFormat) => true; + public bool CanEncode(string videoFormat, Option maybePixelFormat) + { + int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8); + + return (videoFormat, bitDepth) switch + { + // 10-bit h264 encoding is not support by any hardware + (VideoFormat.H264, 10) => false, + + _ => true + }; + } } diff --git a/ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs index e81b57d46..e3845d835 100644 --- a/ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs @@ -28,6 +28,9 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities // some second gen maxwell can decode vp9, otherwise pascal is required VideoFormat.Vp9 => _architecture == 52 && _maxwellGm206.Contains(_model) || _architecture >= 60, + + // no hardware decoding of 10-bit h264 + VideoFormat.H264 when bitDepth == 10 => false, _ => true }; diff --git a/ErsatzTV.FFmpeg/CommandGenerator.cs b/ErsatzTV.FFmpeg/CommandGenerator.cs index 0af256240..dc2c06bfa 100644 --- a/ErsatzTV.FFmpeg/CommandGenerator.cs +++ b/ErsatzTV.FFmpeg/CommandGenerator.cs @@ -2,6 +2,7 @@ using ErsatzTV.FFmpeg.Environment; using ErsatzTV.FFmpeg.Filter; using ErsatzTV.FFmpeg.Option; +using ErsatzTV.FFmpeg.Option.HardwareAcceleration; namespace ErsatzTV.FFmpeg; @@ -39,7 +40,10 @@ public static class CommandGenerator foreach (AudioInputFile audioInputFile in maybeAudioInputFile) { - if (!includedPaths.Contains(audioInputFile.Path)) + bool isVaapiOrQsv = + pipelineSteps.Any(s => s is VaapiHardwareAccelerationOption or QsvHardwareAccelerationOption); + + if (!includedPaths.Contains(audioInputFile.Path) || isVaapiOrQsv) { includedPaths.Add(audioInputFile.Path); diff --git a/ErsatzTV.FFmpeg/Decoder/AvailableDecoders.cs b/ErsatzTV.FFmpeg/Decoder/AvailableDecoders.cs index 8152efc9a..4615fd40b 100644 --- a/ErsatzTV.FFmpeg/Decoder/AvailableDecoders.cs +++ b/ErsatzTV.FFmpeg/Decoder/AvailableDecoders.cs @@ -21,7 +21,7 @@ public static class AvailableDecoders { (HardwareAccelerationMode.Nvenc, VideoFormat.Hevc, _) when hardwareCapabilities.CanDecode(VideoFormat.Hevc, currentState.PixelFormat) => - new DecoderHevcCuvid(ffmpegState), + new DecoderHevcCuvid(ffmpegState.EncoderHardwareAccelerationMode), // nvenc doesn't support hardware decoding of 10-bit content (HardwareAccelerationMode.Nvenc, VideoFormat.H264, PixelFormat.YUV420P10LE or PixelFormat.YUV444P10LE) @@ -33,15 +33,15 @@ public static class AvailableDecoders (HardwareAccelerationMode.Nvenc, VideoFormat.H264, _) when hardwareCapabilities.CanDecode(VideoFormat.H264, currentState.PixelFormat) => - new DecoderH264Cuvid(ffmpegState), + new DecoderH264Cuvid(ffmpegState.EncoderHardwareAccelerationMode), (HardwareAccelerationMode.Nvenc, VideoFormat.Mpeg2Video, _) => new DecoderMpeg2Cuvid( - ffmpegState, + ffmpegState.EncoderHardwareAccelerationMode, desiredState.Deinterlaced), - (HardwareAccelerationMode.Nvenc, VideoFormat.Vc1, _) => new DecoderVc1Cuvid(ffmpegState), + (HardwareAccelerationMode.Nvenc, VideoFormat.Vc1, _) => new DecoderVc1Cuvid(ffmpegState.EncoderHardwareAccelerationMode), (HardwareAccelerationMode.Nvenc, VideoFormat.Vp9, _) when hardwareCapabilities.CanDecode(VideoFormat.Vp9, currentState.PixelFormat) => - new DecoderVp9Cuvid(ffmpegState), - (HardwareAccelerationMode.Nvenc, VideoFormat.Mpeg4, _) => new DecoderMpeg4Cuvid(ffmpegState), + new DecoderVp9Cuvid(ffmpegState.EncoderHardwareAccelerationMode), + (HardwareAccelerationMode.Nvenc, VideoFormat.Mpeg4, _) => new DecoderMpeg4Cuvid(ffmpegState.EncoderHardwareAccelerationMode), // hevc_qsv decoder sometimes causes green lines with 10-bit content (HardwareAccelerationMode.Qsv, VideoFormat.Hevc, PixelFormat.YUV420P10LE) => new DecoderHevc(), diff --git a/ErsatzTV.FFmpeg/Decoder/Cuvid/CuvidDecoder.cs b/ErsatzTV.FFmpeg/Decoder/Cuvid/CuvidDecoder.cs new file mode 100644 index 000000000..872e50b86 --- /dev/null +++ b/ErsatzTV.FFmpeg/Decoder/Cuvid/CuvidDecoder.cs @@ -0,0 +1,34 @@ +namespace ErsatzTV.FFmpeg.Decoder.Cuvid; + +public abstract class CuvidDecoder : DecoderBase +{ + protected CuvidDecoder(HardwareAccelerationMode hardwareAccelerationMode) + { + HardwareAccelerationMode = hardwareAccelerationMode; + } + + public HardwareAccelerationMode HardwareAccelerationMode { get; set; } + + protected override FrameDataLocation OutputFrameDataLocation => + HardwareAccelerationMode == HardwareAccelerationMode.None + ? FrameDataLocation.Software + : FrameDataLocation.Hardware; + + public override IList InputOptions(InputFile inputFile) + { + IList result = base.InputOptions(inputFile); + + if (HardwareAccelerationMode != HardwareAccelerationMode.None) + { + result.Add("-hwaccel_output_format"); + result.Add("cuda"); + } + else + { + result.Add("-hwaccel_output_format"); + result.Add(InputBitDepth(inputFile) == 10 ? "p010le" : "nv12"); + } + + return result; + } +} diff --git a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderH264Cuvid.cs b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderH264Cuvid.cs index a8ad979ba..4e0fd952b 100644 --- a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderH264Cuvid.cs +++ b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderH264Cuvid.cs @@ -1,33 +1,10 @@ namespace ErsatzTV.FFmpeg.Decoder.Cuvid; -public class DecoderH264Cuvid : DecoderBase +public class DecoderH264Cuvid : CuvidDecoder { - private readonly FFmpegState _ffmpegState; - - public DecoderH264Cuvid(FFmpegState ffmpegState) => _ffmpegState = ffmpegState; - - public override string Name => "h264_cuvid"; - - protected override FrameDataLocation OutputFrameDataLocation => - _ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.None - ? FrameDataLocation.Software - : FrameDataLocation.Hardware; - - public override IList InputOptions(InputFile inputFile) + public DecoderH264Cuvid(HardwareAccelerationMode hardwareAccelerationMode) : base(hardwareAccelerationMode) { - IList result = base.InputOptions(inputFile); - - if (_ffmpegState.EncoderHardwareAccelerationMode != HardwareAccelerationMode.None) - { - result.Add("-hwaccel_output_format"); - result.Add("cuda"); - } - else - { - result.Add("-hwaccel_output_format"); - result.Add(InputBitDepth(inputFile) == 10 ? "p010le" : "nv12"); - } - - return result; } + + public override string Name => "h264_cuvid"; } diff --git a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderHevcCuvid.cs b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderHevcCuvid.cs index dc7db6a75..9c2f19d79 100644 --- a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderHevcCuvid.cs +++ b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderHevcCuvid.cs @@ -1,33 +1,10 @@ namespace ErsatzTV.FFmpeg.Decoder.Cuvid; -public class DecoderHevcCuvid : DecoderBase +public class DecoderHevcCuvid : CuvidDecoder { - private readonly FFmpegState _ffmpegState; - - public DecoderHevcCuvid(FFmpegState ffmpegState) => _ffmpegState = ffmpegState; - - public override string Name => "hevc_cuvid"; - - protected override FrameDataLocation OutputFrameDataLocation => - _ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.None - ? FrameDataLocation.Software - : FrameDataLocation.Hardware; - - public override IList InputOptions(InputFile inputFile) + public DecoderHevcCuvid(HardwareAccelerationMode hardwareAccelerationMode) : base(hardwareAccelerationMode) { - IList result = base.InputOptions(inputFile); - - if (_ffmpegState.EncoderHardwareAccelerationMode != HardwareAccelerationMode.None) - { - result.Add("-hwaccel_output_format"); - result.Add("cuda"); - } - else - { - result.Add("-hwaccel_output_format"); - result.Add(InputBitDepth(inputFile) == 10 ? "p010le" : "nv12"); - } - - return result; } + + public override string Name => "hevc_cuvid"; } diff --git a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderMpeg2Cuvid.cs b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderMpeg2Cuvid.cs index fa0fd73d2..c86a907e2 100644 --- a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderMpeg2Cuvid.cs +++ b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderMpeg2Cuvid.cs @@ -1,38 +1,19 @@ namespace ErsatzTV.FFmpeg.Decoder.Cuvid; -public class DecoderMpeg2Cuvid : DecoderBase +public class DecoderMpeg2Cuvid : CuvidDecoder { private readonly bool _contentIsInterlaced; - private readonly FFmpegState _ffmpegState; - public DecoderMpeg2Cuvid(FFmpegState ffmpegState, bool contentIsInterlaced) + public DecoderMpeg2Cuvid(HardwareAccelerationMode hardwareAccelerationMode, bool contentIsInterlaced) + : base(hardwareAccelerationMode) { - _ffmpegState = ffmpegState; _contentIsInterlaced = contentIsInterlaced; } public override string Name => "mpeg2_cuvid"; protected override FrameDataLocation OutputFrameDataLocation => - _contentIsInterlaced || _ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.None + _contentIsInterlaced || HardwareAccelerationMode == HardwareAccelerationMode.None ? FrameDataLocation.Software : FrameDataLocation.Hardware; - - public override IList InputOptions(InputFile inputFile) - { - IList result = base.InputOptions(inputFile); - - if (_ffmpegState.EncoderHardwareAccelerationMode != HardwareAccelerationMode.None) - { - result.Add("-hwaccel_output_format"); - result.Add("cuda"); - } - else - { - result.Add("-hwaccel_output_format"); - result.Add(InputBitDepth(inputFile) == 10 ? "p010le" : "nv12"); - } - - return result; - } } diff --git a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderMpeg4Cuvid.cs b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderMpeg4Cuvid.cs index a2371c6d3..137812eae 100644 --- a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderMpeg4Cuvid.cs +++ b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderMpeg4Cuvid.cs @@ -1,33 +1,12 @@ namespace ErsatzTV.FFmpeg.Decoder.Cuvid; -public class DecoderMpeg4Cuvid : DecoderBase +public class DecoderMpeg4Cuvid : CuvidDecoder { - private readonly FFmpegState _ffmpegState; - - public DecoderMpeg4Cuvid(FFmpegState ffmpegState) => _ffmpegState = ffmpegState; + public DecoderMpeg4Cuvid(HardwareAccelerationMode hardwareAccelerationMode) : base(hardwareAccelerationMode) + { + } public override string Name => "mpeg4_cuvid"; - protected override FrameDataLocation OutputFrameDataLocation => - _ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.None - ? FrameDataLocation.Software - : FrameDataLocation.Hardware; - public override IList InputOptions(InputFile inputFile) - { - IList result = base.InputOptions(inputFile); - - if (_ffmpegState.EncoderHardwareAccelerationMode != HardwareAccelerationMode.None) - { - result.Add("-hwaccel_output_format"); - result.Add("cuda"); - } - else - { - result.Add("-hwaccel_output_format"); - result.Add(InputBitDepth(inputFile) == 10 ? "p010le" : "nv12"); - } - - return result; - } } diff --git a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderVc1Cuvid.cs b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderVc1Cuvid.cs index c671ef7f9..50fc9a71b 100644 --- a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderVc1Cuvid.cs +++ b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderVc1Cuvid.cs @@ -1,25 +1,10 @@ namespace ErsatzTV.FFmpeg.Decoder.Cuvid; -public class DecoderVc1Cuvid : DecoderBase +public class DecoderVc1Cuvid : CuvidDecoder { - private readonly FFmpegState _ffmpegState; - - public DecoderVc1Cuvid(FFmpegState ffmpegState) => _ffmpegState = ffmpegState; - - public override string Name => "vc1_cuvid"; - - protected override FrameDataLocation OutputFrameDataLocation => - _ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.None - ? FrameDataLocation.Software - : FrameDataLocation.Hardware; - - public override IList InputOptions(InputFile inputFile) + public DecoderVc1Cuvid(HardwareAccelerationMode hardwareAccelerationMode) : base(hardwareAccelerationMode) { - IList result = base.InputOptions(inputFile); - - result.Add("-hwaccel_output_format"); - result.Add(_ffmpegState.EncoderHardwareAccelerationMode != HardwareAccelerationMode.None ? "cuda" : "nv12"); - - return result; } + + public override string Name => "vc1_cuvid"; } diff --git a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderVp9Cuvid.cs b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderVp9Cuvid.cs index da7a4ad1d..d6f59d4eb 100644 --- a/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderVp9Cuvid.cs +++ b/ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderVp9Cuvid.cs @@ -1,25 +1,10 @@ namespace ErsatzTV.FFmpeg.Decoder.Cuvid; -public class DecoderVp9Cuvid : DecoderBase +public class DecoderVp9Cuvid : CuvidDecoder { - private readonly FFmpegState _ffmpegState; - - public DecoderVp9Cuvid(FFmpegState ffmpegState) => _ffmpegState = ffmpegState; - - public override string Name => "vp9_cuvid"; - - protected override FrameDataLocation OutputFrameDataLocation => - _ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.None - ? FrameDataLocation.Software - : FrameDataLocation.Hardware; - - public override IList InputOptions(InputFile inputFile) + public DecoderVp9Cuvid(HardwareAccelerationMode hardwareAccelerationMode) : base(hardwareAccelerationMode) { - IList result = base.InputOptions(inputFile); - - result.Add("-hwaccel_output_format"); - result.Add(_ffmpegState.EncoderHardwareAccelerationMode != HardwareAccelerationMode.None ? "cuda" : "nv12"); - - return result; } + + public override string Name => "vp9_cuvid"; } diff --git a/ErsatzTV.FFmpeg/Decoder/DecoderVaapi.cs b/ErsatzTV.FFmpeg/Decoder/DecoderVaapi.cs index 2d22fcd65..1aec3a217 100644 --- a/ErsatzTV.FFmpeg/Decoder/DecoderVaapi.cs +++ b/ErsatzTV.FFmpeg/Decoder/DecoderVaapi.cs @@ -4,7 +4,7 @@ namespace ErsatzTV.FFmpeg.Decoder; public class DecoderVaapi : DecoderBase { - protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Software; + protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Hardware; public override string Name => "implicit_vaapi"; diff --git a/ErsatzTV.FFmpeg/Encoder/AvailableEncoders.cs b/ErsatzTV.FFmpeg/Encoder/AvailableEncoders.cs index 74fd41024..6146d667a 100644 --- a/ErsatzTV.FFmpeg/Encoder/AvailableEncoders.cs +++ b/ErsatzTV.FFmpeg/Encoder/AvailableEncoders.cs @@ -25,53 +25,29 @@ public static class AvailableEncoders (HardwareAccelerationMode.Nvenc, VideoFormat.Hevc) when hardwareCapabilities.CanEncode( VideoFormat.Hevc, desiredState.PixelFormat) => - new EncoderHevcNvenc( - currentState, - maybeWatermarkInputFile, - maybeSubtitleInputFile), + new EncoderHevcNvenc(), (HardwareAccelerationMode.Nvenc, VideoFormat.H264) when hardwareCapabilities.CanEncode( VideoFormat.H264, desiredState.PixelFormat) => - new EncoderH264Nvenc( - currentState, - maybeWatermarkInputFile, - maybeSubtitleInputFile), + new EncoderH264Nvenc(), (HardwareAccelerationMode.Qsv, VideoFormat.Hevc) when hardwareCapabilities.CanEncode( VideoFormat.Hevc, - desiredState.PixelFormat) => - new EncoderHevcQsv( - currentState, - maybeWatermarkInputFile, - maybeSubtitleInputFile, - ffmpegState.QsvExtraHardwareFrames), + desiredState.PixelFormat) => new EncoderHevcQsv(), (HardwareAccelerationMode.Qsv, VideoFormat.H264) when hardwareCapabilities.CanEncode( VideoFormat.H264, - desiredState.PixelFormat) => - new EncoderH264Qsv( - currentState, - maybeWatermarkInputFile, - maybeSubtitleInputFile, - ffmpegState.QsvExtraHardwareFrames), + desiredState.PixelFormat) => new EncoderH264Qsv(), (HardwareAccelerationMode.Vaapi, VideoFormat.Hevc) when hardwareCapabilities.CanEncode( VideoFormat.Hevc, - desiredState.PixelFormat) => - new EncoderHevcVaapi( - currentState, - maybeWatermarkInputFile, - maybeSubtitleInputFile), + desiredState.PixelFormat) => new EncoderHevcVaapi(), (HardwareAccelerationMode.Vaapi, VideoFormat.H264) when hardwareCapabilities.CanEncode( VideoFormat.H264, - desiredState.PixelFormat) => - new EncoderH264Vaapi( - currentState, - maybeWatermarkInputFile, - maybeSubtitleInputFile), + desiredState.PixelFormat) => new EncoderH264Vaapi(), (HardwareAccelerationMode.VideoToolbox, VideoFormat.Hevc) when hardwareCapabilities.CanEncode( VideoFormat.Hevc, - desiredState.PixelFormat) => new EncoderHevcVideoToolbox(), + desiredState.PixelFormat) => new EncoderHevcVideoToolbox(desiredState.BitDepth), (HardwareAccelerationMode.VideoToolbox, VideoFormat.H264) when hardwareCapabilities.CanEncode( VideoFormat.H264, desiredState.PixelFormat) => new EncoderH264VideoToolbox(), diff --git a/ErsatzTV.FFmpeg/Encoder/Nvenc/EncoderH264Nvenc.cs b/ErsatzTV.FFmpeg/Encoder/Nvenc/EncoderH264Nvenc.cs index 94af8e287..9e10e4d13 100644 --- a/ErsatzTV.FFmpeg/Encoder/Nvenc/EncoderH264Nvenc.cs +++ b/ErsatzTV.FFmpeg/Encoder/Nvenc/EncoderH264Nvenc.cs @@ -4,46 +4,10 @@ namespace ErsatzTV.FFmpeg.Encoder.Nvenc; public class EncoderH264Nvenc : EncoderBase { - private readonly FrameState _currentState; - private readonly Option _maybeSubtitleInputFile; - private readonly Option _maybeWatermarkInputFile; - - public EncoderH264Nvenc( - FrameState currentState, - Option maybeWatermarkInputFile, - Option maybeSubtitleInputFile) - { - _currentState = currentState; - _maybeWatermarkInputFile = maybeWatermarkInputFile; - _maybeSubtitleInputFile = maybeSubtitleInputFile; - } - public override string Name => "h264_nvenc"; public override StreamKind Kind => StreamKind.Video; - - // need to upload if we're still in software and a watermark is used - public override string Filter - { - get - { - // only upload to hw if we need to overlay (watermark or subtitle) - if (_currentState.FrameDataLocation == FrameDataLocation.Software) - { - bool isPictureSubtitle = _maybeSubtitleInputFile.Map(s => s.IsImageBased).IfNone(false); - - if (isPictureSubtitle || _maybeWatermarkInputFile.IsSome) - { - return "hwupload_cuda"; - } - } - - return string.Empty; - } - } - public override FrameState NextState(FrameState currentState) => currentState with { - VideoFormat = VideoFormat.H264, - // FrameDataLocation = FrameDataLocation.Hardware + VideoFormat = VideoFormat.H264 }; } diff --git a/ErsatzTV.FFmpeg/Encoder/Nvenc/EncoderHevcNvenc.cs b/ErsatzTV.FFmpeg/Encoder/Nvenc/EncoderHevcNvenc.cs index a2775e491..83273abbc 100644 --- a/ErsatzTV.FFmpeg/Encoder/Nvenc/EncoderHevcNvenc.cs +++ b/ErsatzTV.FFmpeg/Encoder/Nvenc/EncoderHevcNvenc.cs @@ -4,46 +4,10 @@ namespace ErsatzTV.FFmpeg.Encoder.Nvenc; public class EncoderHevcNvenc : EncoderBase { - private readonly FrameState _currentState; - private readonly Option _maybeSubtitleInputFile; - private readonly Option _maybeWatermarkInputFile; - - public EncoderHevcNvenc( - FrameState currentState, - Option maybeWatermarkInputFile, - Option maybeSubtitleInputFile) - { - _currentState = currentState; - _maybeWatermarkInputFile = maybeWatermarkInputFile; - _maybeSubtitleInputFile = maybeSubtitleInputFile; - } - public override string Name => "hevc_nvenc"; public override StreamKind Kind => StreamKind.Video; - - // need to upload if we're still in software and a watermark is used - public override string Filter - { - get - { - // only upload to hw if we need to overlay (watermark or subtitle) - if (_currentState.FrameDataLocation == FrameDataLocation.Software) - { - bool isPictureSubtitle = _maybeSubtitleInputFile.Map(s => s.IsImageBased).IfNone(false); - - if (isPictureSubtitle || _maybeWatermarkInputFile.IsSome) - { - return "hwupload_cuda"; - } - } - - return string.Empty; - } - } - public override FrameState NextState(FrameState currentState) => currentState with { - VideoFormat = VideoFormat.Hevc, - // FrameDataLocation = FrameDataLocation.Hardware + VideoFormat = VideoFormat.Hevc }; } diff --git a/ErsatzTV.FFmpeg/Encoder/Qsv/EncoderH264Qsv.cs b/ErsatzTV.FFmpeg/Encoder/Qsv/EncoderH264Qsv.cs index c3c4cdbe6..dededd3ff 100644 --- a/ErsatzTV.FFmpeg/Encoder/Qsv/EncoderH264Qsv.cs +++ b/ErsatzTV.FFmpeg/Encoder/Qsv/EncoderH264Qsv.cs @@ -4,56 +4,12 @@ namespace ErsatzTV.FFmpeg.Encoder.Qsv; public class EncoderH264Qsv : EncoderBase { - private readonly FrameState _currentState; - private readonly Option _maybeSubtitleInputFile; - private readonly int _extraHardwareFrames; - private readonly Option _maybeWatermarkInputFile; - - public EncoderH264Qsv( - FrameState currentState, - Option maybeWatermarkInputFile, - Option maybeSubtitleInputFile, - int extraHardwareFrames) - { - _currentState = currentState; - _maybeWatermarkInputFile = maybeWatermarkInputFile; - _maybeSubtitleInputFile = maybeSubtitleInputFile; - _extraHardwareFrames = extraHardwareFrames; - } - public override string Name => "h264_qsv"; public override StreamKind Kind => StreamKind.Video; public override IList OutputOptions => new[] { "-c:v", "h264_qsv", "-low_power", "0", "-look_ahead", "0" }; - // need to upload if we're still in software and a watermark is used - public override string Filter - { - get - { - // only upload to hw if we need to overlay (watermark or subtitle) - if (_currentState.FrameDataLocation == FrameDataLocation.Software) - { - bool isPictureSubtitle = _maybeSubtitleInputFile.Map(s => s.IsImageBased).IfNone(false); - - if (isPictureSubtitle || _maybeWatermarkInputFile.IsSome) - { - // pixel format should already be converted to a supported format by QsvHardwareAccelerationOption - foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) - { - return $"format={pixelFormat.FFmpegName},hwupload=extra_hw_frames={_extraHardwareFrames}"; - } - - // default to nv12 - return $"format=nv12,hwupload=extra_hw_frames={_extraHardwareFrames}"; - } - } - - return string.Empty; - } - } - public override FrameState NextState(FrameState currentState) => currentState with { VideoFormat = VideoFormat.H264, diff --git a/ErsatzTV.FFmpeg/Encoder/Qsv/EncoderHevcQsv.cs b/ErsatzTV.FFmpeg/Encoder/Qsv/EncoderHevcQsv.cs index 74f273076..f3668af0b 100644 --- a/ErsatzTV.FFmpeg/Encoder/Qsv/EncoderHevcQsv.cs +++ b/ErsatzTV.FFmpeg/Encoder/Qsv/EncoderHevcQsv.cs @@ -4,56 +4,12 @@ namespace ErsatzTV.FFmpeg.Encoder.Qsv; public class EncoderHevcQsv : EncoderBase { - private readonly FrameState _currentState; - private readonly Option _maybeSubtitleInputFile; - private readonly int _extraHardwareFrames; - private readonly Option _maybeWatermarkInputFile; - - public EncoderHevcQsv( - FrameState currentState, - Option maybeWatermarkInputFile, - Option maybeSubtitleInputFile, - int extraHardwareFrames) - { - _currentState = currentState; - _maybeWatermarkInputFile = maybeWatermarkInputFile; - _maybeSubtitleInputFile = maybeSubtitleInputFile; - _extraHardwareFrames = extraHardwareFrames; - } - public override string Name => "hevc_qsv"; public override StreamKind Kind => StreamKind.Video; public override IList OutputOptions => new[] { "-c:v", "hevc_qsv", "-low_power", "0", "-look_ahead", "0" }; - // need to upload if we're still in software and a watermark is used - public override string Filter - { - get - { - // only upload to hw if we need to overlay (watermark or subtitle) - if (_currentState.FrameDataLocation == FrameDataLocation.Software) - { - bool isPictureSubtitle = _maybeSubtitleInputFile.Map(s => s.IsImageBased).IfNone(false); - - if (isPictureSubtitle || _maybeWatermarkInputFile.IsSome) - { - // pixel format should already be converted to a supported format by QsvHardwareAccelerationOption - foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) - { - return $"format={pixelFormat.FFmpegName},hwupload=extra_hw_frames={_extraHardwareFrames}"; - } - - // default to nv12 - return $"format=nv12,hwupload=extra_hw_frames={_extraHardwareFrames}"; - } - } - - return string.Empty; - } - } - public override FrameState NextState(FrameState currentState) => currentState with { VideoFormat = VideoFormat.Hevc, diff --git a/ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderH264Vaapi.cs b/ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderH264Vaapi.cs index b6be4e5e6..c22a0f048 100644 --- a/ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderH264Vaapi.cs +++ b/ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderH264Vaapi.cs @@ -4,40 +4,8 @@ namespace ErsatzTV.FFmpeg.Encoder.Vaapi; public class EncoderH264Vaapi : EncoderBase { - private readonly FrameState _currentState; - private readonly Option _maybeSubtitleInputFile; - private readonly Option _maybeWatermarkInputFile; - - public EncoderH264Vaapi( - FrameState currentState, - Option maybeWatermarkInputFile, - Option maybeSubtitleInputFile) - { - _currentState = currentState; - _maybeWatermarkInputFile = maybeWatermarkInputFile; - _maybeSubtitleInputFile = maybeSubtitleInputFile; - } - public override string Name => "h264_vaapi"; public override StreamKind Kind => StreamKind.Video; - - // need to upload if we're still in software unless a watermark or subtitle is used - public override string Filter - { - get - { - if (_currentState.FrameDataLocation == FrameDataLocation.Software) - { - if (_maybeWatermarkInputFile.IsNone && _maybeSubtitleInputFile.IsNone) - { - return "format=nv12|vaapi,hwupload"; - } - } - - return string.Empty; - } - } - public override FrameState NextState(FrameState currentState) => currentState with { VideoFormat = VideoFormat.H264 diff --git a/ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderHevcVaapi.cs b/ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderHevcVaapi.cs index ec0501b27..7d1a2c499 100644 --- a/ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderHevcVaapi.cs +++ b/ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderHevcVaapi.cs @@ -4,40 +4,8 @@ namespace ErsatzTV.FFmpeg.Encoder.Vaapi; public class EncoderHevcVaapi : EncoderBase { - private readonly FrameState _currentState; - private readonly Option _maybeSubtitleInputFile; - private readonly Option _maybeWatermarkInputFile; - - public EncoderHevcVaapi( - FrameState currentState, - Option maybeWatermarkInputFile, - Option maybeSubtitleInputFile) - { - _currentState = currentState; - _maybeWatermarkInputFile = maybeWatermarkInputFile; - _maybeSubtitleInputFile = maybeSubtitleInputFile; - } - public override string Name => "hevc_vaapi"; public override StreamKind Kind => StreamKind.Video; - - // need to upload if we're still in software unless a watermark or subtitle is used - public override string Filter - { - get - { - if (_currentState.FrameDataLocation == FrameDataLocation.Software) - { - if (_maybeWatermarkInputFile.IsNone && _maybeSubtitleInputFile.IsNone) - { - return "format=nv12|vaapi,hwupload"; - } - } - - return string.Empty; - } - } - public override FrameState NextState(FrameState currentState) => currentState with { VideoFormat = VideoFormat.Hevc diff --git a/ErsatzTV.FFmpeg/Encoder/VideoToolbox/EncoderHevcVideoToolbox.cs b/ErsatzTV.FFmpeg/Encoder/VideoToolbox/EncoderHevcVideoToolbox.cs index 2d99f98ce..dec84c553 100644 --- a/ErsatzTV.FFmpeg/Encoder/VideoToolbox/EncoderHevcVideoToolbox.cs +++ b/ErsatzTV.FFmpeg/Encoder/VideoToolbox/EncoderHevcVideoToolbox.cs @@ -4,9 +4,20 @@ namespace ErsatzTV.FFmpeg.Encoder.VideoToolbox; public class EncoderHevcVideoToolbox : EncoderBase { + private readonly int _desiredBitDepth; + + public EncoderHevcVideoToolbox(int desiredBitDepth) => _desiredBitDepth = desiredBitDepth; + public override string Name => "hevc_videotoolbox"; public override StreamKind Kind => StreamKind.Video; + public override IList OutputOptions => base.OutputOptions.Concat( + new List + { + "-profile:v", + _desiredBitDepth == 10 ? "main10" : "main" + }).ToList(); + public override FrameState NextState(FrameState currentState) => currentState with { VideoFormat = VideoFormat.Hevc, diff --git a/ErsatzTV.FFmpeg/Filter/AvailableSubtitleOverlayFilters.cs b/ErsatzTV.FFmpeg/Filter/AvailableSubtitleOverlayFilters.cs index 5ac31f5bc..e92255331 100644 --- a/ErsatzTV.FFmpeg/Filter/AvailableSubtitleOverlayFilters.cs +++ b/ErsatzTV.FFmpeg/Filter/AvailableSubtitleOverlayFilters.cs @@ -1,5 +1,6 @@ using ErsatzTV.FFmpeg.Filter.Cuda; using ErsatzTV.FFmpeg.Filter.Qsv; +using ErsatzTV.FFmpeg.Format; namespace ErsatzTV.FFmpeg.Filter; @@ -10,6 +11,6 @@ public static class AvailableSubtitleOverlayFilters { HardwareAccelerationMode.Nvenc => new OverlaySubtitleCudaFilter(), HardwareAccelerationMode.Qsv => new OverlaySubtitleQsvFilter(), - _ => new OverlaySubtitleFilter() + _ => new OverlaySubtitleFilter(new PixelFormatYuv420P()) }; } diff --git a/ErsatzTV.FFmpeg/Filter/AvailableWatermarkOverlayFilters.cs b/ErsatzTV.FFmpeg/Filter/AvailableWatermarkOverlayFilters.cs index a2bb71bb3..cba1dffc4 100644 --- a/ErsatzTV.FFmpeg/Filter/AvailableWatermarkOverlayFilters.cs +++ b/ErsatzTV.FFmpeg/Filter/AvailableWatermarkOverlayFilters.cs @@ -1,5 +1,6 @@ using ErsatzTV.FFmpeg.Filter.Cuda; using ErsatzTV.FFmpeg.Filter.Qsv; +using ErsatzTV.FFmpeg.Format; using ErsatzTV.FFmpeg.State; using Microsoft.Extensions.Logging; @@ -19,6 +20,11 @@ public static class AvailableWatermarkOverlayFilters new OverlayWatermarkCudaFilter(watermarkState, resolution, squarePixelFrameSize, logger), HardwareAccelerationMode.Qsv => new OverlayWatermarkQsvFilter(watermarkState, resolution, squarePixelFrameSize, logger), - _ => new OverlayWatermarkFilter(watermarkState, resolution, squarePixelFrameSize, logger) + _ => new OverlayWatermarkFilter( + watermarkState, + resolution, + squarePixelFrameSize, + new PixelFormatYuv420P(), + logger) }; } diff --git a/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs b/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs index a69e27441..07b828327 100644 --- a/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs @@ -6,19 +6,29 @@ public class ColorspaceFilter : BaseFilter { private readonly VideoStream _videoStream; private readonly IPixelFormat _desiredPixelFormat; + private readonly FrameDataLocation _nextDataLocation; - public ColorspaceFilter(VideoStream videoStream, IPixelFormat desiredPixelFormat) + public ColorspaceFilter( + VideoStream videoStream, + IPixelFormat desiredPixelFormat, + FrameDataLocation nextDataLocation = FrameDataLocation.Software) { _videoStream = videoStream; _desiredPixelFormat = desiredPixelFormat; + _nextDataLocation = nextDataLocation; } - public override FrameState NextState(FrameState currentState) => - currentState with + public override FrameState NextState(FrameState currentState) + { + FrameState nextState = currentState with { FrameDataLocation = _nextDataLocation }; + + if (!_videoStream.ColorParams.IsUnknown && _desiredPixelFormat.BitDepth is 10 or 8) { - PixelFormat = Some(_desiredPixelFormat), - FrameDataLocation = FrameDataLocation.Software - }; + nextState = nextState with { PixelFormat = Some(_desiredPixelFormat) }; + } + + return nextState; + } public override string Filter { @@ -35,8 +45,8 @@ public class ColorspaceFilter : BaseFilter string colorspace = _desiredPixelFormat.BitDepth switch { _ when cp.IsUnknown => "setparams=range=tv:colorspace=bt709:color_trc=bt709:color_primaries=bt709", - 10 when !cp.IsUnknown => $"colorspace={inputOverrides}all=bt709:format=yuv420p10", - 8 when !cp.IsUnknown => $"colorspace={inputOverrides}all=bt709:format=yuv420p", + 10 or 8 when !cp.IsUnknown => + $"colorspace={inputOverrides}all=bt709:format={_desiredPixelFormat.FFmpegName}", _ => string.Empty }; diff --git a/ErsatzTV.FFmpeg/Filter/ComplexFilter.cs b/ErsatzTV.FFmpeg/Filter/ComplexFilter.cs index 0aa7c0be5..cd607ff34 100644 --- a/ErsatzTV.FFmpeg/Filter/ComplexFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/ComplexFilter.cs @@ -17,6 +17,8 @@ public class ComplexFilter : IPipelineStep private readonly Option _maybeWatermarkInputFile; private readonly FrameSize _resolution; private readonly List _outputOptions; + private readonly List _pipelineSteps; + private readonly IList _arguments; public ComplexFilter( FrameState currentState, @@ -42,16 +44,21 @@ public class ComplexFilter : IPipelineStep _logger = logger; _outputOptions = new List(); + _pipelineSteps = new List(); + + _arguments = Arguments(); } public IList EnvironmentVariables => Array.Empty(); public IList GlobalOptions => Array.Empty(); public IList InputOptions(InputFile inputFile) => Array.Empty(); - public IList FilterOptions => Arguments(); + public IList FilterOptions => _arguments; public IList OutputOptions => _outputOptions; + public IList PipelineSteps => _pipelineSteps; + public FrameState NextState(FrameState currentState) => currentState; - private IList Arguments() + private List Arguments() { var state = _currentState; @@ -172,6 +179,8 @@ public class ComplexFilter : IPipelineStep if (overlayFilter.Filter != string.Empty) { + _pipelineSteps.Add(overlayFilter); + state = overlayFilter.NextState(state); string tempVideoLabel = string.IsNullOrWhiteSpace(videoFilterComplex) @@ -188,7 +197,9 @@ public class ComplexFilter : IPipelineStep HardwareAccelerationMode.VideoToolbox && state.VideoFormat == VideoFormat.Hevc)) { - uploadDownloadFilter = new HardwareUploadFilter(_ffmpegState).Filter; + var hardwareUpload = new HardwareUploadFilter(_ffmpegState); + _pipelineSteps.Add(hardwareUpload); + uploadDownloadFilter = hardwareUpload.Filter; state = state with { FrameDataLocation = FrameDataLocation.Hardware }; } @@ -197,7 +208,9 @@ public class ComplexFilter : IPipelineStep _ffmpegState.EncoderHardwareAccelerationMode != HardwareAccelerationMode.VideoToolbox && _ffmpegState.EncoderHardwareAccelerationMode != HardwareAccelerationMode.Amf) { - uploadDownloadFilter = new HardwareDownloadFilter(state).Filter; + var hardwareDownload = new HardwareDownloadFilter(state); + _pipelineSteps.Add(hardwareDownload); + uploadDownloadFilter = hardwareDownload.Filter; state = state with { FrameDataLocation = FrameDataLocation.Software }; } @@ -241,6 +254,7 @@ public class ComplexFilter : IPipelineStep { IPipelineFilterStep overlayFilter = AvailableSubtitleOverlayFilters.ForAcceleration( _ffmpegState.EncoderHardwareAccelerationMode); + _pipelineSteps.Add(overlayFilter); state = overlayFilter.NextState(state); filter = overlayFilter.Filter; } @@ -248,6 +262,7 @@ public class ComplexFilter : IPipelineStep { subtitleLabel = string.Empty; var subtitlesFilter = new SubtitlesFilter(_fontsDir, subtitleInputFile); + _pipelineSteps.Add(subtitlesFilter); state = subtitlesFilter.NextState(state); filter = subtitlesFilter.Filter; } @@ -265,7 +280,9 @@ public class ComplexFilter : IPipelineStep || _ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.VideoToolbox && state.VideoFormat == VideoFormat.Hevc) { - uploadFilter = new HardwareUploadFilter(_ffmpegState).Filter; + var hardwareUpload = new HardwareUploadFilter(_ffmpegState); + _pipelineSteps.Add(hardwareUpload); + uploadFilter = hardwareUpload.Filter; if (!string.IsNullOrWhiteSpace(uploadFilter)) { state = state with { FrameDataLocation = FrameDataLocation.Hardware }; @@ -300,7 +317,9 @@ public class ComplexFilter : IPipelineStep if (!videoStream.ColorParams.IsBt709) { _logger.LogDebug("Adding colorspace filter"); - filter = new ColorspaceFilter(videoStream, pixelFormat).Filter; + var colorspace = new ColorspaceFilter(videoStream, pixelFormat); + _pipelineSteps.Add(colorspace); + filter = colorspace.Filter; } if (state.PixelFormat.Map(f => f.FFmpegName) != pixelFormat.FFmpegName) diff --git a/ErsatzTV.FFmpeg/Filter/Cuda/CudaFormatFilter.cs b/ErsatzTV.FFmpeg/Filter/Cuda/CudaFormatFilter.cs new file mode 100644 index 000000000..97f890243 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Cuda/CudaFormatFilter.cs @@ -0,0 +1,14 @@ +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter.Cuda; + +public class CudaFormatFilter : BaseFilter +{ + private readonly IPixelFormat _pixelFormat; + + public CudaFormatFilter(IPixelFormat pixelFormat) => _pixelFormat = pixelFormat; + + public override FrameState NextState(FrameState currentState) => currentState with { PixelFormat = Some(_pixelFormat) }; + + public override string Filter => $"scale_cuda=format={_pixelFormat.FFmpegName}"; +} diff --git a/ErsatzTV.FFmpeg/Filter/Cuda/CudaHardwareDownloadFilter.cs b/ErsatzTV.FFmpeg/Filter/Cuda/CudaHardwareDownloadFilter.cs new file mode 100644 index 000000000..ce85c9a96 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Cuda/CudaHardwareDownloadFilter.cs @@ -0,0 +1,57 @@ +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter.Cuda; + +public class CudaHardwareDownloadFilter : BaseFilter +{ + private readonly Option _maybePixelFormat; + + public CudaHardwareDownloadFilter(Option maybePixelFormat) => _maybePixelFormat = maybePixelFormat; + + public override string Filter + { + get + { + var hwdownload = "hwdownload"; + foreach (IPixelFormat pixelFormat in _maybePixelFormat) + { + if (!string.IsNullOrWhiteSpace(pixelFormat.FFmpegName)) + { + hwdownload += $",format={pixelFormat.FFmpegName}"; + + if (pixelFormat is PixelFormatNv12 nv12) + { + foreach (IPixelFormat pf in AvailablePixelFormats.ForPixelFormat(nv12.Name, null)) + { + hwdownload += $",format={pf.FFmpegName}"; + } + } + } + } + + return hwdownload; + } + } + + public override FrameState NextState(FrameState currentState) + { + FrameState result = currentState with + { + FrameDataLocation = FrameDataLocation.Software + }; + + foreach (IPixelFormat pixelFormat in _maybePixelFormat) + { + if (pixelFormat is PixelFormatNv12 nv12) + { + result = result with { PixelFormat = AvailablePixelFormats.ForPixelFormat(nv12.Name, null) }; + } + else + { + result = result with { PixelFormat = Some(pixelFormat) }; + } + } + + return result; + } +} diff --git a/ErsatzTV.FFmpeg/Filter/Cuda/OverlayWatermarkCudaFilter.cs b/ErsatzTV.FFmpeg/Filter/Cuda/OverlayWatermarkCudaFilter.cs index c902f1d0c..372f74cdc 100644 --- a/ErsatzTV.FFmpeg/Filter/Cuda/OverlayWatermarkCudaFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Cuda/OverlayWatermarkCudaFilter.cs @@ -1,4 +1,5 @@ -using ErsatzTV.FFmpeg.State; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.State; using Microsoft.Extensions.Logging; namespace ErsatzTV.FFmpeg.Filter.Cuda; @@ -13,6 +14,7 @@ public class OverlayWatermarkCudaFilter : OverlayWatermarkFilter watermarkState, resolution, squarePixelFrameSize, + new PixelFormatUnknown(), logger) { } diff --git a/ErsatzTV.FFmpeg/Filter/Cuda/ScaleCudaFilter.cs b/ErsatzTV.FFmpeg/Filter/Cuda/ScaleCudaFilter.cs index eb611d468..7c499841b 100644 --- a/ErsatzTV.FFmpeg/Filter/Cuda/ScaleCudaFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Cuda/ScaleCudaFilter.cs @@ -78,11 +78,21 @@ public class ScaleCudaFilter : BaseFilter } } - public override FrameState NextState(FrameState currentState) => currentState with + public override FrameState NextState(FrameState currentState) { - ScaledSize = _scaledSize, - PaddedSize = _scaledSize, - FrameDataLocation = FrameDataLocation.Hardware, - IsAnamorphic = false // this filter always outputs square pixels - }; + FrameState result = currentState with + { + ScaledSize = _scaledSize, + PaddedSize = _scaledSize, + FrameDataLocation = FrameDataLocation.Hardware, + IsAnamorphic = false // this filter always outputs square pixels + }; + + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + result = result with { PixelFormat = Some(pixelFormat) }; + } + + return result; + } } diff --git a/ErsatzTV.FFmpeg/Filter/HardwareUploadCudaFilter.cs b/ErsatzTV.FFmpeg/Filter/HardwareUploadCudaFilter.cs new file mode 100644 index 000000000..390dc2845 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/HardwareUploadCudaFilter.cs @@ -0,0 +1,20 @@ +namespace ErsatzTV.FFmpeg.Filter; + +public class HardwareUploadCudaFilter : BaseFilter +{ + private readonly FrameState _currentState; + + public HardwareUploadCudaFilter(FrameState currentState) + { + _currentState = currentState; + } + + public override string Filter => _currentState.FrameDataLocation switch + { + FrameDataLocation.Hardware => string.Empty, + _ => "hwupload_cuda" + }; + + public override FrameState NextState(FrameState currentState) => + currentState with { FrameDataLocation = FrameDataLocation.Hardware }; +} diff --git a/ErsatzTV.FFmpeg/Filter/HardwareUploadFilter.cs b/ErsatzTV.FFmpeg/Filter/HardwareUploadFilter.cs index 65ba13b08..c01275238 100644 --- a/ErsatzTV.FFmpeg/Filter/HardwareUploadFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/HardwareUploadFilter.cs @@ -14,7 +14,7 @@ public class HardwareUploadFilter : BaseFilter HardwareAccelerationMode.None => string.Empty, HardwareAccelerationMode.Nvenc => "hwupload_cuda", HardwareAccelerationMode.Qsv => $"hwupload=extra_hw_frames={_ffmpegState.QsvExtraHardwareFrames}", - HardwareAccelerationMode.Vaapi => "format=nv12|vaapi,hwupload", + HardwareAccelerationMode.Vaapi => "format=nv12|p010le|vaapi,hwupload", _ => "hwupload" }; diff --git a/ErsatzTV.FFmpeg/Filter/NewComplexFilter.cs b/ErsatzTV.FFmpeg/Filter/NewComplexFilter.cs new file mode 100644 index 000000000..9b02b0efb --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/NewComplexFilter.cs @@ -0,0 +1,252 @@ +using ErsatzTV.FFmpeg.Environment; +using ErsatzTV.FFmpeg.Pipeline; + +namespace ErsatzTV.FFmpeg.Filter; + +public class NewComplexFilter : IPipelineStep +{ + private readonly Option _maybeAudioInputFile; + private readonly Option _maybeSubtitleInputFile; + private readonly PipelineContext _pipelineContext; + private readonly FilterChain _filterChain; + private readonly Option _maybeVideoInputFile; + private readonly Option _maybeWatermarkInputFile; + private readonly List _outputOptions; + private readonly IList _arguments; + + public NewComplexFilter( + Option maybeVideoInputFile, + Option maybeAudioInputFile, + Option maybeWatermarkInputFile, + Option maybeSubtitleInputFile, + PipelineContext pipelineContext, + FilterChain filterChain) + { + _maybeVideoInputFile = maybeVideoInputFile; + _maybeAudioInputFile = maybeAudioInputFile; + _maybeWatermarkInputFile = maybeWatermarkInputFile; + _maybeSubtitleInputFile = maybeSubtitleInputFile; + _pipelineContext = pipelineContext; + _filterChain = filterChain; + + _outputOptions = new List(); + + _arguments = Arguments(); + } + + public IList EnvironmentVariables => Array.Empty(); + public IList GlobalOptions => Array.Empty(); + public IList InputOptions(InputFile inputFile) => Array.Empty(); + public IList FilterOptions => _arguments; + public IList OutputOptions => _outputOptions; + + public FrameState NextState(FrameState currentState) => currentState; + + // for testing + public FilterChain FilterChain => _filterChain; + + private List Arguments() + { + var audioLabel = "0:a"; + var videoLabel = "0:v"; + string? watermarkLabel = null; + string? subtitleLabel = null; + + var result = new List(); + + string audioFilterComplex = string.Empty; + string videoFilterComplex = string.Empty; + string watermarkFilterComplex = string.Empty; + string watermarkOverlayFilterComplex = string.Empty; + string subtitleFilterComplex = string.Empty; + string subtitleOverlayFilterComplex = string.Empty; + string pixelFormatFilterComplex = string.Empty; + + var distinctPaths = new List(); + foreach ((string path, _) in _maybeVideoInputFile) + { + if (!distinctPaths.Contains(path)) + { + distinctPaths.Add(path); + } + } + + foreach ((string path, _) in _maybeAudioInputFile) + { + if (!distinctPaths.Contains(path) || + // use audio as a separate input with vaapi/qsv + _pipelineContext.HardwareAccelerationMode is HardwareAccelerationMode.Vaapi + or HardwareAccelerationMode.Qsv) + { + distinctPaths.Add(path); + } + } + + foreach ((string path, _) in _maybeWatermarkInputFile) + { + if (!distinctPaths.Contains(path)) + { + distinctPaths.Add(path); + } + } + + foreach ((string path, _) in _maybeSubtitleInputFile) + { + if (!distinctPaths.Contains(path)) + { + distinctPaths.Add(path); + } + } + + foreach (VideoInputFile videoInputFile in _maybeVideoInputFile) + { + int inputIndex = distinctPaths.IndexOf(videoInputFile.Path); + foreach ((int index, _, _) in videoInputFile.Streams) + { + videoLabel = $"{inputIndex}:{index}"; + if (_filterChain.VideoFilterSteps.Any(f => !string.IsNullOrWhiteSpace(f.Filter))) + { + videoFilterComplex += $"[{inputIndex}:{index}]"; + videoFilterComplex += string.Join( + ",", + _filterChain.VideoFilterSteps.Select(f => f.Filter).Filter(s => !string.IsNullOrWhiteSpace(s))); + videoLabel = "[v]"; + videoFilterComplex += videoLabel; + } + } + } + + foreach (WatermarkInputFile watermarkInputFile in _maybeWatermarkInputFile) + { + int inputIndex = distinctPaths.IndexOf(watermarkInputFile.Path); + foreach ((int index, _, _) in watermarkInputFile.Streams) + { + watermarkLabel = $"{inputIndex}:{index}"; + if (_filterChain.WatermarkFilterSteps.Any(f => !string.IsNullOrWhiteSpace(f.Filter))) + { + watermarkFilterComplex += $"[{inputIndex}:{index}]"; + watermarkFilterComplex += string.Join( + ",", + _filterChain.WatermarkFilterSteps.Select(f => f.Filter) + .Filter(s => !string.IsNullOrWhiteSpace(s))); + watermarkLabel = "[wm]"; + watermarkFilterComplex += watermarkLabel; + } + else + { + watermarkLabel = $"[{watermarkLabel}]"; + } + } + } + + foreach (SubtitleInputFile subtitleInputFile in _maybeSubtitleInputFile.Filter(s => !s.Copy)) + { + int inputIndex = distinctPaths.IndexOf(subtitleInputFile.Path); + foreach ((int index, _, _) in subtitleInputFile.Streams) + { + subtitleLabel = $"{inputIndex}:{index}"; + if (_filterChain.SubtitleFilterSteps.Any(f => !string.IsNullOrWhiteSpace(f.Filter))) + { + subtitleFilterComplex += $"[{inputIndex}:{index}]"; + subtitleFilterComplex += string.Join( + ",", + _filterChain.SubtitleFilterSteps.Select(f => f.Filter).Filter(s => !string.IsNullOrWhiteSpace(s))); + subtitleLabel = "[st]"; + subtitleFilterComplex += subtitleLabel; + } + else + { + subtitleLabel = $"[{subtitleLabel}]"; + } + } + } + + // overlay subtitle + if (!string.IsNullOrWhiteSpace(subtitleLabel) && _filterChain.SubtitleOverlayFilterSteps.Any()) + { + subtitleOverlayFilterComplex += $"{ProperLabel(videoLabel)}{ProperLabel(subtitleLabel)}"; + subtitleOverlayFilterComplex += string.Join( + ",", + _filterChain.SubtitleOverlayFilterSteps.Select(f => f.Filter) + .Filter(s => !string.IsNullOrWhiteSpace(s))); + videoLabel = "[vst]"; + subtitleOverlayFilterComplex += videoLabel; + } + + // overlay watermark + if (!string.IsNullOrWhiteSpace(watermarkLabel) && _filterChain.WatermarkOverlayFilterSteps.Any()) + { + watermarkOverlayFilterComplex += $"{ProperLabel(videoLabel)}{ProperLabel(watermarkLabel)}"; + watermarkOverlayFilterComplex += string.Join( + ",", + _filterChain.WatermarkOverlayFilterSteps.Select(f => f.Filter) + .Filter(s => !string.IsNullOrWhiteSpace(s))); + videoLabel = "[vwm]"; + watermarkOverlayFilterComplex += videoLabel; + } + + // pixel format + if (_filterChain.PixelFormatFilterSteps.Any()) + { + pixelFormatFilterComplex += $"{ProperLabel(videoLabel)}"; + pixelFormatFilterComplex += string.Join( + ",", + _filterChain.PixelFormatFilterSteps.Select(f => f.Filter) + .Filter(s => !string.IsNullOrWhiteSpace(s))); + videoLabel = "[vpf]"; + pixelFormatFilterComplex += videoLabel; + } + + foreach (AudioInputFile audioInputFile in _maybeAudioInputFile) + { + int inputIndex = distinctPaths.LastIndexOf(audioInputFile.Path); + foreach ((int index, _, _) in audioInputFile.Streams) + { + audioLabel = $"{inputIndex}:{index}"; + if (audioInputFile.FilterSteps.Any(f => !string.IsNullOrWhiteSpace(f.Filter))) + { + audioFilterComplex += $"[{inputIndex}:{index}]"; + audioFilterComplex += string.Join( + ",", + audioInputFile.FilterSteps.Select(f => f.Filter).Filter(s => !string.IsNullOrWhiteSpace(s))); + audioLabel = "[a]"; + audioFilterComplex += audioLabel; + } + } + } + + var filterComplex = string.Join( + ";", + new[] + { + audioFilterComplex, + videoFilterComplex, + subtitleFilterComplex, + watermarkFilterComplex, + subtitleOverlayFilterComplex, + watermarkOverlayFilterComplex, + pixelFormatFilterComplex + }.Where(s => !string.IsNullOrWhiteSpace(s))); + + if (!string.IsNullOrWhiteSpace(filterComplex)) + { + result.AddRange(new[] { "-filter_complex", filterComplex }); + } + + result.AddRange(new[] { "-map", audioLabel, "-map", videoLabel }); + + foreach (SubtitleInputFile subtitleInputFile in _maybeSubtitleInputFile.Filter(s => s.Copy)) + { + int inputIndex = distinctPaths.IndexOf(subtitleInputFile.Path); + foreach ((int index, _, _) in subtitleInputFile.Streams) + { + subtitleLabel = $"{inputIndex}:{index}"; + result.AddRange(new[] { "-map", subtitleLabel }); + } + } + + return result; + } + + private string ProperLabel(string label) => label.StartsWith("[") ? label : $"[{label}]"; +} diff --git a/ErsatzTV.FFmpeg/Filter/OverlaySubtitleFilter.cs b/ErsatzTV.FFmpeg/Filter/OverlaySubtitleFilter.cs index f8a925473..682babf99 100644 --- a/ErsatzTV.FFmpeg/Filter/OverlaySubtitleFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/OverlaySubtitleFilter.cs @@ -1,7 +1,13 @@ -namespace ErsatzTV.FFmpeg.Filter; +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter; public class OverlaySubtitleFilter : BaseFilter { - public override string Filter => "overlay=x=(W-w)/2:y=(H-h)/2"; + private readonly IPixelFormat _outputPixelFormat; + + public OverlaySubtitleFilter(IPixelFormat outputPixelFormat) => _outputPixelFormat = outputPixelFormat; + + public override string Filter => $"overlay=x=(W-w)/2:y=(H-h)/2:format={(_outputPixelFormat.BitDepth == 10 ? '1' : '0')}"; public override FrameState NextState(FrameState currentState) => currentState; } diff --git a/ErsatzTV.FFmpeg/Filter/OverlayWatermarkFilter.cs b/ErsatzTV.FFmpeg/Filter/OverlayWatermarkFilter.cs index fe1652331..07e481d10 100644 --- a/ErsatzTV.FFmpeg/Filter/OverlayWatermarkFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/OverlayWatermarkFilter.cs @@ -1,4 +1,5 @@ -using ErsatzTV.FFmpeg.State; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.State; using Microsoft.Extensions.Logging; namespace ErsatzTV.FFmpeg.Filter; @@ -7,6 +8,7 @@ public class OverlayWatermarkFilter : BaseFilter { private readonly FrameSize _resolution; private readonly FrameSize _squarePixelFrameSize; + private readonly IPixelFormat _outputPixelFormat; private readonly ILogger _logger; private readonly WatermarkState _watermarkState; @@ -14,15 +16,17 @@ public class OverlayWatermarkFilter : BaseFilter WatermarkState watermarkState, FrameSize resolution, FrameSize squarePixelFrameSize, + IPixelFormat outputPixelFormat, ILogger logger) { _watermarkState = watermarkState; _resolution = resolution; _squarePixelFrameSize = squarePixelFrameSize; + _outputPixelFormat = outputPixelFormat; _logger = logger; } - public override string Filter => $"overlay={Position}"; + public override string Filter => $"overlay={Position}:format={(_outputPixelFormat.BitDepth == 10 ? '1' : '0')}"; protected string Position { @@ -34,6 +38,7 @@ public class OverlayWatermarkFilter : BaseFilter return _watermarkState.Location switch { + // TODO: can these be pre-calculated (and used with accelerated overlay filters) WatermarkLocation.BottomLeft => $"x={horizontalMargin}:y=H-h-{verticalMargin}", WatermarkLocation.TopLeft => $"x={horizontalMargin}:y={verticalMargin}", WatermarkLocation.TopRight => $"x=W-w-{horizontalMargin}:y={verticalMargin}", diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/HardwareUploadQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/HardwareUploadQsvFilter.cs new file mode 100644 index 000000000..7f5647a98 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Qsv/HardwareUploadQsvFilter.cs @@ -0,0 +1,22 @@ +namespace ErsatzTV.FFmpeg.Filter.Qsv; + +public class HardwareUploadQsvFilter : BaseFilter +{ + private readonly FrameState _currentState; + private readonly FFmpegState _ffmpegState; + + public HardwareUploadQsvFilter(FrameState currentState, FFmpegState ffmpegState) + { + _currentState = currentState; + _ffmpegState = ffmpegState; + } + + public override string Filter => _currentState.FrameDataLocation switch + { + FrameDataLocation.Hardware => string.Empty, + _ => $"hwupload=extra_hw_frames={_ffmpegState.QsvExtraHardwareFrames}", + }; + + public override FrameState NextState(FrameState currentState) => + currentState with { FrameDataLocation = FrameDataLocation.Hardware }; +} diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/OverlayWatermarkQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/OverlayWatermarkQsvFilter.cs index a8c75db19..a0adb3300 100644 --- a/ErsatzTV.FFmpeg/Filter/Qsv/OverlayWatermarkQsvFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Qsv/OverlayWatermarkQsvFilter.cs @@ -1,4 +1,5 @@ -using ErsatzTV.FFmpeg.State; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.State; using Microsoft.Extensions.Logging; namespace ErsatzTV.FFmpeg.Filter.Qsv; @@ -13,6 +14,7 @@ public class OverlayWatermarkQsvFilter : OverlayWatermarkFilter watermarkState, resolution, squarePixelFrameSize, + new PixelFormatUnknown(), logger) { } diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/QsvFormatFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/QsvFormatFilter.cs new file mode 100644 index 000000000..df61bb6ad --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Qsv/QsvFormatFilter.cs @@ -0,0 +1,14 @@ +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter.Qsv; + +public class QsvFormatFilter : BaseFilter +{ + private readonly IPixelFormat _pixelFormat; + + public QsvFormatFilter(IPixelFormat pixelFormat) => _pixelFormat = pixelFormat; + + public override FrameState NextState(FrameState currentState) => currentState with { PixelFormat = Some(_pixelFormat) }; + + public override string Filter => $"vpp_qsv=format={_pixelFormat.FFmpegName}"; +} diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs index 49c189ba0..7678c91b7 100644 --- a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs @@ -78,15 +78,39 @@ public class ScaleQsvFilter : BaseFilter return $"format={initialPixelFormat},hwupload=extra_hw_frames={_extraHardwareFrames},{scale}"; } - return $"format={initialPixelFormat},hwupload=extra_hw_frames={_extraHardwareFrames}"; + return string.Empty; } } - public override FrameState NextState(FrameState currentState) => currentState with + public override FrameState NextState(FrameState currentState) { - ScaledSize = _scaledSize, - PaddedSize = _scaledSize, - FrameDataLocation = FrameDataLocation.Hardware, - IsAnamorphic = false // this filter always outputs square pixels - }; + FrameState result = currentState with + { + ScaledSize = _scaledSize, + PaddedSize = _scaledSize, + FrameDataLocation = FrameDataLocation.Hardware, + IsAnamorphic = false // this filter always outputs square pixels + }; + + if (_currentState.PixelFormat.IsNone && + _currentState.FrameDataLocation == FrameDataLocation.Software && + currentState.PixelFormat.Map(pf => pf is not PixelFormatNv12).IfNone(false)) + { + // wrap in nv12 + result = result with + { + PixelFormat = currentState.PixelFormat + .Map(pf => (IPixelFormat)new PixelFormatNv12(pf.Name)) + }; + } + else + { + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + result = result with { PixelFormat = Some(pixelFormat) }; + } + } + + return result; + } } diff --git a/ErsatzTV.FFmpeg/Filter/SubtitlePixelFormatFilter.cs b/ErsatzTV.FFmpeg/Filter/SubtitlePixelFormatFilter.cs index d71a5516d..6f849c2d9 100644 --- a/ErsatzTV.FFmpeg/Filter/SubtitlePixelFormatFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/SubtitlePixelFormatFilter.cs @@ -3,13 +3,19 @@ public class SubtitlePixelFormatFilter : BaseFilter { private readonly FFmpegState _ffmpegState; + private readonly bool _is10BitOutput; - public SubtitlePixelFormatFilter(FFmpegState ffmpegState) => _ffmpegState = ffmpegState; + public SubtitlePixelFormatFilter(FFmpegState ffmpegState, bool is10BitOutput) + { + _ffmpegState = ffmpegState; + _is10BitOutput = is10BitOutput; + } public override string Filter => MaybeFormat.Match(f => $"format={f}", () => string.Empty); public Option MaybeFormat =>_ffmpegState.EncoderHardwareAccelerationMode switch { + HardwareAccelerationMode.Nvenc when _is10BitOutput => "nv12", HardwareAccelerationMode.Nvenc => "yuva420p", HardwareAccelerationMode.Qsv => "yuva420p", _ => None diff --git a/ErsatzTV.FFmpeg/Filter/Vaapi/DeinterlaceVaapiFilter.cs b/ErsatzTV.FFmpeg/Filter/Vaapi/DeinterlaceVaapiFilter.cs index 72661775c..e8026cc67 100644 --- a/ErsatzTV.FFmpeg/Filter/Vaapi/DeinterlaceVaapiFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Vaapi/DeinterlaceVaapiFilter.cs @@ -9,7 +9,7 @@ public class DeinterlaceVaapiFilter : BaseFilter public override string Filter => _currentState.FrameDataLocation == FrameDataLocation.Hardware ? "deinterlace_vaapi" - : "format=nv12|vaapi,hwupload,deinterlace_vaapi"; + : "format=nv12|p010le|vaapi,hwupload,deinterlace_vaapi"; public override FrameState NextState(FrameState currentState) => currentState with { diff --git a/ErsatzTV.FFmpeg/Filter/Vaapi/HardwareUploadVaapiFilter.cs b/ErsatzTV.FFmpeg/Filter/Vaapi/HardwareUploadVaapiFilter.cs new file mode 100644 index 000000000..f2dc03686 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Vaapi/HardwareUploadVaapiFilter.cs @@ -0,0 +1,17 @@ +namespace ErsatzTV.FFmpeg.Filter.Vaapi; + +public class HardwareUploadVaapiFilter : BaseFilter +{ + private readonly bool _setFormat; + + public HardwareUploadVaapiFilter(bool setFormat) => _setFormat = setFormat; + + public override string Filter => _setFormat switch + { + false => "hwupload", + true => "format=nv12|p010le|vaapi,hwupload" + }; + + public override FrameState NextState(FrameState currentState) => + currentState with { FrameDataLocation = FrameDataLocation.Hardware }; +} diff --git a/ErsatzTV.FFmpeg/Filter/Vaapi/OverlaySubtitleVaapiFilter.cs b/ErsatzTV.FFmpeg/Filter/Vaapi/OverlaySubtitleVaapiFilter.cs new file mode 100644 index 000000000..4477b25a9 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Vaapi/OverlaySubtitleVaapiFilter.cs @@ -0,0 +1,7 @@ +namespace ErsatzTV.FFmpeg.Filter.Vaapi; + +public class OverlaySubtitleVaapiFilter : BaseFilter +{ + public override string Filter => "overlay_vaapi"; + public override FrameState NextState(FrameState currentState) => currentState; +} diff --git a/ErsatzTV.FFmpeg/Filter/Vaapi/ScaleVaapiFilter.cs b/ErsatzTV.FFmpeg/Filter/Vaapi/ScaleVaapiFilter.cs index 8793db08a..f0ab99c83 100644 --- a/ErsatzTV.FFmpeg/Filter/Vaapi/ScaleVaapiFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Vaapi/ScaleVaapiFilter.cs @@ -74,10 +74,10 @@ public class ScaleVaapiFilter : BaseFilter if (!string.IsNullOrWhiteSpace(scale)) { - return $"format=nv12|vaapi,hwupload,{scale}"; + return $"format=nv12|p010le|vaapi,hwupload,{scale}"; } - return "format=nv12|vaapi,hwupload"; + return string.Empty; } } diff --git a/ErsatzTV.FFmpeg/Filter/Vaapi/SubtitleScaleVaapiFilter.cs b/ErsatzTV.FFmpeg/Filter/Vaapi/SubtitleScaleVaapiFilter.cs new file mode 100644 index 000000000..11fb50f3c --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Vaapi/SubtitleScaleVaapiFilter.cs @@ -0,0 +1,16 @@ +namespace ErsatzTV.FFmpeg.Filter.Vaapi; + +public class SubtitleScaleVaapiFilter : BaseFilter +{ + private readonly FrameSize _paddedSize; + + public SubtitleScaleVaapiFilter(FrameSize paddedSize) + { + _paddedSize = paddedSize; + } + + public override string Filter => + $"scale_vaapi={_paddedSize.Width}:{_paddedSize.Height}:force_original_aspect_ratio=decrease"; + + public override FrameState NextState(FrameState currentState) => currentState; +} diff --git a/ErsatzTV.FFmpeg/Filter/Vaapi/VaapiFormatFilter.cs b/ErsatzTV.FFmpeg/Filter/Vaapi/VaapiFormatFilter.cs new file mode 100644 index 000000000..3e234e423 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Vaapi/VaapiFormatFilter.cs @@ -0,0 +1,14 @@ +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter.Vaapi; + +public class VaapiFormatFilter : BaseFilter +{ + private readonly IPixelFormat _pixelFormat; + + public VaapiFormatFilter(IPixelFormat pixelFormat) => _pixelFormat = pixelFormat; + + public override FrameState NextState(FrameState currentState) => currentState with { PixelFormat = Some(_pixelFormat) }; + + public override string Filter => $"scale_vaapi=format={_pixelFormat.FFmpegName}"; +} diff --git a/ErsatzTV.FFmpeg/Filter/WatermarkPixelFormatFilter.cs b/ErsatzTV.FFmpeg/Filter/WatermarkPixelFormatFilter.cs index 93cd2de28..59d6f3262 100644 --- a/ErsatzTV.FFmpeg/Filter/WatermarkPixelFormatFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/WatermarkPixelFormatFilter.cs @@ -6,11 +6,13 @@ public class WatermarkPixelFormatFilter : BaseFilter { private readonly FFmpegState _ffmpegState; private readonly WatermarkState _watermarkState; + private readonly bool _is10BitOutput; - public WatermarkPixelFormatFilter(FFmpegState ffmpegState, WatermarkState watermarkState) + public WatermarkPixelFormatFilter(FFmpegState ffmpegState, WatermarkState watermarkState, bool is10BitOutput) { _ffmpegState = ffmpegState; _watermarkState = watermarkState; + _is10BitOutput = is10BitOutput; } public override string Filter @@ -21,6 +23,7 @@ public class WatermarkPixelFormatFilter : BaseFilter Option maybeFormat = _ffmpegState.EncoderHardwareAccelerationMode switch { + HardwareAccelerationMode.Nvenc when _is10BitOutput => "nv12", HardwareAccelerationMode.Nvenc => "yuva420p", HardwareAccelerationMode.Qsv => "yuva420p", _ when _watermarkState.Opacity != 100 || hasFadePoints => diff --git a/ErsatzTV.FFmpeg/Filter/YadifFilter.cs b/ErsatzTV.FFmpeg/Filter/YadifFilter.cs index c0a6deec4..2e90a819a 100644 --- a/ErsatzTV.FFmpeg/Filter/YadifFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/YadifFilter.cs @@ -6,6 +6,7 @@ public class YadifFilter : BaseFilter { private readonly FrameState _currentState; + // TODO: accept frame data location instead of current state public YadifFilter(FrameState currentState) => _currentState = currentState; public override string Filter diff --git a/ErsatzTV.FFmpeg/FilterChain.cs b/ErsatzTV.FFmpeg/FilterChain.cs new file mode 100644 index 000000000..de5fc409e --- /dev/null +++ b/ErsatzTV.FFmpeg/FilterChain.cs @@ -0,0 +1,18 @@ +namespace ErsatzTV.FFmpeg; + +public record FilterChain( + List VideoFilterSteps, + List WatermarkFilterSteps, + List SubtitleFilterSteps, + List WatermarkOverlayFilterSteps, + List SubtitleOverlayFilterSteps, + List PixelFormatFilterSteps) +{ + public static readonly FilterChain Empty = new( + new List(), + new List(), + new List(), + new List(), + new List(), + new List()); +} diff --git a/ErsatzTV.FFmpeg/Format/FFmpegFormat.cs b/ErsatzTV.FFmpeg/Format/FFmpegFormat.cs index 46c7d1a7c..d627fe1a4 100644 --- a/ErsatzTV.FFmpeg/Format/FFmpegFormat.cs +++ b/ErsatzTV.FFmpeg/Format/FFmpegFormat.cs @@ -2,6 +2,7 @@ public class FFmpegFormat { + public const string ARGB = "argb"; public const string YUV420P = "yuv420p"; public const string YUV444P = "yuv444p"; public const string YUVA420P = "yuva420p"; diff --git a/ErsatzTV.FFmpeg/Format/PixelFormat.cs b/ErsatzTV.FFmpeg/Format/PixelFormat.cs index a5f015314..a30653591 100644 --- a/ErsatzTV.FFmpeg/Format/PixelFormat.cs +++ b/ErsatzTV.FFmpeg/Format/PixelFormat.cs @@ -2,6 +2,7 @@ public static class PixelFormat { + public const string ARGB = "argb"; public const string YUV420P = "yuv420p"; public const string YUV420P10LE = "yuv420p10le"; public const string YUVA420P = "yuva420p"; diff --git a/ErsatzTV.FFmpeg/Format/PixelFormatArgb.cs b/ErsatzTV.FFmpeg/Format/PixelFormatArgb.cs new file mode 100644 index 000000000..430dd6d50 --- /dev/null +++ b/ErsatzTV.FFmpeg/Format/PixelFormatArgb.cs @@ -0,0 +1,8 @@ +namespace ErsatzTV.FFmpeg.Format; + +public class PixelFormatArgb : IPixelFormat +{ + public string Name => PixelFormat.ARGB; + public string FFmpegName => FFmpegFormat.ARGB; + public int BitDepth => 8; +} diff --git a/ErsatzTV.FFmpeg/Format/PixelFormatQsv.cs b/ErsatzTV.FFmpeg/Format/PixelFormatQsv.cs new file mode 100644 index 000000000..f1039d0bf --- /dev/null +++ b/ErsatzTV.FFmpeg/Format/PixelFormatQsv.cs @@ -0,0 +1,11 @@ +namespace ErsatzTV.FFmpeg.Format; + +public class PixelFormatQsv : IPixelFormat +{ + public PixelFormatQsv(string name) => Name = name; + + public string Name { get; } + + public string FFmpegName => "qsv"; + public int BitDepth => throw new NotSupportedException("This is probably an issue"); +} diff --git a/ErsatzTV.FFmpeg/FrameState.cs b/ErsatzTV.FFmpeg/FrameState.cs index 9c81536e9..9ca0fe0bc 100644 --- a/ErsatzTV.FFmpeg/FrameState.cs +++ b/ErsatzTV.FFmpeg/FrameState.cs @@ -18,4 +18,5 @@ public record FrameState( FrameDataLocation FrameDataLocation = FrameDataLocation.Unknown) { public string FFmpegAspectRatio => PaddedSize.Width == 640 ? "4/3" : "16/9"; + public int BitDepth => PixelFormat.Map(pf => pf.BitDepth).IfNone(8); } diff --git a/ErsatzTV.FFmpeg/InputFile.cs b/ErsatzTV.FFmpeg/InputFile.cs index c497e23f6..6229438b2 100644 --- a/ErsatzTV.FFmpeg/InputFile.cs +++ b/ErsatzTV.FFmpeg/InputFile.cs @@ -23,7 +23,8 @@ public record ConcatInputFile(string Url, FrameSize Resolution) : InputFile( string.Empty, string.Empty, Option.None, - false) + false, + ScanKind.Unknown) }) { public void AddOption(IInputOption option) diff --git a/ErsatzTV.FFmpeg/MediaStream.cs b/ErsatzTV.FFmpeg/MediaStream.cs index 44b735dee..f45954a24 100644 --- a/ErsatzTV.FFmpeg/MediaStream.cs +++ b/ErsatzTV.FFmpeg/MediaStream.cs @@ -18,11 +18,14 @@ public record VideoStream( string SampleAspectRatio, string DisplayAspectRatio, Option FrameRate, - bool StillImage) : MediaStream( + bool StillImage, + ScanKind ScanKind) : MediaStream( Index, Codec, StreamKind.Video) { + public int BitDepth => PixelFormat.Map(pf => pf.BitDepth).IfNone(8); + public bool IsAnamorphic { get diff --git a/ErsatzTV.FFmpeg/Option/RealtimeInputOption.cs b/ErsatzTV.FFmpeg/Option/RealtimeInputOption.cs index c9bc67bd2..5413ba776 100644 --- a/ErsatzTV.FFmpeg/Option/RealtimeInputOption.cs +++ b/ErsatzTV.FFmpeg/Option/RealtimeInputOption.cs @@ -6,9 +6,6 @@ public class RealtimeInputOption : IInputOption { public IList EnvironmentVariables => Array.Empty(); - // some builds of ffmpeg seem to hang when realtime input is requested with multithreading, - // so we force a single thread here - // public IList GlobalOptions => new List { "-threads", "1" }; public IList GlobalOptions => Array.Empty(); public IList InputOptions(InputFile inputFile) => new List { "-re" }; diff --git a/ErsatzTV.FFmpeg/Pipeline/IPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/IPipelineBuilder.cs new file mode 100644 index 000000000..34ef796a3 --- /dev/null +++ b/ErsatzTV.FFmpeg/Pipeline/IPipelineBuilder.cs @@ -0,0 +1,8 @@ +namespace ErsatzTV.FFmpeg.Pipeline; + +public interface IPipelineBuilder +{ + FFmpegPipeline Resize(string outputFile, FrameSize scaledSize); + FFmpegPipeline Concat(ConcatInputFile concatInputFile, FFmpegState ffmpegState); + FFmpegPipeline Build(FFmpegState ffmpegState, FrameState desiredState); +} diff --git a/ErsatzTV.FFmpeg/Pipeline/IPipelineBuilderFactory.cs b/ErsatzTV.FFmpeg/Pipeline/IPipelineBuilderFactory.cs new file mode 100644 index 000000000..d010c42ea --- /dev/null +++ b/ErsatzTV.FFmpeg/Pipeline/IPipelineBuilderFactory.cs @@ -0,0 +1,14 @@ +namespace ErsatzTV.FFmpeg.Pipeline; + +public interface IPipelineBuilderFactory +{ + Task GetBuilder( + HardwareAccelerationMode hardwareAccelerationMode, + Option videoInputFile, + Option audioInputFile, + Option watermarkInputFile, + Option subtitleInputFile, + string reportsFolder, + string fontsFolder, + string ffmpegPath); +} diff --git a/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs new file mode 100644 index 000000000..71dd6020b --- /dev/null +++ b/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs @@ -0,0 +1,562 @@ +using ErsatzTV.FFmpeg.Capabilities; +using ErsatzTV.FFmpeg.Decoder; +using ErsatzTV.FFmpeg.Decoder.Cuvid; +using ErsatzTV.FFmpeg.Encoder; +using ErsatzTV.FFmpeg.Encoder.Nvenc; +using ErsatzTV.FFmpeg.Filter; +using ErsatzTV.FFmpeg.Filter.Cuda; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.Option; +using ErsatzTV.FFmpeg.Option.HardwareAcceleration; +using ErsatzTV.FFmpeg.State; +using Microsoft.Extensions.Logging; + +namespace ErsatzTV.FFmpeg.Pipeline; + +public class NvidiaPipelineBuilder : SoftwarePipelineBuilder +{ + private readonly IHardwareCapabilities _hardwareCapabilities; + private readonly ILogger _logger; + + public NvidiaPipelineBuilder( + IHardwareCapabilities hardwareCapabilities, + HardwareAccelerationMode hardwareAccelerationMode, + Option videoInputFile, + Option audioInputFile, + Option watermarkInputFile, + Option subtitleInputFile, + string reportsFolder, + string fontsFolder, + ILogger logger) : base( + hardwareAccelerationMode, + videoInputFile, + audioInputFile, + watermarkInputFile, + subtitleInputFile, + reportsFolder, + fontsFolder, + logger) + { + _hardwareCapabilities = hardwareCapabilities; + _logger = logger; + } + + protected override FFmpegState SetAccelState( + VideoStream videoStream, + FFmpegState ffmpegState, + FrameState desiredState, + PipelineContext context, + ICollection pipelineSteps) + { + bool canDecode = _hardwareCapabilities.CanDecode(videoStream.Codec, videoStream.PixelFormat); + bool canEncode = _hardwareCapabilities.CanEncode(desiredState.VideoFormat, desiredState.PixelFormat); + + // mpeg2_cuvid seems to have issues when yadif_cuda is used, so just use software decoding + if (context.ShouldDeinterlace && videoStream.Codec == VideoFormat.Mpeg2Video) + { + canDecode = false; + } + + if (canDecode || canEncode) + { + pipelineSteps.Add(new CudaHardwareAccelerationOption()); + } + + // disable hw accel if decoder/encoder isn't supported + return ffmpegState with + { + DecoderHardwareAccelerationMode = canDecode + ? HardwareAccelerationMode.Nvenc + : HardwareAccelerationMode.None, + EncoderHardwareAccelerationMode = canEncode + ? HardwareAccelerationMode.Nvenc + : HardwareAccelerationMode.None + }; + } + + protected override void SetDecoder( + VideoInputFile videoInputFile, + VideoStream videoStream, + FFmpegState ffmpegState, + PipelineContext context, + ICollection pipelineSteps) + { + Option maybeDecoder = (ffmpegState.DecoderHardwareAccelerationMode, videoStream.Codec) switch + { + (HardwareAccelerationMode.Nvenc, VideoFormat.Hevc) => new DecoderHevcCuvid(HardwareAccelerationMode.Nvenc), + (HardwareAccelerationMode.Nvenc, VideoFormat.H264) => new DecoderH264Cuvid(HardwareAccelerationMode.Nvenc), + (HardwareAccelerationMode.Nvenc, VideoFormat.Mpeg2Video) => new DecoderMpeg2Cuvid( + HardwareAccelerationMode.Nvenc, + context.ShouldDeinterlace), + (HardwareAccelerationMode.Nvenc, VideoFormat.Vc1) => new DecoderVc1Cuvid(HardwareAccelerationMode.Nvenc), + (HardwareAccelerationMode.Nvenc, VideoFormat.Vp9) => new DecoderVp9Cuvid(HardwareAccelerationMode.Nvenc), + (HardwareAccelerationMode.Nvenc, VideoFormat.Mpeg4) => + new DecoderMpeg4Cuvid(HardwareAccelerationMode.Nvenc), + + _ => GetSoftwareDecoder(videoStream) + }; + + foreach (IDecoder decoder in maybeDecoder) + { + videoInputFile.AddOption(decoder); + } + } + + protected override FilterChain SetVideoFilters( + VideoInputFile videoInputFile, + VideoStream videoStream, + Option watermarkInputFile, + Option subtitleInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + string fontsFolder, + ICollection pipelineSteps) + { + var watermarkOverlayFilterSteps = new List(); + var subtitleOverlayFilterSteps = new List(); + + FrameState currentState = desiredState with + { + ScaledSize = videoStream.FrameSize, + PaddedSize = videoStream.FrameSize, + + PixelFormat = ffmpegState.DecoderHardwareAccelerationMode == HardwareAccelerationMode.Nvenc && videoStream.BitDepth == 8 + ? videoStream.PixelFormat.Map(pf => (IPixelFormat)new PixelFormatNv12(pf.Name)) + : videoStream.PixelFormat, + + IsAnamorphic = videoStream.IsAnamorphic, + FrameDataLocation = ffmpegState.DecoderHardwareAccelerationMode == HardwareAccelerationMode.Nvenc + ? FrameDataLocation.Hardware + : FrameDataLocation.Software + }; + + // if (context.HasSubtitleOverlay || context.HasWatermark) + // { + // IPixelFormat pixelFormat = desiredState.PixelFormat.IfNone( + // context.Is10BitOutput ? new PixelFormatYuv420P10Le() : new PixelFormatYuv420P()); + // desiredState = desiredState with { PixelFormat = Some(pixelFormat) }; + // } + + currentState = SetDeinterlace(videoInputFile, context, currentState); + currentState = SetScale(videoInputFile, videoStream, context, ffmpegState, desiredState, currentState); + currentState = SetPad(videoInputFile, videoStream, desiredState, currentState); + + if (currentState.BitDepth == 8 && context.HasSubtitleOverlay || context.HasWatermark) + { + Option desiredPixelFormat = Some((IPixelFormat)new PixelFormatYuv420P()); + + if (desiredPixelFormat != currentState.PixelFormat) + { + if (currentState.FrameDataLocation == FrameDataLocation.Software) + { + foreach (IPixelFormat pixelFormat in desiredPixelFormat) + { + var filter = new PixelFormatFilter(pixelFormat); + currentState = filter.NextState(currentState); + videoInputFile.FilterSteps.Add(filter); + } + } + else + { + foreach (IPixelFormat pixelFormat in desiredPixelFormat) + { + var filter = new ScaleCudaFilter( + currentState with { PixelFormat = Some(pixelFormat) }, + currentState.ScaledSize, + currentState.PaddedSize, + false); + currentState = filter.NextState(currentState); + videoInputFile.FilterSteps.Add(filter); + } + } + } + } + + // need to upload for any sort of overlay + if (currentState.FrameDataLocation == FrameDataLocation.Software && + currentState.BitDepth == 8 && (context.HasSubtitleOverlay || context.HasWatermark)) + { + var hardwareUpload = new HardwareUploadCudaFilter(currentState); + currentState = hardwareUpload.NextState(currentState); + videoInputFile.FilterSteps.Add(hardwareUpload); + } + + currentState = SetSubtitle( + videoInputFile, + subtitleInputFile, + context, + ffmpegState, + currentState, + desiredState, + fontsFolder, + subtitleOverlayFilterSteps); + + currentState = SetWatermark( + videoStream, + watermarkInputFile, + context, + ffmpegState, + desiredState, + currentState, + watermarkOverlayFilterSteps); + + // after everything else is done, apply the encoder + if (pipelineSteps.OfType().All(e => e.Kind != StreamKind.Video)) + { + Option maybeEncoder = + (ffmpegState.EncoderHardwareAccelerationMode, desiredState.VideoFormat) switch + { + (HardwareAccelerationMode.Nvenc, VideoFormat.Hevc) => new EncoderHevcNvenc(), + (HardwareAccelerationMode.Nvenc, VideoFormat.H264) => new EncoderH264Nvenc(), + + (_, _) => GetSoftwareEncoder(currentState, desiredState) + }; + + foreach (IEncoder encoder in maybeEncoder) + { + pipelineSteps.Add(encoder); + videoInputFile.FilterSteps.Add(encoder); + } + } + + List pixelFormatFilterSteps = SetPixelFormat( + videoStream, + desiredState.PixelFormat, + ffmpegState, + currentState, + context, + pipelineSteps); + + return new FilterChain( + videoInputFile.FilterSteps, + watermarkInputFile.Map(wm => wm.FilterSteps).IfNone(new List()), + subtitleInputFile.Map(st => st.FilterSteps).IfNone(new List()), + watermarkOverlayFilterSteps, + subtitleOverlayFilterSteps, + pixelFormatFilterSteps); + } + + private List SetPixelFormat( + VideoStream videoStream, + Option desiredPixelFormat, + FFmpegState ffmpegState, + FrameState currentState, + PipelineContext context, + ICollection pipelineSteps) + { + var result = new List(); + + foreach (IPixelFormat pixelFormat in desiredPixelFormat) + { + IPixelFormat format = pixelFormat; + + if (pixelFormat is PixelFormatNv12 nv12) + { + foreach (IPixelFormat pf in AvailablePixelFormats.ForPixelFormat(nv12.Name, null)) + { + format = pf; + } + } + + if (!videoStream.ColorParams.IsBt709) + { + _logger.LogDebug("Adding colorspace filter"); + var colorspace = new ColorspaceFilter(videoStream, format, currentState.FrameDataLocation); + + currentState = colorspace.NextState(currentState); + result.Add(colorspace); + } + + if (ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.None) + { + _logger.LogDebug("Using software encoder"); + + if ((context.HasSubtitleOverlay || context.HasWatermark) && + currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + _logger.LogDebug( + "HasSubtitleOverlay || HasWatermark && FrameDataLocation == FrameDataLocation.Hardware"); + + var hardwareDownload = new CudaHardwareDownloadFilter(currentState.PixelFormat); + currentState = hardwareDownload.NextState(currentState); + result.Add(hardwareDownload); + } + } + + if (currentState.PixelFormat.Map(f => f.FFmpegName) != format.FFmpegName) + { + _logger.LogDebug( + "Format {A} doesn't equal {B}", + currentState.PixelFormat.Map(f => f.FFmpegName), + format.FFmpegName); + + if (currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + result.Add(new CudaFormatFilter(format)); + } + else + { + pipelineSteps.Add(new PixelFormatOutputOption(format)); + } + } + + if (currentState.FrameDataLocation == FrameDataLocation.Hardware && + ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.None) + { + var hardwareDownload = new CudaHardwareDownloadFilter(Some(format)); + currentState = hardwareDownload.NextState(currentState); + result.Add(hardwareDownload); + } + } + + return result; + } + + private FrameState SetWatermark( + VideoStream videoStream, + Option watermarkInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + FrameState currentState, + List watermarkOverlayFilterSteps) + { + if (context.HasWatermark) + { + WatermarkInputFile watermark = watermarkInputFile.Head(); + + foreach (VideoStream watermarkStream in watermark.VideoStreams) + { + if (watermarkStream.StillImage == false) + { + watermark.AddOption(new DoNotIgnoreLoopInputOption()); + } + else if (watermark.DesiredState.MaybeFadePoints.Map(fp => fp.Count > 0).IfNone(false)) + { + // looping is required to fade a static image in and out + watermark.AddOption(new InfiniteLoopInputOption(HardwareAccelerationMode.None)); + } + } + + if (watermark.DesiredState.Size == WatermarkSize.Scaled) + { + watermark.FilterSteps.Add( + new WatermarkScaleFilter(watermark.DesiredState, currentState.PaddedSize)); + } + + if (watermark.DesiredState.Opacity != 100) + { + watermark.FilterSteps.Add(new WatermarkOpacityFilter(watermark.DesiredState)); + } + + watermark.FilterSteps.Add(new PixelFormatFilter(new PixelFormatYuva420P())); + + foreach (List fadePoints in watermark.DesiredState.MaybeFadePoints) + { + watermark.FilterSteps.AddRange(fadePoints.Map(fp => new WatermarkFadeFilter(fp))); + } + + watermark.FilterSteps.Add( + new HardwareUploadCudaFilter(currentState with { FrameDataLocation = FrameDataLocation.Software })); + + var watermarkFilter = new OverlayWatermarkCudaFilter( + watermark.DesiredState, + desiredState.PaddedSize, + videoStream.SquarePixelFrameSize(currentState.PaddedSize), + _logger); + watermarkOverlayFilterSteps.Add(watermarkFilter); + } + + return currentState; + } + + private static FrameState SetSubtitle( + VideoInputFile videoInputFile, + Option subtitleInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState currentState, + FrameState desiredState, + string fontsFolder, + ICollection subtitleOverlayFilterSteps) + { + foreach (SubtitleInputFile subtitle in subtitleInputFile) + { + if (context.HasSubtitleText) + { + videoInputFile.AddOption(new CopyTimestampInputOption()); + + if (videoInputFile.FilterSteps.Count == 0 && videoInputFile.InputOptions.OfType().Any()) + { + // change the hw accel output to software so the explicit download isn't needed + foreach (CuvidDecoder decoder in videoInputFile.InputOptions.OfType()) + { + decoder.HardwareAccelerationMode = HardwareAccelerationMode.None; + } + } + else + { + var downloadFilter = new HardwareDownloadFilter(currentState); + currentState = downloadFilter.NextState(currentState); + videoInputFile.FilterSteps.Add(downloadFilter); + } + + var subtitlesFilter = new SubtitlesFilter(fontsFolder, subtitle); + currentState = subtitlesFilter.NextState(currentState); + videoInputFile.FilterSteps.Add(subtitlesFilter); + + if (context.HasWatermark) + { + var subtitleHardwareUpload = new HardwareUploadCudaFilter(currentState); + currentState = subtitleHardwareUpload.NextState(currentState); + videoInputFile.FilterSteps.Add(subtitleHardwareUpload); + } + } + else if (context.HasSubtitleOverlay) + { + var pixelFormatFilter = new PixelFormatFilter(new PixelFormatYuva420P()); + subtitle.FilterSteps.Add(pixelFormatFilter); + + if (currentState.PixelFormat.Map(pf => pf.BitDepth).IfNone(8) == 8) + { + var subtitleHardwareUpload = new HardwareUploadCudaFilter( + currentState with { FrameDataLocation = FrameDataLocation.Software }); + subtitle.FilterSteps.Add(subtitleHardwareUpload); + + // only scale if scaling or padding was used for main video stream + if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleCudaFilter or PadFilter)) + { + var scaleFilter = new SubtitleScaleNppFilter( + currentState, + desiredState.ScaledSize, + desiredState.PaddedSize); + subtitle.FilterSteps.Add(scaleFilter); + } + + var subtitlesFilter = new OverlaySubtitleCudaFilter(); + subtitleOverlayFilterSteps.Add(subtitlesFilter); + } + else + { + if (currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + var cudaDownload = new CudaHardwareDownloadFilter(currentState.PixelFormat); + currentState = cudaDownload.NextState(currentState); + videoInputFile.FilterSteps.Add(cudaDownload); + } + + // only scale if scaling or padding was used for main video stream + if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleCudaFilter or PadFilter)) + { + var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize); + subtitle.FilterSteps.Add(scaleFilter); + } + + var subtitlesFilter = + new OverlaySubtitleFilter(desiredState.PixelFormat.IfNone(new PixelFormatYuv420P())); + subtitleOverlayFilterSteps.Add(subtitlesFilter); + + // overlay produces YUVA420P10, so we need to strip the alpha + if (currentState.BitDepth == 10) + { + subtitleOverlayFilterSteps.Add(new PixelFormatFilter(new PixelFormatYuv420P10Le())); + } + } + } + } + + return currentState; + } + + private static FrameState SetPad( + VideoInputFile videoInputFile, + VideoStream videoStream, + FrameState desiredState, + FrameState currentState) + { + if (currentState.PaddedSize != desiredState.PaddedSize) + { + IPipelineFilterStep padStep = new PadFilter(currentState, desiredState.PaddedSize); + currentState = padStep.NextState(currentState); + videoInputFile.FilterSteps.Add(padStep); + } + + return currentState; + } + + private static FrameState SetScale( + VideoInputFile videoInputFile, + VideoStream videoStream, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + FrameState currentState) + { + IPipelineFilterStep scaleStep; + + if (currentState.ScaledSize != desiredState.ScaledSize && ffmpegState is + { + DecoderHardwareAccelerationMode: HardwareAccelerationMode.None, + EncoderHardwareAccelerationMode: HardwareAccelerationMode.None + } && context is { HasWatermark: false, HasSubtitleOverlay: false, ShouldDeinterlace: false }) + { + scaleStep = new ScaleFilter( + currentState, + desiredState.ScaledSize, + desiredState.PaddedSize, + videoStream.IsAnamorphicEdgeCase); + } + else + { + scaleStep = new ScaleCudaFilter( + currentState with + { + PixelFormat = !context.Is10BitOutput && (context.HasWatermark || + context.HasSubtitleOverlay || + context.ShouldDeinterlace || + (desiredState.ScaledSize != desiredState.PaddedSize) || + context.HasSubtitleText || + ffmpegState is + { + DecoderHardwareAccelerationMode: HardwareAccelerationMode.Nvenc, + EncoderHardwareAccelerationMode: HardwareAccelerationMode.None + }) + ? desiredState.PixelFormat.Map(pf => (IPixelFormat)new PixelFormatNv12(pf.Name)) + : Option.None + }, + desiredState.ScaledSize, + desiredState.PaddedSize, + videoStream.IsAnamorphicEdgeCase); + } + + if (!string.IsNullOrWhiteSpace(scaleStep.Filter)) + { + currentState = scaleStep.NextState(currentState); + videoInputFile.FilterSteps.Add(scaleStep); + } + + return currentState; + } + + private static FrameState SetDeinterlace(VideoInputFile videoInputFile, PipelineContext context, FrameState currentState) + { + if (context.ShouldDeinterlace) + { + if (currentState.FrameDataLocation == FrameDataLocation.Software) + { + var filter = new YadifFilter(currentState); + currentState = filter.NextState(currentState); + videoInputFile.FilterSteps.Add(filter); + } + else + { + var filter = new YadifCudaFilter(currentState); + currentState = filter.NextState(currentState); + videoInputFile.FilterSteps.Add(filter); + } + } + + return currentState; + } +} diff --git a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs new file mode 100644 index 000000000..52c516735 --- /dev/null +++ b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs @@ -0,0 +1,605 @@ +using System.Diagnostics; +using ErsatzTV.FFmpeg.Decoder; +using ErsatzTV.FFmpeg.Encoder; +using ErsatzTV.FFmpeg.Environment; +using ErsatzTV.FFmpeg.Filter; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.Option; +using ErsatzTV.FFmpeg.Option.Metadata; +using ErsatzTV.FFmpeg.OutputFormat; +using ErsatzTV.FFmpeg.Protocol; +using Microsoft.Extensions.Logging; + +namespace ErsatzTV.FFmpeg.Pipeline; + +public abstract class PipelineBuilderBase : IPipelineBuilder +{ + private readonly HardwareAccelerationMode _hardwareAccelerationMode; + private readonly Option _videoInputFile; + private readonly Option _audioInputFile; + private readonly Option _watermarkInputFile; + private readonly Option _subtitleInputFile; + private readonly string _reportsFolder; + private readonly string _fontsFolder; + private readonly ILogger _logger; + + protected PipelineBuilderBase( + HardwareAccelerationMode hardwareAccelerationMode, + Option videoInputFile, + Option audioInputFile, + Option watermarkInputFile, + Option subtitleInputFile, + string reportsFolder, + string fontsFolder, + ILogger logger) + { + _hardwareAccelerationMode = hardwareAccelerationMode; + _videoInputFile = videoInputFile; + _audioInputFile = audioInputFile; + _watermarkInputFile = watermarkInputFile; + _subtitleInputFile = subtitleInputFile; + _reportsFolder = reportsFolder; + _fontsFolder = fontsFolder; + _logger = logger; + } + + public FFmpegPipeline Resize(string outputFile, FrameSize scaledSize) + { + var pipelineSteps = new List + { + new NoStandardInputOption(), + new HideBannerOption(), + new NoStatsOption(), + new LoglevelErrorOption() + }; + + IPipelineFilterStep scaleStep = new ScaleImageFilter(scaledSize); + _videoInputFile.Iter(f => f.FilterSteps.Add(scaleStep)); + + pipelineSteps.Add(new VideoFilter(new[] { scaleStep })); + pipelineSteps.Add(scaleStep); + pipelineSteps.Add(new FileNameOutputOption(outputFile)); + + return new FFmpegPipeline(pipelineSteps); + } + + public FFmpegPipeline Concat(ConcatInputFile concatInputFile, FFmpegState ffmpegState) + { + var pipelineSteps = new List + { + new NoStandardInputOption(), + new HideBannerOption(), + new NoStatsOption(), + new LoglevelErrorOption(), + new StandardFormatFlags(), + new NoDemuxDecodeDelayOutputOption(), + new FastStartOutputOption(), + new ClosedGopOutputOption() + }; + + concatInputFile.AddOption(new ConcatInputFormat()); + concatInputFile.AddOption(new RealtimeInputOption()); + concatInputFile.AddOption(new InfiniteLoopInputOption(HardwareAccelerationMode.None)); + + foreach (int threadCount in ffmpegState.ThreadCount) + { + pipelineSteps.Insert(0, new ThreadCountOption(threadCount)); + } + + pipelineSteps.Add(new NoSceneDetectOutputOption(0)); + pipelineSteps.Add(new EncoderCopyAll()); + + if (ffmpegState.DoNotMapMetadata) + { + pipelineSteps.Add(new DoNotMapMetadataOutputOption()); + } + + pipelineSteps.AddRange( + ffmpegState.MetadataServiceProvider.Map(sp => new MetadataServiceProviderOutputOption(sp))); + + pipelineSteps.AddRange(ffmpegState.MetadataServiceName.Map(sn => new MetadataServiceNameOutputOption(sn))); + + pipelineSteps.Add(new OutputFormatMpegTs()); + pipelineSteps.Add(new PipeProtocol()); + + if (ffmpegState.SaveReport) + { + pipelineSteps.Add(new FFReportVariable(_reportsFolder, concatInputFile)); + } + + return new FFmpegPipeline(pipelineSteps); + } + + public FFmpegPipeline Build(FFmpegState ffmpegState, FrameState desiredState) + { + var pipelineSteps = new List + { + new NoStandardInputOption(), + new HideBannerOption(), + new NoStatsOption(), + new LoglevelErrorOption(), + new StandardFormatFlags(), + new NoDemuxDecodeDelayOutputOption(), + new FastStartOutputOption(), + new ClosedGopOutputOption() + }; + + Debug.Assert(_videoInputFile.IsSome, "Pipeline builder requires exactly one video input file"); + VideoInputFile videoInputFile = _videoInputFile.Head(); + + var allVideoStreams = _videoInputFile.SelectMany(f => f.VideoStreams).ToList(); + Debug.Assert(allVideoStreams.Count == 1, "Pipeline builder requires exactly one video stream"); + VideoStream videoStream = allVideoStreams.Head(); + + var context = new PipelineContext( + HardwareAccelerationMode: _hardwareAccelerationMode, + HasWatermark: _watermarkInputFile.IsSome, + HasSubtitleOverlay: _subtitleInputFile.Map(s => s is { IsImageBased: true, Copy: false }).IfNone(false), + HasSubtitleText: _subtitleInputFile.Map(s => s is { IsImageBased: false }).IfNone(false), + ShouldDeinterlace: desiredState.Deinterlaced, + Is10BitOutput: desiredState.PixelFormat.Map(pf => pf.BitDepth).IfNone(8) == 10); + + SetThreadCount(ffmpegState, desiredState, pipelineSteps); + SetSceneDetect(videoStream, ffmpegState, desiredState, pipelineSteps); + SetFFReport(ffmpegState, pipelineSteps); + SetStreamSeek(ffmpegState, videoInputFile, context, pipelineSteps); + SetTimeLimit(ffmpegState, pipelineSteps); + + FilterChain filterChain = FilterChain.Empty; + + if (desiredState.VideoFormat == VideoFormat.Copy) + { + pipelineSteps.Add(new EncoderCopyVideo()); + } + else + { + filterChain = BuildVideoPipeline( + videoInputFile, + videoStream, + ffmpegState, + desiredState, + context, + pipelineSteps); + } + + if (_audioInputFile.IsNone) + { + pipelineSteps.Add(new EncoderCopyAudio()); + } + else + { + foreach (AudioInputFile audioInputFile in _audioInputFile) + { + BuildAudioPipeline(audioInputFile, pipelineSteps); + } + } + + SetDoNotMapMetadata(ffmpegState, pipelineSteps); + SetMetadataServiceProvider(ffmpegState, pipelineSteps); + SetMetadataServiceName(ffmpegState, pipelineSteps); + SetMetadataAudioLanguage(ffmpegState, pipelineSteps); + SetOutputFormat(ffmpegState, desiredState, pipelineSteps, videoStream); + + var complexFilter = new NewComplexFilter( + _videoInputFile, + _audioInputFile, + _watermarkInputFile, + _subtitleInputFile, + context, + filterChain); + + pipelineSteps.Add(complexFilter); + + return new FFmpegPipeline(pipelineSteps); + } + + protected Option LogUnknownDecoder( + HardwareAccelerationMode hardwareAccelerationMode, + string videoFormat, + string pixelFormat) + { + _logger.LogWarning( + "Unable to determine decoder for {AccelMode} - {VideoFormat} - {PixelFormat}; may have playback issues", + hardwareAccelerationMode, + videoFormat, + pixelFormat); + return Option.None; + } + + protected Option LogUnknownEncoder(HardwareAccelerationMode hardwareAccelerationMode, string videoFormat) + { + _logger.LogWarning( + "Unable to determine video encoder for {AccelMode} - {VideoFormat}; may have playback issues", + hardwareAccelerationMode, + videoFormat); + return Option.None; + } + + private static void SetOutputFormat( + FFmpegState ffmpegState, + FrameState desiredState, + List pipelineSteps, + VideoStream videoStream) + { + switch (ffmpegState.OutputFormat) + { + case OutputFormatKind.MpegTs: + pipelineSteps.Add(new OutputFormatMpegTs()); + pipelineSteps.Add(new PipeProtocol()); + break; + case OutputFormatKind.Hls: + foreach (string playlistPath in ffmpegState.HlsPlaylistPath) + { + foreach (string segmentTemplate in ffmpegState.HlsSegmentTemplate) + { + pipelineSteps.Add( + new OutputFormatHls( + desiredState, + videoStream.FrameRate, + segmentTemplate, + playlistPath)); + } + } + + break; + } + } + + private static void SetMetadataAudioLanguage(FFmpegState ffmpegState, List pipelineSteps) + { + foreach (string desiredAudioLanguage in ffmpegState.MetadataAudioLanguage) + { + pipelineSteps.Add(new MetadataAudioLanguageOutputOption(desiredAudioLanguage)); + } + } + + private static void SetMetadataServiceName(FFmpegState ffmpegState, List pipelineSteps) + { + foreach (string desiredServiceName in ffmpegState.MetadataServiceName) + { + pipelineSteps.Add(new MetadataServiceNameOutputOption(desiredServiceName)); + } + } + + private static void SetMetadataServiceProvider(FFmpegState ffmpegState, List pipelineSteps) + { + foreach (string desiredServiceProvider in ffmpegState.MetadataServiceProvider) + { + pipelineSteps.Add(new MetadataServiceProviderOutputOption(desiredServiceProvider)); + } + } + + private static void SetDoNotMapMetadata(FFmpegState ffmpegState, List pipelineSteps) + { + if (ffmpegState.DoNotMapMetadata) + { + pipelineSteps.Add(new DoNotMapMetadataOutputOption()); + } + } + + private void BuildAudioPipeline(AudioInputFile audioInputFile, IList pipelineSteps) + { + // always need to specify audio codec so ffmpeg doesn't default to a codec we don't want + foreach (IEncoder step in AvailableEncoders.ForAudioFormat(audioInputFile.DesiredState, _logger)) + { + pipelineSteps.Add(step); + } + + SetAudioChannels(audioInputFile, pipelineSteps); + SetAudioBitrate(audioInputFile, pipelineSteps); + SetAudioBufferSize(audioInputFile, pipelineSteps); + SetAudioSampleRate(audioInputFile, pipelineSteps); + SetAudioLoudness(audioInputFile); + SetAudioPad(audioInputFile); + } + + private void SetAudioPad(AudioInputFile audioInputFile) + { + foreach (TimeSpan desiredDuration in audioInputFile.DesiredState.AudioDuration) + { + _audioInputFile.Iter(f => f.FilterSteps.Add(new AudioPadFilter(desiredDuration))); + } + } + + private void SetAudioLoudness(AudioInputFile audioInputFile) + { + if (audioInputFile.DesiredState.NormalizeLoudness) + { + _audioInputFile.Iter(f => f.FilterSteps.Add(new NormalizeLoudnessFilter())); + } + } + + private static void SetAudioSampleRate(AudioInputFile audioInputFile, IList pipelineSteps) + { + foreach (int desiredSampleRate in audioInputFile.DesiredState.AudioSampleRate) + { + pipelineSteps.Add(new AudioSampleRateOutputOption(desiredSampleRate)); + } + } + + private static void SetAudioBufferSize(AudioInputFile audioInputFile, IList pipelineSteps) + { + foreach (int desiredBufferSize in audioInputFile.DesiredState.AudioBufferSize) + { + pipelineSteps.Add(new AudioBufferSizeOutputOption(desiredBufferSize)); + } + } + + private static void SetAudioBitrate(AudioInputFile audioInputFile, IList pipelineSteps) + { + foreach (int desiredBitrate in audioInputFile.DesiredState.AudioBitrate) + { + pipelineSteps.Add(new AudioBitrateOutputOption(desiredBitrate)); + } + } + + private static void SetAudioChannels(AudioInputFile audioInputFile, IList pipelineSteps) + { + foreach (AudioStream audioStream in audioInputFile.AudioStreams.HeadOrNone()) + { + foreach (int desiredAudioChannels in audioInputFile.DesiredState.AudioChannels) + { + pipelineSteps.Add( + new AudioChannelsOutputOption( + audioInputFile.DesiredState.AudioFormat, + audioStream.Channels, + desiredAudioChannels)); + } + } + } + + protected abstract FFmpegState SetAccelState( + VideoStream videoStream, + FFmpegState ffmpegState, + FrameState desiredState, + PipelineContext context, + ICollection pipelineSteps); + + private FilterChain BuildVideoPipeline( + VideoInputFile videoInputFile, + VideoStream videoStream, + FFmpegState ffmpegState, + FrameState desiredState, + PipelineContext context, + ICollection pipelineSteps) + { + if (_subtitleInputFile.Map(s => s.Copy) == Some(true)) + { + pipelineSteps.Add(new EncoderCopySubtitle()); + } + + ffmpegState = SetAccelState(videoStream, ffmpegState, desiredState, context, pipelineSteps); + + SetDecoder(videoInputFile, videoStream, ffmpegState, context, pipelineSteps); + + SetStillImageInfiniteLoop(videoInputFile, videoStream, ffmpegState); + SetRealtimeInput(videoInputFile, desiredState); + SetInfiniteLoop(videoInputFile, videoStream, ffmpegState, desiredState); + SetFrameRateOutput(desiredState, pipelineSteps); + SetVideoTrackTimescaleOutput(desiredState, pipelineSteps); + SetVideoBitrateOutput(desiredState, pipelineSteps); + SetVideoBufferSizeOutput(desiredState, pipelineSteps); + + FilterChain filterChain = SetVideoFilters( + videoInputFile, + videoStream, + _watermarkInputFile, + _subtitleInputFile, + context, + ffmpegState, + desiredState, + _fontsFolder, + pipelineSteps); + + SetOutputTsOffset(ffmpegState, desiredState, pipelineSteps); + + return filterChain; + } + + protected abstract void SetDecoder( + VideoInputFile videoInputFile, + VideoStream videoStream, + FFmpegState ffmpegState, + PipelineContext context, + ICollection pipelineSteps); + + protected Option GetSoftwareDecoder(VideoStream videoStream) => + videoStream.Codec switch + { + VideoFormat.Hevc => new DecoderHevc(), + VideoFormat.H264 => new DecoderH264(), + VideoFormat.Mpeg1Video => new DecoderMpeg1Video(), + VideoFormat.Mpeg2Video => new DecoderMpeg2Video(), + VideoFormat.Vc1 => new DecoderVc1(), + VideoFormat.MsMpeg4V2 => new DecoderMsMpeg4V2(), + VideoFormat.MsMpeg4V3 => new DecoderMsMpeg4V3(), + VideoFormat.Mpeg4 => new DecoderMpeg4(), + VideoFormat.Vp9 => new DecoderVp9(), + + VideoFormat.Undetermined => new DecoderImplicit(), + VideoFormat.Copy => new DecoderImplicit(), + VideoFormat.GeneratedImage => new DecoderImplicit(), + + _ => LogUnknownDecoder( + HardwareAccelerationMode.None, + videoStream.Codec, + videoStream.PixelFormat.Match(pf => pf.Name, () => string.Empty)) + }; + + protected Option GetSoftwareEncoder(FrameState currentState, FrameState desiredState) => + desiredState.VideoFormat switch + { + VideoFormat.Hevc => new EncoderLibx265( + currentState with { FrameDataLocation = FrameDataLocation.Software }), + VideoFormat.H264 => new EncoderLibx264(), + VideoFormat.Mpeg2Video => new EncoderMpeg2Video(), + + VideoFormat.Copy => new EncoderCopyVideo(), + VideoFormat.Undetermined => new EncoderImplicitVideo(), + + _ => LogUnknownEncoder(HardwareAccelerationMode.None, desiredState.VideoFormat) + }; + + protected abstract FilterChain SetVideoFilters( + VideoInputFile videoInputFile, + VideoStream videoStream, + Option watermarkInputFile, + Option subtitleInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + string fontsFolder, + ICollection pipelineSteps); + + private static void SetOutputTsOffset(FFmpegState ffmpegState, FrameState desiredState, ICollection pipelineSteps) + { + if (ffmpegState.PtsOffset > 0) + { + foreach (int videoTrackTimeScale in desiredState.VideoTrackTimeScale) + { + pipelineSteps.Add(new OutputTsOffsetOption(ffmpegState.PtsOffset, videoTrackTimeScale)); + } + } + } + + private static void SetVideoBufferSizeOutput(FrameState desiredState, ICollection pipelineSteps) + { + foreach (int desiredBufferSize in desiredState.VideoBufferSize) + { + pipelineSteps.Add(new VideoBufferSizeOutputOption(desiredBufferSize)); + } + } + + private static void SetVideoBitrateOutput(FrameState desiredState, ICollection pipelineSteps) + { + foreach (int desiredBitrate in desiredState.VideoBitrate) + { + pipelineSteps.Add(new VideoBitrateOutputOption(desiredBitrate)); + } + } + + private static void SetVideoTrackTimescaleOutput(FrameState desiredState, ICollection pipelineSteps) + { + foreach (int desiredTimeScale in desiredState.VideoTrackTimeScale) + { + pipelineSteps.Add(new VideoTrackTimescaleOutputOption(desiredTimeScale)); + } + } + + private static void SetFrameRateOutput(FrameState desiredState, ICollection pipelineSteps) + { + foreach (int desiredFrameRate in desiredState.FrameRate) + { + pipelineSteps.Add(new FrameRateOutputOption(desiredFrameRate)); + } + } + + private void SetInfiniteLoop( + VideoInputFile videoInputFile, + VideoStream videoStream, + FFmpegState ffmpegState, + FrameState desiredState) + { + if (desiredState.InfiniteLoop) + { + _audioInputFile.Iter( + a => a.AddOption(new InfiniteLoopInputOption(ffmpegState.EncoderHardwareAccelerationMode))); + + if (!videoStream.StillImage) + { + videoInputFile.AddOption(new InfiniteLoopInputOption(ffmpegState.EncoderHardwareAccelerationMode)); + } + } + } + + private void SetRealtimeInput(VideoInputFile videoInputFile, FrameState desiredState) + { + if (desiredState.Realtime) + { + _audioInputFile.Iter(a => a.AddOption(new RealtimeInputOption())); + videoInputFile.AddOption(new RealtimeInputOption()); + } + } + + private static void SetStillImageInfiniteLoop( + VideoInputFile videoInputFile, + VideoStream videoStream, + FFmpegState ffmpegState) + { + if (videoStream.StillImage) + { + videoInputFile.AddOption(new InfiniteLoopInputOption(ffmpegState.EncoderHardwareAccelerationMode)); + } + } + + private void SetThreadCount(FFmpegState ffmpegState, FrameState desiredState, IList pipelineSteps) + { + if (ffmpegState.Start.Exists(s => s > TimeSpan.Zero) && desiredState.Realtime) + { + _logger.LogInformation( + "Forcing {Threads} ffmpeg thread due to buggy combination of stream seek and realtime output", + 1); + + pipelineSteps.Insert(0, new ThreadCountOption(1)); + } + else + { + foreach (int threadCount in ffmpegState.ThreadCount) + { + pipelineSteps.Insert(0, new ThreadCountOption(threadCount)); + } + } + } + + private static void SetSceneDetect( + // ReSharper disable once SuggestBaseTypeForParameter + VideoStream videoStream, + FFmpegState ffmpegState, + FrameState desiredState, + ICollection pipelineSteps) + { + // -sc_threshold 0 is unsupported with mpeg2video + if (videoStream.Codec == VideoFormat.Mpeg2Video || desiredState.VideoFormat == VideoFormat.Mpeg2Video || + ffmpegState.DecoderHardwareAccelerationMode == HardwareAccelerationMode.VideoToolbox) + { + pipelineSteps.Add(new NoSceneDetectOutputOption(1_000_000_000)); + } + else + { + pipelineSteps.Add(new NoSceneDetectOutputOption(0)); + } + } + + private void SetFFReport(FFmpegState ffmpegState, ICollection pipelineSteps) + { + if (ffmpegState.SaveReport) + { + pipelineSteps.Add(new FFReportVariable(_reportsFolder, None)); + } + } + + private void SetStreamSeek( + FFmpegState ffmpegState, + VideoInputFile videoInputFile, + PipelineContext context, + ICollection pipelineSteps) + { + foreach (TimeSpan desiredStart in ffmpegState.Start.Filter(s => s > TimeSpan.Zero)) + { + var option = new StreamSeekInputOption(desiredStart); + _audioInputFile.Iter(a => a.AddOption(option)); + videoInputFile.AddOption(option); + + // need to seek text subtitle files + if (context.HasSubtitleText) + { + pipelineSteps.Add(new StreamSeekFilterOption(desiredStart)); + } + } + } + + private static void SetTimeLimit(FFmpegState ffmpegState, List pipelineSteps) + { + pipelineSteps.AddRange(ffmpegState.Finish.Map(finish => new TimeLimitOutputOption(finish))); + } +} diff --git a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderFactory.cs b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderFactory.cs new file mode 100644 index 000000000..a7379f640 --- /dev/null +++ b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderFactory.cs @@ -0,0 +1,83 @@ +using ErsatzTV.FFmpeg.Capabilities; +using ErsatzTV.FFmpeg.Runtime; +using Microsoft.Extensions.Logging; + +namespace ErsatzTV.FFmpeg.Pipeline; + +public class PipelineBuilderFactory : IPipelineBuilderFactory +{ + private readonly IRuntimeInfo _runtimeInfo; + private readonly IHardwareCapabilitiesFactory _hardwareCapabilitiesFactory; + private readonly ILogger _logger; + + public PipelineBuilderFactory( + IRuntimeInfo runtimeInfo, + IHardwareCapabilitiesFactory hardwareCapabilitiesFactory, + ILogger logger) + { + _runtimeInfo = runtimeInfo; + _hardwareCapabilitiesFactory = hardwareCapabilitiesFactory; + _logger = logger; + } + + public async Task GetBuilder( + HardwareAccelerationMode hardwareAccelerationMode, + Option videoInputFile, + Option audioInputFile, + Option watermarkInputFile, + Option subtitleInputFile, + string reportsFolder, + string fontsFolder, + string ffmpegPath) => hardwareAccelerationMode switch + { + HardwareAccelerationMode.Nvenc => new NvidiaPipelineBuilder( + await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, hardwareAccelerationMode), + hardwareAccelerationMode, + videoInputFile, + audioInputFile, + watermarkInputFile, + subtitleInputFile, + reportsFolder, + fontsFolder, + _logger), + HardwareAccelerationMode.Vaapi => new VaapiPipelineBuilder( + await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, hardwareAccelerationMode), + hardwareAccelerationMode, + videoInputFile, + audioInputFile, + watermarkInputFile, + subtitleInputFile, + reportsFolder, + fontsFolder, + _logger), + HardwareAccelerationMode.Qsv => new QsvPipelineBuilder( + await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, hardwareAccelerationMode), + hardwareAccelerationMode, + videoInputFile, + audioInputFile, + watermarkInputFile, + subtitleInputFile, + reportsFolder, + fontsFolder, + _logger), + HardwareAccelerationMode.VideoToolbox => new VideoToolboxPipelineBuilder( + await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, hardwareAccelerationMode), + hardwareAccelerationMode, + videoInputFile, + audioInputFile, + watermarkInputFile, + subtitleInputFile, + reportsFolder, + fontsFolder, + _logger), + _ => new SoftwarePipelineBuilder( + hardwareAccelerationMode, + videoInputFile, + audioInputFile, + watermarkInputFile, + subtitleInputFile, + reportsFolder, + fontsFolder, + _logger) + }; +} diff --git a/ErsatzTV.FFmpeg/Pipeline/PipelineContext.cs b/ErsatzTV.FFmpeg/Pipeline/PipelineContext.cs new file mode 100644 index 000000000..246507413 --- /dev/null +++ b/ErsatzTV.FFmpeg/Pipeline/PipelineContext.cs @@ -0,0 +1,10 @@ +namespace ErsatzTV.FFmpeg.Pipeline; + +public record PipelineContext( + HardwareAccelerationMode HardwareAccelerationMode, + bool HasWatermark, + bool HasSubtitleOverlay, + bool HasSubtitleText, + bool ShouldDeinterlace, + bool Is10BitOutput); + diff --git a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs new file mode 100644 index 000000000..3adad93cf --- /dev/null +++ b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs @@ -0,0 +1,518 @@ +using ErsatzTV.FFmpeg.Capabilities; +using ErsatzTV.FFmpeg.Decoder; +using ErsatzTV.FFmpeg.Decoder.Qsv; +using ErsatzTV.FFmpeg.Encoder; +using ErsatzTV.FFmpeg.Encoder.Qsv; +using ErsatzTV.FFmpeg.Filter; +using ErsatzTV.FFmpeg.Filter.Qsv; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.Option; +using ErsatzTV.FFmpeg.Option.HardwareAcceleration; +using ErsatzTV.FFmpeg.State; +using Microsoft.Extensions.Logging; + +namespace ErsatzTV.FFmpeg.Pipeline; + +public class QsvPipelineBuilder : SoftwarePipelineBuilder +{ + private readonly IHardwareCapabilities _hardwareCapabilities; + private readonly ILogger _logger; + + public QsvPipelineBuilder( + IHardwareCapabilities hardwareCapabilities, + HardwareAccelerationMode hardwareAccelerationMode, + Option videoInputFile, + Option audioInputFile, + Option watermarkInputFile, + Option subtitleInputFile, + string reportsFolder, + string fontsFolder, + ILogger logger) : base( + hardwareAccelerationMode, + videoInputFile, + audioInputFile, + watermarkInputFile, + subtitleInputFile, + reportsFolder, + fontsFolder, + logger) + { + _hardwareCapabilities = hardwareCapabilities; + _logger = logger; + } + + protected override FFmpegState SetAccelState( + VideoStream videoStream, + FFmpegState ffmpegState, + FrameState desiredState, + PipelineContext context, + ICollection pipelineSteps) + { + bool canDecode = _hardwareCapabilities.CanDecode(videoStream.Codec, videoStream.PixelFormat); + bool canEncode = _hardwareCapabilities.CanEncode(desiredState.VideoFormat, desiredState.PixelFormat); + + pipelineSteps.Add(new QsvHardwareAccelerationOption(ffmpegState.VaapiDevice)); + + // 10-bit hevc/h264 qsv decoders have issues, so use software + if (canDecode && videoStream.Codec is VideoFormat.Hevc or VideoFormat.H264 && + videoStream.PixelFormat.Map(pf => pf.BitDepth).IfNone(8) == 10) + { + canDecode = false; + } + + // disable hw accel if decoder/encoder isn't supported + return ffmpegState with + { + DecoderHardwareAccelerationMode = canDecode + ? HardwareAccelerationMode.Qsv + : HardwareAccelerationMode.None, + EncoderHardwareAccelerationMode = canEncode + ? HardwareAccelerationMode.Qsv + : HardwareAccelerationMode.None + }; + } + + protected override void SetDecoder( + VideoInputFile videoInputFile, + VideoStream videoStream, + FFmpegState ffmpegState, + PipelineContext context, + ICollection pipelineSteps) + { + Option maybeDecoder = (ffmpegState.DecoderHardwareAccelerationMode, videoStream.Codec) switch + { + (HardwareAccelerationMode.Qsv, VideoFormat.Hevc) => new DecoderHevcQsv(), + (HardwareAccelerationMode.Qsv, VideoFormat.H264) => new DecoderH264Qsv(), + (HardwareAccelerationMode.Qsv, VideoFormat.Mpeg2Video) => new DecoderMpeg2Qsv(), + (HardwareAccelerationMode.Qsv, VideoFormat.Vc1) => new DecoderVc1Qsv(), + (HardwareAccelerationMode.Qsv, VideoFormat.Vp9) => new DecoderVp9Qsv(), + + _ => GetSoftwareDecoder(videoStream) + }; + + foreach (IDecoder decoder in maybeDecoder) + { + videoInputFile.AddOption(decoder); + } + } + + protected override FilterChain SetVideoFilters( + VideoInputFile videoInputFile, + VideoStream videoStream, + Option watermarkInputFile, + Option subtitleInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + string fontsFolder, + ICollection pipelineSteps) + { + var watermarkOverlayFilterSteps = new List(); + var subtitleOverlayFilterSteps = new List(); + + FrameState currentState = desiredState with + { + ScaledSize = videoStream.FrameSize, + PaddedSize = videoStream.FrameSize, + + // consider hardware frames to be wrapped in nv12 + PixelFormat = ffmpegState.DecoderHardwareAccelerationMode == HardwareAccelerationMode.Qsv + ? videoStream.PixelFormat.Map(pf => (IPixelFormat)new PixelFormatNv12(pf.Name)) + : videoStream.PixelFormat, + + IsAnamorphic = videoStream.IsAnamorphic, + FrameDataLocation = ffmpegState.DecoderHardwareAccelerationMode == HardwareAccelerationMode.Qsv + ? FrameDataLocation.Hardware + : FrameDataLocation.Software + }; + + // easier to use nv12 for overlay + if (context.HasSubtitleOverlay || context.HasWatermark) + { + IPixelFormat pixelFormat = desiredState.PixelFormat.IfNone( + context.Is10BitOutput ? new PixelFormatYuv420P10Le() : new PixelFormatYuv420P()); + desiredState = desiredState with { PixelFormat = new PixelFormatNv12(pixelFormat.Name) }; + } + + // _logger.LogDebug("After decode: {PixelFormat}", currentState.PixelFormat); + currentState = SetDeinterlace(videoInputFile, context, ffmpegState, currentState); + // _logger.LogDebug("After deinterlace: {PixelFormat}", currentState.PixelFormat); + currentState = SetScale(videoInputFile, videoStream, context, ffmpegState, desiredState, currentState); + // _logger.LogDebug("After scale: {PixelFormat}", currentState.PixelFormat); + currentState = SetPad(videoInputFile, videoStream, desiredState, currentState); + // _logger.LogDebug("After pad: {PixelFormat}", currentState.PixelFormat); + + // need to download for any sort of overlay + if (currentState.FrameDataLocation == FrameDataLocation.Hardware && + (context.HasSubtitleOverlay || context.HasWatermark)) + { + var hardwareDownload = new HardwareDownloadFilter(currentState); + currentState = hardwareDownload.NextState(currentState); + videoInputFile.FilterSteps.Add(hardwareDownload); + } + + currentState = SetSubtitle( + videoInputFile, + subtitleInputFile, + context, + ffmpegState, + currentState, + desiredState, + fontsFolder, + subtitleOverlayFilterSteps); + + currentState = SetWatermark( + videoStream, + watermarkInputFile, + context, + ffmpegState, + desiredState, + currentState, + watermarkOverlayFilterSteps); + + // after everything else is done, apply the encoder + if (pipelineSteps.OfType().All(e => e.Kind != StreamKind.Video)) + { + Option maybeEncoder = + (ffmpegState.EncoderHardwareAccelerationMode, desiredState.VideoFormat) switch + { + (HardwareAccelerationMode.Qsv, VideoFormat.Hevc) => new EncoderHevcQsv(), + (HardwareAccelerationMode.Qsv, VideoFormat.H264) => new EncoderH264Qsv(), + + (_, _) => GetSoftwareEncoder(currentState, desiredState) + }; + + foreach (IEncoder encoder in maybeEncoder) + { + pipelineSteps.Add(encoder); + videoInputFile.FilterSteps.Add(encoder); + } + } + + List pixelFormatFilterSteps = SetPixelFormat( + videoStream, + desiredState.PixelFormat, + ffmpegState, + currentState, + context, + pipelineSteps); + + return new FilterChain( + videoInputFile.FilterSteps, + watermarkInputFile.Map(wm => wm.FilterSteps).IfNone(new List()), + subtitleInputFile.Map(st => st.FilterSteps).IfNone(new List()), + watermarkOverlayFilterSteps, + subtitleOverlayFilterSteps, + pixelFormatFilterSteps); + } + + private List SetPixelFormat( + VideoStream videoStream, + Option desiredPixelFormat, + FFmpegState ffmpegState, + FrameState currentState, + PipelineContext context, + ICollection pipelineSteps) + { + var result = new List(); + + foreach (IPixelFormat pixelFormat in desiredPixelFormat) + { + IPixelFormat format = pixelFormat; + + if (pixelFormat is PixelFormatNv12 nv12) + { + foreach (IPixelFormat pf in AvailablePixelFormats.ForPixelFormat(nv12.Name, null)) + { + format = pf; + } + } + + IPixelFormat formatForDownload = pixelFormat; + + if (!videoStream.ColorParams.IsBt709) + { + _logger.LogDebug("Adding colorspace filter"); + var colorspace = new ColorspaceFilter(videoStream, format, currentState.FrameDataLocation); + + // force nv12 if we're still in hardware + if (currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + if (formatForDownload is not PixelFormatNv12) + { + formatForDownload = new PixelFormatNv12(pixelFormat.Name); + } + } + + currentState = colorspace.NextState(currentState); + result.Add(colorspace); + } + + if (ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.None) + { + _logger.LogDebug("Using software encoder"); + + if (currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + _logger.LogDebug("FrameDataLocation == FrameDataLocation.Hardware"); + + var hardwareDownload = + new HardwareDownloadFilter(currentState with { PixelFormat = Some(formatForDownload) }); + currentState = hardwareDownload.NextState(currentState); + result.Add(hardwareDownload); + } + } + + if (currentState.PixelFormat.Map(f => f.FFmpegName) != format.FFmpegName) + { + _logger.LogDebug( + "Format {A} doesn't equal {B}", + currentState.PixelFormat.Map(f => f.FFmpegName), + format.FFmpegName); + + // remind qsv that it uses qsv + if (currentState.FrameDataLocation == FrameDataLocation.Hardware && + result.Count == 1 && result[0] is ColorspaceFilter colorspace) + { + if (colorspace.Filter.StartsWith("setparams=")) + { + result.Insert(0, new QsvFormatFilter(new PixelFormatQsv(format.Name))); + } + } + + // qsv encoders don't like yuv420p + format = format switch + { + PixelFormatYuv420P => new PixelFormatNv12(PixelFormat.YUV420P), + _ => format + }; + + pipelineSteps.Add(new PixelFormatOutputOption(format)); + } + } + + return result; + } + + private FrameState SetWatermark( + VideoStream videoStream, + Option watermarkInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + FrameState currentState, + List watermarkOverlayFilterSteps) + { + if (context.HasWatermark) + { + WatermarkInputFile watermark = watermarkInputFile.Head(); + + foreach (VideoStream watermarkStream in watermark.VideoStreams) + { + if (watermarkStream.StillImage == false) + { + watermark.AddOption(new DoNotIgnoreLoopInputOption()); + } + else if (watermark.DesiredState.MaybeFadePoints.Map(fp => fp.Count > 0).IfNone(false)) + { + // looping is required to fade a static image in and out + watermark.AddOption(new InfiniteLoopInputOption(HardwareAccelerationMode.None)); + } + } + + if (watermark.DesiredState.Size == WatermarkSize.Scaled) + { + watermark.FilterSteps.Add( + new WatermarkScaleFilter(watermark.DesiredState, currentState.PaddedSize)); + } + + if (watermark.DesiredState.Opacity != 100) + { + watermark.FilterSteps.Add(new WatermarkOpacityFilter(watermark.DesiredState)); + } + + IPixelFormat pixelFormat = context.Is10BitOutput + ? new PixelFormatNv12(FFmpegFormat.P010LE) + : new PixelFormatNv12(FFmpegFormat.YUVA420P); + + watermark.FilterSteps.Add(new PixelFormatFilter(pixelFormat)); + + foreach (List fadePoints in watermark.DesiredState.MaybeFadePoints) + { + watermark.FilterSteps.AddRange(fadePoints.Map(fp => new WatermarkFadeFilter(fp))); + } + + foreach (IPixelFormat desiredPixelFormat in desiredState.PixelFormat) + { + IPixelFormat pf = desiredPixelFormat; + if (desiredPixelFormat is PixelFormatNv12 nv12) + { + foreach (IPixelFormat availablePixelFormat in AvailablePixelFormats.ForPixelFormat(nv12.Name, null)) + { + pf = availablePixelFormat; + } + } + + var watermarkFilter = new OverlayWatermarkFilter( + watermark.DesiredState, + desiredState.PaddedSize, + videoStream.SquarePixelFrameSize(currentState.PaddedSize), + pf, + _logger); + watermarkOverlayFilterSteps.Add(watermarkFilter); + } + } + + return currentState; + } + + private static FrameState SetSubtitle( + VideoInputFile videoInputFile, + Option subtitleInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState currentState, + FrameState desiredState, + string fontsFolder, + ICollection subtitleOverlayFilterSteps) + { + foreach (SubtitleInputFile subtitle in subtitleInputFile) + { + if (context.HasSubtitleText) + { + videoInputFile.AddOption(new CopyTimestampInputOption()); + + // if (videoInputFile.FilterSteps.Count == 0 && videoInputFile.InputOptions.OfType().Any()) + // { + // // change the hw accel output to software so the explicit download isn't needed + // foreach (CuvidDecoder decoder in videoInputFile.InputOptions.OfType()) + // { + // decoder.HardwareAccelerationMode = HardwareAccelerationMode.None; + // } + // } + // else + // { + var downloadFilter = new HardwareDownloadFilter(currentState); + currentState = downloadFilter.NextState(currentState); + videoInputFile.FilterSteps.Add(downloadFilter); + // } + + var subtitlesFilter = new SubtitlesFilter(fontsFolder, subtitle); + currentState = subtitlesFilter.NextState(currentState); + videoInputFile.FilterSteps.Add(subtitlesFilter); + } + else if (context.HasSubtitleOverlay) + { + IPixelFormat pixelFormat = new PixelFormatYuva420P(); + + var pixelFormatFilter = new PixelFormatFilter(pixelFormat); + subtitle.FilterSteps.Add(pixelFormatFilter); + + foreach (IPixelFormat desiredPixelFormat in desiredState.PixelFormat) + { + IPixelFormat pf = desiredPixelFormat; + if (desiredPixelFormat is PixelFormatNv12 nv12) + { + foreach (IPixelFormat availablePixelFormat in AvailablePixelFormats.ForPixelFormat(nv12.Name, null)) + { + pf = availablePixelFormat; + } + } + + var subtitlesFilter = new OverlaySubtitleFilter(pf); + subtitleOverlayFilterSteps.Add(subtitlesFilter); + } + } + } + + return currentState; + } + + private static FrameState SetPad( + VideoInputFile videoInputFile, + VideoStream videoStream, + FrameState desiredState, + FrameState currentState) + { + if (currentState.PaddedSize != desiredState.PaddedSize) + { + IPipelineFilterStep padStep = new PadFilter( + currentState, + desiredState.PaddedSize); + currentState = padStep.NextState(currentState); + videoInputFile.FilterSteps.Add(padStep); + } + + return currentState; + } + + private static FrameState SetScale( + VideoInputFile videoInputFile, + VideoStream videoStream, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + FrameState currentState) + { + IPipelineFilterStep scaleStep; + + if (currentState.ScaledSize != desiredState.ScaledSize && ffmpegState is + { + DecoderHardwareAccelerationMode: HardwareAccelerationMode.None, + EncoderHardwareAccelerationMode: HardwareAccelerationMode.None + } && context is { HasWatermark: false, HasSubtitleOverlay: false, ShouldDeinterlace: false }) + { + scaleStep = new ScaleFilter( + currentState, + desiredState.ScaledSize, + desiredState.PaddedSize, + videoStream.IsAnamorphicEdgeCase); + } + else + { + scaleStep = new ScaleQsvFilter( + currentState with + { + PixelFormat = //context.HasWatermark || + //context.HasSubtitleOverlay || + // (desiredState.ScaledSize != desiredState.PaddedSize) || + // context.HasSubtitleText || + ffmpegState is + { + DecoderHardwareAccelerationMode: HardwareAccelerationMode.Nvenc, + EncoderHardwareAccelerationMode: HardwareAccelerationMode.None + } + ? desiredState.PixelFormat.Map(pf => (IPixelFormat)new PixelFormatNv12(pf.Name)) + : Option.None + }, + desiredState.ScaledSize, + ffmpegState.QsvExtraHardwareFrames, + videoStream.IsAnamorphicEdgeCase, + videoStream.SampleAspectRatio); + } + + if (!string.IsNullOrWhiteSpace(scaleStep.Filter)) + { + currentState = scaleStep.NextState(currentState); + videoInputFile.FilterSteps.Add(scaleStep); + } + + return currentState; + } + + private static FrameState SetDeinterlace( + VideoInputFile videoInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState currentState) + { + if (context.ShouldDeinterlace) + { + var filter = new DeinterlaceQsvFilter(currentState, ffmpegState.QsvExtraHardwareFrames); + currentState = filter.NextState(currentState); + videoInputFile.FilterSteps.Add(filter); + } + + return currentState; + } +} diff --git a/ErsatzTV.FFmpeg/Pipeline/SoftwarePipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/SoftwarePipelineBuilder.cs new file mode 100644 index 000000000..9db0f84f1 --- /dev/null +++ b/ErsatzTV.FFmpeg/Pipeline/SoftwarePipelineBuilder.cs @@ -0,0 +1,318 @@ +using ErsatzTV.FFmpeg.Decoder; +using ErsatzTV.FFmpeg.Encoder; +using ErsatzTV.FFmpeg.Filter; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.Option; +using ErsatzTV.FFmpeg.State; +using Microsoft.Extensions.Logging; + +namespace ErsatzTV.FFmpeg.Pipeline; + +public class SoftwarePipelineBuilder : PipelineBuilderBase +{ + private readonly ILogger _logger; + + public SoftwarePipelineBuilder( + HardwareAccelerationMode hardwareAccelerationMode, + Option videoInputFile, + Option audioInputFile, + Option watermarkInputFile, + Option subtitleInputFile, + string reportsFolder, + string fontsFolder, + ILogger logger) : base( + hardwareAccelerationMode, + videoInputFile, + audioInputFile, + watermarkInputFile, + subtitleInputFile, + reportsFolder, + fontsFolder, + logger) + { + _logger = logger; + } + + protected override FFmpegState SetAccelState( + VideoStream videoStream, + FFmpegState ffmpegState, + FrameState desiredState, + PipelineContext context, + ICollection pipelineSteps) => ffmpegState with + { + DecoderHardwareAccelerationMode = HardwareAccelerationMode.None, + EncoderHardwareAccelerationMode = HardwareAccelerationMode.None + }; + + protected override void SetDecoder( + VideoInputFile videoInputFile, + VideoStream videoStream, + FFmpegState ffmpegState, + PipelineContext context, + ICollection pipelineSteps) + { + foreach (IDecoder decoder in GetSoftwareDecoder(videoStream)) + { + videoInputFile.AddOption(decoder); + } + } + + protected virtual Option GetEncoder( + FFmpegState ffmpegState, + FrameState currentState, + FrameState desiredState) + { + return GetSoftwareEncoder(currentState, desiredState); + } + + protected override FilterChain SetVideoFilters( + VideoInputFile videoInputFile, + VideoStream videoStream, + Option watermarkInputFile, + Option subtitleInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + string fontsFolder, + ICollection pipelineSteps) + { + var watermarkOverlayFilterSteps = new List(); + var subtitleOverlayFilterSteps = new List(); + + FrameState currentState = desiredState with + { + PixelFormat = videoStream.PixelFormat, + FrameDataLocation = FrameDataLocation.Software, + IsAnamorphic = videoStream.IsAnamorphic, + ScaledSize = videoStream.FrameSize, + PaddedSize = videoStream.FrameSize + }; + + SetDeinterlace(videoInputFile, context, currentState); + + currentState = SetScale(videoInputFile, videoStream, desiredState, currentState); + currentState = SetPad(videoInputFile, videoStream, desiredState, currentState); + SetSubtitle(videoInputFile, subtitleInputFile, context, desiredState, fontsFolder, subtitleOverlayFilterSteps); + SetWatermark( + videoStream, + watermarkInputFile, + context, + ffmpegState, + desiredState, + currentState, + watermarkOverlayFilterSteps); + + // after everything else is done, apply the encoder + if (pipelineSteps.OfType().All(e => e.Kind != StreamKind.Video)) + { + foreach (IEncoder encoder in GetEncoder(ffmpegState, currentState, desiredState)) + { + pipelineSteps.Add(encoder); + videoInputFile.FilterSteps.Add(encoder); + } + } + + List pixelFormatFilterSteps = SetPixelFormat( + videoStream, + desiredState.PixelFormat, + currentState, + pipelineSteps); + + return new FilterChain( + videoInputFile.FilterSteps, + watermarkInputFile.Map(wm => wm.FilterSteps).IfNone(new List()), + subtitleInputFile.Map(st => st.FilterSteps).IfNone(new List()), + watermarkOverlayFilterSteps, + subtitleOverlayFilterSteps, + pixelFormatFilterSteps); + } + + protected virtual List SetPixelFormat( + VideoStream videoStream, + Option desiredPixelFormat, + FrameState currentState, + ICollection pipelineSteps) + { + var result = new List(); + + foreach (IPixelFormat pixelFormat in desiredPixelFormat) + { + if (!videoStream.ColorParams.IsBt709) + { + _logger.LogDebug("Adding colorspace filter"); + var colorspace = new ColorspaceFilter(videoStream, pixelFormat); + currentState = colorspace.NextState(currentState); + result.Add(colorspace); + } + + if (currentState.PixelFormat.Map(f => f.FFmpegName) != pixelFormat.FFmpegName) + { + _logger.LogDebug( + "Format {A} doesn't equal {B}", + currentState.PixelFormat.Map(f => f.FFmpegName), + pixelFormat.FFmpegName); + + pipelineSteps.Add(new PixelFormatOutputOption(pixelFormat)); + } + } + + return result; + } + + private void SetWatermark( + VideoStream videoStream, + Option watermarkInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + FrameState currentState, + List watermarkOverlayFilterSteps) + { + if (context.HasWatermark) + { + WatermarkInputFile watermark = watermarkInputFile.Head(); + + watermark.FilterSteps.Add( + new WatermarkPixelFormatFilter(ffmpegState, watermark.DesiredState, context.Is10BitOutput)); + + foreach (VideoStream watermarkStream in watermark.VideoStreams) + { + if (watermarkStream.StillImage == false) + { + watermark.AddOption(new DoNotIgnoreLoopInputOption()); + } + else if (watermark.DesiredState.MaybeFadePoints.Map(fp => fp.Count > 0).IfNone(false)) + { + // looping is required to fade a static image in and out + watermark.AddOption(new InfiniteLoopInputOption(HardwareAccelerationMode.None)); + } + } + + if (watermark.DesiredState.Size == WatermarkSize.Scaled) + { + watermark.FilterSteps.Add( + new WatermarkScaleFilter(watermark.DesiredState, currentState.PaddedSize)); + } + + if (watermark.DesiredState.Opacity != 100) + { + watermark.FilterSteps.Add(new WatermarkOpacityFilter(watermark.DesiredState)); + } + + foreach (List fadePoints in watermark.DesiredState.MaybeFadePoints) + { + watermark.FilterSteps.AddRange(fadePoints.Map(fp => new WatermarkFadeFilter(fp))); + } + + foreach (IPixelFormat desiredPixelFormat in desiredState.PixelFormat) + { + IPixelFormat pf = desiredPixelFormat; + if (desiredPixelFormat is PixelFormatNv12 nv12) + { + foreach (IPixelFormat availablePixelFormat in AvailablePixelFormats.ForPixelFormat(nv12.Name, null)) + { + pf = availablePixelFormat; + } + } + + var watermarkFilter = new OverlayWatermarkFilter( + watermark.DesiredState, + desiredState.PaddedSize, + videoStream.SquarePixelFrameSize(currentState.PaddedSize), + pf, + _logger); + watermarkOverlayFilterSteps.Add(watermarkFilter); + } + } + } + + private static void SetSubtitle( + VideoInputFile videoInputFile, + Option subtitleInputFile, + PipelineContext context, + FrameState desiredState, + string fontsFolder, + ICollection subtitleOverlayFilterSteps) + { + foreach (SubtitleInputFile subtitle in subtitleInputFile) + { + if (context.HasSubtitleText) + { + videoInputFile.AddOption(new CopyTimestampInputOption()); + + var subtitlesFilter = new SubtitlesFilter(fontsFolder, subtitle); + videoInputFile.FilterSteps.Add(subtitlesFilter); + } + else if (context.HasSubtitleOverlay) + { + // only scale if scaling or padding was used for main video stream + if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or PadFilter)) + { + var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize); + subtitle.FilterSteps.Add(scaleFilter); + } + + foreach (IPixelFormat desiredPixelFormat in desiredState.PixelFormat) + { + IPixelFormat pf = desiredPixelFormat; + if (desiredPixelFormat is PixelFormatNv12 nv12) + { + foreach (IPixelFormat availablePixelFormat in AvailablePixelFormats.ForPixelFormat(nv12.Name, null)) + { + pf = availablePixelFormat; + } + } + + var subtitlesFilter = new OverlaySubtitleFilter(pf); + subtitleOverlayFilterSteps.Add(subtitlesFilter); + } + } + } + } + + private static FrameState SetPad( + VideoInputFile videoInputFile, + VideoStream videoStream, + FrameState desiredState, + FrameState currentState) + { + if (currentState.PaddedSize != desiredState.PaddedSize) + { + IPipelineFilterStep padStep = new PadFilter(currentState, desiredState.PaddedSize); + currentState = padStep.NextState(currentState); + videoInputFile.FilterSteps.Add(padStep); + } + + return currentState; + } + + private static FrameState SetScale( + VideoInputFile videoInputFile, + VideoStream videoStream, + FrameState desiredState, + FrameState currentState) + { + if (videoStream.FrameSize != desiredState.ScaledSize) + { + IPipelineFilterStep scaleStep = new ScaleFilter( + currentState, + desiredState.ScaledSize, + desiredState.PaddedSize, + videoStream.IsAnamorphicEdgeCase); + + currentState = scaleStep.NextState(currentState); + + videoInputFile.FilterSteps.Add(scaleStep); + } + + return currentState; + } + + private static void SetDeinterlace(VideoInputFile videoInputFile, PipelineContext context, FrameState currentState) + { + if (context.ShouldDeinterlace) + { + videoInputFile.FilterSteps.Add(new YadifFilter(currentState)); + } + } +} diff --git a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs new file mode 100644 index 000000000..85988c886 --- /dev/null +++ b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs @@ -0,0 +1,546 @@ +using ErsatzTV.FFmpeg.Capabilities; +using ErsatzTV.FFmpeg.Decoder; +using ErsatzTV.FFmpeg.Encoder; +using ErsatzTV.FFmpeg.Encoder.Vaapi; +using ErsatzTV.FFmpeg.Filter; +using ErsatzTV.FFmpeg.Filter.Vaapi; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.Option; +using ErsatzTV.FFmpeg.Option.HardwareAcceleration; +using ErsatzTV.FFmpeg.State; +using Microsoft.Extensions.Logging; + +namespace ErsatzTV.FFmpeg.Pipeline; + +public class VaapiPipelineBuilder : SoftwarePipelineBuilder +{ + private readonly IHardwareCapabilities _hardwareCapabilities; + private readonly ILogger _logger; + + public VaapiPipelineBuilder( + IHardwareCapabilities hardwareCapabilities, + HardwareAccelerationMode hardwareAccelerationMode, + Option videoInputFile, + Option audioInputFile, + Option watermarkInputFile, + Option subtitleInputFile, + string reportsFolder, + string fontsFolder, + ILogger logger) : base( + hardwareAccelerationMode, + videoInputFile, + audioInputFile, + watermarkInputFile, + subtitleInputFile, + reportsFolder, + fontsFolder, + logger) + { + _hardwareCapabilities = hardwareCapabilities; + _logger = logger; + } + + protected override FFmpegState SetAccelState( + VideoStream videoStream, + FFmpegState ffmpegState, + FrameState desiredState, + PipelineContext context, + ICollection pipelineSteps) + { + bool canDecode = _hardwareCapabilities.CanDecode(videoStream.Codec, videoStream.PixelFormat); + bool canEncode = _hardwareCapabilities.CanEncode(desiredState.VideoFormat, desiredState.PixelFormat); + + foreach (string vaapiDevice in ffmpegState.VaapiDevice) + { + pipelineSteps.Add(new VaapiHardwareAccelerationOption(vaapiDevice)); + } + + // use software decoding with an extensive pipeline + if (context.HasSubtitleOverlay && context.HasWatermark) + { + canDecode = false; + } + + // disable hw accel if decoder/encoder isn't supported + return ffmpegState with + { + DecoderHardwareAccelerationMode = canDecode + ? HardwareAccelerationMode.Vaapi + : HardwareAccelerationMode.None, + EncoderHardwareAccelerationMode = canEncode + ? HardwareAccelerationMode.Vaapi + : HardwareAccelerationMode.None + }; + } + + protected override void SetDecoder( + VideoInputFile videoInputFile, + VideoStream videoStream, + FFmpegState ffmpegState, + PipelineContext context, + ICollection pipelineSteps) + { + Option maybeDecoder = (ffmpegState.DecoderHardwareAccelerationMode, videoStream.Codec) switch + { + (HardwareAccelerationMode.Vaapi, _) => new DecoderVaapi(), + _ => GetSoftwareDecoder(videoStream) + }; + + foreach (IDecoder decoder in maybeDecoder) + { + videoInputFile.AddOption(decoder); + } + } + + protected override FilterChain SetVideoFilters( + VideoInputFile videoInputFile, + VideoStream videoStream, + Option watermarkInputFile, + Option subtitleInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + string fontsFolder, + ICollection pipelineSteps) + { + var watermarkOverlayFilterSteps = new List(); + var subtitleOverlayFilterSteps = new List(); + + FrameState currentState = desiredState with + { + ScaledSize = videoStream.FrameSize, + PaddedSize = videoStream.FrameSize, + + PixelFormat = videoStream.PixelFormat, + + IsAnamorphic = videoStream.IsAnamorphic, + + FrameDataLocation = ffmpegState.DecoderHardwareAccelerationMode == HardwareAccelerationMode.Vaapi + ? FrameDataLocation.Hardware + : FrameDataLocation.Software + }; + + // easier to use nv12 for overlay + if (context.HasSubtitleOverlay || context.HasWatermark) + { + IPixelFormat pixelFormat = desiredState.PixelFormat.IfNone( + context.Is10BitOutput ? new PixelFormatYuv420P10Le() : new PixelFormatYuv420P()); + desiredState = desiredState with { PixelFormat = new PixelFormatNv12(pixelFormat.Name) }; + } + + // _logger.LogDebug("After decode: {PixelFormat}", currentState.PixelFormat); + + currentState = SetDeinterlace(videoInputFile, context, ffmpegState, currentState); + // _logger.LogDebug("After deinterlace: {PixelFormat}", currentState.PixelFormat); + + currentState = SetScale(videoInputFile, videoStream, context, ffmpegState, desiredState, currentState); + // _logger.LogDebug("After scale: {PixelFormat}", currentState.PixelFormat); + + currentState = SetPad(videoInputFile, videoStream, desiredState, currentState); + // _logger.LogDebug("After pad: {PixelFormat}", currentState.PixelFormat); + + // need to upload for hardware overlay + bool forceSoftwareOverlay = context.HasSubtitleOverlay && context.HasWatermark; + + if (currentState.FrameDataLocation == FrameDataLocation.Software && context.HasSubtitleOverlay && + !forceSoftwareOverlay) + { + var hardwareUpload = new HardwareUploadVaapiFilter(true); + currentState = hardwareUpload.NextState(currentState); + videoInputFile.FilterSteps.Add(hardwareUpload); + } + else if(currentState.FrameDataLocation == FrameDataLocation.Hardware && + (!context.HasSubtitleOverlay || forceSoftwareOverlay) && + context.HasWatermark) + { + // download for watermark (or forced software subtitle) + var hardwareDownload = new HardwareDownloadFilter(currentState); + currentState = hardwareDownload.NextState(currentState); + videoInputFile.FilterSteps.Add(hardwareDownload); + } + + currentState = SetSubtitle( + videoInputFile, + subtitleInputFile, + context, + forceSoftwareOverlay, + ffmpegState, + currentState, + desiredState, + fontsFolder, + subtitleOverlayFilterSteps); + + currentState = SetWatermark( + videoStream, + watermarkInputFile, + context, + ffmpegState, + desiredState, + currentState, + watermarkOverlayFilterSteps); + + // after everything else is done, apply the encoder + if (pipelineSteps.OfType().All(e => e.Kind != StreamKind.Video)) + { + Option maybeEncoder = + (ffmpegState.EncoderHardwareAccelerationMode, desiredState.VideoFormat) switch + { + (HardwareAccelerationMode.Vaapi, VideoFormat.Hevc) => new EncoderHevcVaapi(), + (HardwareAccelerationMode.Vaapi, VideoFormat.H264) => new EncoderH264Vaapi(), + + (_, _) => GetSoftwareEncoder(currentState, desiredState) + }; + + foreach (IEncoder encoder in maybeEncoder) + { + pipelineSteps.Add(encoder); + videoInputFile.FilterSteps.Add(encoder); + } + } + + List pixelFormatFilterSteps = SetPixelFormat( + videoStream, + desiredState.PixelFormat, + ffmpegState, + currentState, + context, + pipelineSteps); + + return new FilterChain( + videoInputFile.FilterSteps, + watermarkInputFile.Map(wm => wm.FilterSteps).IfNone(new List()), + subtitleInputFile.Map(st => st.FilterSteps).IfNone(new List()), + watermarkOverlayFilterSteps, + subtitleOverlayFilterSteps, + pixelFormatFilterSteps); + } + + private List SetPixelFormat( + VideoStream videoStream, + Option desiredPixelFormat, + FFmpegState ffmpegState, + FrameState currentState, + PipelineContext context, + ICollection pipelineSteps) + { + var result = new List(); + + foreach (IPixelFormat pixelFormat in desiredPixelFormat) + { + IPixelFormat format = pixelFormat; + + if (pixelFormat is PixelFormatNv12 nv12) + { + foreach (IPixelFormat pf in AvailablePixelFormats.ForPixelFormat(nv12.Name, null)) + { + format = pf; + } + } + + if (!videoStream.ColorParams.IsBt709) + { + _logger.LogDebug("Adding colorspace filter"); + var colorspace = new ColorspaceFilter(videoStream, format, currentState.FrameDataLocation); + currentState = colorspace.NextState(currentState); + result.Add(colorspace); + } + + if (ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.None) + { + _logger.LogDebug("Using software encoder"); + + if (currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + _logger.LogDebug("FrameDataLocation == FrameDataLocation.Hardware"); + + var hardwareDownload = + new HardwareDownloadFilter(currentState with { PixelFormat = Some(format) }); + currentState = hardwareDownload.NextState(currentState); + result.Add(hardwareDownload); + } + } + + if (currentState.PixelFormat.Map(f => f.FFmpegName) != format.FFmpegName) + { + _logger.LogDebug( + "Format {A} doesn't equal {B}", + currentState.PixelFormat.Map(f => f.FFmpegName), + format.FFmpegName); + + // NV12 is 8-bit + if (format is PixelFormatYuv420P) + { + format = new PixelFormatNv12(format.Name); + } + + if (currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + result.Add(new VaapiFormatFilter(format)); + } + else + { + result.Add(new PixelFormatFilter(format)); + } + } + + if (ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Vaapi && + currentState.FrameDataLocation == FrameDataLocation.Software) + { + bool setFormat = result.All(f => f is not VaapiFormatFilter && f is not PixelFormatFilter); + result.Add(new HardwareUploadVaapiFilter(setFormat)); + } + } + + return result; + } + + private FrameState SetWatermark( + VideoStream videoStream, + Option watermarkInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + FrameState currentState, + List watermarkOverlayFilterSteps) + { + if (context.HasWatermark) + { + WatermarkInputFile watermark = watermarkInputFile.Head(); + + foreach (VideoStream watermarkStream in watermark.VideoStreams) + { + if (watermarkStream.StillImage == false) + { + watermark.AddOption(new DoNotIgnoreLoopInputOption()); + } + else if (watermark.DesiredState.MaybeFadePoints.Map(fp => fp.Count > 0).IfNone(false)) + { + // looping is required to fade a static image in and out + watermark.AddOption(new InfiniteLoopInputOption(HardwareAccelerationMode.None)); + } + } + + if (watermark.DesiredState.Size == WatermarkSize.Scaled) + { + watermark.FilterSteps.Add( + new WatermarkScaleFilter(watermark.DesiredState, currentState.PaddedSize)); + } + + if (watermark.DesiredState.Opacity != 100) + { + watermark.FilterSteps.Add(new WatermarkOpacityFilter(watermark.DesiredState)); + } + + watermark.FilterSteps.Add(new PixelFormatFilter(new PixelFormatYuva420P())); + + foreach (List fadePoints in watermark.DesiredState.MaybeFadePoints) + { + watermark.FilterSteps.AddRange(fadePoints.Map(fp => new WatermarkFadeFilter(fp))); + } + + foreach (IPixelFormat desiredPixelFormat in desiredState.PixelFormat) + { + IPixelFormat pf = desiredPixelFormat; + if (desiredPixelFormat is PixelFormatNv12 nv12) + { + foreach (IPixelFormat availablePixelFormat in AvailablePixelFormats.ForPixelFormat(nv12.Name, null)) + { + pf = availablePixelFormat; + } + } + + var watermarkFilter = new OverlayWatermarkFilter( + watermark.DesiredState, + desiredState.PaddedSize, + videoStream.SquarePixelFrameSize(currentState.PaddedSize), + pf, + _logger); + watermarkOverlayFilterSteps.Add(watermarkFilter); + } + } + + return currentState; + } + + private static FrameState SetSubtitle( + VideoInputFile videoInputFile, + Option subtitleInputFile, + PipelineContext context, + bool forceSoftwareOverlay, + FFmpegState ffmpegState, + FrameState currentState, + FrameState desiredState, + string fontsFolder, + ICollection subtitleOverlayFilterSteps) + { + foreach (SubtitleInputFile subtitle in subtitleInputFile) + { + if (context.HasSubtitleText) + { + videoInputFile.AddOption(new CopyTimestampInputOption()); + + // if (videoInputFile.FilterSteps.Count == 0 && videoInputFile.InputOptions.OfType().Any()) + // { + // // change the hw accel output to software so the explicit download isn't needed + // foreach (CuvidDecoder decoder in videoInputFile.InputOptions.OfType()) + // { + // decoder.HardwareAccelerationMode = HardwareAccelerationMode.None; + // } + // } + // else + // { + var downloadFilter = new HardwareDownloadFilter(currentState); + currentState = downloadFilter.NextState(currentState); + videoInputFile.FilterSteps.Add(downloadFilter); + // } + + var subtitlesFilter = new SubtitlesFilter(fontsFolder, subtitle); + currentState = subtitlesFilter.NextState(currentState); + videoInputFile.FilterSteps.Add(subtitlesFilter); + } + else if (context.HasSubtitleOverlay) + { + var pixelFormatFilter = new PixelFormatFilter(new PixelFormatArgb()); + subtitle.FilterSteps.Add(pixelFormatFilter); + + if (forceSoftwareOverlay) + { + foreach (IPixelFormat pixelFormat in desiredState.PixelFormat) + { + IPixelFormat pf = pixelFormat; + if (pixelFormat is PixelFormatNv12 nv12) + { + foreach (IPixelFormat format in AvailablePixelFormats.ForPixelFormat(nv12.Name, null)) + { + pf = format; + } + } + + var subtitlesFilter = new OverlaySubtitleFilter(pf); + subtitleOverlayFilterSteps.Add(subtitlesFilter); + } + } + else + { + var subtitleHardwareUpload = new HardwareUploadVaapiFilter(false); + subtitle.FilterSteps.Add(subtitleHardwareUpload); + + // always scale - shouldn't really be needed outside of transcoding tests, which use picture subtitles + // that are too big + var scaleFilter = new SubtitleScaleVaapiFilter(desiredState.PaddedSize); + subtitle.FilterSteps.Add(scaleFilter); + + var subtitlesFilter = new OverlaySubtitleVaapiFilter(); + subtitleOverlayFilterSteps.Add(subtitlesFilter); + } + + if (context.HasWatermark && !forceSoftwareOverlay) + { + // download for watermark + var hardwareDownload = new HardwareDownloadFilter(currentState); + currentState = hardwareDownload.NextState(currentState); + subtitleOverlayFilterSteps.Add(hardwareDownload); + } + } + } + + return currentState; + } + + private static FrameState SetPad( + VideoInputFile videoInputFile, + VideoStream videoStream, + FrameState desiredState, + FrameState currentState) + { + if (currentState.PaddedSize != desiredState.PaddedSize) + { + IPipelineFilterStep padStep = new PadFilter( + currentState, + desiredState.PaddedSize); + currentState = padStep.NextState(currentState); + videoInputFile.FilterSteps.Add(padStep); + } + + return currentState; + } + + private static FrameState SetScale( + VideoInputFile videoInputFile, + VideoStream videoStream, + PipelineContext context, + FFmpegState ffmpegState, + FrameState desiredState, + FrameState currentState) + { + IPipelineFilterStep scaleStep; + + if ((currentState.ScaledSize != desiredState.ScaledSize && ffmpegState is + { + DecoderHardwareAccelerationMode: HardwareAccelerationMode.None, + EncoderHardwareAccelerationMode: HardwareAccelerationMode.None + } && context is { HasWatermark: false, HasSubtitleOverlay: false, ShouldDeinterlace: false }) || + ffmpegState.DecoderHardwareAccelerationMode != HardwareAccelerationMode.Vaapi) + { + scaleStep = new ScaleFilter( + currentState, + desiredState.ScaledSize, + desiredState.PaddedSize, + videoStream.IsAnamorphicEdgeCase); + } + else + { + scaleStep = new ScaleVaapiFilter( + currentState with + { + PixelFormat = //context.HasWatermark || + //context.HasSubtitleOverlay || + // (desiredState.ScaledSize != desiredState.PaddedSize) || + // context.HasSubtitleText || + ffmpegState is + { + DecoderHardwareAccelerationMode: HardwareAccelerationMode.Nvenc, + EncoderHardwareAccelerationMode: HardwareAccelerationMode.None + } + ? desiredState.PixelFormat.Map(pf => (IPixelFormat)new PixelFormatNv12(pf.Name)) + : Option.None + }, + desiredState.ScaledSize, + desiredState.PaddedSize, + videoStream.IsAnamorphicEdgeCase); + } + + if (!string.IsNullOrWhiteSpace(scaleStep.Filter)) + { + currentState = scaleStep.NextState(currentState); + videoInputFile.FilterSteps.Add(scaleStep); + } + + return currentState; + } + + private static FrameState SetDeinterlace( + VideoInputFile videoInputFile, + PipelineContext context, + FFmpegState ffmpegState, + FrameState currentState) + { + if (context.ShouldDeinterlace) + { + if (ffmpegState.DecoderHardwareAccelerationMode == HardwareAccelerationMode.Vaapi) + { + var filter = new DeinterlaceVaapiFilter(currentState); + currentState = filter.NextState(currentState); + videoInputFile.FilterSteps.Add(filter); + } + else + { + var filter = new YadifFilter(currentState); + currentState = filter.NextState(currentState); + videoInputFile.FilterSteps.Add(filter); + } + } + + return currentState; + } +} diff --git a/ErsatzTV.FFmpeg/Pipeline/VideoToolboxPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/VideoToolboxPipelineBuilder.cs new file mode 100644 index 000000000..3697e5f2e --- /dev/null +++ b/ErsatzTV.FFmpeg/Pipeline/VideoToolboxPipelineBuilder.cs @@ -0,0 +1,130 @@ +using ErsatzTV.FFmpeg.Capabilities; +using ErsatzTV.FFmpeg.Decoder; +using ErsatzTV.FFmpeg.Encoder; +using ErsatzTV.FFmpeg.Encoder.VideoToolbox; +using ErsatzTV.FFmpeg.Filter; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.Option; +using ErsatzTV.FFmpeg.Option.HardwareAcceleration; +using Microsoft.Extensions.Logging; + +namespace ErsatzTV.FFmpeg.Pipeline; + +public class VideoToolboxPipelineBuilder : SoftwarePipelineBuilder +{ + private readonly IHardwareCapabilities _hardwareCapabilities; + private readonly ILogger _logger; + + public VideoToolboxPipelineBuilder( + IHardwareCapabilities hardwareCapabilities, + HardwareAccelerationMode hardwareAccelerationMode, + Option videoInputFile, + Option audioInputFile, + Option watermarkInputFile, + Option subtitleInputFile, + string reportsFolder, + string fontsFolder, + ILogger logger) : base( + hardwareAccelerationMode, + videoInputFile, + audioInputFile, + watermarkInputFile, + subtitleInputFile, + reportsFolder, + fontsFolder, + logger) + { + _hardwareCapabilities = hardwareCapabilities; + _logger = logger; + } + + protected override FFmpegState SetAccelState( + VideoStream videoStream, + FFmpegState ffmpegState, + FrameState desiredState, + PipelineContext context, + ICollection pipelineSteps) + { + bool canDecode = _hardwareCapabilities.CanDecode(videoStream.Codec, videoStream.PixelFormat); + bool canEncode = _hardwareCapabilities.CanEncode(desiredState.VideoFormat, desiredState.PixelFormat); + + pipelineSteps.Add(new VideoToolboxHardwareAccelerationOption()); + + // disable hw accel if decoder/encoder isn't supported + return ffmpegState with + { + DecoderHardwareAccelerationMode = canDecode + ? HardwareAccelerationMode.VideoToolbox + : HardwareAccelerationMode.None, + EncoderHardwareAccelerationMode = canEncode + ? HardwareAccelerationMode.VideoToolbox + : HardwareAccelerationMode.None + }; + } + + protected override void SetDecoder( + VideoInputFile videoInputFile, + VideoStream videoStream, + FFmpegState ffmpegState, + PipelineContext context, + ICollection pipelineSteps) + { + Option maybeDecoder = (ffmpegState.DecoderHardwareAccelerationMode, videoStream.Codec) switch + { + (HardwareAccelerationMode.VideoToolbox, _) => new DecoderVideoToolbox(), + + _ => GetSoftwareDecoder(videoStream) + }; + + foreach (IDecoder decoder in maybeDecoder) + { + videoInputFile.AddOption(decoder); + } + } + + protected override Option GetEncoder(FFmpegState ffmpegState, FrameState currentState, FrameState desiredState) + { + return (ffmpegState.EncoderHardwareAccelerationMode, desiredState.VideoFormat) switch + { + (HardwareAccelerationMode.VideoToolbox, VideoFormat.Hevc) => + new EncoderHevcVideoToolbox(desiredState.BitDepth), + (HardwareAccelerationMode.VideoToolbox, VideoFormat.H264) => + new EncoderH264VideoToolbox(), + + _ => GetSoftwareEncoder(currentState, desiredState) + }; + } + + protected override List SetPixelFormat( + VideoStream videoStream, + Option desiredPixelFormat, + FrameState currentState, + ICollection pipelineSteps) + { + var result = new List(); + + foreach (IPixelFormat pixelFormat in desiredPixelFormat) + { + if (!videoStream.ColorParams.IsBt709) + { + _logger.LogDebug("Adding colorspace filter"); + var colorspace = new ColorspaceFilter(videoStream, pixelFormat); + currentState = colorspace.NextState(currentState); + result.Add(colorspace); + } + + if (currentState.PixelFormat.Map(f => f.FFmpegName) != pixelFormat.FFmpegName) + { + _logger.LogDebug( + "Format {A} doesn't equal {B}", + currentState.PixelFormat.Map(f => f.FFmpegName), + pixelFormat.FFmpegName); + + //result.Add(new PixelFormatFilter(pixelFormat)); + pipelineSteps.Add(new PixelFormatOutputOption(pixelFormat)); + } + } + + return result; + } +} diff --git a/ErsatzTV.FFmpeg/PipelineBuilder.cs b/ErsatzTV.FFmpeg/PipelineBuilder.cs index a30e0285b..b2b542968 100644 --- a/ErsatzTV.FFmpeg/PipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/PipelineBuilder.cs @@ -9,6 +9,7 @@ using ErsatzTV.FFmpeg.Option; using ErsatzTV.FFmpeg.Option.HardwareAcceleration; using ErsatzTV.FFmpeg.Option.Metadata; using ErsatzTV.FFmpeg.OutputFormat; +using ErsatzTV.FFmpeg.Pipeline; using ErsatzTV.FFmpeg.Protocol; using ErsatzTV.FFmpeg.Runtime; using ErsatzTV.FFmpeg.State; @@ -16,7 +17,7 @@ using Microsoft.Extensions.Logging; namespace ErsatzTV.FFmpeg; -public class PipelineBuilder +public class PipelineBuilder : IPipelineBuilder { private readonly Option _audioInputFile; private readonly string _fontsFolder; @@ -121,9 +122,10 @@ public class PipelineBuilder return new FFmpegPipeline(_pipelineSteps); } - public FFmpegPipeline Build(FFmpegState ffmpegState, FrameState desiredState) + public virtual FFmpegPipeline Build(FFmpegState ffmpegState, FrameState desiredState) { - var originalDesiredPixelFormat = desiredState.PixelFormat; + Option originalDesiredPixelFormat = desiredState.PixelFormat; + bool is10BitOutput = desiredState.PixelFormat.Map(pf => pf.BitDepth).IfNone(8) == 10; if (ffmpegState.Start.Exists(s => s > TimeSpan.Zero) && desiredState.Realtime) { @@ -253,19 +255,21 @@ public class PipelineBuilder } } - // nvenc requires yuv420p background with yuva420p overlay - if (ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Nvenc && hasOverlay) + if (ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Nvenc && hasOverlay && + is10BitOutput) { - desiredState = desiredState with { PixelFormat = new PixelFormatYuv420P() }; - } - - // qsv should stay nv12 - if (ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Qsv && hasOverlay) - { - IPixelFormat pixelFormat = desiredState.PixelFormat.IfNone(new PixelFormatYuv420P()); + IPixelFormat pixelFormat = desiredState.PixelFormat.IfNone(new PixelFormatYuv420P10Le()); desiredState = desiredState with { PixelFormat = new PixelFormatNv12(pixelFormat.Name) }; } + // + // // qsv should stay nv12 + // if (ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Qsv && hasOverlay) + // { + // IPixelFormat pixelFormat = desiredState.PixelFormat.IfNone(new PixelFormatYuv420P()); + // desiredState = desiredState with { PixelFormat = new PixelFormatNv12(pixelFormat.Name) }; + // } + foreach (string desiredVaapiDriver in ffmpegState.VaapiDriver) { IPipelineStep step = new LibvaDriverNameVariable(desiredVaapiDriver); @@ -597,7 +601,7 @@ public class PipelineBuilder _videoInputFile.Iter(f => f.FilterSteps.Add(downloadFilter)); } - var pixelFormatFilter = new SubtitlePixelFormatFilter(ffmpegState); + var pixelFormatFilter = new SubtitlePixelFormatFilter(ffmpegState, is10BitOutput); subtitleInputFile.FilterSteps.Add(pixelFormatFilter); subtitleInputFile.FilterSteps.Add(new SubtitleHardwareUploadFilter(currentState, ffmpegState)); @@ -669,7 +673,7 @@ public class PipelineBuilder } watermarkInputFile.FilterSteps.Add( - new WatermarkPixelFormatFilter(ffmpegState, watermarkInputFile.DesiredState)); + new WatermarkPixelFormatFilter(ffmpegState, watermarkInputFile.DesiredState, is10BitOutput)); foreach (VideoStream watermarkStream in watermarkInputFile.VideoStreams) { diff --git a/ErsatzTV.FFmpeg/ScanKind.cs b/ErsatzTV.FFmpeg/ScanKind.cs new file mode 100644 index 000000000..26694015f --- /dev/null +++ b/ErsatzTV.FFmpeg/ScanKind.cs @@ -0,0 +1,8 @@ +namespace ErsatzTV.FFmpeg; + +public enum ScanKind +{ + Unknown = 0, + Progressive = 1, + Interlaced = 2 +} diff --git a/ErsatzTV/Startup.cs b/ErsatzTV/Startup.cs index d21d7e84f..ef27cd108 100644 --- a/ErsatzTV/Startup.cs +++ b/ErsatzTV/Startup.cs @@ -34,6 +34,7 @@ using ErsatzTV.Core.Plex; using ErsatzTV.Core.Scheduling; using ErsatzTV.Core.Trakt; using ErsatzTV.FFmpeg.Capabilities; +using ErsatzTV.FFmpeg.Pipeline; using ErsatzTV.FFmpeg.Runtime; using ErsatzTV.Formatters; using ErsatzTV.Infrastructure.Data; @@ -460,6 +461,7 @@ public class Startup MultiEpisodeShuffleCollectionEnumeratorFactory>(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped();