From 70c37df59666a28ba0c7daaa6fc5a392dd5818f6 Mon Sep 17 00:00:00 2001 From: Jason Dove Date: Sat, 26 Feb 2022 14:00:23 -0600 Subject: [PATCH] fix vaapi watermarks with new transcoder (#657) --- .../Filter/AvailableOverlayFilters.cs | 8 ++-- ErsatzTV.FFmpeg/Filter/ComplexFilter.cs | 18 ++++++++- .../Filter/Cuda/OverlayCudaFilter.cs | 5 ++- .../Filter/HardwareDownloadFilter.cs | 37 +++++++++++++++++++ .../Filter/HardwareUploadFilter.cs | 1 + ErsatzTV.FFmpeg/Filter/OverlayFilter.cs | 30 +++++++++++++-- .../Filter/Qsv/OverlayQsvFilter.cs | 5 ++- .../Filter/WatermarkHardwareUploadFilter.cs | 28 ++++++++++++++ ErsatzTV.FFmpeg/PipelineBuilder.cs | 11 +++++- 9 files changed, 131 insertions(+), 12 deletions(-) create mode 100644 ErsatzTV.FFmpeg/Filter/HardwareDownloadFilter.cs create mode 100644 ErsatzTV.FFmpeg/Filter/WatermarkHardwareUploadFilter.cs diff --git a/ErsatzTV.FFmpeg/Filter/AvailableOverlayFilters.cs b/ErsatzTV.FFmpeg/Filter/AvailableOverlayFilters.cs index a8c5590f..5387cac7 100644 --- a/ErsatzTV.FFmpeg/Filter/AvailableOverlayFilters.cs +++ b/ErsatzTV.FFmpeg/Filter/AvailableOverlayFilters.cs @@ -8,13 +8,13 @@ public static class AvailableOverlayFilters { public static IPipelineFilterStep ForAcceleration( HardwareAccelerationMode accelMode, + FrameState currentState, WatermarkState watermarkState, FrameSize resolution) => accelMode switch { - HardwareAccelerationMode.Nvenc => new OverlayCudaFilter(watermarkState, resolution), - HardwareAccelerationMode.Qsv => new OverlayQsvFilter(watermarkState, resolution), - // HardwareAccelerationMode.Vaapi => new DeinterlaceVaapiFilter(), - _ => new OverlayFilter(watermarkState, resolution) + HardwareAccelerationMode.Nvenc => new OverlayCudaFilter(currentState, watermarkState, resolution), + HardwareAccelerationMode.Qsv => new OverlayQsvFilter(currentState, watermarkState, resolution), + _ => new OverlayFilter(currentState, watermarkState, resolution) }; } diff --git a/ErsatzTV.FFmpeg/Filter/ComplexFilter.cs b/ErsatzTV.FFmpeg/Filter/ComplexFilter.cs index 06c038af..1284b828 100644 --- a/ErsatzTV.FFmpeg/Filter/ComplexFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/ComplexFilter.cs @@ -5,6 +5,7 @@ namespace ErsatzTV.FFmpeg.Filter; public class ComplexFilter : IPipelineStep { + private readonly FrameState _currentState; private readonly FFmpegState _ffmpegState; private readonly Option _maybeVideoInputFile; private readonly Option _maybeAudioInputFile; @@ -12,12 +13,14 @@ public class ComplexFilter : IPipelineStep private readonly FrameSize _resolution; public ComplexFilter( + FrameState currentState, FFmpegState ffmpegState, Option maybeVideoInputFile, Option maybeAudioInputFile, Option maybeWatermarkInputFile, FrameSize resolution) { + _currentState = currentState; _ffmpegState = ffmpegState; _maybeVideoInputFile = maybeVideoInputFile; _maybeAudioInputFile = maybeAudioInputFile; @@ -117,6 +120,7 @@ public class ComplexFilter : IPipelineStep IPipelineFilterStep overlayFilter = AvailableOverlayFilters.ForAcceleration( _ffmpegState.HardwareAccelerationMode, + _currentState, watermarkInputFile.DesiredState, _resolution); @@ -125,8 +129,20 @@ public class ComplexFilter : IPipelineStep string tempVideoLabel = string.IsNullOrWhiteSpace(videoFilterComplex) ? $"[{videoLabel}]" : videoLabel; + + // vaapi uses software overlay and needs to upload + string uploadFilter = string.Empty; + if (_ffmpegState.HardwareAccelerationMode == HardwareAccelerationMode.Vaapi) + { + uploadFilter = new HardwareUploadFilter(_ffmpegState).Filter; + } + + if (!string.IsNullOrWhiteSpace(uploadFilter)) + { + uploadFilter = "," + uploadFilter; + } - overlayFilterComplex = $"{tempVideoLabel}{watermarkLabel}{overlayFilter.Filter}[vf]"; + overlayFilterComplex = $"{tempVideoLabel}{watermarkLabel}{overlayFilter.Filter}{uploadFilter}[vf]"; // change the mapped label videoLabel = "[vf]"; diff --git a/ErsatzTV.FFmpeg/Filter/Cuda/OverlayCudaFilter.cs b/ErsatzTV.FFmpeg/Filter/Cuda/OverlayCudaFilter.cs index abd947d2..c9327d91 100644 --- a/ErsatzTV.FFmpeg/Filter/Cuda/OverlayCudaFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Cuda/OverlayCudaFilter.cs @@ -4,7 +4,10 @@ namespace ErsatzTV.FFmpeg.Filter.Cuda; public class OverlayCudaFilter : OverlayFilter { - public OverlayCudaFilter(WatermarkState watermarkState, FrameSize resolution) : base(watermarkState, resolution) + public OverlayCudaFilter(FrameState currentState, WatermarkState watermarkState, FrameSize resolution) : base( + currentState, + watermarkState, + resolution) { } diff --git a/ErsatzTV.FFmpeg/Filter/HardwareDownloadFilter.cs b/ErsatzTV.FFmpeg/Filter/HardwareDownloadFilter.cs new file mode 100644 index 00000000..24f32931 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/HardwareDownloadFilter.cs @@ -0,0 +1,37 @@ +using ErsatzTV.FFmpeg.Format; + +namespace ErsatzTV.FFmpeg.Filter; + +public class HardwareDownloadFilter : BaseFilter +{ + private readonly FrameState _currentState; + + public HardwareDownloadFilter(FrameState currentState) + { + _currentState = currentState; + } + + public override FrameState NextState(FrameState currentState) => + currentState with { FrameDataLocation = FrameDataLocation.Software }; + + public override string Filter + { + get + { + string hwdownload = string.Empty; + if (_currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + hwdownload = "hwdownload"; + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + if (!string.IsNullOrWhiteSpace(pixelFormat.FFmpegName)) + { + hwdownload = $"hwdownload,format={pixelFormat.FFmpegName}"; + } + } + } + + return hwdownload; + } + } +} diff --git a/ErsatzTV.FFmpeg/Filter/HardwareUploadFilter.cs b/ErsatzTV.FFmpeg/Filter/HardwareUploadFilter.cs index 2ad1f62b..9338a6ce 100644 --- a/ErsatzTV.FFmpeg/Filter/HardwareUploadFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/HardwareUploadFilter.cs @@ -16,6 +16,7 @@ public class HardwareUploadFilter : BaseFilter HardwareAccelerationMode.None => string.Empty, HardwareAccelerationMode.Nvenc => "hwupload_cuda", HardwareAccelerationMode.Qsv => "hwupload=extra_hw_frames=64", + HardwareAccelerationMode.Vaapi => "format=nv12|vaapi,hwupload", _ => "hwupload" }; } diff --git a/ErsatzTV.FFmpeg/Filter/OverlayFilter.cs b/ErsatzTV.FFmpeg/Filter/OverlayFilter.cs index b6c72fdf..62dbe121 100644 --- a/ErsatzTV.FFmpeg/Filter/OverlayFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/OverlayFilter.cs @@ -1,22 +1,44 @@ -using ErsatzTV.FFmpeg.State; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.FFmpeg.State; namespace ErsatzTV.FFmpeg.Filter; public class OverlayFilter : BaseFilter { + private readonly FrameState _currentState; private readonly WatermarkState _watermarkState; private readonly FrameSize _resolution; - public OverlayFilter(WatermarkState watermarkState, FrameSize resolution) + public OverlayFilter(FrameState currentState, WatermarkState watermarkState, FrameSize resolution) { + _currentState = currentState; _watermarkState = watermarkState; _resolution = resolution; } public override FrameState NextState(FrameState currentState) => currentState; - public override string Filter => $"overlay={Position}"; - + public override string Filter + { + get + { + string hwdownload = string.Empty; + if (_currentState.FrameDataLocation == FrameDataLocation.Hardware) + { + hwdownload = "hwdownload,"; + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + if (pixelFormat.FFmpegName == FFmpegFormat.NV12) + { + hwdownload = "hwdownload,format=nv12,"; + } + } + } + + return $"{hwdownload}overlay={Position}"; + } + } + protected string Position { get diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/OverlayQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/OverlayQsvFilter.cs index 2971e1ca..9211ea90 100644 --- a/ErsatzTV.FFmpeg/Filter/Qsv/OverlayQsvFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Qsv/OverlayQsvFilter.cs @@ -4,7 +4,10 @@ namespace ErsatzTV.FFmpeg.Filter.Qsv; public class OverlayQsvFilter : OverlayFilter { - public OverlayQsvFilter(WatermarkState watermarkState, FrameSize resolution) : base(watermarkState, resolution) + public OverlayQsvFilter(FrameState currentState, WatermarkState watermarkState, FrameSize resolution) : base( + currentState, + watermarkState, + resolution) { } diff --git a/ErsatzTV.FFmpeg/Filter/WatermarkHardwareUploadFilter.cs b/ErsatzTV.FFmpeg/Filter/WatermarkHardwareUploadFilter.cs new file mode 100644 index 00000000..9b5e4bec --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/WatermarkHardwareUploadFilter.cs @@ -0,0 +1,28 @@ +namespace ErsatzTV.FFmpeg.Filter; + +public class WatermarkHardwareUploadFilter : BaseFilter +{ + private readonly FrameState _currentState; + private readonly FFmpegState _ffmpegState; + + public WatermarkHardwareUploadFilter(FrameState currentState, FFmpegState ffmpegState) + { + _currentState = currentState; + _ffmpegState = ffmpegState; + } + + public override FrameState NextState(FrameState currentState) => currentState; + + public override string Filter => _ffmpegState.HardwareAccelerationMode switch + { + HardwareAccelerationMode.None => string.Empty, + HardwareAccelerationMode.Nvenc => "hwupload_cuda", + HardwareAccelerationMode.Qsv => "hwupload=extra_hw_frames=64", + + // leave vaapi in software since we don't (yet) use overlay_vaapi + HardwareAccelerationMode.Vaapi when _currentState.FrameDataLocation == FrameDataLocation.Software => + string.Empty, + + _ => "hwupload" + }; +} diff --git a/ErsatzTV.FFmpeg/PipelineBuilder.cs b/ErsatzTV.FFmpeg/PipelineBuilder.cs index 10f1cc2b..9ba323cd 100644 --- a/ErsatzTV.FFmpeg/PipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/PipelineBuilder.cs @@ -443,6 +443,14 @@ public class PipelineBuilder foreach (WatermarkInputFile watermarkInputFile in _watermarkInputFile) { + // vaapi uses a software overlay, so we need to ensure the background is already in software + if (ffmpegState.HardwareAccelerationMode == HardwareAccelerationMode.Vaapi) + { + var downloadFilter = new HardwareDownloadFilter(currentState); + currentState = downloadFilter.NextState(currentState); + _videoInputFile.Iter(f => f.FilterSteps.Add(downloadFilter)); + } + watermarkInputFile.FilterSteps.Add( new WatermarkPixelFormatFilter(ffmpegState, watermarkInputFile.DesiredState)); @@ -475,7 +483,7 @@ public class PipelineBuilder watermarkInputFile.FilterSteps.AddRange(fadePoints.Map(fp => new WatermarkFadeFilter(fp))); } - watermarkInputFile.FilterSteps.Add(new HardwareUploadFilter(ffmpegState)); + watermarkInputFile.FilterSteps.Add(new WatermarkHardwareUploadFilter(currentState, ffmpegState)); } if (ffmpegState.DoNotMapMetadata) @@ -524,6 +532,7 @@ public class PipelineBuilder } var complexFilter = new ComplexFilter( + currentState, ffmpegState, _videoInputFile, _audioInputFile,