From dfdfa6f3495707356c84a5434e3748d622a213d1 Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Thu, 12 Jun 2025 11:13:43 -0500 Subject: [PATCH] use hardware-accelerated tonemapping with vaapi (#2028) * add tonemap_vaapi filter * let vaapi pipeline handle hdr content * use tonemap_opencl with vaapi * update changelog --- CHANGELOG.md | 3 +- .../Capabilities/FFmpegCapabilities.cs | 1 + .../Capabilities/FFmpegKnownFilter.cs | 4 ++- .../FFmpegKnownHardwareAcceleration.cs | 4 ++- .../Filter/Vaapi/TonemapVaapiFilter.cs | 12 +++++++ ErsatzTV.FFmpeg/HardwareAccelerationMode.cs | 3 +- .../Pipeline/PipelineBuilderFactory.cs | 19 ++++++---- .../Pipeline/VaapiPipelineBuilder.cs | 35 +++++++++++++++++++ ErsatzTV.sln.DotSettings | 1 + 9 files changed, 71 insertions(+), 11 deletions(-) create mode 100644 ErsatzTV.FFmpeg/Filter/Vaapi/TonemapVaapiFilter.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b8bc97e..e18b3820 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - `Strict` will add nearly 24h (23:58) of unscheduled time so that it can start exactly at 6:00 AM the next day - `Flexible` will NOT add unscheduled time, and will schedule its item at 6:02 AM (which may also affect the scheduling of later items) - Add basic HDR transcoding support - - For this initial implementation, HDR content will *always* use a software pipeline + - VAAPI may use hardware-accelerated tone mapping + - In all other cases, HDR content will use a software pipeline ### Changed - Start to make UI minimally responsive (functional on smaller screens) diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs index 0178c0ef..df2842e0 100644 --- a/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegCapabilities.cs @@ -40,6 +40,7 @@ public class FFmpegCapabilities : IFFmpegCapabilities HardwareAccelerationMode.Qsv => FFmpegKnownHardwareAcceleration.Qsv, HardwareAccelerationMode.Vaapi => FFmpegKnownHardwareAcceleration.Vaapi, HardwareAccelerationMode.VideoToolbox => FFmpegKnownHardwareAcceleration.VideoToolbox, + HardwareAccelerationMode.OpenCL => FFmpegKnownHardwareAcceleration.OpenCL, _ => Option.None }; diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs index 66a159bb..d559efd9 100644 --- a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownFilter.cs @@ -3,6 +3,7 @@ namespace ErsatzTV.FFmpeg.Capabilities; public record FFmpegKnownFilter { public static readonly FFmpegKnownFilter ScaleNpp = new("scale_npp"); + public static readonly FFmpegKnownFilter TonemapOpenCL = new("tonemap_opencl"); private FFmpegKnownFilter(string Name) => this.Name = Name; @@ -11,6 +12,7 @@ public record FFmpegKnownFilter public static IList AllFilters => new[] { - ScaleNpp.Name + ScaleNpp.Name, + TonemapOpenCL.Name, }; } diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownHardwareAcceleration.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownHardwareAcceleration.cs index 477180b8..12da1488 100644 --- a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownHardwareAcceleration.cs +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownHardwareAcceleration.cs @@ -7,6 +7,7 @@ public record FFmpegKnownHardwareAcceleration public static readonly FFmpegKnownHardwareAcceleration Qsv = new("qsv"); public static readonly FFmpegKnownHardwareAcceleration Vaapi = new("vaapi"); public static readonly FFmpegKnownHardwareAcceleration VideoToolbox = new("videotoolbox"); + public static readonly FFmpegKnownHardwareAcceleration OpenCL = new("opencl"); private FFmpegKnownHardwareAcceleration(string Name) => this.Name = Name; @@ -19,6 +20,7 @@ public record FFmpegKnownHardwareAcceleration Cuda.Name, Qsv.Name, Vaapi.Name, - VideoToolbox.Name + VideoToolbox.Name, + OpenCL.Name }; } diff --git a/ErsatzTV.FFmpeg/Filter/Vaapi/TonemapVaapiFilter.cs b/ErsatzTV.FFmpeg/Filter/Vaapi/TonemapVaapiFilter.cs new file mode 100644 index 00000000..931209fc --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/Vaapi/TonemapVaapiFilter.cs @@ -0,0 +1,12 @@ +namespace ErsatzTV.FFmpeg.Filter.Vaapi; + +public class TonemapVaapiFilter : BaseFilter +{ + public override string Filter => "hwupload=derive_device=vaapi,hwmap=derive_device=opencl,tonemap_opencl,hwmap=derive_device=vaapi:reverse=1"; + + public override FrameState NextState(FrameState currentState) => + currentState with + { + FrameDataLocation = FrameDataLocation.Hardware + }; +} diff --git a/ErsatzTV.FFmpeg/HardwareAccelerationMode.cs b/ErsatzTV.FFmpeg/HardwareAccelerationMode.cs index 1c2c4d2f..98dd88f1 100644 --- a/ErsatzTV.FFmpeg/HardwareAccelerationMode.cs +++ b/ErsatzTV.FFmpeg/HardwareAccelerationMode.cs @@ -7,5 +7,6 @@ public enum HardwareAccelerationMode Nvenc = 2, Vaapi = 3, VideoToolbox = 4, - Amf = 5 + Amf = 5, + OpenCL = 6 } diff --git a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderFactory.cs b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderFactory.cs index 80ecb4fd..cac09224 100644 --- a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderFactory.cs +++ b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderFactory.cs @@ -44,10 +44,10 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory return hardwareAccelerationMode switch { - // force software pipeline when content is HDR - _ when isHdrContent => new SoftwarePipelineBuilder( + HardwareAccelerationMode.Vaapi when capabilities is not NoHardwareCapabilities => new VaapiPipelineBuilder( ffmpegCapabilities, - HardwareAccelerationMode.None, + capabilities, + hardwareAccelerationMode, videoInputFile, audioInputFile, watermarkInputFile, @@ -57,10 +57,10 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory fontsFolder, _logger), - HardwareAccelerationMode.Nvenc when capabilities is not NoHardwareCapabilities => new NvidiaPipelineBuilder( + // force software pipeline when content is HDR (and not VAAPI) + _ when isHdrContent => new SoftwarePipelineBuilder( ffmpegCapabilities, - capabilities, - hardwareAccelerationMode, + HardwareAccelerationMode.None, videoInputFile, audioInputFile, watermarkInputFile, @@ -69,7 +69,8 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory reportsFolder, fontsFolder, _logger), - HardwareAccelerationMode.Vaapi when capabilities is not NoHardwareCapabilities => new VaapiPipelineBuilder( + + HardwareAccelerationMode.Nvenc when capabilities is not NoHardwareCapabilities => new NvidiaPipelineBuilder( ffmpegCapabilities, capabilities, hardwareAccelerationMode, @@ -81,6 +82,7 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory reportsFolder, fontsFolder, _logger), + HardwareAccelerationMode.Qsv when capabilities is not NoHardwareCapabilities => new QsvPipelineBuilder( ffmpegCapabilities, capabilities, @@ -93,6 +95,7 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory reportsFolder, fontsFolder, _logger), + HardwareAccelerationMode.VideoToolbox when capabilities is not NoHardwareCapabilities => new VideoToolboxPipelineBuilder( ffmpegCapabilities, @@ -106,6 +109,7 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory reportsFolder, fontsFolder, _logger), + HardwareAccelerationMode.Amf when capabilities is not NoHardwareCapabilities => new AmfPipelineBuilder( ffmpegCapabilities, capabilities, @@ -118,6 +122,7 @@ public class PipelineBuilderFactory : IPipelineBuilderFactory reportsFolder, fontsFolder, _logger), + _ => new SoftwarePipelineBuilder( ffmpegCapabilities, HardwareAccelerationMode.None, diff --git a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs index 465e0d5b..46db3ea9 100644 --- a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs @@ -17,6 +17,7 @@ namespace ErsatzTV.FFmpeg.Pipeline; public class VaapiPipelineBuilder : SoftwarePipelineBuilder { + private readonly IFFmpegCapabilities _ffmpegCapabilities; private readonly IHardwareCapabilities _hardwareCapabilities; private readonly ILogger _logger; @@ -43,6 +44,7 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder fontsFolder, logger) { + _ffmpegCapabilities = ffmpegCapabilities; _hardwareCapabilities = hardwareCapabilities; _logger = logger; } @@ -168,6 +170,8 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder // _logger.LogDebug("After decode: {PixelFormat}", currentState.PixelFormat); + currentState = SetTonemap(videoInputFile, videoStream, ffmpegState, desiredState, currentState); + currentState = SetDeinterlace(videoInputFile, context, ffmpegState, currentState); // _logger.LogDebug("After deinterlace: {PixelFormat}", currentState.PixelFormat); @@ -619,4 +623,35 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder return currentState; } + + private FrameState SetTonemap( + VideoInputFile videoInputFile, + VideoStream videoStream, + FFmpegState ffmpegState, + FrameState desiredState, + FrameState currentState) + { + if (videoStream.ColorParams.IsHdr) + { + foreach (IPixelFormat pixelFormat in desiredState.PixelFormat) + { + if (ffmpegState.DecoderHardwareAccelerationMode == HardwareAccelerationMode.Vaapi && _ffmpegCapabilities.HasFilter(FFmpegKnownFilter.TonemapOpenCL)) + { + var filter = new TonemapVaapiFilter(); + currentState = filter.NextState(currentState); + videoStream.ResetColorParams(ColorParams.Default); + videoInputFile.FilterSteps.Add(filter); + } + else + { + var filter = new TonemapFilter(currentState, pixelFormat); + currentState = filter.NextState(currentState); + videoStream.ResetColorParams(ColorParams.Default); + videoInputFile.FilterSteps.Add(filter); + } + } + } + + return currentState; + } } diff --git a/ErsatzTV.sln.DotSettings b/ErsatzTV.sln.DotSettings index db69c1a2..44670318 100644 --- a/ErsatzTV.sln.DotSettings +++ b/ErsatzTV.sln.DotSettings @@ -1,5 +1,6 @@  True + CL DTO EPG FF