use cuvid to check nvidia decode capabilities (#2461)

* detect nvidia decode capabilities

* use cuvid to check b-ref mode

CHANGELOG.md

@@ -74,7 +74,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix extracted text subtitles getting into invalid state after media server deep scans
   - Targeted deep scans will now extract text subtitles for the scanned show
 - Fix playlist preview
-- Use NvEnc API to detect encoder capability instead of heuristic based on GPU model/architecture
+- Use NVIDIA NvEnc API to detect encoder capability instead of heuristic based on GPU model/architecture
+- Use NVIDIA Cuvid API to detect decoder capability instead of heuristic based on GPU model/architecture
 
 ### Changed
 - Filler presets: use separate text fields for `hours`, `minutes` and `seconds` duration

ErsatzTV.FFmpeg/Capabilities/Nvidia/CudaDevice.cs

@@ -1,5 +1,19 @@
+using Lennox.NvEncSharp;
+
 namespace ErsatzTV.FFmpeg.Capabilities.Nvidia;
 
-public record CudaDevice(int Handle, string Model, Version Version, IReadOnlyList<CudaCodec> Codecs);
+public record CudaDevice(
+    int Handle,
+    string Model,
+    Version Version,
+    IReadOnlyList<CudaCodec> Encoders,
+    IReadOnlyList<CudaDecoder> Decoders);
+
+public record CudaCodec(
+    string Name,
+    Guid CodecGuid,
+    IReadOnlyList<Guid> ProfileGuids,
+    IReadOnlyList<int> BitDepths,
+    bool BFrames);
 
-public record CudaCodec(string Name, Guid CodecGuid, IReadOnlyList<Guid> ProfileGuids, IReadOnlyList<int> BitDepths);
+public record CudaDecoder(string Name, CuVideoCodec VideoCodec, int BitDepth);

ErsatzTV.FFmpeg/Capabilities/Nvidia/CudaHelper.cs

@@ -12,12 +12,34 @@ internal static class CudaHelper
     private static bool _initialized;
     private static readonly Lock Lock = new();
 
-    private static readonly Dictionary<string, Guid> AllCodecs = new()
+    private static readonly Dictionary<string, Guid> AllEncoders = new()
     {
         [VideoFormat.H264] = NvEncCodecGuids.H264,
         [VideoFormat.Hevc] = NvEncCodecGuids.Hevc
     };
 
+    private sealed record Decoder(CuVideoChromaFormat ChromaFormat, CuVideoCodec VideoCodec, int BitDepth);
+
+    private static readonly Dictionary<string, Decoder> AllDecoders = new()
+    {
+        [$"{VideoFormat.Mpeg2Video} 8-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.MPEG2, 8),
+        [$"{VideoFormat.Mpeg2Video} 10-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.MPEG2, 10),
+        [$"{VideoFormat.Mpeg4} 8-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.MPEG4, 8),
+        [$"{VideoFormat.Mpeg4} 10-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.MPEG4, 10),
+        [$"{VideoFormat.Vc1} 8-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.VC1, 8),
+        [$"{VideoFormat.Vc1} 10-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.VC1, 10),
+        [$"{VideoFormat.H264} 8-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.H264, 8),
+        [$"{VideoFormat.H264} 10-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.H264, 10),
+        [$"{VideoFormat.Hevc} 8-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.HEVC, 8),
+        [$"{VideoFormat.Hevc} 10-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.HEVC, 10),
+        [$"{VideoFormat.Vp8} 8-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.VP8, 8),
+        [$"{VideoFormat.Vp8} 10-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.VP8, 10),
+        [$"{VideoFormat.Vp9} 8-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.VP9, 8),
+        [$"{VideoFormat.Vp9} 10-bit"] = new Decoder(CuVideoChromaFormat.YUV420, CuVideoCodec.VP9, 10),
+        [$"{VideoFormat.Av1} 8-bit"] = new Decoder(CuVideoChromaFormat.YUV420, (CuVideoCodec)11, 8), // AV1
+        [$"{VideoFormat.Av1} 10-bit"] = new Decoder(CuVideoChromaFormat.YUV420, (CuVideoCodec)11, 10) // AV1
+    };
+
     private static bool EnsureInit()
     {
         if (_initialized)
@@ -78,6 +100,34 @@ internal static class CudaHelper
                 int minor = device.GetAttribute(CuDeviceAttribute.ComputeCapabilityMinor);
 
                 using var context = device.CreateContext();
+
+                var encoders = new List<CudaCodec>();
+                var decoders = new List<CudaDecoder>();
+
+                foreach ((string decoderName, Decoder decoder) in AllDecoders)
+                {
+                    var caps = new CuVideoDecodeCaps
+                    {
+                        BitDepthMinus8 = decoder.BitDepth - 8,
+                        ChromaFormat = decoder.ChromaFormat,
+                        CodecType = decoder.VideoCodec
+                    };
+
+                    var decoderCaps = LibCuVideo.GetDecoderCaps(ref caps);
+                    if (decoderCaps != CuResult.Success)
+                    {
+                        Console.WriteLine("Failed to check decode capability: " + result);
+                        continue;
+                    }
+
+                    if (!caps.IsSupported)
+                    {
+                        continue;
+                    }
+
+                    decoders.Add(new CudaDecoder(decoderName, decoder.VideoCodec, decoder.BitDepth));
+                }
+
                 var sessionParams = new NvEncOpenEncodeSessionExParams
                 {
                     Version = LibNvEnc.NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER,
@@ -86,12 +136,11 @@ internal static class CudaHelper
                     DeviceType = NvEncDeviceType.Cuda
                 };
 
-                var codecs = new List<CudaCodec>();
                 var encoder = LibNvEnc.OpenEncoder(ref sessionParams);
                 try
                 {
                     IReadOnlyList<Guid> codecGuids = encoder.GetEncodeGuids();
-                    foreach ((string codecName, Guid codecGuid) in AllCodecs)
+                    foreach ((string codecName, Guid codecGuid) in AllEncoders)
                     {
                         if (codecGuids.Contains(codecGuid))
                         {
@@ -107,13 +156,19 @@ internal static class CudaHelper
                                 bitDepths.Add(10);
                             }
 
+                            cap = new NvEncCapsParam { CapsToQuery = NvEncCaps.SupportBframeRefMode };
+                            capsVal = 0;
+                            encoder.GetEncodeCaps(codecGuid, ref cap, ref capsVal);
+                            bool bFrameRefMode = capsVal > 0;
+
                             var cudaCodec = new CudaCodec(
                                 codecName,
                                 codecGuid,
                                 codecProfileGuids,
-                                bitDepths.ToImmutableList());
+                                bitDepths.ToImmutableList(),
+                                bFrameRefMode);
 
-                            codecs.Add(cudaCodec);
+                            encoders.Add(cudaCodec);
                         }
                     }
                 }
@@ -122,7 +177,7 @@ internal static class CudaHelper
                     encoder.DestroyEncoder();
                 }
 
-                result.Add(new CudaDevice(device.Handle, name, new Version(major, minor), codecs));
+                result.Add(new CudaDevice(device.Handle, name, new Version(major, minor), encoders, decoders));
             }
             catch (Exception)
             {
@@ -137,17 +192,27 @@ internal static class CudaHelper
     {
         var sb = new StringBuilder();
 
+        sb.AppendLine();
         sb.AppendLine("  Encoding:");
-        foreach (CudaCodec cudaCodec in device.Codecs)
+        foreach (CudaCodec cudaCodec in device.Encoders)
         {
-            sb.AppendLine(CultureInfo.InvariantCulture, $"    - Supports {cudaCodec.Name} 8-bit");
+            string bFrames = cudaCodec.BFrames ? " (with B-frames)" : string.Empty;
+
+            sb.AppendLine(CultureInfo.InvariantCulture, $"    - Supports {cudaCodec.Name} 8-bit{bFrames}");
 
             if (cudaCodec.BitDepths.Contains(10))
             {
-                sb.AppendLine(CultureInfo.InvariantCulture, $"    - Supports {cudaCodec.Name} 10-bit");
+                sb.AppendLine(CultureInfo.InvariantCulture, $"    - Supports {cudaCodec.Name} 10-bit{bFrames}");
             }
         }
 
+        sb.AppendLine();
+        sb.AppendLine("  Decoding:");
+        foreach (CudaDecoder cudaDecoder in device.Decoders)
+        {
+            sb.AppendLine(CultureInfo.InvariantCulture, $"    - Supports {cudaDecoder.Name}");
+        }
+
         return sb.ToString();
     }
 }

ErsatzTV.FFmpeg/Capabilities/Nvidia/NvEncSharpRedirector.cs

@@ -36,6 +36,14 @@ public static class NvEncSharpRedirector
                 return NativeLibrary.Load("nvcuda.dll", assembly, searchPath);
         }
 
+        if (libraryName.Equals("nvcuvid.dll", StringComparison.OrdinalIgnoreCase))
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+                return NativeLibrary.Load("libnvcuvid.so.1", assembly, searchPath);
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+                return NativeLibrary.Load("nvcuvid.dll", assembly, searchPath);
+        }
+
         return IntPtr.Zero;
     }
 

ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs

@@ -5,28 +5,11 @@ using Microsoft.Extensions.Logging;
 
 namespace ErsatzTV.FFmpeg.Capabilities;
 
-public class NvidiaHardwareCapabilities : IHardwareCapabilities
+public class NvidiaHardwareCapabilities(CudaDevice cudaDevice, IFFmpegCapabilities ffmpegCapabilities, ILogger logger)
+    : IHardwareCapabilities
 {
-    private readonly CudaDevice _cudaDevice;
-    private readonly IFFmpegCapabilities _ffmpegCapabilities;
-    private readonly ILogger _logger;
-    private readonly List<string> _maxwellGm206 = ["GTX 750", "GTX 950", "GTX 960", "GTX 965M"];
-    private readonly Version _maxwell = new(5, 2);
-    private readonly Version _pascal = new(6, 0);
-    private readonly Version _ampere = new(8, 6);
-
-    public NvidiaHardwareCapabilities(
-        CudaDevice cudaDevice,
-        IFFmpegCapabilities ffmpegCapabilities,
-        ILogger logger)
-    {
-        _cudaDevice = cudaDevice;
-        _ffmpegCapabilities = ffmpegCapabilities;
-        _logger = logger;
-    }
-
-    // this fails with some 1650 cards, so let's try greater than 75
-    public bool HevcBFrames => _cudaDevice.Version >= new Version(7, 5);
+    public bool HevcBFrames(int bitDepth) =>
+        cudaDevice.Encoders.Any(e => e.CodecGuid == NvEncCodecGuids.Hevc && e.BFrames);
 
     public FFmpegCapability CanDecode(
         string videoFormat,
@@ -34,53 +17,54 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities
         Option<IPixelFormat> maybePixelFormat,
         bool isHdr)
     {
-        // we use vulkan for hdr, so only support h264, hevc and av1 when isHdr == true
-
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 
-        bool isHardware = videoFormat switch
-        {
-            // some second gen maxwell can decode hevc, otherwise pascal is required
-            VideoFormat.Hevc => _cudaDevice.Version == _maxwell && _maxwellGm206.Contains(_cudaDevice.Model) || _cudaDevice.Version >= _pascal,
-
-            // pascal is required to decode vp9 10-bit
-            VideoFormat.Vp9 when bitDepth == 10 => !isHdr && _cudaDevice.Version >= _pascal,
-
-            // some second gen maxwell can decode vp9, otherwise pascal is required
-            VideoFormat.Vp9 => !isHdr && _cudaDevice.Version == _maxwell && _maxwellGm206.Contains(_cudaDevice.Model) || _cudaDevice.Version >= _pascal,
-
-            // no hardware decoding of 10-bit h264
-            VideoFormat.H264 => bitDepth < 10,
-
-            VideoFormat.Mpeg2Video => !isHdr,
-
-            VideoFormat.Vc1 => !isHdr,
-
-            // too many issues with odd mpeg4 content, so use software
-            VideoFormat.Mpeg4 => false,
-
-            // ampere is required for av1 decoding
-            VideoFormat.Av1 => _cudaDevice.Version >= _ampere,
+        logger.LogDebug(
+            "Checking NVIDIA decode {Format} / {Profile} / {BitDepth}-bit",
+            videoFormat,
+            videoProfile,
+            bitDepth);
 
-            // generated images are decoded into software
-            VideoFormat.GeneratedImage => false,
+        var isHardware = false;
 
-            _ => false
+        CuVideoCodec? codecType = videoFormat switch
+        {
+            VideoFormat.Mpeg2Video => CuVideoCodec.MPEG2,
+            VideoFormat.Mpeg4 => CuVideoCodec.MPEG4,
+            VideoFormat.Vc1 => CuVideoCodec.VC1,
+            VideoFormat.H264 => CuVideoCodec.H264,
+            VideoFormat.Hevc => CuVideoCodec.HEVC,
+            VideoFormat.Vp8 => CuVideoCodec.VP8,
+            VideoFormat.Vp9 => CuVideoCodec.VP9,
+            VideoFormat.Av1 => (CuVideoCodec)11, // confirmed in dynlink_cuviddec.h
+            _ => null
         };
 
+        if (codecType.HasValue)
+        {
+            isHardware = cudaDevice.Decoders.Any(d => d.VideoCodec == codecType.Value && d.BitDepth == bitDepth);
+            if (!isHardware)
+            {
+                logger.LogWarning(
+                    "NVIDIA decode {Format} / {BitDepth} is not supported; will use software decode",
+                    videoFormat,
+                    bitDepth);
+            }
+        }
+
         if (isHardware)
         {
             return videoFormat switch
             {
                 VideoFormat.Mpeg2Video => CheckHardwareCodec(
                     FFmpegKnownDecoder.Mpeg2Cuvid,
-                    _ffmpegCapabilities.HasDecoder),
-                VideoFormat.Mpeg4 => CheckHardwareCodec(FFmpegKnownDecoder.Mpeg4Cuvid, _ffmpegCapabilities.HasDecoder),
-                VideoFormat.Vc1 => CheckHardwareCodec(FFmpegKnownDecoder.Vc1Cuvid, _ffmpegCapabilities.HasDecoder),
-                VideoFormat.H264 => CheckHardwareCodec(FFmpegKnownDecoder.H264Cuvid, _ffmpegCapabilities.HasDecoder),
-                VideoFormat.Hevc => CheckHardwareCodec(FFmpegKnownDecoder.HevcCuvid, _ffmpegCapabilities.HasDecoder),
-                VideoFormat.Vp9 => CheckHardwareCodec(FFmpegKnownDecoder.Vp9Cuvid, _ffmpegCapabilities.HasDecoder),
-                VideoFormat.Av1 => CheckHardwareCodec(FFmpegKnownDecoder.Av1Cuvid, _ffmpegCapabilities.HasDecoder),
+                    ffmpegCapabilities.HasDecoder),
+                VideoFormat.Mpeg4 => CheckHardwareCodec(FFmpegKnownDecoder.Mpeg4Cuvid, ffmpegCapabilities.HasDecoder),
+                VideoFormat.Vc1 => CheckHardwareCodec(FFmpegKnownDecoder.Vc1Cuvid, ffmpegCapabilities.HasDecoder),
+                VideoFormat.H264 => CheckHardwareCodec(FFmpegKnownDecoder.H264Cuvid, ffmpegCapabilities.HasDecoder),
+                VideoFormat.Hevc => CheckHardwareCodec(FFmpegKnownDecoder.HevcCuvid, ffmpegCapabilities.HasDecoder),
+                VideoFormat.Vp9 => CheckHardwareCodec(FFmpegKnownDecoder.Vp9Cuvid, ffmpegCapabilities.HasDecoder),
+                VideoFormat.Av1 => CheckHardwareCodec(FFmpegKnownDecoder.Av1Cuvid, ffmpegCapabilities.HasDecoder),
                 _ => FFmpegCapability.Software
             };
         }
@@ -95,19 +79,19 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 
-        _logger.LogDebug(
-            "Checking NvEnc {Format} / {Profile} / {BitDepth}-bit",
+        logger.LogDebug(
+            "Checking NVIDIA encode {Format} / {Profile} / {BitDepth}-bit",
             videoFormat,
             videoProfile,
             bitDepth);
 
-        var codec = _cudaDevice.Codecs.FirstOrDefault(c => c.Name.Equals(
+        var codec = cudaDevice.Encoders.FirstOrDefault(c => c.Name.Equals(
             videoFormat,
             StringComparison.OrdinalIgnoreCase));
 
         if (codec == null)
         {
-            _logger.LogWarning("NvEnc {Format} is not supported; will use software encode", videoFormat);
+            logger.LogWarning("NVIDIA encode {Format} is not supported; will use software encode", videoFormat);
             return FFmpegCapability.Software;
         }
 
@@ -126,8 +110,8 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities
 
         if (!codec.ProfileGuids.Contains(profileGuid))
         {
-            _logger.LogWarning(
-                "NvEnc {Format} / {Profile} is not supported; will use software encode",
+            logger.LogWarning(
+                "NVIDIA encode {Format} / {Profile} is not supported; will use software encode",
                 videoFormat,
                 videoProfile);
             return FFmpegCapability.Software;
@@ -135,8 +119,8 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities
 
         if (!codec.BitDepths.Contains(bitDepth))
         {
-            _logger.LogWarning(
-                "NvEnc {Format} / {Profile} / {BitDepth}-bit is not supported; will use software encode",
+            logger.LogWarning(
+                "NVIDIA encode {Format} / {Profile} / {BitDepth}-bit is not supported; will use software encode",
                 videoFormat,
                 videoProfile,
                 bitDepth);
@@ -156,7 +140,7 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities
             return FFmpegCapability.Hardware;
         }
 
-        _logger.LogWarning("FFmpeg does not contain codec {Codec}; will fall back to software codec", codec.Name);
+        logger.LogWarning("FFmpeg does not contain codec {Codec}; will fall back to software codec", codec.Name);
         return FFmpegCapability.Software;
     }
 }

ErsatzTV.FFmpeg/Encoder/Nvenc/EncoderHevcNvenc.cs

@@ -9,13 +9,17 @@ public class EncoderHevcNvenc : EncoderBase
     private readonly Option<string> _maybeVideoPreset;
     private readonly bool _allowBFrames;
 
-    public EncoderHevcNvenc(IHardwareCapabilities hardwareCapabilities, Option<string> maybeVideoPreset, bool allowBFrames)
+    public EncoderHevcNvenc(
+        IHardwareCapabilities hardwareCapabilities,
+        Option<string> maybeVideoPreset,
+        int bitDepth,
+        bool allowBFrames)
     {
         _maybeVideoPreset = maybeVideoPreset;
         _allowBFrames = allowBFrames;
         if (hardwareCapabilities is NvidiaHardwareCapabilities nvidia)
         {
-            _bFrames = nvidia.HevcBFrames;
+            _bFrames = nvidia.HevcBFrames(bitDepth);
         }
     }
 

ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs

@@ -307,7 +307,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
                 (ffmpegState.EncoderHardwareAccelerationMode, desiredState.VideoFormat) switch
                 {
                     (HardwareAccelerationMode.Nvenc, VideoFormat.Hevc) =>
-                        new EncoderHevcNvenc(_hardwareCapabilities, desiredState.VideoPreset, desiredState.AllowBFrames),
+                        new EncoderHevcNvenc(_hardwareCapabilities, desiredState.VideoPreset, desiredState.BitDepth, desiredState.AllowBFrames),
                     (HardwareAccelerationMode.Nvenc, VideoFormat.H264) =>
                         new EncoderH264Nvenc(desiredState.VideoProfile, desiredState.VideoPreset),