try multiple qsv init options

CHANGELOG.md

@@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
   - A warning will be logged when this scenario is detected
 - AMD VAAPI: work around buggy ffmpeg behavior where hevc_vaapi encoder with RadeonSI driver incorrectly outputs height of 1088 instead of 1080
 - Optimize graphics engine to generate element frames in parallel and to eliminate redundant frame copies
+- QSV: try multiple variations of ffmpeg init syntax to find one that works for the specific GPU
 
 ## [25.9.0] - 2025-11-29
 ### Added

ErsatzTV.Application/Troubleshooting/Queries/GetTroubleshootingInfoHandler.cs

@@ -102,14 +102,27 @@ public class GetTroubleshootingInfoHandler : IRequestHandler<GetTroubleshootingI
                     vaapiDisplays = ["drm"];
                 }
 
+                List<QsvInitMode> qsvInitModes = [QsvInitMode.None];
+                if (_runtimeInfo.IsOSPlatform(OSPlatform.Windows))
+                {
+                    qsvInitModes.Add(QsvInitMode.D3d11Va);
+                }
+                qsvInitModes.Add(QsvInitMode.Qsv);
+
                 foreach (string qsvDevice in vaapiDevices)
                 {
-                    QsvOutput output = await _hardwareCapabilitiesFactory.GetQsvOutput(ffmpegPath.Value, qsvDevice);
-                    qsvCapabilities.AppendLine(CultureInfo.InvariantCulture, $"Checking device {qsvDevice}");
-                    qsvCapabilities.AppendLine(CultureInfo.InvariantCulture, $"Exit Code: {output.ExitCode}");
-                    qsvCapabilities.AppendLine();
-                    qsvCapabilities.AppendLine(output.Output);
-                    qsvCapabilities.AppendLine();
+                    foreach (var qsvInitMode in qsvInitModes)
+                    {
+                        QsvOutput output = await _hardwareCapabilitiesFactory.GetQsvOutput(
+                            ffmpegPath.Value,
+                            qsvDevice,
+                            qsvInitMode);
+                        qsvCapabilities.AppendLine(CultureInfo.InvariantCulture, $"Checking device {qsvDevice} with init mode {qsvInitMode}");
+                        qsvCapabilities.AppendLine(CultureInfo.InvariantCulture, $"Exit Code: {output.ExitCode}");
+                        qsvCapabilities.AppendLine();
+                        qsvCapabilities.AppendLine(output.Output);
+                        qsvCapabilities.AppendLine();
+                    }
                 }
 
                 if (_runtimeInfo.IsOSPlatform(OSPlatform.Linux))

ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs

@@ -32,7 +32,9 @@ public partial class HardwareCapabilitiesFactory(
     private static readonly CompositeFormat VaapiGenerationCacheKeyFormat =
         CompositeFormat.Parse("ffmpeg.hardware.vaapi.generation.{0}.{1}.{2}");
 
+    private static readonly CompositeFormat QsvVaapiCacheKeyFormat = CompositeFormat.Parse("ffmpeg.hardware.qsv.vaapi.{0}");
     private static readonly CompositeFormat QsvCacheKeyFormat = CompositeFormat.Parse("ffmpeg.hardware.qsv.{0}");
+
     private static readonly CompositeFormat FFmpegCapabilitiesCacheKeyFormat = CompositeFormat.Parse("ffmpeg.{0}");
 
     private static readonly string[] QsvArguments =
@@ -179,14 +181,14 @@ public partial class HardwareCapabilitiesFactory(
         return output;
     }
 
-    public async Task<QsvOutput> GetQsvOutput(string ffmpegPath, Option<string> qsvDevice)
+    public async Task<QsvOutput> GetQsvOutput(string ffmpegPath, Option<string> qsvDevice, QsvInitMode qsvInitMode)
     {
         if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
         {
             return new QsvOutput(0, string.Empty);
         }
 
-        var option = new QsvHardwareAccelerationOption(qsvDevice, FFmpegCapability.Software);
+        var option = new QsvHardwareAccelerationOption(qsvDevice, FFmpegCapability.Software, qsvInitMode);
         var arguments = option.GlobalOptions.ToList();
 
         arguments.AddRange(QsvArguments);
@@ -558,21 +560,50 @@ public partial class HardwareCapabilitiesFactory(
             }
 
             string device = qsvDevice.IfNone(string.Empty);
-            var cacheKey = string.Format(CultureInfo.InvariantCulture, QsvCacheKeyFormat, device);
+            var qsvCacheKey = string.Format(CultureInfo.InvariantCulture, QsvCacheKeyFormat, device);
+            var qsvVaapiCacheKey = string.Format(CultureInfo.InvariantCulture, QsvVaapiCacheKeyFormat, device);
 
-            if (memoryCache.TryGetValue(cacheKey, out List<VaapiProfileEntrypoint>? profileEntrypoints) &&
-                profileEntrypoints is not null)
+            List<VaapiProfileEntrypoint>? profileEntrypoints = null;
+            Option<VaapiHardwareCapabilities> vaapiCapabilities = None;
+
+            if (memoryCache.TryGetValue(qsvCacheKey, out QsvInitMode? qsvInitMode) &&
+                qsvInitMode is not null)
             {
-                return new VaapiHardwareCapabilities(profileEntrypoints, string.Empty, logger);
+                if (memoryCache.TryGetValue(qsvVaapiCacheKey, out profileEntrypoints) && profileEntrypoints is not null)
+                {
+                    vaapiCapabilities = new VaapiHardwareCapabilities(profileEntrypoints, string.Empty, logger);
+                }
+
+                return new QsvHardwareCapabilities(vaapiCapabilities, qsvInitMode.Value);
             }
 
-            QsvOutput output = await GetQsvOutput(ffmpegPath, qsvDevice);
+            var initModesToTest = new Queue<QsvInitMode>();
+            initModesToTest.Enqueue(QsvInitMode.None);
+            if (runtimeInfo.IsOSPlatform(OSPlatform.Windows))
+            {
+                initModesToTest.Enqueue(QsvInitMode.D3d11Va);
+            }
+            initModesToTest.Enqueue(QsvInitMode.Qsv);
+
+            QsvOutput output;
+            QsvInitMode initMode = initModesToTest.Dequeue();
+            do
+            {
+                output = await GetQsvOutput(ffmpegPath, qsvDevice, initMode);
+                if (output.ExitCode == 0)
+                {
+                    break;
+                }
+            } while (initModesToTest.Count > 0);
+
             if (output.ExitCode != 0)
             {
                 logger.LogWarning("QSV test failed; some hardware accelerated features will be unavailable");
                 return new NoHardwareCapabilities();
             }
 
+            memoryCache.Set(qsvCacheKey, initMode);
+
             if (runtimeInfo.IsOSPlatform(OSPlatform.Linux))
             {
                 if (!memoryCache.TryGetValue("ffmpeg.vaapi_displays", out List<string>? vaapiDisplays))
@@ -589,7 +620,7 @@ public partial class HardwareCapabilitiesFactory(
                     if (vaapiOutput.IsNone)
                     {
                         logger.LogWarning("Unable to determine QSV capabilities; please install vainfo");
-                        return new DefaultHardwareCapabilities();
+                        return new QsvHardwareCapabilities(vaapiCapabilities, initMode);
                     }
 
                     foreach (string o in vaapiOutput)
@@ -604,14 +635,14 @@ public partial class HardwareCapabilitiesFactory(
                             profileEntrypoints.Count,
                             device);
 
-                        memoryCache.Set(cacheKey, profileEntrypoints);
-                        return new VaapiHardwareCapabilities(profileEntrypoints, string.Empty, logger);
+                        memoryCache.Set(qsvVaapiCacheKey, profileEntrypoints);
+                        vaapiCapabilities = new VaapiHardwareCapabilities(profileEntrypoints, string.Empty, logger);
                     }
                 }
             }
 
             // not sure how to check capabilities on windows
-            return new DefaultHardwareCapabilities();
+            return new QsvHardwareCapabilities(vaapiCapabilities, initMode);
         }
         catch (Exception ex)
         {

ErsatzTV.FFmpeg/Capabilities/IHardwareCapabilitiesFactory.cs

@@ -18,7 +18,7 @@ public interface IHardwareCapabilitiesFactory
 
     Task<string> GetNvidiaOutput(string ffmpegPath);
 
-    Task<QsvOutput> GetQsvOutput(string ffmpegPath, Option<string> qsvDevice);
+    Task<QsvOutput> GetQsvOutput(string ffmpegPath, Option<string> qsvDevice, QsvInitMode qsvInitMode);
 
     Task<Option<string>> GetVaapiOutput(string display, Option<string> vaapiDriver, string vaapiDevice);
 

ErsatzTV.FFmpeg/Capabilities/Qsv/QsvInitMode.cs

@@ -0,0 +1,8 @@
+namespace ErsatzTV.FFmpeg.Capabilities.Qsv;
+
+public enum QsvInitMode
+{
+    None = 0,
+    Qsv = 1,
+    D3d11Va = 2,
+}

ErsatzTV.FFmpeg/Capabilities/QsvHardwareCapabilities.cs

@@ -0,0 +1,56 @@
+using ErsatzTV.FFmpeg.Capabilities.Qsv;
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Capabilities;
+
+public class QsvHardwareCapabilities(Option<VaapiHardwareCapabilities> vaapiHardwareCapabilities, QsvInitMode initMode)
+    : IHardwareCapabilities
+{
+    public QsvInitMode InitMode { get; } = initMode;
+
+    public FFmpegCapability CanDecode(
+        string videoFormat,
+        Option<string> videoProfile,
+        Option<IPixelFormat> maybePixelFormat,
+        bool isHdr)
+    {
+        foreach (var vaapi in vaapiHardwareCapabilities)
+        {
+            return vaapi.CanDecode(videoFormat, videoProfile, maybePixelFormat, isHdr);
+        }
+
+        int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
+
+        return (videoFormat, bitDepth) switch
+        {
+            // 10-bit h264 decoding is likely not support by any hardware
+            (VideoFormat.H264, 10) => FFmpegCapability.Software,
+
+            _ => FFmpegCapability.Hardware
+        };
+    }
+
+    public FFmpegCapability CanEncode(
+        string videoFormat,
+        Option<string> videoProfile,
+        Option<IPixelFormat> maybePixelFormat)
+    {
+        foreach (var vaapi in vaapiHardwareCapabilities)
+        {
+            return vaapi.CanEncode(videoFormat, videoProfile, maybePixelFormat);
+        }
+
+        int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
+
+        return (videoFormat, bitDepth) switch
+        {
+            // 10-bit h264 encoding is not support by any hardware
+            (VideoFormat.H264, 10) => FFmpegCapability.Software,
+
+            _ => FFmpegCapability.Hardware
+        };
+    }
+
+    public Option<RateControlMode> GetRateControlMode(string videoFormat, Option<IPixelFormat> maybePixelFormat) =>
+        Option<RateControlMode>.None;
+}

ErsatzTV.FFmpeg/GlobalOption/HardwareAcceleration/QsvHardwareAccelerationOption.cs

@@ -1,22 +1,32 @@
 using ErsatzTV.FFmpeg.Capabilities;
+using ErsatzTV.FFmpeg.Capabilities.Qsv;
 using ErsatzTV.FFmpeg.Format;
 
 namespace ErsatzTV.FFmpeg.GlobalOption.HardwareAcceleration;
 
-public class QsvHardwareAccelerationOption(Option<string> device, FFmpegCapability decodeCapability) : GlobalOption
+public class QsvHardwareAccelerationOption(
+    Option<string> device,
+    FFmpegCapability decodeCapability,
+    QsvInitMode qsvInitMode) : GlobalOption
 {
     // TODO: read this from ffmpeg output
-    private readonly List<string> _supportedFFmpegFormats = new()
-    {
+    private readonly List<string> _supportedFFmpegFormats =
+    [
         FFmpegFormat.NV12,
         FFmpegFormat.P010LE
-    };
+    ];
 
     public override string[] GlobalOptions
     {
         get
         {
-            string[] initDevices = ["-init_hw_device", "qsv=hw", "-filter_hw_device", "hw"];
+            string[] initDevices = qsvInitMode switch
+            {
+                QsvInitMode.None => [],
+                QsvInitMode.Qsv => ["-init_hw_device", "qsv=hw", "-filter_hw_device", "hw"],
+                QsvInitMode.D3d11Va => ["-init_hw_device", "d3d11va=hw:,vendor=0x8086", "-filter_hw_device", "hw"],
+                _ => throw new NotSupportedException()
+            };
 
             var result = new List<string>
             {

ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs

@@ -1,5 +1,6 @@
 using System.Globalization;
 using ErsatzTV.FFmpeg.Capabilities;
+using ErsatzTV.FFmpeg.Capabilities.Qsv;
 using ErsatzTV.FFmpeg.Decoder;
 using ErsatzTV.FFmpeg.Decoder.Qsv;
 using ErsatzTV.FFmpeg.Encoder;
@@ -95,7 +96,13 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
         // give a bogus value so no cuda devices are visible to ffmpeg
         pipelineSteps.Add(new CudaVisibleDevicesVariable("999"));
 
-        pipelineSteps.Add(new QsvHardwareAccelerationOption(ffmpegState.VaapiDevice, decodeCapability));
+        QsvInitMode initMode = QsvInitMode.Qsv;
+        if (_hardwareCapabilities is QsvHardwareCapabilities qsvHardwareCapabilities)
+        {
+            initMode = qsvHardwareCapabilities.InitMode;
+        }
+
+        pipelineSteps.Add(new QsvHardwareAccelerationOption(ffmpegState.VaapiDevice, decodeCapability, initMode));
 
         // disable hw accel if decoder/encoder isn't supported
         return ffmpegState with