detect some videotoolbox decoders (#2240)

CHANGELOG.md

@@ -34,8 +34,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     - If the playlist has 3 items and none set to play all, it will schedule 3 items when `Count = 1`
     - If the playlist has 3 items and none set to play all, it will schedule 6 items when `Count = 2`
   - Using the same playlist in the same schedule for anything other than filler may cause undesired behavior
-- Detect supported VideoToolbox hardware encoders
+- Detect supported VideoToolbox hardware decoders and encoders
-  - Software encoders will automatically be used when hardware encoders are unavailable
+  - Software decoders/encoders will automatically be used when hardware versions are unavailable
 - Add VideoToolbox Capabilities to Troubleshooting page
 
 ### Fixed

ErsatzTV.Application/Troubleshooting/Queries/GetTroubleshootingInfoHandler.cs

@@ -135,6 +135,16 @@ public class GetTroubleshootingInfoHandler : IRequestHandler<GetTroubleshootingI
 
                 if (_runtimeInfo.IsOSPlatform(OSPlatform.OSX))
                 {
+                    var decoders = _hardwareCapabilitiesFactory.GetVideoToolboxDecoders();
+                    videoToolboxCapabilities.AppendLine("VideoToolbox Decoders: ");
+                    videoToolboxCapabilities.AppendLine();
+                    foreach (var decoder in decoders)
+                    {
+                        videoToolboxCapabilities.AppendLine(CultureInfo.InvariantCulture, $"\t{decoder}");
+                    }
+                    videoToolboxCapabilities.AppendLine();
+                    videoToolboxCapabilities.AppendLine();
+
                     var encoders = _hardwareCapabilitiesFactory.GetVideoToolboxEncoders();
                     videoToolboxCapabilities.AppendLine("VideoToolbox Encoders: ");
                     videoToolboxCapabilities.AppendLine();

ErsatzTV.FFmpeg/Capabilities/FourCC.cs

@@ -0,0 +1,15 @@
+namespace ErsatzTV.FFmpeg.Capabilities;
+
+public static class FourCC
+{
+    public static readonly List<string> AllVideoToolbox =
+    [
+        H264,
+        Hevc,
+        Vp9
+    ];
+
+    public const string H264 = "avc1";
+    public const string Hevc = "hvc1";
+    public const string Vp9 = "vp90";
+}

ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs

@@ -261,9 +261,24 @@ public class HardwareCapabilitiesFactory : IHardwareCapabilitiesFactory
         return [];
     }
 
+    public List<string> GetVideoToolboxDecoders()
+    {
+        var result = new List<string>();
+
+        foreach (var fourCC in FourCC.AllVideoToolbox)
+        {
+            if (VideoToolboxUtil.IsHardwareDecoderSupported(fourCC, _logger))
+            {
+                result.Add(fourCC);
+            }
+        }
+
+        return result;
+    }
+
     public List<string> GetVideoToolboxEncoders()
     {
-        return VideoToolboxUtil.GetAvailableEncoders();
+        return VideoToolboxUtil.GetAvailableEncoders(_logger);
     }
 
     private async Task<IReadOnlySet<string>> GetFFmpegCapabilities(

ErsatzTV.FFmpeg/Capabilities/IHardwareCapabilitiesFactory.cs

@@ -26,5 +26,7 @@ public interface IHardwareCapabilitiesFactory
 
     List<VideoControllerModel> GetVideoControllerList();
 
+    List<string> GetVideoToolboxDecoders();
+
     List<string> GetVideoToolboxEncoders();
 }

ErsatzTV.FFmpeg/Capabilities/VideoToolbox/VideoToolboxUtil.cs

@@ -1,5 +1,6 @@
 using System.Runtime.InteropServices;
 using System.Text;
+using Microsoft.Extensions.Logging;
 
 namespace ErsatzTV.FFmpeg.Capabilities.VideoToolbox;
 
@@ -34,6 +35,10 @@ internal static partial class VideoToolboxUtil
     [LibraryImport(VideoToolbox)]
     private static partial int VTCopyVideoEncoderList(IntPtr options, out IntPtr listOfEncoders);
 
+    [LibraryImport(VideoToolbox)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    private static partial bool VTIsHardwareDecodeSupported(uint codecType);
+
     [LibraryImport(LibSystem, StringMarshalling = StringMarshalling.Utf8)]
     private static partial IntPtr dlopen(string path, int mode);
 
@@ -95,7 +100,17 @@ internal static partial class VideoToolboxUtil
         return null;
     }
 
-    internal static List<string> GetAvailableEncoders()
+    private static uint FourCCToUInt32(string fourCC)
+    {
+        if (fourCC.Length != 4)
+        {
+            throw new ArgumentException("FourCC must be 4 characters long.", nameof(fourCC));
+        }
+
+        return ((uint)fourCC[0] << 24) | ((uint)fourCC[1] << 16) | ((uint)fourCC[2] << 8) | (uint)fourCC[3];
+    }
+
+    internal static List<string> GetAvailableEncoders(ILogger logger)
     {
         var encoderNames = new List<string>();
 
@@ -107,7 +122,7 @@ internal static partial class VideoToolboxUtil
         IntPtr kVTVideoEncoderList_EncoderName = GetCFString(VideoToolbox, "kVTVideoEncoderList_EncoderName");
         if (kVTVideoEncoderList_EncoderName == IntPtr.Zero)
         {
-            Console.Error.WriteLine("Failed to load kVTVideoEncoderList_EncoderName symbol.");
+            logger.LogWarning("Failed to load kVTVideoEncoderList_EncoderName symbol.");
             return encoderNames;
         }
 
@@ -118,7 +133,7 @@ internal static partial class VideoToolboxUtil
 
             if (status != 0 || encoderList == IntPtr.Zero)
             {
-                Console.Error.WriteLine($"VTCopyVideoEncoderList failed with status: {status}");
+                logger.LogWarning("VTCopyVideoEncoderList failed with status: {Status}", status);
                 return encoderNames;
             }
 
@@ -126,7 +141,10 @@ internal static partial class VideoToolboxUtil
             for (int i = 0; i < count; i++)
             {
                 IntPtr encoderDict = CFArrayGetValueAtIndex(encoderList, i);
-                if (encoderDict == IntPtr.Zero) continue;
+                if (encoderDict == IntPtr.Zero)
+                {
+                    continue;
+                }
 
                 IntPtr encoderNameCfString = CFDictionaryGetValue(encoderDict, kVTVideoEncoderList_EncoderName);
                 string? encoderName = CFStringToString(encoderNameCfString);
@@ -147,4 +165,18 @@ internal static partial class VideoToolboxUtil
 
         return encoderNames;
     }
+
+    internal static bool IsHardwareDecoderSupported(string codecFourCC, ILogger logger)
+    {
+        try
+        {
+            uint codecType = FourCCToUInt32(codecFourCC);
+            return VTIsHardwareDecodeSupported(codecType);
+        }
+        catch (Exception ex)
+        {
+            logger.LogWarning(ex, "Unexpected error checking decoder support for {CodecFourCC}", codecFourCC);
+            return false;
+        }
+    }
 }

ErsatzTV.FFmpeg/Capabilities/VideoToolboxHardwareCapabilities.cs

@@ -9,6 +9,7 @@ namespace ErsatzTV.FFmpeg.Capabilities;
 public class VideoToolboxHardwareCapabilities : IHardwareCapabilities
 {
     private static readonly ConcurrentDictionary<string, bool> Encoders = new ();
+    private static readonly ConcurrentDictionary<string, bool> Decoders = new ();
 
     private readonly IFFmpegCapabilities _ffmpegCapabilities;
     private readonly ILogger _logger;
@@ -21,13 +22,31 @@ public class VideoToolboxHardwareCapabilities : IHardwareCapabilities
 
     public FFmpegCapability CanDecode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat, bool isHdr)
     {
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) && Decoders.IsEmpty)
+        {
+            if (VideoToolboxUtil.IsHardwareDecoderSupported(FourCC.H264, _logger))
+            {
+                Decoders.AddOrUpdate(VideoFormat.H264, true, (_, _) => true);
+            }
+
+            if (VideoToolboxUtil.IsHardwareDecoderSupported(FourCC.Hevc, _logger))
+            {
+                Decoders.AddOrUpdate(VideoFormat.Hevc, true, (_, _) => true);
+            }
+
+            if (VideoToolboxUtil.IsHardwareDecoderSupported(FourCC.Vp9, _logger))
+            {
+                Decoders.AddOrUpdate(VideoFormat.Vp9, true, (_, _) => true);
+            }
+        }
+
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
         return (videoFormat, bitDepth) switch
         {
             // 10-bit h264 decoding is likely not support by any hardware
             (VideoFormat.H264, 10) => FFmpegCapability.Software,
 
-            _ => FFmpegCapability.Hardware
+            _ => Decoders.ContainsKey(videoFormat) ? FFmpegCapability.Hardware : FFmpegCapability.Software
         };
     }
 
@@ -35,7 +54,7 @@ public class VideoToolboxHardwareCapabilities : IHardwareCapabilities
     {
         if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) && Encoders.IsEmpty)
         {
-            var encoderList = VideoToolboxUtil.GetAvailableEncoders();
+            var encoderList = VideoToolboxUtil.GetAvailableEncoders(_logger);
             _logger.LogDebug("VideoToolbox reports {Count} encoders", encoderList.Count);
 
             // we only really care about h264 and hevc hardware encoders

ErsatzTV.FFmpeg/Pipeline/VideoToolboxPipelineBuilder.cs

@@ -59,7 +59,10 @@ public class VideoToolboxPipelineBuilder : SoftwarePipelineBuilder
             desiredState.VideoProfile,
             desiredState.PixelFormat);
 
-        pipelineSteps.Add(new VideoToolboxHardwareAccelerationOption());
+        if (decodeCapability is FFmpegCapability.Hardware)
+        {
+            pipelineSteps.Add(new VideoToolboxHardwareAccelerationOption());
+        }
 
         // disable hw accel if decoder/encoder isn't supported
         return ffmpegState with