only use packed headers with vaapi when supported by encoder (#2706)

CHANGELOG.md

@@ -44,6 +44,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix bug with mirror channels where seemingly random content would be played every ~40 seconds
 - Fix chronological sorting for Other Videos that have release date metadata
 - Fix playout sorting after using channel number editor
+- VAAPI: Only include `-sei a53_cc` flags when misc packed headers are supported by the encoder
+  - This should fix playback in some cases, e.g. AMD VAAPI h264 encoder
 
 ### Changed
 - No longer round framerate to nearest integer when normalizing framerate

ErsatzTV.FFmpeg/Capabilities/Vaapi/VaapiCapabilityParser.cs

@@ -57,6 +57,13 @@ public static partial class VaapiCapabilityParser
                             break;
                     }
                 }
+
+                // check for enc packed header misc
+                match = ProfileEncPackedHeaderMiscRegex().Match(line);
+                if (match.Success)
+                {
+                    profile.AddPackedHeaderMisc();
+                }
             }
         }
 
@@ -89,6 +96,9 @@ public static partial class VaapiCapabilityParser
     [GeneratedRegex(@".*VA_RC_(\w*).*")]
     private static partial Regex ProfileRateControlRegex();
 
+    [GeneratedRegex(@".*VA_ENC_PACKED_HEADER_MISC.*")]
+    private static partial Regex ProfileEncPackedHeaderMiscRegex();
+
     [GeneratedRegex(@"Driver version:.*\(radeonsi, (\w+)")]
     private static partial Regex RadeonSiGenerationRegex();
 

ErsatzTV.FFmpeg/Capabilities/Vaapi/VaapiProfileEntrypoint.cs

@@ -2,9 +2,13 @@ namespace ErsatzTV.FFmpeg.Capabilities.Vaapi;
 
 public record VaapiProfileEntrypoint(string VaapiProfile, string VaapiEntrypoint)
 {
-    private readonly System.Collections.Generic.HashSet<RateControlMode> _rateControlModes = new();
+    private readonly System.Collections.Generic.HashSet<RateControlMode> _rateControlModes = [];
 
     public IReadOnlyCollection<RateControlMode> RateControlModes => _rateControlModes;
 
+    public bool PackedHeaderMisc { get; private set; }
+
     public bool AddRateControlMode(RateControlMode mode) => _rateControlModes.Add(mode);
+
+    public void AddPackedHeaderMisc() => PackedHeaderMisc = true;
 }

ErsatzTV.FFmpeg/Capabilities/VaapiHardwareCapabilities.cs

@@ -163,49 +163,7 @@ public class VaapiHardwareCapabilities(
         Option<IPixelFormat> maybePixelFormat)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
-
-        bool isHardware = videoFormat switch
-        {
-            // vaapi cannot encode 10-bit h264
-            VideoFormat.H264 when bitDepth == 10 => false,
-
-            VideoFormat.H264 =>
-                profileEntrypoints.Any(e => e is
-                {
-                    VaapiProfile: VaapiProfile.H264Main,
-                    VaapiEntrypoint: VaapiEntrypoint.Encode or VaapiEntrypoint.EncodeLowPower
-                }),
-
-            VideoFormat.Hevc when bitDepth == 10 =>
-                profileEntrypoints.Any(e => e is
-                {
-                    VaapiProfile: VaapiProfile.HevcMain10,
-                    VaapiEntrypoint: VaapiEntrypoint.Encode or VaapiEntrypoint.EncodeLowPower
-                }),
-
-            VideoFormat.Hevc =>
-                profileEntrypoints.Any(e => e is
-                {
-                    VaapiProfile: VaapiProfile.HevcMain,
-                    VaapiEntrypoint: VaapiEntrypoint.Encode or VaapiEntrypoint.EncodeLowPower
-                }),
-
-            VideoFormat.Av1 =>
-                profileEntrypoints.Any(e => e is
-                {
-                    VaapiProfile: VaapiProfile.Av1Profile0,
-                    VaapiEntrypoint: VaapiEntrypoint.Encode or VaapiEntrypoint.EncodeLowPower
-                }),
-
-            VideoFormat.Mpeg2Video =>
-                profileEntrypoints.Any(e => e is
-                {
-                    VaapiProfile: VaapiProfile.Mpeg2Main,
-                    VaapiEntrypoint: VaapiEntrypoint.Encode or VaapiEntrypoint.EncodeLowPower
-                }),
-
-            _ => false
-        };
+        bool isHardware = GetEntrypoint(videoFormat, bitDepth).IsSome;
 
         if (!isHardware)
         {
@@ -221,7 +179,36 @@ public class VaapiHardwareCapabilities(
     public Option<RateControlMode> GetRateControlMode(string videoFormat, Option<IPixelFormat> maybePixelFormat)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
-        Option<VaapiProfileEntrypoint> maybeEntrypoint = videoFormat switch
+        foreach (VaapiProfileEntrypoint entrypoint in GetEntrypoint(videoFormat, bitDepth))
+        {
+            if (entrypoint.RateControlModes.Contains(RateControlMode.VBR) ||
+                entrypoint.RateControlModes.Contains(RateControlMode.CBR))
+            {
+                return Option<RateControlMode>.None;
+            }
+
+            if (entrypoint.RateControlModes.Contains(RateControlMode.CQP))
+            {
+                return RateControlMode.CQP;
+            }
+        }
+
+        return Option<RateControlMode>.None;
+    }
+
+    public bool GetPackedHeaderMisc(string videoFormat, Option<IPixelFormat> maybePixelFormat)
+    {
+        int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
+        foreach (VaapiProfileEntrypoint entrypoint in GetEntrypoint(videoFormat, bitDepth))
+        {
+            return entrypoint.PackedHeaderMisc;
+        }
+
+        return false;
+    }
+
+    private Option<VaapiProfileEntrypoint> GetEntrypoint(string videoFormat, int bitDepth) =>
+        videoFormat switch
         {
             // vaapi cannot encode 10-bit h264
             VideoFormat.H264 when bitDepth == 10 => None,
@@ -250,6 +237,14 @@ public class VaapiHardwareCapabilities(
                     })
                     .HeadOrNone(),
 
+            VideoFormat.Av1 =>
+                profileEntrypoints.Where(e => e is
+                {
+                    VaapiProfile: VaapiProfile.Av1Profile0,
+                    VaapiEntrypoint: VaapiEntrypoint.Encode or VaapiEntrypoint.EncodeLowPower
+                })
+                .HeadOrNone(),
+
             VideoFormat.Mpeg2Video =>
                 profileEntrypoints.Where(e => e is
                     {
@@ -260,21 +255,4 @@ public class VaapiHardwareCapabilities(
 
             _ => None
         };
-
-        foreach (VaapiProfileEntrypoint entrypoint in maybeEntrypoint)
-        {
-            if (entrypoint.RateControlModes.Contains(RateControlMode.VBR) ||
-                entrypoint.RateControlModes.Contains(RateControlMode.CBR))
-            {
-                return Option<RateControlMode>.None;
-            }
-
-            if (entrypoint.RateControlModes.Contains(RateControlMode.CQP))
-            {
-                return RateControlMode.CQP;
-            }
-        }
-
-        return Option<RateControlMode>.None;
-    }
 }

ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderAv1Vaapi.cs

@@ -20,9 +20,6 @@ public class EncoderAv1Vaapi(RateControlMode rateControlMode) : EncoderBase
                 result.Add("1");
             }
 
-            result.Add("-sei");
-            result.Add("-a53_cc");
-
             return result.ToArray();
         }
     }

ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderH264Vaapi.cs

@@ -2,7 +2,8 @@
 
 namespace ErsatzTV.FFmpeg.Encoder.Vaapi;
 
-public class EncoderH264Vaapi(Option<string> maybeVideoProfile, RateControlMode rateControlMode) : EncoderBase
+public class EncoderH264Vaapi(Option<string> maybeVideoProfile, RateControlMode rateControlMode, bool packedHeaderMisc)
+    : EncoderBase
 {
     public override string Name => "h264_vaapi";
 
@@ -20,8 +21,11 @@ public class EncoderH264Vaapi(Option<string> maybeVideoProfile, RateControlMode
                 result.Add("1");
             }
 
-            result.Add("-sei");
-            result.Add("-a53_cc");
+            if (packedHeaderMisc)
+            {
+                result.Add("-sei");
+                result.Add("-a53_cc");
+            }
 
             foreach (string videoProfile in maybeVideoProfile)
             {

ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderHevcVaapi.cs

@@ -2,12 +2,8 @@
 
 namespace ErsatzTV.FFmpeg.Encoder.Vaapi;
 
-public class EncoderHevcVaapi : EncoderBase
+public class EncoderHevcVaapi(RateControlMode rateControlMode, bool packedHeaderMisc) : EncoderBase
 {
-    private readonly RateControlMode _rateControlMode;
-
-    public EncoderHevcVaapi(RateControlMode rateControlMode) => _rateControlMode = rateControlMode;
-
     public override string Name => "hevc_vaapi";
 
     public override StreamKind Kind => StreamKind.Video;
@@ -18,14 +14,17 @@ public class EncoderHevcVaapi : EncoderBase
         {
             var result = new List<string>(base.OutputOptions);
 
-            if (_rateControlMode == RateControlMode.CQP)
+            if (rateControlMode == RateControlMode.CQP)
             {
                 result.Add("-rc_mode");
                 result.Add("1");
             }
 
-            result.Add("-sei");
-            result.Add("-a53_cc");
+            if (packedHeaderMisc)
+            {
+                result.Add("-sei");
+                result.Add("-a53_cc");
+            }
 
             return result.ToArray();
         }

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -250,6 +250,14 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
         // after everything else is done, apply the encoder
         if (pipelineSteps.OfType<IEncoder>().All(e => e.Kind != StreamKind.Video))
         {
+            bool packedHeaderMisc = false;
+            if (_hardwareCapabilities is VaapiHardwareCapabilities vaapiHardwareCapabilities)
+            {
+                packedHeaderMisc = vaapiHardwareCapabilities.GetPackedHeaderMisc(
+                    desiredState.VideoFormat,
+                    desiredState.PixelFormat);
+            }
+
             RateControlMode rateControlMode =
                 _hardwareCapabilities.GetRateControlMode(desiredState.VideoFormat, desiredState.PixelFormat)
                     .IfNone(RateControlMode.VBR);
@@ -260,9 +268,9 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
                     (HardwareAccelerationMode.Vaapi, VideoFormat.Av1) =>
                         new EncoderAv1Vaapi(rateControlMode),
                     (HardwareAccelerationMode.Vaapi, VideoFormat.Hevc) =>
-                        new EncoderHevcVaapi(rateControlMode),
+                        new EncoderHevcVaapi(rateControlMode, packedHeaderMisc),
                     (HardwareAccelerationMode.Vaapi, VideoFormat.H264) =>
-                        new EncoderH264Vaapi(desiredState.VideoProfile, rateControlMode),
+                        new EncoderH264Vaapi(desiredState.VideoProfile, rateControlMode, packedHeaderMisc),
                     (HardwareAccelerationMode.Vaapi, VideoFormat.Mpeg2Video) =>
                         new EncoderMpeg2Vaapi(rateControlMode),