fix bt2020 playback (#2714)

* fix bt2020 playback

* update pixel format

* update changelog

CHANGELOG.md

@@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
   - work around buggy ffmpeg behavior where hevc_vaapi encoder with RadeonSI driver incorrectly outputs height of 1088 instead of 1080
   - fix green padding when encoding h264 using main profile
 - Automatically kill playback troubleshooting ffmpeg process if it hasn't completed after two minutes
+- Fix playback of certain BT.2020 content
 
 ### Changed
 - No longer round framerate to nearest integer when normalizing framerate

ErsatzTV.FFmpeg/Capabilities/AmfHardwareCapabilities.cs

@@ -8,7 +8,7 @@ public class AmfHardwareCapabilities : IHardwareCapabilities
         string videoFormat,
         Option<string> videoProfile,
         Option<IPixelFormat> maybePixelFormat,
-        bool isHdr) => FFmpegCapability.Software;
+        ColorParams colorParams) => FFmpegCapability.Software;
 
     public FFmpegCapability CanEncode(
         string videoFormat,

ErsatzTV.FFmpeg/Capabilities/DefaultHardwareCapabilities.cs

@@ -8,7 +8,7 @@ public class DefaultHardwareCapabilities : IHardwareCapabilities
         string videoFormat,
         Option<string> videoProfile,
         Option<IPixelFormat> maybePixelFormat,
-        bool isHdr)
+        ColorParams colorParams)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 
@@ -17,6 +17,8 @@ public class DefaultHardwareCapabilities : IHardwareCapabilities
             // 10-bit h264 decoding is likely not support by any hardware
             (VideoFormat.H264, 10) => FFmpegCapability.Software,
 
+            (_, _) when colorParams.IsBt2020Ten => FFmpegCapability.Software,
+
             _ => FFmpegCapability.Hardware
         };
     }

ErsatzTV.FFmpeg/Capabilities/IHardwareCapabilities.cs

@@ -8,7 +8,7 @@ public interface IHardwareCapabilities
         string videoFormat,
         Option<string> videoProfile,
         Option<IPixelFormat> maybePixelFormat,
-        bool isHdr);
+        ColorParams colorParams);
 
     FFmpegCapability CanEncode(
         string videoFormat,

ErsatzTV.FFmpeg/Capabilities/NoHardwareCapabilities.cs

@@ -8,7 +8,7 @@ public class NoHardwareCapabilities : IHardwareCapabilities
         string videoFormat,
         Option<string> videoProfile,
         Option<IPixelFormat> maybePixelFormat,
-        bool isHdr) =>
+        ColorParams colorParams) =>
         FFmpegCapability.Software;
 
     public FFmpegCapability CanEncode(

ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs

@@ -15,7 +15,7 @@ public class NvidiaHardwareCapabilities(CudaDevice cudaDevice, IFFmpegCapabiliti
         string videoFormat,
         Option<string> videoProfile,
         Option<IPixelFormat> maybePixelFormat,
-        bool isHdr)
+        ColorParams colorParams)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 
@@ -52,6 +52,11 @@ public class NvidiaHardwareCapabilities(CudaDevice cudaDevice, IFFmpegCapabiliti
             }
         }
 
+        if (colorParams.IsBt2020Ten)
+        {
+            isHardware = false;
+        }
+
         if (isHardware)
         {
             return videoFormat switch

ErsatzTV.FFmpeg/Capabilities/RkmppHardwareCapabilities.cs

@@ -8,7 +8,7 @@ public class RkmppHardwareCapabilities : IHardwareCapabilities
         string videoFormat,
         Option<string> videoProfile,
         Option<IPixelFormat> maybePixelFormat,
-        bool isHdr) => FFmpegCapability.Hardware;
+        ColorParams colorParams) => FFmpegCapability.Hardware;
 
     public FFmpegCapability CanEncode(
         string videoFormat,

ErsatzTV.FFmpeg/Capabilities/V4l2m2mHardwareCapabilities.cs

@@ -8,12 +8,14 @@ public class V4l2m2mHardwareCapabilities(IFFmpegCapabilities ffmpegCapabilities)
         string videoFormat,
         Option<string> videoProfile,
         Option<IPixelFormat> maybePixelFormat,
-        bool isHdr)
+        ColorParams colorParams)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 
         return (videoFormat, bitDepth) switch
         {
+            (_, _) when colorParams.IsBt2020Ten => FFmpegCapability.Software,
+
             (VideoFormat.H264, 8) => ffmpegCapabilities.HasDecoder(FFmpegKnownDecoder.H264V4l2m2m)
                 ? FFmpegCapability.Hardware
                 : FFmpegCapability.Software,

ErsatzTV.FFmpeg/Capabilities/VaapiHardwareCapabilities.cs

@@ -18,12 +18,14 @@ public class VaapiHardwareCapabilities(
         string videoFormat,
         Option<string> videoProfile,
         Option<IPixelFormat> maybePixelFormat,
-        bool isHdr)
+        ColorParams colorParams)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 
         bool isHardware = (videoFormat, videoProfile.IfNone(string.Empty).ToLowerInvariant()) switch
         {
+            (_, _) when colorParams.IsBt2020Ten => false,
+
             // no hardware decoding of 10-bit h264
             (VideoFormat.H264, _) when bitDepth == 10 => false,
 

ErsatzTV.FFmpeg/Capabilities/VideoToolboxHardwareCapabilities.cs

@@ -24,7 +24,7 @@ public class VideoToolboxHardwareCapabilities : IHardwareCapabilities
         string videoFormat,
         Option<string> videoProfile,
         Option<IPixelFormat> maybePixelFormat,
-        bool isHdr)
+        ColorParams colorParams)
     {
         if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) && Decoders.IsEmpty)
         {
@@ -65,6 +65,8 @@ public class VideoToolboxHardwareCapabilities : IHardwareCapabilities
             // 10-bit h264 decoding is likely not support by any hardware
             (VideoFormat.H264, 10) => FFmpegCapability.Software,
 
+            (_, _) when colorParams.IsBt2020Ten => FFmpegCapability.Software,
+
             _ => Decoders.ContainsKey(videoFormat) ? FFmpegCapability.Hardware : FFmpegCapability.Software
         };
     }

ErsatzTV.FFmpeg/ColorParams.cs

@@ -30,4 +30,6 @@ public record ColorParams(string ColorRange, string ColorSpace, string ColorTran
                                       && string.IsNullOrWhiteSpace(ColorPrimaries);
         }
     }
+
+    public bool IsBt2020Ten => ColorTransfer is "bt2020-10";
 }

ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs

@@ -75,7 +75,7 @@ public class ColorspaceFilter : BaseFilter
             }
 
             string inputOverrides = string.Empty;
-            if (cp.IsMixed || _forceInputOverrides)
+            if (!cp.IsBt2020Ten && (cp.IsMixed || _forceInputOverrides))
             {
                 string range = string.IsNullOrWhiteSpace(cp.ColorRange)
                     ? string.Empty

ErsatzTV.FFmpeg/Pipeline/AmfPipelineBuilder.cs

@@ -54,7 +54,7 @@ public class AmfPipelineBuilder : SoftwarePipelineBuilder
             videoStream.Codec,
             videoStream.Profile,
             videoStream.PixelFormat,
-            videoStream.ColorParams.IsHdr);
+            videoStream.ColorParams);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,

ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs

@@ -62,7 +62,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
             videoStream.Codec,
             videoStream.Profile,
             videoStream.PixelFormat,
-            videoStream.ColorParams.IsHdr);
+            videoStream.ColorParams);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs

@@ -589,6 +589,20 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
             ? None
             : SetDecoder(videoInputFile, videoStream, ffmpegState, context);
 
+        Option<IPixelFormat> videoStreamPixelFormat = videoStream.PixelFormat;
+
+        // bt.2020 needs to immediately change to bt709 and a new format
+        if (videoStream.ColorParams.IsBt2020Ten)
+        {
+            IPixelFormat targetFormat = desiredState.PixelFormat.IfNone(new PixelFormatYuv420P());
+
+            videoInputFile.FilterSteps.Add(new ColorspaceFilter(desiredState, videoStream, targetFormat));
+
+            // update pipeline to have accurate color params and pixel format
+            videoStream.ResetColorParams(ColorParams.Default);
+            videoStreamPixelFormat = Some(targetFormat);
+        }
+
         //SetStillImageInfiniteLoop(videoInputFile, videoStream, ffmpegState);
         SetRealtimeInput(videoInputFile, desiredState);
         SetInfiniteLoop(videoInputFile, videoStream, ffmpegState, desiredState);
@@ -603,7 +617,7 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
 
         FilterChain filterChain = SetVideoFilters(
             videoInputFile,
-            videoStream,
+            videoStream with { PixelFormat = videoStreamPixelFormat },
             _watermarkInputFile,
             _subtitleInputFile,
             _graphicsEngineInput,

ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs

@@ -65,7 +65,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
             videoStream.Codec,
             videoStream.Profile,
             videoStream.PixelFormat,
-            videoStream.ColorParams.IsHdr);
+            videoStream.ColorParams);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,

ErsatzTV.FFmpeg/Pipeline/RkmppPipelineBuilder.cs

@@ -57,7 +57,7 @@ public class RkmppPipelineBuilder : SoftwarePipelineBuilder
             videoStream.Codec,
             videoStream.Profile,
             videoStream.PixelFormat,
-            videoStream.ColorParams.IsHdr);
+            videoStream.ColorParams);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,

ErsatzTV.FFmpeg/Pipeline/V4l2m2mPipelineBuilder.cs

@@ -56,7 +56,7 @@ public class V4l2m2mPipelineBuilder : SoftwarePipelineBuilder
             videoStream.Codec,
             videoStream.Profile,
             videoStream.PixelFormat,
-            videoStream.ColorParams.IsHdr);
+            videoStream.ColorParams);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -68,7 +68,7 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
             videoStream.Codec,
             videoStream.Profile,
             videoStream.PixelFormat,
-            videoStream.ColorParams.IsHdr);
+            videoStream.ColorParams);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,

ErsatzTV.FFmpeg/Pipeline/VideoToolboxPipelineBuilder.cs

@@ -56,7 +56,7 @@ public class VideoToolboxPipelineBuilder : SoftwarePipelineBuilder
             videoStream.Codec,
             videoStream.Profile,
             videoStream.PixelFormat,
-            videoStream.ColorParams.IsHdr);
+            videoStream.ColorParams);
         FFmpegCapability encodeCapability = _hardwareCapabilities.CanEncode(
             desiredState.VideoFormat,
             desiredState.VideoProfile,