From cd0f1e98ccf0f0435f4354e90171d404a904f85a Mon Sep 17 00:00:00 2001
From: Jason Dove <jason@jasondove.me>
Date: Thu, 8 Dec 2022 08:17:59 -0600
Subject: [PATCH] fix qsv color normalization (#1046)

---
 ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs        |  9 +++++++--
 ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs |  2 +-
 ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs    | 12 +++++++++++-
 ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs  |  2 +-
 4 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs b/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs
index 07b828327..7973cf0de 100644
--- a/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs
+++ b/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs
@@ -6,15 +6,18 @@ public class ColorspaceFilter : BaseFilter
 {
     private readonly VideoStream _videoStream;
     private readonly IPixelFormat _desiredPixelFormat;
+    private readonly bool _forceInputOverrides;
     private readonly FrameDataLocation _nextDataLocation;
 
     public ColorspaceFilter(
         VideoStream videoStream,
         IPixelFormat desiredPixelFormat,
+        bool forceInputOverrides = false,
         FrameDataLocation nextDataLocation = FrameDataLocation.Software)
     {
         _videoStream = videoStream;
         _desiredPixelFormat = desiredPixelFormat;
+        _forceInputOverrides = forceInputOverrides;
         _nextDataLocation = nextDataLocation;
     }
 
@@ -36,10 +39,12 @@ public class ColorspaceFilter : BaseFilter
         {
             string inputOverrides = string.Empty;
             ColorParams cp = _videoStream.ColorParams;
-            if (cp.IsMixed)
+            if (cp.IsMixed || _forceInputOverrides)
             {
+                string range = string.IsNullOrWhiteSpace(cp.ColorRange) ? "tv" : cp.ColorRange;
+
                 inputOverrides =
-                    $"irange={cp.ColorRange}:ispace={cp.ColorSpace}:itrc={cp.ColorTransfer}:iprimaries={cp.ColorPrimaries}:";
+                    $"irange={range}:ispace={cp.ColorSpace}:itrc={cp.ColorTransfer}:iprimaries={cp.ColorPrimaries}:";
             }
 
             string colorspace = _desiredPixelFormat.BitDepth switch
diff --git a/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs
index 71dd6020b..4b7f10c72 100644
--- a/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs
+++ b/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs
@@ -262,7 +262,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
             if (!videoStream.ColorParams.IsBt709)
             {
                 _logger.LogDebug("Adding colorspace filter");
-                var colorspace = new ColorspaceFilter(videoStream, format, currentState.FrameDataLocation);
+                var colorspace = new ColorspaceFilter(videoStream, format, false, currentState.FrameDataLocation);
 
                 currentState = colorspace.NextState(currentState);
                 result.Add(colorspace);
diff --git a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs
index 3adad93cf..7f6b66bf8 100644
--- a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs
+++ b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs
@@ -190,6 +190,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
         }
         
         List<IPipelineFilterStep> pixelFormatFilterSteps = SetPixelFormat(
+            videoInputFile,
             videoStream,
             desiredState.PixelFormat,
             ffmpegState,
@@ -207,6 +208,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
     }
     
     private List<IPipelineFilterStep> SetPixelFormat(
+        VideoInputFile videoInputFile,
         VideoStream videoStream,
         Option<IPixelFormat> desiredPixelFormat,
         FFmpegState ffmpegState,
@@ -233,7 +235,15 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
             if (!videoStream.ColorParams.IsBt709)
             {
                 _logger.LogDebug("Adding colorspace filter");
-                var colorspace = new ColorspaceFilter(videoStream, format, currentState.FrameDataLocation);
+
+                // vpp_qsv seems to strip color info, so if we use that at all, force overriding input color info
+                bool forceInputOverrides = videoInputFile.FilterSteps.Any(f => f is QsvFormatFilter or ScaleQsvFilter);
+
+                var colorspace = new ColorspaceFilter(
+                    videoStream,
+                    format,
+                    forceInputOverrides,
+                    currentState.FrameDataLocation);
 
                 // force nv12 if we're still in hardware
                 if (currentState.FrameDataLocation == FrameDataLocation.Hardware)
diff --git a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs
index 85988c886..bc9de633b 100644
--- a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs
+++ b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs
@@ -240,7 +240,7 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
             if (!videoStream.ColorParams.IsBt709)
             {
                 _logger.LogDebug("Adding colorspace filter");
-                var colorspace = new ColorspaceFilter(videoStream, format, currentState.FrameDataLocation);
+                var colorspace = new ColorspaceFilter(videoStream, format, false, currentState.FrameDataLocation);
                 currentState = colorspace.NextState(currentState);
                 result.Add(colorspace);
             }