pipeline fixes when colorspace filter is used (#1068)

* fix colorspace filter with missing input transfer or input primaries

* properly download before applying colorspace filter

* fix extra hwupload/hwdownload with nvidia pipeline

* colorspace tests

* update dependencies

ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj

@@ -16,7 +16,7 @@
       <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
       <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
       <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="7.0.0" />
-      <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
+      <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
       <PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.4.27">
         <PrivateAssets>all</PrivateAssets>
         <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs

@@ -35,7 +35,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg;
 public class TranscodingTests
 {
     private static readonly ILoggerFactory LoggerFactory;
-    private static readonly MemoryCache _memoryCache;
+    private static readonly MemoryCache MemoryCache;
 
     static TranscodingTests()
     {
@@ -46,7 +46,7 @@ public class TranscodingTests
 
         LoggerFactory = new LoggerFactory().AddSerilog(Log.Logger);
 
-        _memoryCache = new MemoryCache(new MemoryCacheOptions());
+        MemoryCache = new MemoryCache(new MemoryCacheOptions());
     }
 
     [Test]
@@ -61,7 +61,13 @@ public class TranscodingTests
         Assert.Pass();
     }
 
-    public record InputFormat(string Encoder, string PixelFormat);
+    public record InputFormat(
+        string Encoder,
+        string PixelFormat,
+        string ColorRange = "tv",
+        string ColorSpace = "bt709",
+        string ColorTransfer = "bt709",
+        string ColorPrimaries = "bt709");
 
     public enum Padding
     {
@@ -93,45 +99,47 @@ public class TranscodingTests
     {
         public static Watermark[] Watermarks =
         {
-            Watermark.None,
+            // Watermark.None,
             Watermark.PermanentOpaqueScaled,
-            Watermark.PermanentOpaqueActualSize,
+            // Watermark.PermanentOpaqueActualSize,
-            Watermark.PermanentTransparentScaled,
+            // Watermark.PermanentTransparentScaled,
-            Watermark.PermanentTransparentActualSize
+            // Watermark.PermanentTransparentActualSize
         };
 
         public static Subtitle[] Subtitles =
         {
-            Subtitle.None,
+            // Subtitle.None,
-            Subtitle.Picture,
+            // Subtitle.Picture,
             Subtitle.Text
         };
 
         public static Padding[] Paddings =
         {
-            Padding.NoPadding,
+            // Padding.NoPadding,
             Padding.WithPadding
         };
 
         public static VideoScanKind[] VideoScanKinds =
         {
             VideoScanKind.Progressive,
-            VideoScanKind.Interlaced
+            // VideoScanKind.Interlaced
         };
 
         public static InputFormat[] InputFormats =
         {
-            new("libx264", "yuv420p"),
+            // example format that requires colorspace filter
+            new("libx264", "yuv420p", "tv", "smpte170m", "bt709", "smpte170m"),
+
             // new("libx264", "yuvj420p"),
-            new("libx264", "yuv420p10le"),
+            // new("libx264", "yuv420p10le"),
             // new("libx264", "yuv444p10le"),
 
             // new("mpeg1video", "yuv420p"),
             //
             // new("mpeg2video", "yuv420p"),
 
-            new("libx265", "yuv420p"),
+            // new("libx265", "yuv420p"),
-            new("libx265", "yuv420p10le"),
+            // new("libx265", "yuv420p10le"),
 
             // new("mpeg4", "yuv420p"),
             //
@@ -148,14 +156,14 @@ public class TranscodingTests
 
         public static Resolution[] Resolutions =
         {
-            new() { Width = 1920, Height = 1080 },
+            // new() { Width = 1920, Height = 1080 },
             new() { Width = 1280, Height = 720 }
         };
 
         public static FFmpegProfileBitDepth[] BitDepths =
         {
             FFmpegProfileBitDepth.EightBit,
-            FFmpegProfileBitDepth.TenBit
+            // FFmpegProfileBitDepth.TenBit
         };
 
         public static HardwareAccelerationKind[] NoAcceleration =
@@ -166,7 +174,7 @@ public class TranscodingTests
         public static FFmpegProfileVideoFormat[] VideoFormats =
         {
             FFmpegProfileVideoFormat.H264,
-            FFmpegProfileVideoFormat.Hevc
+            // FFmpegProfileVideoFormat.Hevc
         };
 
         public static HardwareAccelerationKind[] NvidiaAcceleration =
@@ -215,8 +223,8 @@ public class TranscodingTests
             [ValueSource(typeof(TestData), nameof(TestData.VideoFormats))]
             FFmpegProfileVideoFormat profileVideoFormat,
             // [ValueSource(typeof(TestData), nameof(TestData.NoAcceleration))] HardwareAccelerationKind profileAcceleration)
-            // [ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration)
+            [ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration)
-        [ValueSource(typeof(TestData), nameof(TestData.VaapiAcceleration))] HardwareAccelerationKind profileAcceleration)
+        // [ValueSource(typeof(TestData), nameof(TestData.VaapiAcceleration))] HardwareAccelerationKind profileAcceleration)
         // [ValueSource(typeof(TestData), nameof(TestData.QsvAcceleration))] HardwareAccelerationKind profileAcceleration)
         // [ValueSource(typeof(TestData), nameof(TestData.VideoToolboxAcceleration))] HardwareAccelerationKind profileAcceleration)
         // [ValueSource(typeof(TestData), nameof(TestData.AmfAcceleration))] HardwareAccelerationKind profileAcceleration)
@@ -230,7 +238,7 @@ public class TranscodingTests
             }
         }
 
-        string name = GetStringSha256Hash($"{inputFormat.Encoder}_{inputFormat.PixelFormat}_{videoScanKind}_{padding}_{subtitle}");
+        string name = GetStringSha256Hash($"{inputFormat}_{videoScanKind}_{padding}_{subtitle}");
 
         string file = Path.Combine(TestContext.CurrentContext.TestDirectory, $"{name}.mkv");
         if (!File.Exists(file))
@@ -242,8 +250,24 @@ public class TranscodingTests
                 : string.Empty;
             string flags = videoScanKind == VideoScanKind.Interlaced ? "-field_order tt -flags +ildct+ilme" : string.Empty;
 
+            string colorRange = !string.IsNullOrWhiteSpace(inputFormat.ColorRange)
+                ? $" -color_range {inputFormat.ColorRange}"
+                : string.Empty;
+
+            string colorSpace = !string.IsNullOrWhiteSpace(inputFormat.ColorSpace)
+                ? $" -colorspace {inputFormat.ColorSpace}"
+                : string.Empty;
+
+            string colorTransfer = !string.IsNullOrWhiteSpace(inputFormat.ColorTransfer)
+                ? $" -color_trc {inputFormat.ColorTransfer}"
+                : string.Empty;
+
+            string colorPrimaries = !string.IsNullOrWhiteSpace(inputFormat.ColorPrimaries)
+                ? $" -color_primaries {inputFormat.ColorPrimaries}"
+                : string.Empty;
+            
             string args =
-                $"-y -f lavfi -i anoisesrc=color=brown -f lavfi -i testsrc=duration=1:size={resolution}:rate=30 {videoFilter} -c:a aac -c:v {inputFormat.Encoder} -shortest -pix_fmt {inputFormat.PixelFormat} -strict -2 {flags} {file}";
+                $"-y -f lavfi -i anoisesrc=color=brown -f lavfi -i testsrc=duration=1:size={resolution}:rate=30 {videoFilter} -c:a aac -c:v {inputFormat.Encoder}{colorRange}{colorSpace}{colorTransfer}{colorPrimaries} -shortest -pix_fmt {inputFormat.PixelFormat} -strict -2 {flags} {file}";
             var p1 = new Process
             {
                 StartInfo = new ProcessStartInfo
@@ -321,7 +345,7 @@ public class TranscodingTests
             imageCache.Object,
             new Mock<ITempFilePool>().Object,
             new Mock<IClient>().Object,
-            _memoryCache,
+            MemoryCache,
             LoggerFactory.CreateLogger<FFmpegProcessService>());
 
         var service = new FFmpegLibraryProcessService(
@@ -333,7 +357,7 @@ public class TranscodingTests
                 new RuntimeInfo(),
                 //new FakeNvidiaCapabilitiesFactory(),
                 new HardwareCapabilitiesFactory(
-                    _memoryCache,
+                    MemoryCache,
                     LoggerFactory.CreateLogger<HardwareCapabilitiesFactory>()),
                 LoggerFactory.CreateLogger<PipelineBuilderFactory>()),
             LoggerFactory.CreateLogger<FFmpegLibraryProcessService>());

ErsatzTV.Core/ErsatzTV.Core.csproj

@@ -9,7 +9,7 @@
     <ItemGroup>
       <PackageReference Include="Bugsnag" Version="3.1.0" />
       <PackageReference Include="Destructurama.Attributed" Version="3.0.0" />
-      <PackageReference Include="Flurl" Version="3.0.6" />
+      <PackageReference Include="Flurl" Version="3.0.7" />
       <PackageReference Include="LanguageExt.Core" Version="4.4.0" />
       <PackageReference Include="LanguageExt.Transformers" Version="4.4.0" />
       <PackageReference Include="MediatR" Version="11.1.0" />

ErsatzTV.FFmpeg.Tests/ErsatzTV.FFmpeg.Tests.csproj

@@ -9,7 +9,7 @@
 
     <ItemGroup>
         <PackageReference Include="FluentAssertions" Version="6.8.0" />
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
         <PackageReference Include="Moq" Version="4.18.3" />
         <PackageReference Include="NUnit" Version="3.13.3" />
         <PackageReference Include="NUnit3TestAdapter" Version="4.3.1" />

ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs

@@ -4,17 +4,20 @@ namespace ErsatzTV.FFmpeg.Filter;
 
 public class ColorspaceFilter : BaseFilter
 {
+    private readonly FrameState _currentState;
     private readonly VideoStream _videoStream;
     private readonly IPixelFormat _desiredPixelFormat;
     private readonly bool _forceInputOverrides;
     private readonly FrameDataLocation _nextDataLocation;
 
     public ColorspaceFilter(
+        FrameState currentState,
         VideoStream videoStream,
         IPixelFormat desiredPixelFormat,
         bool forceInputOverrides = false,
         FrameDataLocation nextDataLocation = FrameDataLocation.Software)
     {
+        _currentState = currentState;
         _videoStream = videoStream;
         _desiredPixelFormat = desiredPixelFormat;
         _forceInputOverrides = forceInputOverrides;
@@ -37,21 +40,33 @@ public class ColorspaceFilter : BaseFilter
     {
         get
         {
+            string hwdownload = string.Empty;
+            if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)
+            {
+                hwdownload = "hwdownload";
+            }
+
             string inputOverrides = string.Empty;
             ColorParams cp = _videoStream.ColorParams;
             if (cp.IsMixed || _forceInputOverrides)
             {
                 string range = string.IsNullOrWhiteSpace(cp.ColorRange) ? "tv" : cp.ColorRange;
+                string transfer = string.IsNullOrWhiteSpace(cp.ColorTransfer)
+                    ? "bt709"
+                    : cp.ColorTransfer;
+                string primaries = string.IsNullOrWhiteSpace(cp.ColorPrimaries)
+                    ? "bt709"
+                    : cp.ColorPrimaries;
 
                 inputOverrides =
-                    $"irange={range}:ispace={cp.ColorSpace}:itrc={cp.ColorTransfer}:iprimaries={cp.ColorPrimaries}:";
+                    $"irange={range}:ispace={cp.ColorSpace}:itrc={transfer}:iprimaries={primaries}:";
             }
 
             string colorspace = _desiredPixelFormat.BitDepth switch
             {
                 _ when cp.IsUnknown => "setparams=range=tv:colorspace=bt709:color_trc=bt709:color_primaries=bt709",
                 10 or 8 when !cp.IsUnknown =>
-                    $"colorspace={inputOverrides}all=bt709:format={_desiredPixelFormat.FFmpegName}",
+                    $"{hwdownload},colorspace={inputOverrides}all=bt709:format={_desiredPixelFormat.FFmpegName}",
                 _ => string.Empty
             };
 

ErsatzTV.FFmpeg/Filter/ComplexFilter.cs

@@ -317,7 +317,7 @@ public class ComplexFilter : IPipelineStep
             if (!videoStream.ColorParams.IsBt709)
             {
                 _logger.LogDebug("Adding colorspace filter");
-                var colorspace = new ColorspaceFilter(videoStream, pixelFormat);
+                var colorspace = new ColorspaceFilter(_currentState, videoStream, pixelFormat);
                 _pipelineSteps.Add(colorspace);
                 filter = colorspace.Filter;
             }

ErsatzTV.FFmpeg/Pipeline/AmfPipelineBuilder.cs

@@ -93,7 +93,7 @@ public class AmfPipelineBuilder : SoftwarePipelineBuilder
             if (!videoStream.ColorParams.IsBt709)
             {
                 _logger.LogDebug("Adding colorspace filter");
-                var colorspace = new ColorspaceFilter(videoStream, pixelFormat);
+                var colorspace = new ColorspaceFilter(currentState, videoStream, pixelFormat);
                 currentState = colorspace.NextState(currentState);
                 result.Add(colorspace);
             }

ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs

@@ -181,7 +181,8 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
 
         // need to upload for any sort of overlay
         if (currentState.FrameDataLocation == FrameDataLocation.Software &&
-            currentState.BitDepth == 8 && (context.HasSubtitleOverlay || context.HasWatermark))
+            currentState.BitDepth == 8 && context.HasSubtitleText == false
+            && (context.HasSubtitleOverlay || context.HasWatermark))
         {
             var hardwareUpload = new HardwareUploadCudaFilter(currentState);
             currentState = hardwareUpload.NextState(currentState);
@@ -268,7 +269,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
             if (!videoStream.ColorParams.IsBt709)
             {
                 _logger.LogDebug("Adding colorspace filter");
-                var colorspace = new ColorspaceFilter(videoStream, format, false, currentState.FrameDataLocation);
+                var colorspace = new ColorspaceFilter(currentState, videoStream, format, false, currentState.FrameDataLocation);
 
                 currentState = colorspace.NextState(currentState);
                 result.Add(colorspace);

ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs

@@ -246,6 +246,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
                 bool forceInputOverrides = videoInputFile.FilterSteps.Any(f => f is QsvFormatFilter or ScaleQsvFilter);
 
                 var colorspace = new ColorspaceFilter(
+                    currentState,
                     videoStream,
                     format,
                     forceInputOverrides,

ErsatzTV.FFmpeg/Pipeline/SoftwarePipelineBuilder.cs

@@ -140,7 +140,7 @@ public class SoftwarePipelineBuilder : PipelineBuilderBase
             if (!videoStream.ColorParams.IsBt709)
             {
                 _logger.LogDebug("Adding colorspace filter");
-                var colorspace = new ColorspaceFilter(videoStream, pixelFormat);
+                var colorspace = new ColorspaceFilter(currentState, videoStream, pixelFormat);
                 currentState = colorspace.NextState(currentState);
                 result.Add(colorspace);
             }

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -246,7 +246,12 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
             if (!videoStream.ColorParams.IsBt709)
             {
                 _logger.LogDebug("Adding colorspace filter");
-                var colorspace = new ColorspaceFilter(videoStream, format, false, currentState.FrameDataLocation);
+                var colorspace = new ColorspaceFilter(
+                    currentState,
+                    videoStream,
+                    format,
+                    false,
+                    currentState.FrameDataLocation);
                 currentState = colorspace.NextState(currentState);
                 result.Add(colorspace);
             }

ErsatzTV.FFmpeg/Pipeline/VideoToolboxPipelineBuilder.cs

@@ -114,7 +114,7 @@ public class VideoToolboxPipelineBuilder : SoftwarePipelineBuilder
             if (!videoStream.ColorParams.IsBt709)
             {
                 _logger.LogDebug("Adding colorspace filter");
-                var colorspace = new ColorspaceFilter(videoStream, pixelFormat);
+                var colorspace = new ColorspaceFilter(currentState, videoStream, pixelFormat);
                 currentState = colorspace.NextState(currentState);
                 result.Add(colorspace);
             }

ErsatzTV.Infrastructure.Tests/ErsatzTV.Infrastructure.Tests.csproj

@@ -10,7 +10,7 @@
 
     <ItemGroup>
         <PackageReference Include="FluentAssertions" Version="6.8.0" />
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
         <PackageReference Include="Moq" Version="4.18.3" />
         <PackageReference Include="NUnit" Version="3.13.3" />
         <PackageReference Include="NUnit3TestAdapter" Version="4.3.1" />

ErsatzTV/ErsatzTV.csproj

@@ -70,7 +70,7 @@
         <PrivateAssets>all</PrivateAssets>
         <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       </PackageReference>
-      <PackageReference Include="MudBlazor" Version="6.1.5" />
+      <PackageReference Include="MudBlazor" Version="6.1.6" />
       <PackageReference Include="NaturalSort.Extension" Version="4.0.0" />
       <PackageReference Include="PPioli.FluentValidation.Blazor" Version="11.1.0" />
       <PackageReference Include="Refit.HttpClientFactory" Version="6.3.2" />