colorspace fixes; song playback fixes (#1072)

* fix colorspace bug, vaapi song playback

* more colorspace fixes, nvidia fixes

* nvidia colorspace fixes

* fix some qsv output color metadata

* update changelog

* update changelog

CHANGELOG.md

@@ -5,7 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 ### Fixed
-- Fix some transcoding failures caused by the colorspace filter  
+- Fix many transcoding failures caused by the colorspace filter
+- Fix song playback with VAAPI
 
 ### Changed
 - Upgrade to dotnet 7

ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs

@@ -99,30 +99,30 @@ public class TranscodingTests
     {
         public static Watermark[] Watermarks =
         {
-            // Watermark.None,
+            Watermark.None,
             Watermark.PermanentOpaqueScaled,
             // Watermark.PermanentOpaqueActualSize,
-            // Watermark.PermanentTransparentScaled,
+            Watermark.PermanentTransparentScaled,
             // Watermark.PermanentTransparentActualSize
         };
 
         public static Subtitle[] Subtitles =
         {
-            // Subtitle.None,
+            Subtitle.None,
-            // Subtitle.Picture,
+            Subtitle.Picture,
             Subtitle.Text
         };
 
         public static Padding[] Paddings =
         {
-            // Padding.NoPadding,
+            Padding.NoPadding,
             Padding.WithPadding
         };
 
         public static VideoScanKind[] VideoScanKinds =
         {
             VideoScanKind.Progressive,
-            // VideoScanKind.Interlaced
+            VideoScanKind.Interlaced
         };
 
         public static InputFormat[] InputFormats =
@@ -131,15 +131,15 @@ public class TranscodingTests
             new("libx264", "yuv420p", "tv", "smpte170m", "bt709", "smpte170m"),
 
             // new("libx264", "yuvj420p"),
-            // new("libx264", "yuv420p10le"),
+            new("libx264", "yuv420p10le"),
             // new("libx264", "yuv444p10le"),
 
             // new("mpeg1video", "yuv420p"),
             //
             // new("mpeg2video", "yuv420p"),
 
-            // new("libx265", "yuv420p"),
+            new("libx265", "yuv420p"),
-            // new("libx265", "yuv420p10le"),
+            new("libx265", "yuv420p10le"),
 
             // new("mpeg4", "yuv420p"),
             //
@@ -156,14 +156,14 @@ public class TranscodingTests
 
         public static Resolution[] Resolutions =
         {
-            // new() { Width = 1920, Height = 1080 },
+            new() { Width = 1920, Height = 1080 },
             new() { Width = 1280, Height = 720 }
         };
 
         public static FFmpegProfileBitDepth[] BitDepths =
         {
             FFmpegProfileBitDepth.EightBit,
-            // FFmpegProfileBitDepth.TenBit
+            FFmpegProfileBitDepth.TenBit
         };
 
         public static HardwareAccelerationKind[] NoAcceleration =
@@ -201,6 +201,14 @@ public class TranscodingTests
         {
             HardwareAccelerationKind.Qsv
         };
+
+        public static HardwareAccelerationKind[] LinuxTestAccelerations =
+        {
+            HardwareAccelerationKind.None,
+            HardwareAccelerationKind.Nvenc,
+            HardwareAccelerationKind.Vaapi,
+            HardwareAccelerationKind.Qsv
+        };
     }
 
     [Test]
@@ -222,8 +230,9 @@ public class TranscodingTests
             Subtitle subtitle,
             [ValueSource(typeof(TestData), nameof(TestData.VideoFormats))]
             FFmpegProfileVideoFormat profileVideoFormat,
+            [ValueSource(typeof(TestData), nameof(TestData.LinuxTestAccelerations))] HardwareAccelerationKind profileAcceleration)
             // [ValueSource(typeof(TestData), nameof(TestData.NoAcceleration))] HardwareAccelerationKind profileAcceleration)
-            [ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration)
+            // [ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration)
         // [ValueSource(typeof(TestData), nameof(TestData.VaapiAcceleration))] HardwareAccelerationKind profileAcceleration)
         // [ValueSource(typeof(TestData), nameof(TestData.QsvAcceleration))] HardwareAccelerationKind profileAcceleration)
         // [ValueSource(typeof(TestData), nameof(TestData.VideoToolboxAcceleration))] HardwareAccelerationKind profileAcceleration)

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -218,7 +218,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             playbackSettings.RealtimeOutput,
             false, // TODO: fallback filler needs to loop
             videoFormat,
-            videoStream.Profile,
+            Optional(videoStream.Profile),
             Optional(desiredPixelFormat),
             ffmpegVideoStream.SquarePixelFrameSize(
                 new FrameSize(channel.FFmpegProfile.Resolution.Width, channel.FFmpegProfile.Resolution.Height)),

ErsatzTV.FFmpeg/Capabilities/AmfHardwareCapabilities.cs

@@ -4,9 +4,9 @@ namespace ErsatzTV.FFmpeg.Capabilities;
 
 public class AmfHardwareCapabilities : IHardwareCapabilities
 {
-    public bool CanDecode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat) => false;
+    public bool CanDecode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat) => false;
 
-    public bool CanEncode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat)
+    public bool CanEncode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 

ErsatzTV.FFmpeg/Capabilities/DefaultHardwareCapabilities.cs

@@ -4,9 +4,9 @@ namespace ErsatzTV.FFmpeg.Capabilities;
 
 public class DefaultHardwareCapabilities : IHardwareCapabilities
 {
-    public bool CanDecode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat) => true;
+    public bool CanDecode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat) => true;
 
-    public bool CanEncode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat)
+    public bool CanEncode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 

ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs

@@ -103,7 +103,7 @@ public class HardwareCapabilitiesFactory : IHardwareCapabilitiesFactory
 
             if (profileEntrypoints.Any())
             {
-                _logger.LogWarning(
+                _logger.LogInformation(
                     "Detected {Count} VAAPI profile entrypoints for using {Driver} {Device}",
                     profileEntrypoints.Count,
                     driver,

ErsatzTV.FFmpeg/Capabilities/IHardwareCapabilities.cs

@@ -4,6 +4,6 @@ namespace ErsatzTV.FFmpeg.Capabilities;
 
 public interface IHardwareCapabilities
 {
-    public bool CanDecode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat);
+    public bool CanDecode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat);
-    public bool CanEncode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat);
+    public bool CanEncode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat);
 }

ErsatzTV.FFmpeg/Capabilities/NoHardwareCapabilities.cs

@@ -4,6 +4,6 @@ namespace ErsatzTV.FFmpeg.Capabilities;
 
 public class NoHardwareCapabilities : IHardwareCapabilities
 {
-    public bool CanDecode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat) => false;
+    public bool CanDecode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat) => false;
-    public bool CanEncode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat) => false;
+    public bool CanEncode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat) => false;
 }

ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs

@@ -14,7 +14,7 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities
         _model = model;
     }
 
-    public bool CanDecode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat)
+    public bool CanDecode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 
@@ -36,7 +36,7 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities
         };
     }
 
-    public bool CanEncode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat)
+    public bool CanEncode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 

ErsatzTV.FFmpeg/Capabilities/VaapiHardwareCapabilities.cs

@@ -15,11 +15,11 @@ public class VaapiHardwareCapabilities : IHardwareCapabilities
         _logger = logger;
     }
 
-    public bool CanDecode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat)
+    public bool CanDecode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 
-        bool result = (videoFormat, videoProfile.ToLowerInvariant()) switch
+        bool result = (videoFormat, videoProfile.IfNone(string.Empty).ToLowerInvariant()) switch
         {
             // no hardware decoding of 10-bit h264
             (VideoFormat.H264, _) when bitDepth == 10 => false,
@@ -86,7 +86,7 @@ public class VaapiHardwareCapabilities : IHardwareCapabilities
         return result;
     }
 
-    public bool CanEncode(string videoFormat, string videoProfile, Option<IPixelFormat> maybePixelFormat)
+    public bool CanEncode(string videoFormat, Option<string> videoProfile, Option<IPixelFormat> maybePixelFormat)
     {
         int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8);
 

ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs

@@ -8,29 +8,30 @@ public class ColorspaceFilter : BaseFilter
     private readonly VideoStream _videoStream;
     private readonly IPixelFormat _desiredPixelFormat;
     private readonly bool _forceInputOverrides;
-    private readonly FrameDataLocation _nextDataLocation;
 
     public ColorspaceFilter(
         FrameState currentState,
         VideoStream videoStream,
         IPixelFormat desiredPixelFormat,
-        bool forceInputOverrides = false,
+        bool forceInputOverrides = false)
-        FrameDataLocation nextDataLocation = FrameDataLocation.Software)
     {
         _currentState = currentState;
         _videoStream = videoStream;
         _desiredPixelFormat = desiredPixelFormat;
         _forceInputOverrides = forceInputOverrides;
-        _nextDataLocation = nextDataLocation;
     }
 
     public override FrameState NextState(FrameState currentState)
     {
-        FrameState nextState = currentState with { FrameDataLocation = _nextDataLocation };
+        FrameState nextState = currentState;
 
         if (!_videoStream.ColorParams.IsUnknown && _desiredPixelFormat.BitDepth is 10 or 8)
         {
-            nextState = nextState with { PixelFormat = Some(_desiredPixelFormat) };
+            nextState = nextState with
+            {
+                FrameDataLocation = FrameDataLocation.Software,
+                PixelFormat = Some(_desiredPixelFormat)
+            };
         }
 
         return nextState;
@@ -43,7 +44,14 @@ public class ColorspaceFilter : BaseFilter
             string hwdownload = string.Empty;
             if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)
             {
-                hwdownload = "hwdownload";
+                hwdownload = "hwdownload,";
+                foreach (IPixelFormat pixelFormat in _currentState.PixelFormat)
+                {
+                    if (!string.IsNullOrWhiteSpace(pixelFormat.FFmpegName))
+                    {
+                        hwdownload = $"hwdownload,format={pixelFormat.FFmpegName},";
+                    }
+                }
             }
 
             string inputOverrides = string.Empty;
@@ -65,8 +73,10 @@ public class ColorspaceFilter : BaseFilter
             string colorspace = _desiredPixelFormat.BitDepth switch
             {
                 _ when cp.IsUnknown => "setparams=range=tv:colorspace=bt709:color_trc=bt709:color_primaries=bt709",
-                10 or 8 when !cp.IsUnknown =>
+                10 when !cp.IsUnknown =>
-                    $"{hwdownload},colorspace={inputOverrides}all=bt709:format={_desiredPixelFormat.FFmpegName}",
+                    $"{hwdownload}colorspace={inputOverrides}all=bt709:format=yuv420p10",
+                8 when !cp.IsUnknown =>
+                    $"{hwdownload}colorspace={inputOverrides}all=bt709:format=yuv420p",
                 _ => string.Empty
             };
 

ErsatzTV.FFmpeg/FrameState.cs

@@ -6,7 +6,7 @@ public record FrameState(
     bool Realtime,
     bool InfiniteLoop,
     string VideoFormat,
-    string VideoProfile,
+    Option<string> VideoProfile,
     Option<IPixelFormat> PixelFormat,
     FrameSize ScaledSize,
     FrameSize PaddedSize,

ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs

@@ -269,7 +269,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
             if (!videoStream.ColorParams.IsBt709)
             {
                 _logger.LogDebug("Adding colorspace filter");
-                var colorspace = new ColorspaceFilter(currentState, videoStream, format, false, currentState.FrameDataLocation);
+                var colorspace = new ColorspaceFilter(currentState, videoStream, format, false);
 
                 currentState = colorspace.NextState(currentState);
                 result.Add(colorspace);
@@ -300,7 +300,20 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
 
                 if (currentState.FrameDataLocation == FrameDataLocation.Hardware)
                 {
-                    result.Add(new CudaFormatFilter(format));
+                    bool noPipelineFilters = !pipelineSteps.OfType<IPipelineFilterStep>().Any();
+                    bool softwareColorspace = result is [ColorspaceFilter colorspace] &&
+                                              !colorspace.Filter.StartsWith("setparams=");
+
+                    bool softwareDecoder = ffmpegState.DecoderHardwareAccelerationMode == HardwareAccelerationMode.None;
+                    
+                    if (softwareDecoder || (noPipelineFilters && softwareColorspace))
+                    {
+                        result.Add(new CudaFormatFilter(format));
+                    }
+                    else
+                    {
+                        pipelineSteps.Add(new PixelFormatOutputOption(format));
+                    }
                 }
                 else
                 {

ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs

@@ -237,20 +237,18 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
             }
 
             IPixelFormat formatForDownload = pixelFormat;
-            
+
-            if (!videoStream.ColorParams.IsBt709)
+            bool usesVppQsv = videoInputFile.FilterSteps.Any(f => f is QsvFormatFilter or ScaleQsvFilter);
+            if (!videoStream.ColorParams.IsBt709 || usesVppQsv)
             {
                 _logger.LogDebug("Adding colorspace filter");
 
                 // vpp_qsv seems to strip color info, so if we use that at all, force overriding input color info
-                bool forceInputOverrides = videoInputFile.FilterSteps.Any(f => f is QsvFormatFilter or ScaleQsvFilter);
-
                 var colorspace = new ColorspaceFilter(
                     currentState,
                     videoStream,
                     format,
-                    forceInputOverrides,
+                    forceInputOverrides: usesVppQsv);
-                    currentState.FrameDataLocation);
 
                 // force nv12 if we're still in hardware
                 if (currentState.FrameDataLocation == FrameDataLocation.Hardware)
@@ -289,7 +287,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
 
                 // remind qsv that it uses qsv
                 if (currentState.FrameDataLocation == FrameDataLocation.Hardware &&
-                    result.Count == 1 && result[0] is ColorspaceFilter colorspace)
+                    result is [ColorspaceFilter colorspace])
                 {
                     if (colorspace.Filter.StartsWith("setparams="))
                     {

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -250,8 +250,7 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
                     currentState,
                     videoStream,
                     format,
-                    false,
+                    false);
-                    currentState.FrameDataLocation);
                 currentState = colorspace.NextState(currentState);
                 result.Add(colorspace);
             }