nvidia 10 bit fixes (#413)

ErsatzTV.Core/FFmpeg/FFmpegComplexFilter.cs

@@ -1,4 +1,4 @@
 namespace ErsatzTV.Core.FFmpeg
 {
-    public record FFmpegComplexFilter(string ComplexFilter, string VideoLabel, string AudioLabel);
+    public record FFmpegComplexFilter(string ComplexFilter, string VideoLabel, string AudioLabel, string PixelFormat);
 }

ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs

@@ -234,6 +234,8 @@ namespace ErsatzTV.Core.FFmpeg
 
             _padToSize.IfSome(size => videoFilterQueue.Add($"pad={size.Width}:{size.Height}:(ow-iw)/2:(oh-ih)/2"));
 
+            string outputPixelFormat = null;
+            
             if (usesSoftwareFilters && acceleration != HardwareAccelerationKind.None &&
                 string.IsNullOrWhiteSpace(watermarkOverlay))
             {
@@ -245,6 +247,19 @@ namespace ErsatzTV.Core.FFmpeg
                 videoFilterQueue.Add(upload);
             }
 
+            if (!usesSoftwareFilters && string.IsNullOrWhiteSpace(watermarkOverlay))
+            {
+                switch (acceleration, _videoEncoder, _pixelFormat)
+                {
+                    case (HardwareAccelerationKind.Nvenc, "h264_nvenc", "yuv420p10le"):
+                        outputPixelFormat = "yuv420p";
+                        break;
+                    case (HardwareAccelerationKind.Nvenc, "h264_nvenc", "yuv444p10le"):
+                        outputPixelFormat = "yuv444p";
+                        break;
+                }
+            }
+
             bool hasAudioFilters = audioFilterQueue.Any();
             if (hasAudioFilters)
             {
@@ -307,7 +322,7 @@ namespace ErsatzTV.Core.FFmpeg
             var filterResult = complexFilter.ToString();
             return string.IsNullOrWhiteSpace(filterResult)
                 ? Option<FFmpegComplexFilter>.None
-                : new FFmpegComplexFilter(filterResult, videoLabel, audioLabel);
+                : new FFmpegComplexFilter(filterResult, videoLabel, audioLabel, outputPixelFormat);
         }
     }
 }

ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs

@@ -47,6 +47,7 @@ namespace ErsatzTV.Core.FFmpeg
         private bool _isConcat;
         private VaapiDriver _vaapiDriver;
         private HardwareAccelerationKind _hwAccel;
+        private string _outputPixelFormat;
 
         public FFmpegProcessBuilder(string ffmpegPath, bool saveReports, ILogger logger)
         {
@@ -72,7 +73,7 @@ namespace ErsatzTV.Core.FFmpeg
             return this;
         }
 
-        public FFmpegProcessBuilder WithHardwareAcceleration(HardwareAccelerationKind hwAccel, string pixelFormat)
+        public FFmpegProcessBuilder WithHardwareAcceleration(HardwareAccelerationKind hwAccel, string pixelFormat, string encoder)
         {
             _hwAccel = hwAccel;
 
@@ -85,10 +86,11 @@ namespace ErsatzTV.Core.FFmpeg
                     _arguments.Add("qsv=qsv:MFX_IMPL_hw_any");
                     break;
                 case HardwareAccelerationKind.Nvenc:
-                    string outputFormat = pixelFormat switch
+                    string outputFormat = (encoder, pixelFormat) switch
                     {
-                        "yuv420p10le" => "p010le",
-                        // "yuv444p10le" => "p016le",
+                        ("hevc_nvenc", "yuv420p10le") => "p010le",
+                        ("h264_nvenc", "yuv420p10le") => "p010le",
+                        // ("hevc_nvenc", "yuv444p10le") => "p016le",
                         _ => "cuda"
                     };
                     
@@ -371,6 +373,11 @@ namespace ErsatzTV.Core.FFmpeg
                 "-sc_threshold", "0" // disable scene change detection
             };
 
+            if (!string.IsNullOrWhiteSpace(_outputPixelFormat))
+            {
+                arguments.AddRange(new[] { "-pix_fmt", _outputPixelFormat });
+            }
+
             string[] videoBitrateArgs = playbackSettings.VideoBitrate.Match(
                 bitrate =>
                     new[]
@@ -478,6 +485,11 @@ namespace ErsatzTV.Core.FFmpeg
                     _arguments.Add(filter.ComplexFilter);
                     videoLabel = filter.VideoLabel;
                     audioLabel = filter.AudioLabel;
+
+                    if (!string.IsNullOrWhiteSpace(filter.PixelFormat))
+                    {
+                        _outputPixelFormat = filter.PixelFormat;
+                    }
                 });
 
             _arguments.Add("-map");

ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs

@@ -64,7 +64,7 @@ namespace ErsatzTV.Core.FFmpeg
 
             FFmpegProcessBuilder builder = new FFmpegProcessBuilder(ffmpegPath, saveReports, _logger)
                 .WithThreads(playbackSettings.ThreadCount)
-                .WithHardwareAcceleration(playbackSettings.HardwareAcceleration, videoStream.PixelFormat)
+                .WithHardwareAcceleration(playbackSettings.HardwareAcceleration, videoStream.PixelFormat, playbackSettings.VideoCodec)
                 .WithVaapiDriver(maybeVaapiDriver)
                 .WithQuiet()
                 .WithFormatFlags(playbackSettings.FormatFlags)