more nvidia 10-bit fixes (#2426)

* fix playback with invalid ffmpeg profile

* fix 10 bit output with nvidia and graphics engine

CHANGELOG.md

@@ -23,6 +23,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Fixed
 - Fix green output when libplacebo tonemapping is used with NVIDIA acceleration and 10-bit output in FFmpeg Profile
+- Fix playback when invalid video preset has been saved in FFmpegProfile
+  - This can happen when NVIDIA accel falls back to libx264 software encoder for 10-bit h264 output
+- Fix 10-bit output when using NVIDIA and graphics engine (watermark or other overlays)
 
 ## [25.6.0] - 2025-09-14
 ### Added

ErsatzTV.Application/FFmpegProfiles/Commands/UpdateFFmpegProfileHandler.cs

@@ -1,6 +1,10 @@
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.FFmpeg;
 using ErsatzTV.Core.Interfaces.Search;
+using ErsatzTV.FFmpeg;
+using ErsatzTV.FFmpeg.Format;
+using ErsatzTV.FFmpeg.Preset;
 using ErsatzTV.Infrastructure.Data;
 using ErsatzTV.Infrastructure.Extensions;
 using Microsoft.EntityFrameworkCore;
@@ -64,6 +68,18 @@ public class
         p.AudioSampleRate = update.AudioSampleRate;
         p.NormalizeFramerate = update.NormalizeFramerate;
         p.DeinterlaceVideo = update.DeinterlaceVideo;
+
+        // don't save invalid preset
+        ICollection<string> presets = FFmpegLibraryHelper.PresetsForFFmpegProfile(
+            p.HardwareAcceleration,
+            p.VideoFormat,
+            p.BitDepth);
+
+        if (!presets.Contains(p.VideoPreset))
+        {
+            p.VideoPreset = VideoPreset.Unset;
+        }
+
         await dbContext.SaveChangesAsync(cancellationToken);
 
         _searchTargets.SearchTargetsChanged();

ErsatzTV.Core/FFmpeg/FFmpegLibraryHelper.cs

@@ -0,0 +1,45 @@
+using ErsatzTV.Core.Domain;
+using ErsatzTV.FFmpeg;
+using ErsatzTV.FFmpeg.Format;
+using ErsatzTV.FFmpeg.Preset;
+
+namespace ErsatzTV.Core.FFmpeg;
+
+public static class FFmpegLibraryHelper
+{
+    public static ICollection<string> PresetsForFFmpegProfile(
+        HardwareAccelerationKind hardwareAccelerationKind,
+        FFmpegProfileVideoFormat videoFormat,
+        FFmpegProfileBitDepth bitDepth) => AvailablePresets.ForAccelAndFormat(
+        MapAccel(hardwareAccelerationKind),
+        MapVideoFormat(videoFormat),
+        MapBitDepth(bitDepth));
+
+    public static HardwareAccelerationMode MapAccel(HardwareAccelerationKind kind) =>
+        kind switch
+        {
+            HardwareAccelerationKind.Amf => HardwareAccelerationMode.Amf,
+            HardwareAccelerationKind.Nvenc => HardwareAccelerationMode.Nvenc,
+            HardwareAccelerationKind.Qsv => HardwareAccelerationMode.Qsv,
+            HardwareAccelerationKind.Vaapi => HardwareAccelerationMode.Vaapi,
+            HardwareAccelerationKind.VideoToolbox => HardwareAccelerationMode.VideoToolbox,
+            HardwareAccelerationKind.V4l2m2m => HardwareAccelerationMode.V4l2m2m,
+            HardwareAccelerationKind.Rkmpp => HardwareAccelerationMode.Rkmpp,
+            _ => HardwareAccelerationMode.None
+        };
+
+    public static string MapVideoFormat(FFmpegProfileVideoFormat format) =>
+        format switch
+        {
+            FFmpegProfileVideoFormat.H264 => VideoFormat.H264,
+            FFmpegProfileVideoFormat.Hevc => VideoFormat.Hevc,
+            _ => VideoFormat.Mpeg2Video
+        };
+
+    public static int MapBitDepth(FFmpegProfileBitDepth bitDepth) =>
+        bitDepth switch
+        {
+            FFmpegProfileBitDepth.EightBit => 8,
+            _ => 10
+        };
+}

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -338,7 +338,11 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
         string videoFormat = GetVideoFormat(playbackSettings);
         Option<string> maybeVideoProfile = GetVideoProfile(videoFormat, channel.FFmpegProfile.VideoProfile);
-        Option<string> maybeVideoPreset = GetVideoPreset(hwAccel, videoFormat, channel.FFmpegProfile.VideoPreset);
+        Option<string> maybeVideoPreset = GetVideoPreset(
+            hwAccel,
+            videoFormat,
+            channel.FFmpegProfile.VideoPreset,
+            FFmpegLibraryHelper.MapBitDepth(channel.FFmpegProfile.BitDepth));
 
         Option<string> hlsPlaylistPath = outputFormat == OutputFormatKind.Hls
             ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live.m3u8")
@@ -765,7 +769,11 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
         string videoFormat = GetVideoFormat(playbackSettings);
         Option<string> maybeVideoProfile = GetVideoProfile(videoFormat, channel.FFmpegProfile.VideoProfile);
-        Option<string> maybeVideoPreset = GetVideoPreset(hwAccel, videoFormat, channel.FFmpegProfile.VideoPreset);
+        Option<string> maybeVideoPreset = GetVideoPreset(
+            hwAccel,
+            videoFormat,
+            channel.FFmpegProfile.VideoPreset,
+            FFmpegLibraryHelper.MapBitDepth(channel.FFmpegProfile.BitDepth));
 
         Option<string> hlsPlaylistPath = Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live.m3u8");
 
@@ -1112,9 +1120,10 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
     private static Option<string> GetVideoPreset(
         HardwareAccelerationMode hardwareAccelerationMode,
         string videoFormat,
-        string videoPreset) =>
+        string videoPreset,
+        int bitDepth) =>
         AvailablePresets
-            .ForAccelAndFormat(hardwareAccelerationMode, videoFormat)
+            .ForAccelAndFormat(hardwareAccelerationMode, videoFormat, bitDepth)
             .Find(p => string.Equals(p, videoPreset, StringComparison.OrdinalIgnoreCase));
 
     private static HardwareAccelerationMode GetHardwareAccelerationMode(

ErsatzTV.FFmpeg/Filter/OverlayGraphicsEngineFilter.cs

@@ -4,8 +4,14 @@ namespace ErsatzTV.FFmpeg.Filter;
 
 public class OverlayGraphicsEngineFilter(IPixelFormat outputPixelFormat) : BaseFilter
 {
-    public override string Filter =>
-        $"overlay=format={(outputPixelFormat.BitDepth == 10 ? '1' : '0')}";
+    public override string Filter
+    {
+        get
+        {
+            string extraFormat = outputPixelFormat.BitDepth == 10 ? ",format=p010le" : string.Empty;
+            return $"overlay=format={(outputPixelFormat.BitDepth == 10 ? "yuv420p10" : "0")}{extraFormat}";
+        }
+    }
 
     public override FrameState NextState(FrameState currentState) =>
         currentState with { FrameDataLocation = FrameDataLocation.Software };

ErsatzTV.FFmpeg/Format/FFmpegFormat.cs

@@ -6,6 +6,7 @@ public class FFmpegFormat
     public const string YUV420P = "yuv420p";
     public const string YUV444P = "yuv444p";
     public const string YUVA420P = "yuva420p";
+    public const string YUVA420P10 = "yuva420p10";
     public const string P010LE = "p010le";
     public const string NV12 = "nv12";
     public const string NV15 = "nv15";

ErsatzTV.FFmpeg/Format/PixelFormat.cs

@@ -6,6 +6,7 @@ public static class PixelFormat
     public const string YUV420P = "yuv420p";
     public const string YUV420P10LE = "yuv420p10le";
     public const string YUVA420P = "yuva420p";
+    public const string YUVA420P10LE = "yuva420p10le";
     public const string YUVJ420P = "yuvj420p";
     public const string YUV444P = "yuv444p";
     public const string YUV444P10LE = "yuv444p10le";

ErsatzTV.FFmpeg/Format/PixelFormatYuva420P10Le.cs

@@ -0,0 +1,8 @@
+namespace ErsatzTV.FFmpeg.Format;
+
+public class PixelFormatYuva420P10Le : IPixelFormat
+{
+    public string Name => PixelFormat.YUVA420P10LE;
+    public string FFmpegName => FFmpegFormat.YUVA420P10;
+    public int BitDepth => 10;
+}

ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs

@@ -684,6 +684,8 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
                         }
                     }
 
+                    graphicsEngine.FilterSteps.Add(new PixelFormatFilter(new PixelFormatYuva420P10Le()));
+
                     var graphicsEngineFilter = new OverlayGraphicsEngineFilter(pf);
                     graphicsEngineOverlayFilterSteps.Add(graphicsEngineFilter);
                     currentState = graphicsEngineFilter.NextState(currentState);

ErsatzTV.FFmpeg/Preset/AvailablePresets.cs

@@ -6,20 +6,32 @@ public static class AvailablePresets
 {
     public static ICollection<string> ForAccelAndFormat(
         HardwareAccelerationMode hardwareAccelerationMode,
-        string videoFormat) =>
-        (hardwareAccelerationMode, videoFormat) switch
+        string videoFormat,
+        int bitDepth) =>
+        (hardwareAccelerationMode, videoFormat, bitDepth) switch
         {
-            (HardwareAccelerationMode.Nvenc, VideoFormat.H264 or VideoFormat.Hevc) =>
+            // 10-bit h264 always uses libx264
+            (_, VideoFormat.H264, 10) =>
+            [
+                VideoPreset.VeryFast
+            ],
+
+            (HardwareAccelerationMode.Nvenc, VideoFormat.H264, 8) =>
+            [
+                VideoPreset.LowLatencyHighPerformance, VideoPreset.LowLatencyHighQuality
+            ],
+
+            (HardwareAccelerationMode.Nvenc, VideoFormat.Hevc, _) =>
             [
                 VideoPreset.LowLatencyHighPerformance, VideoPreset.LowLatencyHighQuality
             ],
 
-            (HardwareAccelerationMode.Qsv, VideoFormat.H264 or VideoFormat.Hevc) =>
+            (HardwareAccelerationMode.Qsv, VideoFormat.H264 or VideoFormat.Hevc, _) =>
             [
                 VideoPreset.VeryFast
             ],
 
-            (HardwareAccelerationMode.None, VideoFormat.H264 or VideoFormat.Hevc) =>
+            (HardwareAccelerationMode.None, VideoFormat.H264 or VideoFormat.Hevc, _) =>
             [
                 VideoPreset.VeryFast
             ],

ErsatzTV.Infrastructure.MySql/Migrations/20250916141549_Add_FixH264VideoPreset.cs

@@ -0,0 +1,22 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ErsatzTV.Infrastructure.MySql.Migrations
+{
+    /// <inheritdoc />
+    public partial class Add_FixH264VideoPreset : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            // clear video preset on all h264/10 bit ffmpeg profiles
+            migrationBuilder.Sql("UPDATE `FFmpegProfile` SET `VideoPreset`='' WHERE `VideoFormat` = 1 AND `BitDepth` = 1");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+        }
+    }
+}

ErsatzTV.Infrastructure.Sqlite/Migrations/20250916134827_Add_FixH264VideoPreset.cs

@@ -0,0 +1,22 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ErsatzTV.Infrastructure.Sqlite.Migrations
+{
+    /// <inheritdoc />
+    public partial class Add_FixH264VideoPreset : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            // clear video preset on all h264/10 bit ffmpeg profiles
+            migrationBuilder.Sql("UPDATE `FFmpegProfile` SET `VideoPreset`='' WHERE `VideoFormat` = 1 AND `BitDepth` = 1");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+        }
+    }
+}

ErsatzTV/Pages/FFmpegEditor.razor

@@ -4,9 +4,7 @@
 @using ErsatzTV.Application.FFmpegProfiles
 @using ErsatzTV.Application.Resolutions
 @using ErsatzTV.Core.FFmpeg
-@using ErsatzTV.FFmpeg
 @using ErsatzTV.FFmpeg.Format
-@using ErsatzTV.FFmpeg.Preset
 @using ErsatzTV.Validators
 @using FluentValidation.Results
 @using Microsoft.Extensions.Caching.Memory
@@ -89,7 +87,10 @@
                 <MudText>Preset</MudText>
             </div>
             @{
-                ICollection<string> presets = AvailablePresets.ForAccelAndFormat(MapAccel(_model.HardwareAcceleration), MapVideoFormat(_model.VideoFormat));
+                ICollection<string> presets = FFmpegLibraryHelper.PresetsForFFmpegProfile(
+                    _model.HardwareAcceleration,
+                    _model.VideoFormat,
+                    _model.BitDepth);
             }
             <MudSelect @bind-Value="_model.VideoPreset"
                        For="@(() => _model.VideoPreset)"
@@ -421,26 +422,4 @@
             // do nothing
         }
     }
-
-    private static HardwareAccelerationMode MapAccel(HardwareAccelerationKind kind) =>
-        kind switch
-        {
-            HardwareAccelerationKind.Amf => HardwareAccelerationMode.Amf,
-            HardwareAccelerationKind.Nvenc => HardwareAccelerationMode.Nvenc,
-            HardwareAccelerationKind.Qsv => HardwareAccelerationMode.Qsv,
-            HardwareAccelerationKind.Vaapi => HardwareAccelerationMode.Vaapi,
-            HardwareAccelerationKind.VideoToolbox => HardwareAccelerationMode.VideoToolbox,
-            HardwareAccelerationKind.V4l2m2m => HardwareAccelerationMode.V4l2m2m,
-            HardwareAccelerationKind.Rkmpp => HardwareAccelerationMode.Rkmpp,
-            _ => HardwareAccelerationMode.None
-        };
-
-    private static string MapVideoFormat(FFmpegProfileVideoFormat format) =>
-        format switch
-        {
-            FFmpegProfileVideoFormat.H264 => VideoFormat.H264,
-            FFmpegProfileVideoFormat.Hevc => VideoFormat.Hevc,
-            _ => VideoFormat.Mpeg2Video
-        };
-
 }

ErsatzTV/Program.cs

@@ -81,6 +81,7 @@ public class Program
 
             // streaming
             .MinimumLevel.Override("ErsatzTV.Application.Streaming", LoggingLevelSwitches.StreamingLevelSwitch)
+            .MinimumLevel.Override("ErsatzTV.Application.Troubleshooting", LoggingLevelSwitches.StreamingLevelSwitch)
             .MinimumLevel.Override("ErsatzTV.FFmpeg", LoggingLevelSwitches.StreamingLevelSwitch)
             .MinimumLevel.Override("ErsatzTV.Core.FFmpeg", LoggingLevelSwitches.StreamingLevelSwitch)
             .MinimumLevel.Override("ErsatzTV.Controllers.IptvController", LoggingLevelSwitches.StreamingLevelSwitch)