add picture subtitle transcoding tests, and make them all pass with nvenc (#734)

CHANGELOG.md

@@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
+### Fixed
+- Fix subtitles edge case with NVENC
 
 ## [0.5.0-beta] - 2022-04-13
 ### Fixed

ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj

@@ -8,9 +8,10 @@
 
     <ItemGroup>
       <PackageReference Include="Bugsnag" Version="3.0.1" />
-      <PackageReference Include="CliWrap" Version="3.4.2" />
+      <PackageReference Include="CliWrap" Version="3.4.1" />
       <PackageReference Include="FluentAssertions" Version="6.6.0" />
       <PackageReference Include="LanguageExt.Core" Version="4.0.4" />
+      <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="6.0.1" />
       <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
       <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
       <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
@@ -36,6 +37,9 @@
       <Content Include="Resources\ErsatzTV.png">
         <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       </Content>
+      <Content Include="Resources\test.sup">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </Content>
     </ItemGroup>
 
 </Project>

ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs

@@ -65,6 +65,12 @@ public class TranscodingTests
         // TODO: animated vs static
     }
 
+    public enum Subtitle
+    {
+        None,
+        Picture
+    }
+
     private class TestData
     {
         public static Watermark[] Watermarks =
@@ -73,6 +79,12 @@ public class TranscodingTests
             Watermark.PermanentOpaque,
             Watermark.PermanentTransparent
         };
+
+        public static Subtitle[] Subtitles =
+        {
+            Subtitle.None,
+            Subtitle.Picture
+        };
             
         public static Padding[] Paddings =
         {
@@ -158,6 +170,7 @@ public class TranscodingTests
         [ValueSource(typeof(TestData), nameof(TestData.Paddings))] Padding padding,
         [ValueSource(typeof(TestData), nameof(TestData.VideoScanKinds))] VideoScanKind videoScanKind,
         [ValueSource(typeof(TestData), nameof(TestData.Watermarks))] Watermark watermark,
+        [ValueSource(typeof(TestData), nameof(TestData.Subtitles))] Subtitle subtitle,
         [ValueSource(typeof(TestData), nameof(TestData.VideoFormats))] FFmpegProfileVideoFormat profileVideoFormat,
         // [ValueSource(typeof(TestData), nameof(TestData.NoAcceleration))] HardwareAccelerationKind profileAcceleration)
         [ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration)
@@ -175,7 +188,7 @@ public class TranscodingTests
         }
 
         string name = GetStringSha256Hash(
-            $"{inputFormat.Encoder}_{inputFormat.PixelFormat}_{videoScanKind}_{padding}_{profileResolution}_{profileVideoFormat}_{profileAcceleration}");
+            $"{inputFormat.Encoder}_{inputFormat.PixelFormat}_{videoScanKind}_{padding}_{watermark}_{subtitle}_{profileResolution}_{profileVideoFormat}_{profileAcceleration}");
 
         string file = Path.Combine(TestContext.CurrentContext.TestDirectory, $"{name}.mkv");
         if (!File.Exists(file))
@@ -203,6 +216,46 @@ public class TranscodingTests
             // ReSharper disable once MethodHasAsyncOverload
             p1.WaitForExit();
             p1.ExitCode.Should().Be(0);
+
+            switch (subtitle)
+            {
+                case Subtitle.Picture:
+                    string sourceFile = Path.GetTempFileName() + ".mkv";
+                    File.Move(file, sourceFile, true);
+                    
+                    string tempFileName = Path.GetTempFileName() + ".mkv";
+                    string subPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "test.sup");
+                    var p2 = new Process
+                    {
+                        StartInfo = new ProcessStartInfo
+                        {
+                            FileName = ExecutableName("mkvmerge"),
+                            Arguments = $"-o {tempFileName} {sourceFile} {subPath}"
+                        }
+                    };
+
+                    p2.Start();
+                    await p2.WaitForExitAsync();
+                    // ReSharper disable once MethodHasAsyncOverload
+                    p2.WaitForExit();
+                    if (p2.ExitCode != 0)
+                    {
+                        if (File.Exists(sourceFile))
+                        {
+                            File.Delete(sourceFile);
+                        }
+
+                        if (File.Exists(file))
+                        {
+                            File.Delete(file);
+                        }
+                    }
+
+                    p2.ExitCode.Should().Be(0);
+
+                    File.Move(tempFileName, file, true);
+                    break;
+            }
         }
 
         var imageCache = new Mock<IImageCache>();
@@ -221,7 +274,7 @@ public class TranscodingTests
             imageCache.Object,
             new Mock<ITempFilePool>().Object,
             new Mock<IClient>().Object,
-            new Mock<IMemoryCache>().Object,
+            new MemoryCache(new MemoryCacheOptions()),
             LoggerFactory.CreateLogger<FFmpegProcessService>());
 
         var service = new FFmpegLibraryProcessService(
@@ -318,6 +371,12 @@ public class TranscodingTests
                 break;
         }
 
+        ChannelSubtitleMode subtitleMode = subtitle switch
+        {
+            Subtitle.Picture => ChannelSubtitleMode.Any,
+            _ => ChannelSubtitleMode.None 
+        };
+
         using Process process = await service.ForPlayoutItem(
             ExecutableName("ffmpeg"),
             ExecutableName("ffprobe"),
@@ -329,9 +388,11 @@ public class TranscodingTests
                 {
                     HardwareAcceleration = profileAcceleration,
                     VideoFormat = profileVideoFormat,
-                    AudioFormat = FFmpegProfileAudioFormat.Aac
+                    AudioFormat = FFmpegProfileAudioFormat.Aac,
+                    DeinterlaceVideo = true
                 },
-                StreamingMode = StreamingMode.TransportStream
+                StreamingMode = StreamingMode.TransportStream,
+                SubtitleMode = subtitleMode
             },
             v,
             v,
@@ -368,6 +429,7 @@ public class TranscodingTests
             result = await Cli.Wrap(process.StartInfo.FileName)
                 .WithArguments(process.StartInfo.ArgumentList)
                 .WithValidation(CommandResultValidation.None)
+                .WithStandardOutputPipe(PipeTarget.Null)
                 .WithStandardErrorPipe(PipeTarget.ToStringBuilder(sb))
                 .ExecuteAsync(timeoutSignal.Token);
         }
@@ -425,7 +487,7 @@ public class TranscodingTests
             Optional(version.Streams.First(s => s.MediaStreamKind == MediaStreamKind.Audio)).AsTask();
 
         public Task<Option<MediaStream>> SelectSubtitleStream(Channel channel, MediaVersion version) =>
-            Optional(version.Streams.First(s => s.MediaStreamKind == MediaStreamKind.Subtitle)).AsTask();
+            Optional(version.Streams.Find(s => s.MediaStreamKind == MediaStreamKind.Subtitle)).AsTask();
     }
 
     private static string ExecutableName(string baseName) =>

ErsatzTV.FFmpeg/Encoder/AvailableEncoders.cs

@@ -15,15 +15,18 @@ public static class AvailableEncoders
         FrameState currentState,
         FrameState desiredState,
         Option<WatermarkInputFile> maybeWatermarkInputFile,
+        Option<SubtitleInputFile> maybeSubtitleInputFile,
         ILogger logger) =>
         (ffmpegState.HardwareAccelerationMode, desiredState.VideoFormat) switch
         {
             (HardwareAccelerationMode.Nvenc, VideoFormat.Hevc) => new EncoderHevcNvenc(
                 currentState,
-                maybeWatermarkInputFile),
+                maybeWatermarkInputFile,
+                maybeSubtitleInputFile),
             (HardwareAccelerationMode.Nvenc, VideoFormat.H264) => new EncoderH264Nvenc(
                 currentState,
-                maybeWatermarkInputFile),
+                maybeWatermarkInputFile,
+                maybeSubtitleInputFile),
 
             (HardwareAccelerationMode.Qsv, VideoFormat.Hevc) => new EncoderHevcQsv(
                 currentState,

ErsatzTV.FFmpeg/Encoder/Nvenc/EncoderH264Nvenc.cs

@@ -6,11 +6,16 @@ public class EncoderH264Nvenc : EncoderBase
 {
     private readonly FrameState _currentState;
     private readonly Option<WatermarkInputFile> _maybeWatermarkInputFile;
+    private readonly Option<SubtitleInputFile> _maybeSubtitleInputFile;
 
-    public EncoderH264Nvenc(FrameState currentState, Option<WatermarkInputFile> maybeWatermarkInputFile)
+    public EncoderH264Nvenc(
+        FrameState currentState,
+        Option<WatermarkInputFile> maybeWatermarkInputFile,
+        Option<SubtitleInputFile> maybeSubtitleInputFile)
     {
         _currentState = currentState;
         _maybeWatermarkInputFile = maybeWatermarkInputFile;
+        _maybeSubtitleInputFile = maybeSubtitleInputFile;
     }
 
     public override FrameState NextState(FrameState currentState) => currentState with
@@ -27,10 +32,15 @@ public class EncoderH264Nvenc : EncoderBase
     {
         get
         {
-            // only upload to hw if we need to overlay a watermark
+            // only upload to hw if we need to overlay (watermark or subtitle)
-            if (_maybeWatermarkInputFile.IsSome && _currentState.FrameDataLocation == FrameDataLocation.Software)
+            if (_currentState.FrameDataLocation == FrameDataLocation.Software)
             {
-                return "hwupload_cuda";
+                bool isPictureSubtitle = _maybeSubtitleInputFile.Map(s => s.IsImageBased).IfNone(false);
+                
+                if (isPictureSubtitle || _maybeWatermarkInputFile.IsSome)
+                {
+                    return "hwupload_cuda";
+                }
             }
 
             return string.Empty;

ErsatzTV.FFmpeg/Encoder/Nvenc/EncoderHevcNvenc.cs

@@ -6,11 +6,16 @@ public class EncoderHevcNvenc : EncoderBase
 {
     private readonly FrameState _currentState;
     private readonly Option<WatermarkInputFile> _maybeWatermarkInputFile;
+    private readonly Option<SubtitleInputFile> _maybeSubtitleInputFile;
 
-    public EncoderHevcNvenc(FrameState currentState, Option<WatermarkInputFile> maybeWatermarkInputFile)
+    public EncoderHevcNvenc(
+        FrameState currentState,
+        Option<WatermarkInputFile> maybeWatermarkInputFile,
+        Option<SubtitleInputFile> maybeSubtitleInputFile)
     {
         _currentState = currentState;
         _maybeWatermarkInputFile = maybeWatermarkInputFile;
+        _maybeSubtitleInputFile = maybeSubtitleInputFile;
     }
 
     public override FrameState NextState(FrameState currentState) => currentState with
@@ -27,10 +32,15 @@ public class EncoderHevcNvenc : EncoderBase
     {
         get
         {
-            // only upload to hw if we need to overlay a watermark
+            // only upload to hw if we need to overlay (watermark or subtitle)
-            if (_maybeWatermarkInputFile.IsSome && _currentState.FrameDataLocation == FrameDataLocation.Software)
+            if (_currentState.FrameDataLocation == FrameDataLocation.Software)
             {
-                return "hwupload_cuda";
+                bool isPictureSubtitle = _maybeSubtitleInputFile.Map(s => s.IsImageBased).IfNone(false);
+                
+                if (isPictureSubtitle || _maybeWatermarkInputFile.IsSome)
+                {
+                    return "hwupload_cuda";
+                }
             }
 
             return string.Empty;

ErsatzTV.FFmpeg/ErsatzTV.FFmpeg.csproj

@@ -7,7 +7,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="CliWrap" Version="3.4.2" />
+      <PackageReference Include="CliWrap" Version="3.4.1" />
       <PackageReference Include="LanguageExt.Core" Version="4.0.4" />
       <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
     </ItemGroup>

ErsatzTV.FFmpeg/PipelineBuilder.cs

@@ -474,6 +474,7 @@ public class PipelineBuilder
                                  currentState,
                                  desiredState,
                                  _watermarkInputFile,
+                                 _subtitleInputFile,
                                  _logger))
                     {
                         encoder = e;