scale subtitles with all accels (#1311)

* properly scale subtitles with qsv and vaapi

* fixes

CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Fixed
 - Skip checking for subtitles to extract when subtitles are not enabled on a channel/schedule item
+- Properly scale subtitles with QSV and some VAAPI configurations; now all configurations will scale subtitles
 
 ### Changed
 - `HLS Direct` streaming mode

ErsatzTV.FFmpeg/Filter/AvailableDeinterlaceFilters.cs

@@ -1,28 +0,0 @@
-using ErsatzTV.FFmpeg.Filter.Cuda;
-using ErsatzTV.FFmpeg.Filter.Vaapi;
-
-namespace ErsatzTV.FFmpeg.Filter;
-
-public static class AvailableDeinterlaceFilters
-{
-    public static IPipelineFilterStep ForAcceleration(
-        HardwareAccelerationMode accelMode,
-        FrameState currentState,
-        FrameState desiredState,
-        Option<WatermarkInputFile> watermarkInputFile,
-        Option<SubtitleInputFile> subtitleInputFile) =>
-        accelMode switch
-        {
-            HardwareAccelerationMode.Nvenc => new YadifCudaFilter(currentState),
-
-            // deinterlace_qsv seems to create timestamp issues
-            // HardwareAccelerationMode.Qsv => new DeinterlaceQsvFilter(currentState),
-
-            // fall back to software deinterlace with watermark and no scaling
-            HardwareAccelerationMode.Vaapi when watermarkInputFile.IsNone && subtitleInputFile.IsNone ||
-                                                currentState.ScaledSize != desiredState.ScaledSize =>
-                new DeinterlaceVaapiFilter(currentState),
-
-            _ => new YadifFilter(currentState)
-        };
-}

ErsatzTV.FFmpeg/Filter/AvailableScaleFilters.cs

@@ -1,38 +0,0 @@
-using ErsatzTV.FFmpeg.Filter.Cuda;
-using ErsatzTV.FFmpeg.Filter.Qsv;
-using ErsatzTV.FFmpeg.Filter.Vaapi;
-using ErsatzTV.FFmpeg.Runtime;
-
-namespace ErsatzTV.FFmpeg.Filter;
-
-public static class AvailableScaleFilters
-{
-    public static IPipelineFilterStep ForAcceleration(
-        IRuntimeInfo _,
-        HardwareAccelerationMode accelMode,
-        FrameState currentState,
-        FrameSize scaledSize,
-        FrameSize paddedSize,
-        int extraHardwareFrames,
-        bool isAnamorphicEdgeCase,
-        string sampleAspectRatio) =>
-        accelMode switch
-        {
-            HardwareAccelerationMode.Nvenc =>
-                new ScaleCudaFilter(currentState, scaledSize, paddedSize, isAnamorphicEdgeCase),
-            HardwareAccelerationMode.Qsv when currentState.FrameDataLocation == FrameDataLocation.Hardware ||
-                                              scaledSize == paddedSize =>
-                new ScaleQsvFilter(
-                    currentState,
-                    scaledSize,
-                    extraHardwareFrames,
-                    isAnamorphicEdgeCase,
-                    sampleAspectRatio),
-            HardwareAccelerationMode.Vaapi => new ScaleVaapiFilter(
-                currentState,
-                scaledSize,
-                paddedSize,
-                isAnamorphicEdgeCase),
-            _ => new ScaleFilter(currentState, scaledSize, paddedSize, isAnamorphicEdgeCase)
-        };
-}

ErsatzTV.FFmpeg/Filter/AvailableSubtitleOverlayFilters.cs

@@ -1,16 +0,0 @@
-using ErsatzTV.FFmpeg.Filter.Cuda;
-using ErsatzTV.FFmpeg.Filter.Qsv;
-using ErsatzTV.FFmpeg.Format;
-
-namespace ErsatzTV.FFmpeg.Filter;
-
-public static class AvailableSubtitleOverlayFilters
-{
-    public static IPipelineFilterStep ForAcceleration(HardwareAccelerationMode accelMode) =>
-        accelMode switch
-        {
-            HardwareAccelerationMode.Nvenc => new OverlaySubtitleCudaFilter(),
-            HardwareAccelerationMode.Qsv => new OverlaySubtitleQsvFilter(),
-            _ => new OverlaySubtitleFilter(new PixelFormatYuv420P())
-        };
-}

ErsatzTV.FFmpeg/Filter/AvailableSubtitleScaleFilters.cs

@@ -1,24 +0,0 @@
-using ErsatzTV.FFmpeg.Filter.Cuda;
-using ErsatzTV.FFmpeg.Filter.Qsv;
-
-namespace ErsatzTV.FFmpeg.Filter;
-
-public static class AvailableSubtitleScaleFilters
-{
-    public static IPipelineFilterStep ForAcceleration(
-        HardwareAccelerationMode accelMode,
-        FrameState currentState,
-        FrameSize scaledSize,
-        FrameSize paddedSize,
-        int extraHardwareFrames) =>
-        accelMode switch
-        {
-            HardwareAccelerationMode.Nvenc => new SubtitleScaleNppFilter(currentState, scaledSize, paddedSize),
-            HardwareAccelerationMode.Qsv => new SubtitleScaleQsvFilter(
-                currentState,
-                scaledSize,
-                paddedSize,
-                extraHardwareFrames),
-            _ => new ScaleImageFilter(paddedSize)
-        };
-}

ErsatzTV.FFmpeg/Filter/AvailableWatermarkOverlayFilters.cs

@@ -1,30 +0,0 @@
-using ErsatzTV.FFmpeg.Filter.Cuda;
-using ErsatzTV.FFmpeg.Filter.Qsv;
-using ErsatzTV.FFmpeg.Format;
-using ErsatzTV.FFmpeg.State;
-using Microsoft.Extensions.Logging;
-
-namespace ErsatzTV.FFmpeg.Filter;
-
-public static class AvailableWatermarkOverlayFilters
-{
-    public static IPipelineFilterStep ForAcceleration(
-        HardwareAccelerationMode accelMode,
-        WatermarkState watermarkState,
-        FrameSize resolution,
-        FrameSize squarePixelFrameSize,
-        ILogger logger) =>
-        accelMode switch
-        {
-            HardwareAccelerationMode.Nvenc =>
-                new OverlayWatermarkCudaFilter(watermarkState, resolution, squarePixelFrameSize, logger),
-            HardwareAccelerationMode.Qsv =>
-                new OverlayWatermarkQsvFilter(watermarkState, resolution, squarePixelFrameSize, logger),
-            _ => new OverlayWatermarkFilter(
-                watermarkState,
-                resolution,
-                squarePixelFrameSize,
-                new PixelFormatYuv420P(),
-                logger)
-        };
-}

ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs

@@ -491,6 +491,13 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
                             pf = availablePixelFormat;
                         }
                     }
+                    
+                    // only scale if scaling or padding was used for main video stream
+                    if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleFilter or PadFilter))
+                    {
+                        var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize);
+                        subtitle.FilterSteps.Add(scaleFilter);
+                    }
 
                     var subtitlesFilter = new OverlaySubtitleFilter(pf);
                     subtitleOverlayFilterSteps.Add(subtitlesFilter);

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -418,6 +418,10 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
 
                 if (forceSoftwareOverlay)
                 {
+                    var downloadFilter = new HardwareDownloadFilter(currentState);
+                    currentState = downloadFilter.NextState(currentState);
+                    videoInputFile.FilterSteps.Add(downloadFilter);
+                    
                     foreach (IPixelFormat pixelFormat in desiredState.PixelFormat)
                     {
                         IPixelFormat pf = pixelFormat;
@@ -429,6 +433,13 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
                             }
                         }
 
+                        // only scale if scaling or padding was used for main video stream
+                        if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleVaapiFilter or PadFilter))
+                        {
+                            var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize);
+                            subtitle.FilterSteps.Add(scaleFilter);
+                        }
+
                         var subtitlesFilter = new OverlaySubtitleFilter(pf);
                         subtitleOverlayFilterSteps.Add(subtitlesFilter);
                     }
@@ -438,10 +449,12 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
                     var subtitleHardwareUpload = new HardwareUploadVaapiFilter(false);
                     subtitle.FilterSteps.Add(subtitleHardwareUpload);
 
-                    // always scale - shouldn't really be needed outside of transcoding tests, which use picture subtitles
-                    // that are too big
-                    var scaleFilter = new SubtitleScaleVaapiFilter(desiredState.PaddedSize);
-                    subtitle.FilterSteps.Add(scaleFilter);
+                    // only scale if scaling or padding was used for main video stream
+                    if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleVaapiFilter or PadFilter))
+                    {
+                        var scaleFilter = new SubtitleScaleVaapiFilter(desiredState.PaddedSize);
+                        subtitle.FilterSteps.Add(scaleFilter);
+                    }
 
                     var subtitlesFilter = new OverlaySubtitleVaapiFilter();
                     subtitleOverlayFilterSteps.Add(subtitlesFilter);

ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs

@@ -21,7 +21,6 @@ using ErsatzTV.FFmpeg.Filter.Vaapi;
 using ErsatzTV.FFmpeg.Format;
 using ErsatzTV.FFmpeg.Pipeline;
 using ErsatzTV.FFmpeg.State;
-using ErsatzTV.Infrastructure.Data.Repositories;
 using ErsatzTV.Infrastructure.Images;
 using ErsatzTV.Infrastructure.Runtime;
 using ErsatzTV.Scanner.Core.Interfaces.Metadata;
@@ -164,7 +163,7 @@ public class TranscodingTests
             // // // av1    yuv420p10le    51
             // //
             // new("msmpeg4v2", "yuv420p"),
-            new("msmpeg4v3", "yuv420p")
+            // new("msmpeg4v3", "yuv420p")
             //
             // // wmv3    yuv420p    1
         };
@@ -190,10 +189,10 @@ public class TranscodingTests
 
         public static HardwareAccelerationKind[] TestAccelerations =
         {
-            // HardwareAccelerationKind.None,
+            HardwareAccelerationKind.None,
             // HardwareAccelerationKind.Nvenc,
             HardwareAccelerationKind.Vaapi,
-            HardwareAccelerationKind.Qsv
+            // HardwareAccelerationKind.Qsv
             // HardwareAccelerationKind.VideoToolbox,
             // HardwareAccelerationKind.Amf
         };
@@ -251,7 +250,7 @@ public class TranscodingTests
                     MemoryCache,
                     LoggerFactory.CreateLogger<HardwareCapabilitiesFactory>()),
                 LoggerFactory.CreateLogger<PipelineBuilderFactory>()),
-            new Mock<ConfigElementRepository>().Object,
+            new Mock<IConfigElementRepository>().Object,
             LoggerFactory.CreateLogger<FFmpegLibraryProcessService>());
 
         var songVideoGenerator = new SongVideoGenerator(tempFilePool, mockImageCache.Object, service);
@@ -352,7 +351,7 @@ public class TranscodingTests
             now,
             Option<ChannelWatermark>.None,
             GetWatermark(watermark),
-            VaapiDriver.Default,
+            VaapiDriver.RadeonSI,
             "/dev/dri/renderD128",
             Option<int>.None,
             false,
@@ -372,6 +371,7 @@ public class TranscodingTests
             profileBitDepth,
             profileVideoFormat,
             profileAcceleration,
+            VaapiDriver.RadeonSI,
             localStatisticsProvider,
             () => videoVersion);
     }
@@ -618,7 +618,7 @@ public class TranscodingTests
             now,
             Option<ChannelWatermark>.None,
             channelWatermark,
-            VaapiDriver.Default,
+            VaapiDriver.RadeonSI,
             "/dev/dri/renderD128",
             Option<int>.None,
             false,
@@ -638,6 +638,7 @@ public class TranscodingTests
             profileBitDepth,
             profileVideoFormat,
             profileAcceleration,
+            VaapiDriver.RadeonSI,
             localStatisticsProvider,
             () => v);
     }
@@ -739,20 +740,16 @@ public class TranscodingTests
 
         var args =
             $"-y -f lavfi -i anoisesrc=color=brown -f lavfi -i testsrc=duration=1:size={resolution}:rate=30 {videoFilter} -c:a aac -c:v {inputFormat.Encoder}{colorRange}{colorSpace}{colorTransfer}{colorPrimaries} -shortest -pix_fmt {inputFormat.PixelFormat} -strict -2 {flags} {file}";
-        var p1 = new Process
-        {
-            StartInfo = new ProcessStartInfo
-            {
-                FileName = ExecutableName("ffmpeg"),
-                Arguments = args
-            }
-        };
+        BufferedCommandResult p1 = await Cli.Wrap(ExecutableName("ffmpeg"))
+            .WithArguments(args)
+            .WithValidation(CommandResultValidation.None)
+            .ExecuteBufferedAsync();
+        
+        string output = string.IsNullOrWhiteSpace(p1.StandardOutput)
+            ? p1.StandardError
+            : p1.StandardOutput;
 
-        p1.Start();
-        await p1.WaitForExitAsync();
-        // ReSharper disable once MethodHasAsyncOverload
-        p1.WaitForExit();
-        p1.ExitCode.Should().Be(0);
+        p1.ExitCode.Should().Be(0, output);
 
         switch (subtitle)
         {
@@ -871,7 +868,7 @@ public class TranscodingTests
                     MemoryCache,
                     LoggerFactory.CreateLogger<HardwareCapabilitiesFactory>()),
                 LoggerFactory.CreateLogger<PipelineBuilderFactory>()),
-            new Mock<ConfigElementRepository>().Object,
+            new Mock<IConfigElementRepository>().Object,
             LoggerFactory.CreateLogger<FFmpegLibraryProcessService>());
 
         return service;
@@ -883,6 +880,7 @@ public class TranscodingTests
         FFmpegProfileBitDepth profileBitDepth,
         FFmpegProfileVideoFormat profileVideoFormat,
         HardwareAccelerationKind profileAcceleration,
+        VaapiDriver vaapiDriver,
         ILocalStatisticsProvider localStatisticsProvider,
         Func<MediaVersion> getFinalMediaVersion)
     {
@@ -995,8 +993,10 @@ public class TranscodingTests
 
                 // AMF doesn't seem to set this metadata properly
                 // MPEG2Video doesn't always seem to set this properly
+                // RADEONSI driver doesn't set this properly
                 if (profileAcceleration != HardwareAccelerationKind.Amf &&
-                    profileVideoFormat != FFmpegProfileVideoFormat.Mpeg2Video)
+                    profileVideoFormat != FFmpegProfileVideoFormat.Mpeg2Video &&
+                    (profileAcceleration != HardwareAccelerationKind.Vaapi || vaapiDriver != VaapiDriver.RadeonSI))
                 {
                     colorParams.IsBt709.Should().BeTrue($"{colorParams}");
                 }