more scaling fixes (#948)

* remove force_original_aspect_ratio from scale_cuda

* remove force_original_aspect_ratio from scale_cuda

* fix qsv scaling

* fix qsv scaling on linux

* fix vaapi scaling edge cases

* update changelog

CHANGELOG.md

@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix scanner crash caused by invalid mtime
 - `VAAPI`: Downgrade libva from 2.15 to 2.14
 - Fix bug with XMLTV that caused some filler to display with primary content details
+- Multiple fixes for content scaling with `Nvidia`, `Qsv` and `Vaapi` accelerations
 
 ### Added
 - Add `Preferred Audio Title` feature

ErsatzTV.Core.Tests/Emby/EmbyPathReplacementServiceTests.cs

@@ -2,7 +2,7 @@
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Emby;
 using ErsatzTV.Core.Interfaces.Repositories;
-using ErsatzTV.Core.Interfaces.Runtime;
+using ErsatzTV.FFmpeg.Runtime;
 using FluentAssertions;
 using Microsoft.Extensions.Logging;
 using Moq;

ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs

@@ -13,6 +13,7 @@ using ErsatzTV.Core.Metadata;
 using ErsatzTV.FFmpeg;
 using ErsatzTV.FFmpeg.Capabilities;
 using ErsatzTV.FFmpeg.State;
+using ErsatzTV.Infrastructure.Runtime;
 using FluentAssertions;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
@@ -314,6 +315,7 @@ public class TranscodingTests
             // new HardwareCapabilitiesFactory(
             //     new MemoryCache(new MemoryCacheOptions()),
             //     LoggerFactory.CreateLogger<HardwareCapabilitiesFactory>()),
+            new RuntimeInfo(),
             LoggerFactory.CreateLogger<FFmpegLibraryProcessService>());
 
         var v = new MediaVersion

ErsatzTV.Core.Tests/Jellyfin/JellyfinPathReplacementServiceTests.cs

@@ -1,7 +1,7 @@
 using System.Runtime.InteropServices;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Repositories;
-using ErsatzTV.Core.Interfaces.Runtime;
+using ErsatzTV.FFmpeg.Runtime;
 using ErsatzTV.Core.Jellyfin;
 using FluentAssertions;
 using Microsoft.Extensions.Logging;

ErsatzTV.Core.Tests/Plex/PlexPathReplacementServiceTests.cs

@@ -1,7 +1,7 @@
 using System.Runtime.InteropServices;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Repositories;
-using ErsatzTV.Core.Interfaces.Runtime;
+using ErsatzTV.FFmpeg.Runtime;
 using ErsatzTV.Core.Plex;
 using FluentAssertions;
 using Microsoft.Extensions.Logging;

ErsatzTV.Core/Emby/EmbyPathReplacementService.cs

@@ -2,7 +2,7 @@
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Emby;
 using ErsatzTV.Core.Interfaces.Repositories;
-using ErsatzTV.Core.Interfaces.Runtime;
+using ErsatzTV.FFmpeg.Runtime;
 using Microsoft.Extensions.Logging;
 
 namespace ErsatzTV.Core.Emby;

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -7,6 +7,7 @@ using ErsatzTV.FFmpeg.Capabilities;
 using ErsatzTV.FFmpeg.Environment;
 using ErsatzTV.FFmpeg.Format;
 using ErsatzTV.FFmpeg.OutputFormat;
+using ErsatzTV.FFmpeg.Runtime;
 using ErsatzTV.FFmpeg.State;
 using Microsoft.Extensions.Logging;
 using MediaStream = ErsatzTV.Core.Domain.MediaStream;
@@ -18,6 +19,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
     private readonly FFmpegProcessService _ffmpegProcessService;
     private readonly IFFmpegStreamSelector _ffmpegStreamSelector;
     private readonly IHardwareCapabilitiesFactory _hardwareCapabilitiesFactory;
+    private readonly IRuntimeInfo _runtimeInfo;
     private readonly ILogger<FFmpegLibraryProcessService> _logger;
     private readonly FFmpegPlaybackSettingsCalculator _playbackSettingsCalculator;
     private readonly ITempFilePool _tempFilePool;
@@ -28,6 +30,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         IFFmpegStreamSelector ffmpegStreamSelector,
         ITempFilePool tempFilePool,
         IHardwareCapabilitiesFactory hardwareCapabilitiesFactory,
+        IRuntimeInfo runtimeInfo,
         ILogger<FFmpegLibraryProcessService> logger)
     {
         _ffmpegProcessService = ffmpegProcessService;
@@ -35,6 +38,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         _ffmpegStreamSelector = ffmpegStreamSelector;
         _tempFilePool = tempFilePool;
         _hardwareCapabilitiesFactory = hardwareCapabilitiesFactory;
+        _runtimeInfo = runtimeInfo;
         _logger = logger;
     }
 
@@ -240,6 +244,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         _logger.LogDebug("FFmpeg desired state {FrameState}", desiredState);
 
         var pipelineBuilder = new PipelineBuilder(
+            _runtimeInfo,
             await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, hwAccel),
             videoInputFile,
             audioInputFile,
@@ -372,6 +377,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         _logger.LogDebug("FFmpeg desired error state {FrameState}", desiredState);
 
         var pipelineBuilder = new PipelineBuilder(
+            _runtimeInfo,
             await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, hwAccel),
             videoInputFile,
             audioInputFile,
@@ -400,6 +406,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             resolution);
 
         var pipelineBuilder = new PipelineBuilder(
+            _runtimeInfo,
             await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, HardwareAccelerationMode.None),
             None,
             None,
@@ -426,6 +433,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             new List<VideoStream> { new(0, string.Empty, None, FrameSize.Unknown, string.Empty, None, true) });
 
         var pipelineBuilder = new PipelineBuilder(
+            _runtimeInfo,
             await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, HardwareAccelerationMode.None),
             videoInputFile,
             None,

ErsatzTV.Core/Jellyfin/JellyfinPathReplacementService.cs

@@ -2,7 +2,7 @@
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Jellyfin;
 using ErsatzTV.Core.Interfaces.Repositories;
-using ErsatzTV.Core.Interfaces.Runtime;
+using ErsatzTV.FFmpeg.Runtime;
 using Microsoft.Extensions.Logging;
 
 namespace ErsatzTV.Core.Jellyfin;

ErsatzTV.Core/Plex/PlexPathReplacementService.cs

@@ -2,7 +2,7 @@
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Plex;
 using ErsatzTV.Core.Interfaces.Repositories;
-using ErsatzTV.Core.Interfaces.Runtime;
+using ErsatzTV.FFmpeg.Runtime;
 using Microsoft.Extensions.Logging;
 
 namespace ErsatzTV.Core.Plex;

ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs

@@ -4,6 +4,7 @@ using ErsatzTV.FFmpeg.Capabilities;
 using ErsatzTV.FFmpeg.Encoder;
 using ErsatzTV.FFmpeg.Format;
 using ErsatzTV.FFmpeg.OutputFormat;
+using ErsatzTV.FFmpeg.Runtime;
 using ErsatzTV.FFmpeg.State;
 using FluentAssertions;
 using LanguageExt;
@@ -72,6 +73,7 @@ public class PipelineGeneratorTests
             Option<int>.None);
 
         var builder = new PipelineBuilder(
+            new Mock<IRuntimeInfo>().Object,
             new DefaultHardwareCapabilities(),
             videoInputFile,
             audioInputFile,
@@ -143,6 +145,7 @@ public class PipelineGeneratorTests
             Option<int>.None);
 
         var builder = new PipelineBuilder(
+            new Mock<IRuntimeInfo>().Object,
             new DefaultHardwareCapabilities(),
             videoInputFile,
             audioInputFile,
@@ -167,7 +170,16 @@ public class PipelineGeneratorTests
         var resolution = new FrameSize(1920, 1080);
         var concatInputFile = new ConcatInputFile("http://localhost:8080/ffmpeg/concat/1", resolution);
 
-        var builder = new PipelineBuilder(new DefaultHardwareCapabilities(), None, None, None, None, "", "", _logger);
+        var builder = new PipelineBuilder(
+            new Mock<IRuntimeInfo>().Object,
+            new DefaultHardwareCapabilities(),
+            None,
+            None,
+            None,
+            None,
+            "",
+            "",
+            _logger);
         FFmpegPipeline result = builder.Concat(concatInputFile, FFmpegState.Concat(false, "Some Channel"));
 
         result.PipelineSteps.Should().HaveCountGreaterThan(0);
@@ -231,6 +243,7 @@ public class PipelineGeneratorTests
             Option<int>.None);
 
         var builder = new PipelineBuilder(
+            new Mock<IRuntimeInfo>().Object,
             new DefaultHardwareCapabilities(),
             videoInputFile,
             audioInputFile,
@@ -294,6 +307,7 @@ public class PipelineGeneratorTests
             Option<int>.None);
 
         var builder = new PipelineBuilder(
+            new Mock<IRuntimeInfo>().Object,
             new DefaultHardwareCapabilities(),
             videoInputFile,
             audioInputFile,
@@ -327,6 +341,7 @@ public class PipelineGeneratorTests
             });
 
         var pipelineBuilder = new PipelineBuilder(
+            new Mock<IRuntimeInfo>().Object,
             new DefaultHardwareCapabilities(),
             videoInputFile,
             Option<AudioInputFile>.None,

ErsatzTV.FFmpeg/Filter/AvailableScaleFilters.cs

@@ -1,12 +1,14 @@
 using ErsatzTV.FFmpeg.Filter.Cuda;
 using ErsatzTV.FFmpeg.Filter.Qsv;
 using ErsatzTV.FFmpeg.Filter.Vaapi;
+using ErsatzTV.FFmpeg.Runtime;
 
 namespace ErsatzTV.FFmpeg.Filter;
 
 public static class AvailableScaleFilters
 {
     public static IPipelineFilterStep ForAcceleration(
+        IRuntimeInfo runtimeInfo,
         HardwareAccelerationMode accelMode,
         FrameState currentState,
         FrameSize scaledSize,
@@ -16,7 +18,7 @@ public static class AvailableScaleFilters
             HardwareAccelerationMode.Nvenc => new ScaleCudaFilter(currentState, scaledSize, paddedSize),
             HardwareAccelerationMode.Qsv when currentState.FrameDataLocation == FrameDataLocation.Hardware ||
                                               scaledSize == paddedSize =>
-                new ScaleQsvFilter(currentState, scaledSize),
+                new ScaleQsvFilter(runtimeInfo, currentState, scaledSize, paddedSize),
             HardwareAccelerationMode.Vaapi => new ScaleVaapiFilter(currentState, scaledSize, paddedSize),
             _ => new ScaleFilter(currentState, scaledSize, paddedSize)
         };

ErsatzTV.FFmpeg/Filter/Cuda/ScaleCudaFilter.cs

@@ -37,13 +37,7 @@ public class ScaleCudaFilter : BaseFilter
                     format = $":format={pixelFormat.FFmpegName}";
                 }
 
-                string aspectRatio = string.Empty;
-                if (_scaledSize != _paddedSize)
-                {
-                    aspectRatio = ":force_original_aspect_ratio=1";
-                }
-
-                scale = $"scale_cuda={targetSize}{aspectRatio}{format}";
+                scale = $"scale_cuda={targetSize}{format}";
             }
 
             // TODO: this might not always upload to hardware, so NextState could be inaccurate

ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs

@@ -1,16 +1,22 @@
-using ErsatzTV.FFmpeg.Format;
+using System.Runtime.InteropServices;
+using ErsatzTV.FFmpeg.Format;
+using ErsatzTV.FFmpeg.Runtime;
 
 namespace ErsatzTV.FFmpeg.Filter.Qsv;
 
 public class ScaleQsvFilter : BaseFilter
 {
+    private readonly IRuntimeInfo _runtimeInfo;
     private readonly FrameState _currentState;
     private readonly FrameSize _scaledSize;
+    private readonly FrameSize _paddedSize;
 
-    public ScaleQsvFilter(FrameState currentState, FrameSize scaledSize)
+    public ScaleQsvFilter(IRuntimeInfo runtimeInfo, FrameState currentState, FrameSize scaledSize, FrameSize paddedSize)
     {
+        _runtimeInfo = runtimeInfo;
         _currentState = currentState;
         _scaledSize = scaledSize;
+        _paddedSize = paddedSize;
     }
 
     public override string Filter
@@ -37,7 +43,9 @@ public class ScaleQsvFilter : BaseFilter
                     format = $":format={pixelFormat.FFmpegName}";
                 }
 
-                string targetSize = $"w={_scaledSize.Width}:h={_scaledSize.Height}";
+                string targetSize = _runtimeInfo.IsOSPlatform(OSPlatform.Windows)
+                    ? $"w={_paddedSize.Width}:h={_paddedSize.Height}"
+                    : $"w={_scaledSize.Width}:h={_scaledSize.Height}";
                 scale = $"vpp_qsv={targetSize}{format}";
             }
 

ErsatzTV.FFmpeg/Filter/Vaapi/ScaleVaapiFilter.cs

@@ -37,14 +37,8 @@ public class ScaleVaapiFilter : BaseFilter
                     format = $":format={pixelFormat.FFmpegName}";
                 }
 
-                string aspectRatio = string.Empty;
-                if (_scaledSize != _paddedSize)
-                {
-                    aspectRatio = ":force_original_aspect_ratio=1";
-                }
-
                 string targetSize = $"{_paddedSize.Width}:{_paddedSize.Height}";
-                scale = $"scale_vaapi={targetSize}{aspectRatio}:force_divisible_by=2{format}";
+                scale = $"scale_vaapi={targetSize}:force_divisible_by=2{format}";
             }
 
             if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)

ErsatzTV.FFmpeg/PipelineBuilder.cs

@@ -10,6 +10,7 @@ using ErsatzTV.FFmpeg.Option.HardwareAcceleration;
 using ErsatzTV.FFmpeg.Option.Metadata;
 using ErsatzTV.FFmpeg.OutputFormat;
 using ErsatzTV.FFmpeg.Protocol;
+using ErsatzTV.FFmpeg.Runtime;
 using ErsatzTV.FFmpeg.State;
 using Microsoft.Extensions.Logging;
 
@@ -19,6 +20,7 @@ public class PipelineBuilder
 {
     private readonly Option<AudioInputFile> _audioInputFile;
     private readonly string _fontsFolder;
+    private readonly IRuntimeInfo _runtimeInfo;
     private readonly IHardwareCapabilities _hardwareCapabilities;
     private readonly ILogger _logger;
     private readonly List<IPipelineStep> _pipelineSteps;
@@ -28,6 +30,7 @@ public class PipelineBuilder
     private readonly Option<WatermarkInputFile> _watermarkInputFile;
 
     public PipelineBuilder(
+        IRuntimeInfo runtimeInfo,
         IHardwareCapabilities hardwareCapabilities,
         Option<VideoInputFile> videoInputFile,
         Option<AudioInputFile> audioInputFile,
@@ -49,6 +52,7 @@ public class PipelineBuilder
             new ClosedGopOutputOption()
         };
 
+        _runtimeInfo = runtimeInfo;
         _hardwareCapabilities = hardwareCapabilities;
         _videoInputFile = videoInputFile;
         _audioInputFile = audioInputFile;
@@ -392,6 +396,7 @@ public class PipelineBuilder
                 else if (currentState.ScaledSize != desiredState.ScaledSize)
                 {
                     IPipelineFilterStep scaleFilter = AvailableScaleFilters.ForAcceleration(
+                        _runtimeInfo,
                         ffmpegState.EncoderHardwareAccelerationMode,
                         currentState,
                         desiredState.ScaledSize,
@@ -406,8 +411,9 @@ public class PipelineBuilder
                         currentState = padStep.NextState(currentState);
                         _videoInputFile.Iter(f => f.FilterSteps.Add(padStep));
                     }
-                    
-                    if (videoStream.DisplayAspectRatio == desiredState.DisplayAspectRatio)
+
+                    if (videoStream.DisplayAspectRatio == desiredState.DisplayAspectRatio ||
+                        ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Qsv)
                     {
                         IPipelineFilterStep darStep = new SetDarFilter(desiredState.DisplayAspectRatio);
                         currentState = darStep.NextState(currentState);
@@ -417,6 +423,7 @@ public class PipelineBuilder
                 else if (currentState.PaddedSize != desiredState.PaddedSize)
                 {
                     IPipelineFilterStep scaleFilter = AvailableScaleFilters.ForAcceleration(
+                        _runtimeInfo,
                         ffmpegState.EncoderHardwareAccelerationMode,
                         currentState,
                         desiredState.ScaledSize,
@@ -430,8 +437,9 @@ public class PipelineBuilder
                         currentState = padStep.NextState(currentState);
                         _videoInputFile.Iter(f => f.FilterSteps.Add(padStep));
                     }
-                    
-                    if (videoStream.DisplayAspectRatio == desiredState.DisplayAspectRatio)
+
+                    if (videoStream.DisplayAspectRatio == desiredState.DisplayAspectRatio ||
+                        ffmpegState.EncoderHardwareAccelerationMode == HardwareAccelerationMode.Qsv)
                     {
                         IPipelineFilterStep darStep = new SetDarFilter(desiredState.DisplayAspectRatio);
                         currentState = darStep.NextState(currentState);
@@ -470,6 +478,7 @@ public class PipelineBuilder
                                 currentState = currentState with { PixelFormat = desiredState.PixelFormat };
 
                                 IPipelineFilterStep scaleFilter = AvailableScaleFilters.ForAcceleration(
+                                    _runtimeInfo,
                                     ffmpegState.EncoderHardwareAccelerationMode,
                                     currentState,
                                     desiredState.ScaledSize,
@@ -499,6 +508,7 @@ public class PipelineBuilder
                             currentState = currentState with { PixelFormat = desiredState.PixelFormat };
 
                             IPipelineFilterStep scaleFilter = AvailableScaleFilters.ForAcceleration(
+                                _runtimeInfo,
                                 ffmpegState.EncoderHardwareAccelerationMode,
                                 currentState,
                                 desiredState.ScaledSize,
@@ -625,6 +635,7 @@ public class PipelineBuilder
                         _videoInputFile.Map(f => f.FilterSteps.Count).IfNone(1) == 0)
                     {
                         IPipelineFilterStep scaleFilter = AvailableScaleFilters.ForAcceleration(
+                            _runtimeInfo,
                             ffmpegState.EncoderHardwareAccelerationMode,
                             currentState,
                             desiredState.ScaledSize,

ErsatzTV.FFmpeg/Runtime/IRuntimeInfo.cs

@@ -1,7 +1,7 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Runtime.InteropServices;
 
-namespace ErsatzTV.Core.Interfaces.Runtime;
+namespace ErsatzTV.FFmpeg.Runtime;
 
 public interface IRuntimeInfo
 {

ErsatzTV.Infrastructure/Health/Checks/HardwareAccelerationHealthCheck.cs

@@ -4,7 +4,7 @@ using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Health;
 using ErsatzTV.Core.Health.Checks;
 using ErsatzTV.Core.Interfaces.Repositories;
-using ErsatzTV.Core.Interfaces.Runtime;
+using ErsatzTV.FFmpeg.Runtime;
 using ErsatzTV.Infrastructure.Data;
 using LanguageExt.UnsafeValueAccess;
 using Microsoft.EntityFrameworkCore;

ErsatzTV.Infrastructure/Runtime/RuntimeInfo.cs

@@ -1,5 +1,5 @@
 using System.Runtime.InteropServices;
-using ErsatzTV.Core.Interfaces.Runtime;
+using ErsatzTV.FFmpeg.Runtime;
 
 namespace ErsatzTV.Infrastructure.Runtime;
 

ErsatzTV/Services/RunOnce/PlatformSettingsService.cs

@@ -1,6 +1,6 @@
 using System.Runtime.InteropServices;
 using ErsatzTV.Core.Interfaces.Metadata;
-using ErsatzTV.Core.Interfaces.Runtime;
+using ErsatzTV.FFmpeg.Runtime;
 using ErsatzTV.Infrastructure.Data;
 using Microsoft.Extensions.Caching.Memory;
 

ErsatzTV/Startup.cs

@@ -23,7 +23,6 @@ using ErsatzTV.Core.Interfaces.Metadata.Nfo;
 using ErsatzTV.Core.Interfaces.Plex;
 using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Core.Interfaces.Repositories.Caching;
-using ErsatzTV.Core.Interfaces.Runtime;
 using ErsatzTV.Core.Interfaces.Scheduling;
 using ErsatzTV.Core.Interfaces.Search;
 using ErsatzTV.Core.Interfaces.Trakt;
@@ -34,6 +33,7 @@ using ErsatzTV.Core.Plex;
 using ErsatzTV.Core.Scheduling;
 using ErsatzTV.Core.Trakt;
 using ErsatzTV.FFmpeg.Capabilities;
+using ErsatzTV.FFmpeg.Runtime;
 using ErsatzTV.Formatters;
 using ErsatzTV.Infrastructure.Data;
 using ErsatzTV.Infrastructure.Data.Repositories;