add crop scaling behavior (#1443)

* add scaling behavior - crop

* fix ffmpeg version check on windows (snapshot)

* update dependencies

CHANGELOG.md

@@ -7,7 +7,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Added
 - Add `Scaling Behavior` option to FFmpeg Profile
   - `Scale and Pad`: the default behavior and will maintain aspect ratio of all content
-  - `Stretch`: a new mode that will NOT maintain aspect ratio when normalizing source content to the desired resolution 
+  - `Stretch`: a new mode that will NOT maintain aspect ratio when normalizing source content to the desired resolution
+  - `Crop`: a new mode that will scale beyond the desired resolution (maintaining aspect ratio), and crop to desired resolution
+    - **This mode does NOT detect black and intelligently crop**
+    - The goal is to fill the canvas by over-scaling and cropping, instead of minimally scaling and padding
 
 ### Changed
 - Upgrade ffmpeg to 6.1, which is now *required* for all installs

ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj

@@ -10,7 +10,7 @@
       <PackageReference Include="Bugsnag" Version="3.1.0" />
       <PackageReference Include="CliWrap" Version="3.6.4" />
       <PackageReference Include="FluentAssertions" Version="6.12.0" />
-      <PackageReference Include="LanguageExt.Core" Version="4.4.4" />
+      <PackageReference Include="LanguageExt.Core" Version="4.4.5" />
       <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0" />
       <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="7.0.0" />
       <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />

ErsatzTV.Core/Domain/ScalingBehavior.cs

@@ -3,5 +3,6 @@ namespace ErsatzTV.Core.Domain;
 public enum ScalingBehavior
 {
     ScaleAndPad = 0,
-    Stretch = 1
+    Stretch = 1,
+    Crop = 2
 }

ErsatzTV.Core/ErsatzTV.Core.csproj

@@ -12,8 +12,8 @@
       <PackageReference Include="Bugsnag" Version="3.1.0" />
       <PackageReference Include="Destructurama.Attributed" Version="3.1.0" />
       <PackageReference Include="Flurl" Version="3.0.7" />
-      <PackageReference Include="LanguageExt.Core" Version="4.4.4" />
+      <PackageReference Include="LanguageExt.Core" Version="4.4.5" />
-      <PackageReference Include="LanguageExt.Transformers" Version="4.4.4" />
+      <PackageReference Include="LanguageExt.Transformers" Version="4.4.5" />
       <PackageReference Include="MediatR" Version="12.1.1" />
       <PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="7.0.0" />
       <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="7.0.0" />

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -313,10 +313,22 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             channel.FFmpegProfile.Resolution.Width,
             channel.FFmpegProfile.Resolution.Height);
 
+        Option<FrameSize> cropSize = Option<FrameSize>.None;
+
         if (channel.FFmpegProfile.ScalingBehavior is ScalingBehavior.Stretch)
         {
             scaledSize = paddedSize;
         }
+
+        if (channel.FFmpegProfile.ScalingBehavior is ScalingBehavior.Crop)
+        {
+            paddedSize = ffmpegVideoStream.SquarePixelFrameSizeForCrop(
+                new FrameSize(channel.FFmpegProfile.Resolution.Width, channel.FFmpegProfile.Resolution.Height));
+
+            cropSize = new FrameSize(
+                channel.FFmpegProfile.Resolution.Width,
+                channel.FFmpegProfile.Resolution.Height);
+        }
         
         var desiredState = new FrameState(
             playbackSettings.RealtimeOutput,
@@ -326,6 +338,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             Optional(playbackSettings.PixelFormat),
             scaledSize,
             paddedSize,
+            cropSize,
             false,
             playbackSettings.FrameRate,
             playbackSettings.VideoBitrate,
@@ -429,6 +442,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             new PixelFormatYuv420P(),
             new FrameSize(desiredResolution.Width, desiredResolution.Height),
             new FrameSize(desiredResolution.Width, desiredResolution.Height),
+            Option<FrameSize>.None,
             false,
             playbackSettings.FrameRate,
             playbackSettings.VideoBitrate,

ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs

@@ -61,6 +61,7 @@ public class PipelineBuilderBaseTests
             new PixelFormatYuv420P(),
             new FrameSize(1920, 1080),
             new FrameSize(1920, 1080),
+            Option<FrameSize>.None,
             false,
             Option<int>.None,
             2000,
@@ -149,6 +150,7 @@ public class PipelineBuilderBaseTests
             new PixelFormatYuv420P(),
             new FrameSize(1920, 1080),
             new FrameSize(1920, 1080),
+            Option<FrameSize>.None,
             false,
             Option<int>.None,
             2000,
@@ -291,6 +293,7 @@ public class PipelineBuilderBaseTests
             Option<IPixelFormat>.None,
             new FrameSize(1920, 1080),
             new FrameSize(1920, 1080),
+            Option<FrameSize>.None,
             false,
             Option<int>.None,
             2000,
@@ -373,6 +376,7 @@ public class PipelineBuilderBaseTests
             new PixelFormatYuv420P(),
             new FrameSize(1920, 1080),
             new FrameSize(1920, 1080),
+            Option<FrameSize>.None,
             false,
             Option<int>.None,
             2000,

ErsatzTV.FFmpeg/ErsatzTV.FFmpeg.csproj

@@ -10,7 +10,7 @@
 
     <ItemGroup>
       <PackageReference Include="CliWrap" Version="3.6.4" />
-      <PackageReference Include="LanguageExt.Core" Version="4.4.4" />
+      <PackageReference Include="LanguageExt.Core" Version="4.4.5" />
       <PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="7.0.0" />
       <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.1" />
     </ItemGroup>

ErsatzTV.FFmpeg/Filter/CropFilter.cs

@@ -0,0 +1,49 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Filter;
+
+public class CropFilter : BaseFilter
+{
+    private readonly FrameState _currentState;
+    private readonly FrameSize _croppedSize;
+
+    public CropFilter(FrameState currentState, FrameSize croppedSize)
+    {
+        _currentState = currentState;
+        _croppedSize = croppedSize;
+    }
+
+    public override string Filter
+    {
+        get
+        {
+            var crop = $"crop=w={_croppedSize.Width}:h={_croppedSize.Height}";
+
+            if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)
+            {
+                foreach (IPixelFormat pixelFormat in _currentState.PixelFormat)
+                {
+                    if (pixelFormat is PixelFormatVaapi)
+                    {
+                        foreach (IPixelFormat pf in AvailablePixelFormats.ForPixelFormat(pixelFormat.Name, null))
+                        {
+                            return $"hwdownload,format=vaapi|{pf.FFmpegName},{crop}";
+                        }
+                    }
+
+                    return $"hwdownload,format={pixelFormat.FFmpegName},{crop}";
+                }
+
+                return $"hwdownload,{crop}";
+            }
+
+            return crop;
+        }
+    }
+
+    public override FrameState NextState(FrameState currentState) => currentState with
+    {
+        PaddedSize = _croppedSize,
+        FrameDataLocation = FrameDataLocation.Software
+    };
+}

ErsatzTV.FFmpeg/Filter/Cuda/ScaleCudaFilter.cs

@@ -7,17 +7,20 @@ public class ScaleCudaFilter : BaseFilter
     private readonly FrameState _currentState;
     private readonly bool _isAnamorphicEdgeCase;
     private readonly FrameSize _paddedSize;
+    private readonly Option<FrameSize> _croppedSize;
     private readonly FrameSize _scaledSize;
 
     public ScaleCudaFilter(
         FrameState currentState,
         FrameSize scaledSize,
         FrameSize paddedSize,
+        Option<FrameSize> croppedSize,
         bool isAnamorphicEdgeCase)
     {
         _currentState = currentState;
         _scaledSize = scaledSize;
         _paddedSize = paddedSize;
+        _croppedSize = croppedSize;
         _isAnamorphicEdgeCase = isAnamorphicEdgeCase;
     }
 
@@ -39,7 +42,9 @@ public class ScaleCudaFilter : BaseFilter
                 string aspectRatio = string.Empty;
                 if (_scaledSize != _paddedSize)
                 {
-                    aspectRatio = ":force_original_aspect_ratio=decrease";
+                    aspectRatio = _croppedSize.IsSome
+                        ? ":force_original_aspect_ratio=increase"
+                        : ":force_original_aspect_ratio=decrease";
                 }
 
                 string squareScale = string.Empty;

ErsatzTV.FFmpeg/Filter/ScaleFilter.cs

@@ -7,13 +7,20 @@ public class ScaleFilter : BaseFilter
     private readonly FrameState _currentState;
     private readonly bool _isAnamorphicEdgeCase;
     private readonly FrameSize _paddedSize;
+    private readonly Option<FrameSize> _croppedSize;
     private readonly FrameSize _scaledSize;
 
-    public ScaleFilter(FrameState currentState, FrameSize scaledSize, FrameSize paddedSize, bool isAnamorphicEdgeCase)
+    public ScaleFilter(
+        FrameState currentState,
+        FrameSize scaledSize,
+        FrameSize paddedSize,
+        Option<FrameSize> croppedSize,
+        bool isAnamorphicEdgeCase)
     {
         _currentState = currentState;
         _scaledSize = scaledSize;
         _paddedSize = paddedSize;
+        _croppedSize = croppedSize;
         _isAnamorphicEdgeCase = isAnamorphicEdgeCase;
     }
 
@@ -29,7 +36,9 @@ public class ScaleFilter : BaseFilter
             string aspectRatio = string.Empty;
             if (_scaledSize != _paddedSize)
             {
-                aspectRatio = ":force_original_aspect_ratio=decrease";
+                aspectRatio = _croppedSize.IsSome
+                    ? ":force_original_aspect_ratio=increase"
+                    : ":force_original_aspect_ratio=decrease";
             }
 
             string scale;

ErsatzTV.FFmpeg/Filter/Vaapi/ScaleVaapiFilter.cs

@@ -7,17 +7,20 @@ public class ScaleVaapiFilter : BaseFilter
     private readonly FrameState _currentState;
     private readonly bool _isAnamorphicEdgeCase;
     private readonly FrameSize _paddedSize;
+    private readonly Option<FrameSize> _croppedSize;
     private readonly FrameSize _scaledSize;
 
     public ScaleVaapiFilter(
         FrameState currentState,
         FrameSize scaledSize,
         FrameSize paddedSize,
+        Option<FrameSize> croppedSize,
         bool isAnamorphicEdgeCase)
     {
         _currentState = currentState;
         _scaledSize = scaledSize;
         _paddedSize = paddedSize;
+        _croppedSize = croppedSize;
         _isAnamorphicEdgeCase = isAnamorphicEdgeCase;
     }
 
@@ -40,7 +43,9 @@ public class ScaleVaapiFilter : BaseFilter
                 string aspectRatio = string.Empty;
                 if (_scaledSize != _paddedSize)
                 {
-                    aspectRatio = ":force_original_aspect_ratio=decrease";
+                    aspectRatio = _croppedSize.IsSome
+                        ? ":force_original_aspect_ratio=increase"
+                        : ":force_original_aspect_ratio=decrease";
                 }
 
                 string squareScale = string.Empty;

ErsatzTV.FFmpeg/FrameState.cs

@@ -10,6 +10,7 @@ public record FrameState(
     Option<IPixelFormat> PixelFormat,
     FrameSize ScaledSize,
     FrameSize PaddedSize,
+    Option<FrameSize> CroppedSize,
     bool IsAnamorphic,
     Option<int> FrameRate,
     Option<int> VideoBitrate,

ErsatzTV.FFmpeg/MediaStream.cs

@@ -120,6 +120,36 @@ public record VideoStream(
 
         return result;
     }
+    
+    public FrameSize SquarePixelFrameSizeForCrop(FrameSize resolution)
+    {
+        int width = FrameSize.Width;
+        int height = FrameSize.Height;
+
+        if (IsAnamorphic)
+        {
+            double sar = GetSAR();
+            bool edgeCase = IsAnamorphicEdgeCase;
+
+            width = edgeCase
+                ? FrameSize.Width
+                : (int)Math.Floor(FrameSize.Width * sar);
+
+            height = edgeCase
+                ? (int)Math.Floor(FrameSize.Height * sar)
+                : FrameSize.Height;
+        }
+
+        double widthPercent = (double)resolution.Width / width;
+        double heightPercent = (double)resolution.Height / height;
+        double maxPercent = Math.Max(widthPercent, heightPercent);
+
+        var result = new FrameSize(
+            (int)Math.Floor(width * maxPercent),
+            (int)Math.Floor(height * maxPercent));
+
+        return result;
+    }
 
     private double GetSAR()
     {

ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs

@@ -182,6 +182,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
                             currentState with { PixelFormat = Some(pixelFormat) },
                             currentState.ScaledSize,
                             currentState.PaddedSize,
+                            Option<FrameSize>.None,
                             false);
                         currentState = filter.NextState(currentState);
                         videoInputFile.FilterSteps.Add(filter);
@@ -517,7 +518,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
         FrameState desiredState,
         FrameState currentState)
     {
-        if (currentState.PaddedSize != desiredState.PaddedSize)
+        if (desiredState.CroppedSize.IsNone && currentState.PaddedSize != desiredState.PaddedSize)
         {
             IPipelineFilterStep padStep = new PadFilter(currentState, desiredState.PaddedSize);
             currentState = padStep.NextState(currentState);
@@ -555,6 +556,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
                 currentState,
                 desiredState.ScaledSize,
                 desiredState.PaddedSize,
+                desiredState.CroppedSize,
                 VideoStream.IsAnamorphicEdgeCase);
         }
         else
@@ -579,6 +581,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
                 },
                 desiredState.ScaledSize,
                 desiredState.PaddedSize,
+                desiredState.CroppedSize,
                 VideoStream.IsAnamorphicEdgeCase);
         }
 

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs

@@ -504,6 +504,21 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
         FrameState desiredState,
         string fontsFolder,
         ICollection<IPipelineStep> pipelineSteps);
+    
+    protected static FrameState SetCrop(
+        VideoInputFile videoInputFile,
+        FrameState desiredState,
+        FrameState currentState)
+    {
+        foreach (FrameSize croppedSize in currentState.CroppedSize)
+        {
+            IPipelineFilterStep cropStep = new CropFilter(currentState, croppedSize);
+            currentState = cropStep.NextState(currentState);
+            videoInputFile.FilterSteps.Add(cropStep);
+        }
+
+        return currentState;
+    }
 
     private static void SetOutputTsOffset(
         FFmpegState ffmpegState,

ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs

@@ -521,7 +521,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
         FrameState desiredState,
         FrameState currentState)
     {
-        if (currentState.PaddedSize != desiredState.PaddedSize)
+        if (desiredState.CroppedSize.IsNone && currentState.PaddedSize != desiredState.PaddedSize)
         {
             IPipelineFilterStep padStep = new PadFilter(
                 currentState,
@@ -553,6 +553,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
                 currentState,
                 desiredState.ScaledSize,
                 desiredState.PaddedSize,
+                desiredState.CroppedSize,
                 VideoStream.IsAnamorphicEdgeCase);
         }
         else

ErsatzTV.FFmpeg/Pipeline/SoftwarePipelineBuilder.cs

@@ -102,6 +102,7 @@ public class SoftwarePipelineBuilder : PipelineBuilderBase
 
             currentState = SetScale(videoInputFile, videoStream, desiredState, currentState);
             currentState = SetPad(videoInputFile, videoStream, desiredState, currentState);
+            currentState = SetCrop(videoInputFile, desiredState, currentState);
             SetSubtitle(
                 videoInputFile,
                 subtitleInputFile,
@@ -301,7 +302,7 @@ public class SoftwarePipelineBuilder : PipelineBuilderBase
         FrameState desiredState,
         FrameState currentState)
     {
-        if (currentState.PaddedSize != desiredState.PaddedSize)
+        if (desiredState.CroppedSize.IsNone && currentState.PaddedSize != desiredState.PaddedSize)
         {
             IPipelineFilterStep padStep = new PadFilter(currentState, desiredState.PaddedSize);
             currentState = padStep.NextState(currentState);
@@ -323,6 +324,7 @@ public class SoftwarePipelineBuilder : PipelineBuilderBase
                 currentState,
                 desiredState.ScaledSize,
                 desiredState.PaddedSize,
+                desiredState.CroppedSize,
                 VideoStream.IsAnamorphicEdgeCase);
 
             currentState = scaleStep.NextState(currentState);

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -155,6 +155,8 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
         currentState = SetPad(videoInputFile, desiredState, currentState);
         // _logger.LogDebug("After pad: {PixelFormat}", currentState.PixelFormat);
 
+        currentState = SetCrop(videoInputFile, desiredState, currentState);
+
         // need to upload for hardware overlay
         bool forceSoftwareOverlay = context is { HasSubtitleOverlay: true, HasWatermark: true }
                                     || ffmpegState.VaapiDriver == "radeonsi";
@@ -482,7 +484,7 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
         FrameState desiredState,
         FrameState currentState)
     {
-        if (currentState.PaddedSize != desiredState.PaddedSize)
+        if (desiredState.CroppedSize.IsNone && currentState.PaddedSize != desiredState.PaddedSize)
         {
             IPipelineFilterStep padStep = new PadFilter(
                 currentState,
@@ -515,6 +517,7 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
                 currentState,
                 desiredState.ScaledSize,
                 desiredState.PaddedSize,
+                desiredState.CroppedSize,
                 VideoStream.IsAnamorphicEdgeCase);
         }
         else
@@ -536,6 +539,7 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
                 },
                 desiredState.ScaledSize,
                 desiredState.PaddedSize,
+                desiredState.CroppedSize,
                 VideoStream.IsAnamorphicEdgeCase);
         }
 

ErsatzTV.Infrastructure/Health/Checks/FFmpegVersionHealthCheck.cs

@@ -10,6 +10,7 @@ public class FFmpegVersionHealthCheck : BaseHealthCheck, IFFmpegVersionHealthChe
 {
     private const string BundledVersion = "N-112071-g00a837c70c";
     private const string BundledVersionVaapi = "N-112071-g00a837c70c";
+    private const string WindowsVersionPrefix = "2023-09-07-git-9c9f48e7f2";
     private readonly IConfigElementRepository _configElementRepository;
 
     public FFmpegVersionHealthCheck(IConfigElementRepository configElementRepository) =>
@@ -73,16 +74,19 @@ public class FFmpegVersionHealthCheck : BaseHealthCheck, IFFmpegVersionHealthChe
     private Option<HealthCheckResult> ValidateVersion(string version, string app)
     {
         if (version.StartsWith("3.", StringComparison.OrdinalIgnoreCase) ||
-            version.StartsWith("4.", StringComparison.OrdinalIgnoreCase))
+            version.StartsWith("4.", StringComparison.OrdinalIgnoreCase) ||
+            version.StartsWith("5.", StringComparison.OrdinalIgnoreCase))
         {
-            return FailResult($"{app} version {version} is too old; please install 6.0!");
+            return FailResult($"{app} version {version} is too old; please install 6.1 (snapshot)!");
         }
 
-        if (!version.StartsWith("6.0", StringComparison.OrdinalIgnoreCase) && version != BundledVersion &&
+        if (!version.StartsWith("6.1", StringComparison.OrdinalIgnoreCase) &&
+            !version.StartsWith(WindowsVersionPrefix, StringComparison.OrdinalIgnoreCase) &&
+            version != BundledVersion &&
             version != BundledVersionVaapi)
         {
             return WarningResult(
-                $"{app} version {version} is unexpected and may have problems; please install 6.0!");
+                $"{app} version {version} is unexpected and may have problems; please install 6.1 (snapshot)!");
         }
 
         return None;

ErsatzTV.Scanner.Tests/ErsatzTV.Scanner.Tests.csproj

@@ -10,7 +10,7 @@
 
     <ItemGroup>
         <PackageReference Include="FluentAssertions" Version="6.12.0" />
-        <PackageReference Include="LanguageExt.Core" Version="4.4.4" />
+        <PackageReference Include="LanguageExt.Core" Version="4.4.5" />
         <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
         <PackageReference Include="NSubstitute" Version="5.1.0" />
         <PackageReference Include="NUnit" Version="3.13.3" />

ErsatzTV.Scanner/ErsatzTV.Scanner.csproj

@@ -22,7 +22,7 @@
     <ItemGroup>
       <PackageReference Include="CliWrap" Version="3.6.4" />
       <PackageReference Include="Humanizer.Core" Version="2.14.1" />
-      <PackageReference Include="LanguageExt.Core" Version="4.4.4" />
+      <PackageReference Include="LanguageExt.Core" Version="4.4.5" />
       <PackageReference Include="MediatR" Version="12.1.1" />
       <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0" />
       <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />

ErsatzTV/ErsatzTV.csproj

@@ -60,7 +60,7 @@
       <PackageReference Include="FluentValidation" Version="11.7.1" />
       <PackageReference Include="FluentValidation.AspNetCore" Version="11.3.0" />
       <PackageReference Include="HtmlSanitizer" Version="8.0.692" />
-      <PackageReference Include="LanguageExt.Core" Version="4.4.4" />
+      <PackageReference Include="LanguageExt.Core" Version="4.4.5" />
       <PackageReference Include="Markdig" Version="0.33.0" />
       <PackageReference Include="MediatR.Courier.DependencyInjection" Version="5.0.0" />
       <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="7.0.11" />

ErsatzTV/Pages/FFmpegEditor.razor

@@ -46,6 +46,7 @@
                                         <MudSelect Label="Scaling Behavior" @bind-Value="_model.ScalingBehavior" For="@(() => _model.ScalingBehavior)">
                                             <MudSelectItem Value="@ScalingBehavior.ScaleAndPad">Scale and Pad</MudSelectItem>
                                             <MudSelectItem Value="@ScalingBehavior.Stretch">Stretch</MudSelectItem>
+                                            <MudSelectItem Value="@ScalingBehavior.Crop">Crop</MudSelectItem>
                                         </MudSelect>
                                     </MudElement>
                                 </MudItem>