prioritize container aspect ratio over stream aspect ratio (#930)

* prioritize container aspect ratio over stream aspect ratio

* use setdar filter

CHANGELOG.md

@@ -6,8 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [Unreleased]
 ### Fixed
 - Use MIME Type `application/x-mpegurl` for all playlists instead of `application/vnd.apple.mpegurl`
-- Remove `setsar` filter which caused issues scaling between two different aspect ratios
-  - For example, some 4:3 content would appear stretched when scaled to a 16:9 resolution
+- Replace `setsar` filter with `setdar` filter
+  - `setsar` caused issues scaling between two different aspect ratios
+    - For example, some 4:3 content would appear stretched when scaled to a 16:9 resolution
+  - `setdar` is now only used when aspect ratios match 
+- Prioritize aspect ratio from container when video stream contains conflicting aspect ratio
+  - This is usually caused by bad authoring, but the change should improve scaling behavior for edge cases
 
 ### Added
 - Support DSD audio file formats (DFF and DSF) in local song libraries

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -140,6 +140,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             videoStream.Codec,
             AvailablePixelFormats.ForPixelFormat(videoStream.PixelFormat, _logger),
             new FrameSize(videoVersion.Width, videoVersion.Height),
+            videoVersion.DisplayAspectRatio,
             videoVersion.RFrameRate,
             videoPath != audioPath); // still image when paths are different
 
@@ -209,6 +210,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             await playbackSettings.ScaledSize.Map(ss => new FrameSize(ss.Width, ss.Height))
                 .IfNoneAsync(new FrameSize(videoVersion.Width, videoVersion.Height)),
             new FrameSize(channel.FFmpegProfile.Resolution.Width, channel.FFmpegProfile.Resolution.Height),
+            channel.FFmpegProfile.Resolution.Width == 640 ? "4:3" : "16:9",
             playbackSettings.FrameRate,
             playbackSettings.VideoBitrate,
             playbackSettings.VideoBufferSize,
@@ -302,6 +304,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             new PixelFormatYuv420P(),
             new FrameSize(desiredResolution.Width, desiredResolution.Height),
             new FrameSize(desiredResolution.Width, desiredResolution.Height),
+            desiredResolution.Width == 640 ? "4:3" : "16:9",
             playbackSettings.FrameRate,
             playbackSettings.VideoBitrate,
             playbackSettings.VideoBufferSize,
@@ -329,6 +332,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             VideoFormat.GeneratedImage,
             new PixelFormatUnknown(), // leave this unknown so we convert to desired yuv420p
             new FrameSize(videoVersion.Width, videoVersion.Height),
+            videoVersion.DisplayAspectRatio,
             None,
             true);
 
@@ -417,7 +421,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
     {
         var videoInputFile = new VideoInputFile(
             inputFile,
-            new List<VideoStream> { new(0, string.Empty, None, FrameSize.Unknown, None, true) });
+            new List<VideoStream> { new(0, string.Empty, None, FrameSize.Unknown, string.Empty, None, true) });
 
         var pipelineBuilder = new PipelineBuilder(
             await _hardwareCapabilitiesFactory.GetHardwareCapabilities(ffmpegPath, HardwareAccelerationMode.None),
@@ -497,6 +501,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
                                     "unknown",
                                     new PixelFormatUnknown(),
                                     new FrameSize(1, 1),
+                                    string.Empty,
                                     Option<string>.None,
                                     !options.IsAnimated)
                             },

ErsatzTV.Core/Metadata/LocalStatisticsProvider.cs

@@ -3,6 +3,8 @@ using System.Globalization;
 using System.Text;
 using System.Text.RegularExpressions;
 using Bugsnag;
+using CliWrap;
+using CliWrap.Buffered;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Extensions;
 using ErsatzTV.Core.Interfaces.Metadata;
@@ -157,42 +159,48 @@ public class LocalStatisticsProvider : ILocalStatisticsProvider
         return await _metadataRepository.UpdateLocalStatistics(mediaItem, version) && durationChange;
     }
 
-    private Task<Either<BaseError, FFprobe>> GetProbeOutput(string ffprobePath, string filePath)
+    private async Task<Either<BaseError, FFprobe>> GetProbeOutput(string ffprobePath, string filePath)
     {
-        var startInfo = new ProcessStartInfo
+        string[] arguments =
         {
-            FileName = ffprobePath,
-            RedirectStandardOutput = true,
-            RedirectStandardError = true,
-            UseShellExecute = false,
-            StandardOutputEncoding = Encoding.UTF8,
-            StandardErrorEncoding = Encoding.UTF8
+            "-hide_banner",
+            "-print_format", "json",
+            "-show_format",
+            "-show_streams",
+            "-show_chapters",
+            "-i", filePath
         };
 
-        startInfo.ArgumentList.Add("-v");
-        startInfo.ArgumentList.Add("quiet");
-        startInfo.ArgumentList.Add("-print_format");
-        startInfo.ArgumentList.Add("json");
-        startInfo.ArgumentList.Add("-show_format");
-        startInfo.ArgumentList.Add("-show_streams");
-        startInfo.ArgumentList.Add("-show_chapters");
-        startInfo.ArgumentList.Add("-i");
-        startInfo.ArgumentList.Add(filePath);
-
-        var probe = new Process
+        BufferedCommandResult probe = await Cli.Wrap(ffprobePath)
+            .WithArguments(arguments)
+            .WithValidation(CommandResultValidation.None)
+            .ExecuteBufferedAsync(Encoding.UTF8);
+
+        if (probe.ExitCode != 0)
         {
-            StartInfo = startInfo
-        };
+            return BaseError.New($"FFprobe at {ffprobePath} exited with code {probe.ExitCode}");
+        }
 
-        probe.Start();
-        return probe.StandardOutput.ReadToEndAsync().MapAsync<string, Either<BaseError, FFprobe>>(
-            async output =>
+        FFprobe ffprobe = JsonConvert.DeserializeObject<FFprobe>(probe.StandardOutput);
+
+        if (ffprobe != null)
+        {
+            const string PATTERN = @"\[SAR\s+([0-9]+:[0-9]+)\s+DAR\s+([0-9]+:[0-9]+)\]";
+            Match match = Regex.Match(probe.StandardError, PATTERN);
+            if (match.Success)
             {
-                await probe.WaitForExitAsync();
-                return probe.ExitCode == 0
-                    ? JsonConvert.DeserializeObject<FFprobe>(output)
-                    : BaseError.New($"FFprobe at {ffprobePath} exited with code {probe.ExitCode}");
-            });
+                string sar = match.Groups[1].Value;
+                string dar = match.Groups[2].Value;
+                foreach (FFprobeStream stream in ffprobe.streams.Where(s => s.codec_type == "video").ToList())
+                {
+                    FFprobeStream replacement = stream with { sample_aspect_ratio = sar, display_aspect_ratio = dar };
+                    ffprobe.streams.Remove(stream);
+                    ffprobe.streams.Add(replacement);
+                }
+            }
+        }
+
+        return ffprobe;
     }
 
     private async Task AnalyzeDuration(string ffmpegPath, string path, MediaVersion version)

ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs

@@ -25,7 +25,7 @@ public class PipelineGeneratorTests
         var videoInputFile = new VideoInputFile(
             "/tmp/whatever.mkv",
             new List<VideoStream>
-                { new(0, VideoFormat.H264, new PixelFormatYuv420P(), new FrameSize(1920, 1080), "24", false) });
+                { new(0, VideoFormat.H264, new PixelFormatYuv420P(), new FrameSize(1920, 1080), "16:9", "24", false) });
 
         var audioInputFile = new AudioInputFile(
             "/tmp/whatever.mkv",
@@ -46,6 +46,7 @@ public class PipelineGeneratorTests
             new PixelFormatYuv420P(),
             new FrameSize(1920, 1080),
             new FrameSize(1920, 1080),
+            "16:9",
             Option<int>.None,
             2000,
             4000,
@@ -95,7 +96,7 @@ public class PipelineGeneratorTests
         var videoInputFile = new VideoInputFile(
             "/tmp/whatever.mkv",
             new List<VideoStream>
-                { new(0, VideoFormat.H264, new PixelFormatYuv420P(), new FrameSize(1920, 1080), "24", false) });
+                { new(0, VideoFormat.H264, new PixelFormatYuv420P(), new FrameSize(1920, 1080), "16:9", "24", false) });
 
         var audioInputFile = new AudioInputFile(
             "/tmp/whatever.mkv",
@@ -116,6 +117,7 @@ public class PipelineGeneratorTests
             new PixelFormatYuv420P(),
             new FrameSize(1920, 1080),
             new FrameSize(1920, 1080),
+            "16:9",
             Option<int>.None,
             2000,
             4000,
@@ -182,7 +184,7 @@ public class PipelineGeneratorTests
         var videoInputFile = new VideoInputFile(
             "/tmp/whatever.mkv",
             new List<VideoStream>
-                { new(0, VideoFormat.H264, new PixelFormatYuv420P(), new FrameSize(1920, 1080), "24", false) });
+                { new(0, VideoFormat.H264, new PixelFormatYuv420P(), new FrameSize(1920, 1080), "16:9", "24", false) });
 
         var audioInputFile = new AudioInputFile(
             "/tmp/whatever.mkv",
@@ -203,6 +205,7 @@ public class PipelineGeneratorTests
             new PixelFormatYuv420P(),
             new FrameSize(1920, 1080),
             new FrameSize(1920, 1080),
+            "16:9",
             Option<int>.None,
             2000,
             4000,
@@ -257,7 +260,7 @@ public class PipelineGeneratorTests
             "/test/input/file.png",
             new List<VideoStream>
             {
-                new(0, string.Empty, Option<IPixelFormat>.None, FrameSize.Unknown, Option<string>.None, true)
+                new(0, string.Empty, Option<IPixelFormat>.None, FrameSize.Unknown, string.Empty, Option<string>.None, true)
             });
 
         var pipelineBuilder = new PipelineBuilder(

ErsatzTV.FFmpeg/Filter/SetDarFilter.cs

@@ -0,0 +1,14 @@
+namespace ErsatzTV.FFmpeg.Filter;
+
+public class SetDarFilter : BaseFilter
+{
+    private readonly string _displayAspectRatio;
+
+    public SetDarFilter(string displayAspectRatio)
+    {
+        _displayAspectRatio = displayAspectRatio;
+    }
+
+    public override string Filter => $"setdar=dar={_displayAspectRatio.Replace(':', '/')}";
+    public override FrameState NextState(FrameState currentState) => currentState;
+}

ErsatzTV.FFmpeg/Filter/SetSarFilter.cs

@@ -1,7 +0,0 @@
-namespace ErsatzTV.FFmpeg.Filter;
-
-public class SetSarFilter : BaseFilter
-{
-    public override string Filter => "setsar=1";
-    public override FrameState NextState(FrameState currentState) => currentState;
-}

ErsatzTV.FFmpeg/FrameState.cs

@@ -9,6 +9,7 @@ public record FrameState(
     Option<IPixelFormat> PixelFormat,
     FrameSize ScaledSize,
     FrameSize PaddedSize,
+    string DisplayAspectRatio,
     Option<int> FrameRate,
     Option<int> VideoBitrate,
     Option<int> VideoBufferSize,

ErsatzTV.FFmpeg/InputFile.cs

@@ -19,6 +19,7 @@ public record ConcatInputFile(string Url, FrameSize Resolution) : InputFile(
             string.Empty,
             Option<IPixelFormat>.None,
             Resolution,
+            string.Empty,
             Option<string>.None,
             false)
     })

ErsatzTV.FFmpeg/MediaStream.cs

@@ -14,6 +14,7 @@ public record VideoStream(
     string Codec,
     Option<IPixelFormat> PixelFormat,
     FrameSize FrameSize,
+    string DisplayAspectRatio,
     Option<string> FrameRate,
     bool StillImage) : MediaStream(
     Index,

ErsatzTV.FFmpeg/PipelineBuilder.cs

@@ -188,6 +188,7 @@ public class PipelineBuilder
                 videoStream.PixelFormat,
                 videoStream.FrameSize,
                 videoStream.FrameSize,
+                videoStream.DisplayAspectRatio,
                 initialFrameRate,
                 Option<int>.None,
                 Option<int>.None,
@@ -379,6 +380,13 @@ public class PipelineBuilder
                         IPipelineFilterStep padStep = new PadFilter(currentState, desiredState.PaddedSize);
                         currentState = padStep.NextState(currentState);
                         _videoInputFile.Iter(f => f.FilterSteps.Add(padStep));
+
+                        if (videoStream.DisplayAspectRatio == desiredState.DisplayAspectRatio)
+                        {
+                            IPipelineFilterStep darStep = new SetDarFilter(desiredState.DisplayAspectRatio);
+                            currentState = darStep.NextState(currentState);
+                            _videoInputFile.Iter(f => f.FilterSteps.Add(darStep));
+                        }
                     }
                 }
                 else if (currentState.ScaledSize != desiredState.ScaledSize)
@@ -398,6 +406,13 @@ public class PipelineBuilder
                         currentState = padStep.NextState(currentState);
                         _videoInputFile.Iter(f => f.FilterSteps.Add(padStep));
                     }
+                    
+                    if (videoStream.DisplayAspectRatio == desiredState.DisplayAspectRatio)
+                    {
+                        IPipelineFilterStep darStep = new SetDarFilter(desiredState.DisplayAspectRatio);
+                        currentState = darStep.NextState(currentState);
+                        _videoInputFile.Iter(f => f.FilterSteps.Add(darStep));
+                    }
                 }
                 else if (currentState.PaddedSize != desiredState.PaddedSize)
                 {
@@ -415,6 +430,13 @@ public class PipelineBuilder
                         currentState = padStep.NextState(currentState);
                         _videoInputFile.Iter(f => f.FilterSteps.Add(padStep));
                     }
+                    
+                    if (videoStream.DisplayAspectRatio == desiredState.DisplayAspectRatio)
+                    {
+                        IPipelineFilterStep darStep = new SetDarFilter(desiredState.DisplayAspectRatio);
+                        currentState = darStep.NextState(currentState);
+                        _videoInputFile.Iter(f => f.FilterSteps.Add(darStep));
+                    }
                 }
 
                 if (hasOverlay && currentState.PixelFormat.Map(pf => pf.FFmpegName) !=