fix placing watermarks within source content (#2297)

* fix placing watermarks within source content

* formatting

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -375,6 +375,78 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             graphicsElementContexts.AddRange(watermarks.Values);
         }
 
+        HardwareAccelerationMode hwAccel = GetHardwareAccelerationMode(playbackSettings, fillerKind);
+
+        string videoFormat = GetVideoFormat(playbackSettings);
+        Option<string> maybeVideoProfile = GetVideoProfile(videoFormat, channel.FFmpegProfile.VideoProfile);
+        Option<string> maybeVideoPreset = GetVideoPreset(hwAccel, videoFormat, channel.FFmpegProfile.VideoPreset);
+
+        Option<string> hlsPlaylistPath = outputFormat == OutputFormatKind.Hls
+            ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live.m3u8")
+            : Option<string>.None;
+
+        Option<string> hlsSegmentTemplate = outputFormat == OutputFormatKind.Hls
+            ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live%06d.ts")
+            : Option<string>.None;
+
+        FrameSize scaledSize = ffmpegVideoStream.SquarePixelFrameSize(
+            new FrameSize(channel.FFmpegProfile.Resolution.Width, channel.FFmpegProfile.Resolution.Height));
+
+        var paddedSize = new FrameSize(
+            channel.FFmpegProfile.Resolution.Width,
+            channel.FFmpegProfile.Resolution.Height);
+
+        Option<FrameSize> cropSize = Option<FrameSize>.None;
+
+        if (channel.FFmpegProfile.ScalingBehavior is ScalingBehavior.Stretch)
+        {
+            scaledSize = paddedSize;
+        }
+
+        if (channel.FFmpegProfile.ScalingBehavior is ScalingBehavior.Crop)
+        {
+            bool isTooSmallToCrop = videoVersion.Height < channel.FFmpegProfile.Resolution.Height ||
+                                    videoVersion.Width < channel.FFmpegProfile.Resolution.Width;
+
+            // if any dimension is smaller than the crop, scale beyond the crop (beyond the target resolution)
+            if (isTooSmallToCrop)
+            {
+                foreach (IDisplaySize size in playbackSettings.ScaledSize)
+                {
+                    scaledSize = new FrameSize(size.Width, size.Height);
+                }
+
+                paddedSize = scaledSize;
+            }
+            else
+            {
+                paddedSize = ffmpegVideoStream.SquarePixelFrameSizeForCrop(
+                    new FrameSize(channel.FFmpegProfile.Resolution.Width, channel.FFmpegProfile.Resolution.Height));
+            }
+
+            cropSize = new FrameSize(
+                channel.FFmpegProfile.Resolution.Width,
+                channel.FFmpegProfile.Resolution.Height);
+        }
+
+        var desiredState = new FrameState(
+            playbackSettings.RealtimeOutput,
+            fillerKind == FillerKind.Fallback,
+            videoFormat,
+            maybeVideoProfile,
+            maybeVideoPreset,
+            channel.FFmpegProfile.AllowBFrames,
+            Optional(playbackSettings.PixelFormat),
+            scaledSize,
+            paddedSize,
+            cropSize,
+            false,
+            playbackSettings.FrameRate,
+            playbackSettings.VideoBitrate,
+            playbackSettings.VideoBufferSize,
+            playbackSettings.VideoTrackTimeScale,
+            playbackSettings.Deinterlace);
+
         foreach (var playoutItemGraphicsElement in graphicsElements)
         {
             switch (playoutItemGraphicsElement.GraphicsElement.Kind)
@@ -445,6 +517,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             graphicsEngineContext = new GraphicsEngineContext(
                 audioVersion.MediaItem,
                 graphicsElementContexts,
+                new Resolution { Width = desiredState.ScaledSize.Width, Height = desiredState.ScaledSize.Height },
                 channel.FFmpegProfile.Resolution,
                 await playbackSettings.FrameRate.IfNoneAsync(24),
                 ChannelStartTime: channelStartTime,
@@ -453,78 +526,6 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
                 finish - now);
         }
 
-        HardwareAccelerationMode hwAccel = GetHardwareAccelerationMode(playbackSettings, fillerKind);
-
-        string videoFormat = GetVideoFormat(playbackSettings);
-        Option<string> maybeVideoProfile = GetVideoProfile(videoFormat, channel.FFmpegProfile.VideoProfile);
-        Option<string> maybeVideoPreset = GetVideoPreset(hwAccel, videoFormat, channel.FFmpegProfile.VideoPreset);
-
-        Option<string> hlsPlaylistPath = outputFormat == OutputFormatKind.Hls
-            ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live.m3u8")
-            : Option<string>.None;
-
-        Option<string> hlsSegmentTemplate = outputFormat == OutputFormatKind.Hls
-            ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live%06d.ts")
-            : Option<string>.None;
-
-        FrameSize scaledSize = ffmpegVideoStream.SquarePixelFrameSize(
-            new FrameSize(channel.FFmpegProfile.Resolution.Width, channel.FFmpegProfile.Resolution.Height));
-
-        var paddedSize = new FrameSize(
-            channel.FFmpegProfile.Resolution.Width,
-            channel.FFmpegProfile.Resolution.Height);
-
-        Option<FrameSize> cropSize = Option<FrameSize>.None;
-
-        if (channel.FFmpegProfile.ScalingBehavior is ScalingBehavior.Stretch)
-        {
-            scaledSize = paddedSize;
-        }
-
-        if (channel.FFmpegProfile.ScalingBehavior is ScalingBehavior.Crop)
-        {
-            bool isTooSmallToCrop = videoVersion.Height < channel.FFmpegProfile.Resolution.Height ||
-                                    videoVersion.Width < channel.FFmpegProfile.Resolution.Width;
-
-            // if any dimension is smaller than the crop, scale beyond the crop (beyond the target resolution)
-            if (isTooSmallToCrop)
-            {
-                foreach (IDisplaySize size in playbackSettings.ScaledSize)
-                {
-                    scaledSize = new FrameSize(size.Width, size.Height);
-                }
-
-                paddedSize = scaledSize;
-            }
-            else
-            {
-                paddedSize = ffmpegVideoStream.SquarePixelFrameSizeForCrop(
-                    new FrameSize(channel.FFmpegProfile.Resolution.Width, channel.FFmpegProfile.Resolution.Height));
-            }
-
-            cropSize = new FrameSize(
-                channel.FFmpegProfile.Resolution.Width,
-                channel.FFmpegProfile.Resolution.Height);
-        }
-
-        var desiredState = new FrameState(
-            playbackSettings.RealtimeOutput,
-            fillerKind == FillerKind.Fallback,
-            videoFormat,
-            maybeVideoProfile,
-            maybeVideoPreset,
-            channel.FFmpegProfile.AllowBFrames,
-            Optional(playbackSettings.PixelFormat),
-            scaledSize,
-            paddedSize,
-            cropSize,
-            false,
-            playbackSettings.FrameRate,
-            playbackSettings.VideoBitrate,
-            playbackSettings.VideoBufferSize,
-            playbackSettings.VideoTrackTimeScale,
-            playbackSettings.Deinterlace);
-
         var ffmpegState = new FFmpegState(
             saveReports,
             hwAccel,

ErsatzTV.Core/Interfaces/Streaming/GraphicsEngineContext.cs

@@ -7,6 +7,7 @@ namespace ErsatzTV.Core.Interfaces.Streaming;
 public record GraphicsEngineContext(
     MediaItem MediaItem,
     List<GraphicsElementContext> Elements,
+    Resolution SquarePixelFrameSize,
     Resolution FrameSize,
     int FrameRate,
     DateTimeOffset ChannelStartTime,

ErsatzTV.Infrastructure/Streaming/GraphicsEngine.cs

@@ -61,7 +61,8 @@ public class GraphicsEngine(ITemplateDataRepository templateDataRepository, ILog
         }
 
         // initialize all elements
-        await Task.WhenAll(elements.Select(e => e.InitializeAsync(context.FrameSize, context.FrameRate, cancellationToken)));
+        await Task.WhenAll(elements.Select(e =>
+            e.InitializeAsync(context.SquarePixelFrameSize, context.FrameSize, context.FrameRate, cancellationToken)));
 
         long frameCount = 0;
         var totalFrames = (long)(context.Duration.TotalSeconds * context.FrameRate);

ErsatzTV.Infrastructure/Streaming/IGraphicsElement.cs

@@ -8,7 +8,11 @@ public interface IGraphicsElement
 
     bool IsFailed { get; set; }
 
-    Task InitializeAsync(Resolution frameSize, int frameRate, CancellationToken cancellationToken);
+    Task InitializeAsync(
+        Resolution squarePixelFrameSize,
+        Resolution frameSize,
+        int frameRate,
+        CancellationToken cancellationToken);
 
     ValueTask<Option<PreparedElementImage>> PrepareImage(
         TimeSpan timeOfDay,

ErsatzTV.Infrastructure/Streaming/ImageElement.cs

@@ -24,7 +24,11 @@ public class ImageElement(ImageGraphicsElement imageGraphicsElement, ILogger log
 
     public bool IsFailed { get; set; }
 
-    public async Task InitializeAsync(Resolution frameSize, int frameRate, CancellationToken cancellationToken)
+    public async Task InitializeAsync(
+        Resolution squarePixelFrameSize,
+        Resolution frameSize,
+        int frameRate,
+        CancellationToken cancellationToken)
     {
         try
         {

ErsatzTV.Infrastructure/Streaming/TextElement.cs

@@ -24,7 +24,11 @@ public class TextElement(TextGraphicsElement textElement, Dictionary<string, obj
 
     public bool IsFailed { get; set; }
 
-    public async Task InitializeAsync(Resolution frameSize, int frameRate, CancellationToken cancellationToken)
+    public async Task InitializeAsync(
+        Resolution squarePixelFrameSize,
+        Resolution frameSize,
+        int frameRate,
+        CancellationToken cancellationToken)
     {
         try
         {

ErsatzTV.Infrastructure/Streaming/WatermarkElement.cs

@@ -46,7 +46,7 @@ public class WatermarkElement : IGraphicsElement, IDisposable
 
     public bool IsFailed { get; set; }
 
-    public async Task InitializeAsync(Resolution frameSize, int frameRate, CancellationToken cancellationToken)
+    public async Task InitializeAsync(Resolution squarePixelFrameSize, Resolution frameSize, int frameRate, CancellationToken cancellationToken)
     {
         try
         {
@@ -102,8 +102,9 @@ public class WatermarkElement : IGraphicsElement, IDisposable
                 scaledHeight = (int)(scaledWidth * aspectRatio);
             }
 
-            int horizontalMargin = (int)Math.Round(_watermark.HorizontalMarginPercent / 100.0 * frameSize.Width);
+            (int horizontalMargin, int verticalMargin) = _watermark.PlaceWithinSourceContent
-            int verticalMargin = (int)Math.Round(_watermark.VerticalMarginPercent / 100.0 * frameSize.Height);
+                ? SourceContentMargins(squarePixelFrameSize, frameSize)
+                : NormalMargins(frameSize);
 
             _location = CalculatePosition(
                 _watermark.Location,
@@ -192,8 +193,6 @@ public class WatermarkElement : IGraphicsElement, IDisposable
         int horizontalMargin,
         int verticalMargin)
     {
-        // TODO: source content margins
-
         return location switch
         {
             WatermarkLocation.BottomLeft => new Point(horizontalMargin, frameHeight - imageHeight - verticalMargin),
@@ -213,6 +212,31 @@ public class WatermarkElement : IGraphicsElement, IDisposable
         };
     }
 
+    private WatermarkMargins NormalMargins(Resolution frameSize)
+    {
+        double horizontalMargin = Math.Round(_watermark.HorizontalMarginPercent / 100.0 * frameSize.Width);
+        double verticalMargin = Math.Round(_watermark.VerticalMarginPercent / 100.0 * frameSize.Height);
+
+        return new WatermarkMargins((int)Math.Round(horizontalMargin), (int)Math.Round(verticalMargin));
+    }
+
+    private WatermarkMargins SourceContentMargins(Resolution squarePixelFrameSize, Resolution frameSize)
+    {
+        int horizontalPadding = frameSize.Width - squarePixelFrameSize.Width;
+        int verticalPadding = frameSize.Height - squarePixelFrameSize.Height;
+
+        double horizontalMargin = Math.Round(
+            _watermark.HorizontalMarginPercent / 100.0 * squarePixelFrameSize.Width
+            + horizontalPadding / 2.0);
+        double verticalMargin = Math.Round(
+            _watermark.VerticalMarginPercent / 100.0 * squarePixelFrameSize.Height
+            + verticalPadding / 2.0);
+
+        return new WatermarkMargins((int)Math.Round(horizontalMargin), (int)Math.Round(verticalMargin));
+    }
+
+    private sealed record WatermarkMargins(int HorizontalMargin, int VerticalMargin);
+
     public void Dispose()
     {
         GC.SuppressFinalize(this);