From 4f02bedf696a8fd6a15085dc834c25722fb1b7b5 Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Fri, 15 Aug 2025 16:30:54 -0500 Subject: [PATCH] fix image loading regression in graphics engine (#2322) --- .../ErsatzTV.Infrastructure.csproj | 1 + .../Streaming/Graphics/Image/ImageElement.cs | 135 +--------- .../Graphics/Image/ImageElementBase.cs | 234 ++++++++++++++++++ .../Graphics/Image/WatermarkElement.cs | 165 ++---------- 4 files changed, 265 insertions(+), 270 deletions(-) create mode 100644 ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElementBase.cs diff --git a/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj b/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj index ea7ec6d92..af5b0d1e2 100644 --- a/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj +++ b/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj @@ -35,6 +35,7 @@ + diff --git a/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElement.cs b/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElement.cs index d7ede7f0d..4278b5e0d 100644 --- a/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElement.cs +++ b/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElement.cs @@ -6,16 +6,10 @@ using SkiaSharp; namespace ErsatzTV.Infrastructure.Streaming.Graphics; -public class ImageElement(ImageGraphicsElement imageGraphicsElement, ILogger logger) : GraphicsElement, IDisposable +public class ImageElement(ImageGraphicsElement imageGraphicsElement, ILogger logger) : ImageElementBase { - private readonly List _scaledFrames = []; - private readonly List _frameDelays = []; - private Option _maybeOpacityExpression; private float _opacity; - private int _animatedDurationMs; - private SKCodec _sourceCodec; - private SKPointI _location; public override async Task InitializeAsync( Resolution squarePixelFrameSize, @@ -43,90 +37,17 @@ public class ImageElement(ImageGraphicsElement imageGraphicsElement, ILogger log expression.EvaluateFunction += OpacityExpressionHelper.EvaluateFunction; } - Stream imageStream; - bool isRemoteUri = Uri.TryCreate(imageGraphicsElement.Image, UriKind.Absolute, out var uriResult) - && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps); - - if (isRemoteUri) - { - using var client = new HttpClient(); - imageStream = new MemoryStream(await client.GetByteArrayAsync(uriResult, cancellationToken)); - } - else - { - imageStream = new FileStream(imageGraphicsElement.Image!, FileMode.Open, FileAccess.Read); - } - - _sourceCodec = SKCodec.Create(imageStream); - - int sourceWidth = _sourceCodec.Info.Width; - int sourceHeight = _sourceCodec.Info.Height; - - int scaledWidth = sourceWidth; - int scaledHeight = sourceHeight; - if (imageGraphicsElement.Scale) - { - scaledWidth = (int)Math.Round((imageGraphicsElement.ScaleWidthPercent ?? 100) / 100.0 * frameSize.Width); - double aspectRatio = (double)sourceHeight / sourceWidth; - scaledHeight = (int)(scaledWidth * aspectRatio); - } - - int horizontalMargin = (int)Math.Round((imageGraphicsElement.HorizontalMarginPercent ?? 0) / 100.0 * frameSize.Width); - int verticalMargin = (int)Math.Round((imageGraphicsElement.VerticalMarginPercent ?? 0) / 100.0 * frameSize.Height); - - _location = CalculatePosition( + await LoadImage( + squarePixelFrameSize, + frameSize, + imageGraphicsElement.Image, imageGraphicsElement.Location, - frameSize.Width, - frameSize.Height, - scaledWidth, - scaledHeight, - horizontalMargin, - verticalMargin); - - _animatedDurationMs = 0; - - var scaledImageInfo = new SKImageInfo(scaledWidth, scaledHeight, SKColorType.Bgra8888, SKAlphaType.Premul); - - if (_sourceCodec.FrameCount == 0) - { - // static image - using var frameBitmap = SKBitmap.Decode(_sourceCodec); - if (frameBitmap != null) - { - var scaledBitmap = new SKBitmap(scaledImageInfo); - frameBitmap.ScalePixels(scaledBitmap, SKSamplingOptions.Default); - _scaledFrames.Add(scaledBitmap); - } - } - else - { - // animated image - for (var i = 0; i < _sourceCodec.FrameCount; i++) - { - _sourceCodec.GetFrameInfo(i, out var frameInfo); - int frameDuration = frameInfo.Duration; - if (frameDuration == 0) - { - frameDuration = 100; - } - - using var frameBitmap = new SKBitmap(_sourceCodec.Info); - var pointer = frameBitmap.GetPixels(); - _sourceCodec.GetPixels(_sourceCodec.Info, pointer, new SKCodecOptions(i)); - - var scaledBitmap = new SKBitmap(scaledImageInfo); - frameBitmap.ScalePixels(scaledBitmap, SKSamplingOptions.Default); - _scaledFrames.Add(scaledBitmap); - - _animatedDurationMs += frameDuration; - _frameDelays.Add(frameDuration); - } - } - - if (_sourceCodec.FrameCount > 0 && _animatedDurationMs == 0) - { - _animatedDurationMs = int.MaxValue; - } + imageGraphicsElement.Scale, + imageGraphicsElement.ScaleWidthPercent, + imageGraphicsElement.HorizontalMarginPercent, + imageGraphicsElement.VerticalMarginPercent, + placeWithinSourceContent: false, + cancellationToken); } catch (Exception ex) { @@ -159,36 +80,6 @@ public class ImageElement(ImageGraphicsElement imageGraphicsElement, ILogger log } SKBitmap frameForTimestamp = GetFrameForTimestamp(contentTime); - return ValueTask.FromResult(Optional(new PreparedElementImage(frameForTimestamp, _location, opacity, false))); - } - - private SKBitmap GetFrameForTimestamp(TimeSpan timestamp) - { - if (_scaledFrames.Count <= 1) - { - return _scaledFrames[0]; - } - - long currentTimeMs = (long)timestamp.TotalMilliseconds % _animatedDurationMs; - - long frameTime = 0; - for (var i = 0; i < _sourceCodec.FrameCount; i++) - { - frameTime += _frameDelays[i]; - if (currentTimeMs <= frameTime) - { - return _scaledFrames[i]; - } - } - - return _scaledFrames.Last(); - } - - public void Dispose() - { - GC.SuppressFinalize(this); - - _sourceCodec?.Dispose(); - _scaledFrames?.ForEach(f => f.Dispose()); + return ValueTask.FromResult(Optional(new PreparedElementImage(frameForTimestamp, Location, opacity, false))); } -} +} \ No newline at end of file diff --git a/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElementBase.cs b/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElementBase.cs new file mode 100644 index 000000000..f75570c07 --- /dev/null +++ b/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElementBase.cs @@ -0,0 +1,234 @@ +using System.Runtime.InteropServices; +using ErsatzTV.Core.Domain; +using ErsatzTV.FFmpeg.State; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.Formats.Gif; +using SixLabors.ImageSharp.Formats.Png; +using SixLabors.ImageSharp.Formats.Webp; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Processing; +using SkiaSharp; +using Image = SixLabors.ImageSharp.Image; + +namespace ErsatzTV.Infrastructure.Streaming.Graphics; + +public abstract class ImageElementBase : GraphicsElement, IDisposable +{ + private readonly List _scaledFrames = []; + private readonly List _frameDelays = []; + + private Image _sourceImage; + private double _animatedDurationSeconds; + + protected SKPointI Location { get; private set; } + + protected async Task LoadImage( + Resolution squarePixelFrameSize, + Resolution frameSize, + string image, + WatermarkLocation location, + bool scale, + double? scaleWidthPercent, + double? horizontalMarginPercent, + double? verticalMarginPercent, + bool placeWithinSourceContent, + CancellationToken cancellationToken) + { + bool isRemoteUri = Uri.TryCreate(image, UriKind.Absolute, out var uriResult) + && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps); + + if (isRemoteUri) + { + using var client = new HttpClient(); + await using Stream imageStream = await client.GetStreamAsync(uriResult, cancellationToken); + _sourceImage = await Image.LoadAsync(imageStream, cancellationToken); + } + else + { + _sourceImage = await Image.LoadAsync(image!, cancellationToken); + } + + int scaledWidth = _sourceImage.Width; + int scaledHeight = _sourceImage.Height; + if (scale) + { + scaledWidth = (int)Math.Round((scaleWidthPercent ?? 100) / 100.0 * frameSize.Width); + double aspectRatio = (double)_sourceImage.Height / _sourceImage.Width; + scaledHeight = (int)(scaledWidth * aspectRatio); + } + + (int horizontalMargin, int verticalMargin) = placeWithinSourceContent + ? SourceContentMargins( + squarePixelFrameSize, + frameSize, + horizontalMarginPercent ?? 0, + verticalMarginPercent ?? 0) + : NormalMargins(frameSize, horizontalMarginPercent ?? 0, verticalMarginPercent ?? 0); + + Location = CalculatePosition( + location, + frameSize.Width, + frameSize.Height, + scaledWidth, + scaledHeight, + horizontalMargin, + verticalMargin); + + _animatedDurationSeconds = 0; + + for (int i = 0; i < _sourceImage.Frames.Count; i++) + { + Image frame = _sourceImage.Frames.CloneFrame(i); + frame.Mutate(ctx => ctx.Resize(scaledWidth, scaledHeight)); + _scaledFrames.Add(ToSkiaBitmap(frame)); + + var frameDelay = GetFrameDelaySeconds(_sourceImage, i); + _animatedDurationSeconds += frameDelay; + _frameDelays.Add(frameDelay); + } + } + + protected static async Task GetImageStream(string image, CancellationToken cancellationToken) + { + Stream imageStream; + + bool isRemoteUri = Uri.TryCreate(image, UriKind.Absolute, out var uriResult) + && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps); + + if (isRemoteUri) + { + using var client = new HttpClient(); + imageStream = new MemoryStream(await client.GetByteArrayAsync(uriResult, cancellationToken)); + } + else + { + imageStream = new FileStream(image!, FileMode.Open, FileAccess.Read); + } + + return imageStream; + } + + protected static SKBitmap ToSkiaBitmap(Image image) + { + // Force a known pixel format + using Image rgbaImage = image.CloneAs(); + + int width = rgbaImage.Width; + int height = rgbaImage.Height; + + // Allocate destination SKBitmap with RGBA8888 (straight alpha) + var info = new SKImageInfo(width, height, SKColorType.Rgba8888, SKAlphaType.Unpremul); + var skBitmap = new SKBitmap(info); + if (!skBitmap.TryAllocPixels(info)) + { + skBitmap.Dispose(); + throw new InvalidOperationException("Failed to allocate pixels for SKBitmap."); + } + + // Pull pixel data from ImageSharp into a managed byte[] + // (Rgba32 is 4 bytes, so length = width * height * 4) + var pixelArray = new Rgba32[width * height]; + rgbaImage.CopyPixelDataTo(pixelArray); + + // Convert the Rgba32[] to a byte[] without unsafe code + var bytes = new byte[pixelArray.Length * 4]; + MemoryMarshal.AsBytes(pixelArray.AsSpan()).CopyTo(bytes); + + // Copy into Skia's pixel buffer + IntPtr dstPtr = skBitmap.GetPixels(out _); + Marshal.Copy(bytes, 0, dstPtr, bytes.Length); + + return skBitmap; + } + + protected static double GetFrameDelaySeconds(Image image, int frameIndex) + { + var format = image.Metadata.DecodedImageFormat; + var frameMeta = image.Frames[frameIndex].Metadata; + + if (format == GifFormat.Instance) + { + // GIF frame delay is in hundredths of a second + var gifMeta = frameMeta.GetFormatMetadata(GifFormat.Instance); + return gifMeta.FrameDelay / 100.0; + } + + if (format == PngFormat.Instance) + { + // PNG animated frame delay is in seconds (as double) + var pngMeta = frameMeta.GetFormatMetadata(PngFormat.Instance); + return pngMeta.FrameDelay.ToDouble(); + } + + if (format == WebpFormat.Instance) + { + // WEBP animated frame delay is in milliseconds + var webpMeta = frameMeta.GetFormatMetadata(WebpFormat.Instance); + return webpMeta.FrameDelay / 1000.0; + } + + // Default: assume 1/60th second (~16.67 ms) if unknown + return 1.0 / 60.0; + } + + protected SKBitmap GetFrameForTimestamp(TimeSpan timestamp) + { + if (_scaledFrames.Count <= 1) + { + return _scaledFrames[0]; + } + + double currentTime = timestamp.TotalSeconds % _animatedDurationSeconds; + + double frameTime = 0; + for (int i = 0; i < _sourceImage.Frames.Count; i++) + { + frameTime += _frameDelays[i]; + if (currentTime <= frameTime) + { + return _scaledFrames[i]; + } + } + + return _scaledFrames.Last(); + } + + private static WatermarkMargins NormalMargins( + Resolution frameSize, + double horizontalMarginPercent, + double verticalMarginPercent) + { + double horizontalMargin = Math.Round(horizontalMarginPercent / 100.0 * frameSize.Width); + double verticalMargin = Math.Round(verticalMarginPercent / 100.0 * frameSize.Height); + + return new WatermarkMargins((int)Math.Round(horizontalMargin), (int)Math.Round(verticalMargin)); + } + + private static WatermarkMargins SourceContentMargins( + Resolution squarePixelFrameSize, + Resolution frameSize, + double horizontalMarginPercent, + double verticalMarginPercent) + { + int horizontalPadding = frameSize.Width - squarePixelFrameSize.Width; + int verticalPadding = frameSize.Height - squarePixelFrameSize.Height; + + double horizontalMargin = Math.Round( + horizontalMarginPercent / 100.0 * squarePixelFrameSize.Width + + horizontalPadding / 2.0); + double verticalMargin = Math.Round( + verticalMarginPercent / 100.0 * squarePixelFrameSize.Height + + verticalPadding / 2.0); + + return new WatermarkMargins((int)Math.Round(horizontalMargin), (int)Math.Round(verticalMargin)); + } + + private sealed record WatermarkMargins(int HorizontalMargin, int VerticalMargin); + + public virtual void Dispose() + { + GC.SuppressFinalize(this); + _sourceImage?.Dispose(); + _scaledFrames?.ForEach(f => f.Dispose()); + } +} diff --git a/ErsatzTV.Infrastructure/Streaming/Graphics/Image/WatermarkElement.cs b/ErsatzTV.Infrastructure/Streaming/Graphics/Image/WatermarkElement.cs index 0c4a0c4c1..2df0608dc 100644 --- a/ErsatzTV.Infrastructure/Streaming/Graphics/Image/WatermarkElement.cs +++ b/ErsatzTV.Infrastructure/Streaming/Graphics/Image/WatermarkElement.cs @@ -7,19 +7,14 @@ using SkiaSharp; namespace ErsatzTV.Infrastructure.Streaming.Graphics; -public class WatermarkElement : GraphicsElement, IDisposable +public class WatermarkElement : ImageElementBase { private readonly ILogger _logger; private readonly string _imagePath; private readonly ChannelWatermark _watermark; - private readonly List _scaledFrames = []; - private readonly List _frameDelays = []; private Option _maybeOpacityExpression; private float _opacity; - private int _animatedDurationMs; - private SKCodec _sourceCodec; - private SKPointI _location; public WatermarkElement(WatermarkOptions watermarkOptions, ILogger logger) { @@ -39,7 +34,11 @@ public class WatermarkElement : GraphicsElement, IDisposable public bool IsValid => _imagePath != null && _watermark != null; - public override async Task InitializeAsync(Resolution squarePixelFrameSize, Resolution frameSize, int frameRate, CancellationToken cancellationToken) + public override async Task InitializeAsync( + Resolution squarePixelFrameSize, + Resolution frameSize, + int frameRate, + CancellationToken cancellationToken) { try { @@ -72,92 +71,17 @@ public class WatermarkElement : GraphicsElement, IDisposable expression.EvaluateFunction += OpacityExpressionHelper.EvaluateFunction; } - Stream imageStream; - bool isRemoteUri = Uri.TryCreate(_imagePath, UriKind.Absolute, out var uriResult) - && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps); - - if (isRemoteUri) - { - using var client = new HttpClient(); - imageStream = new MemoryStream(await client.GetByteArrayAsync(uriResult, cancellationToken)); - } - else - { - imageStream = new FileStream(_imagePath!, FileMode.Open, FileAccess.Read); - } - - _sourceCodec = SKCodec.Create(imageStream); - - int sourceWidth = _sourceCodec.Info.Width; - int sourceHeight = _sourceCodec.Info.Height; - - int scaledWidth = sourceWidth; - int scaledHeight = sourceHeight; - if (_watermark.Size == WatermarkSize.Scaled) - { - scaledWidth = (int)Math.Round(_watermark.WidthPercent / 100.0 * frameSize.Width); - double aspectRatio = (double)sourceHeight / sourceWidth; - scaledHeight = (int)(scaledWidth * aspectRatio); - } - - (int horizontalMargin, int verticalMargin) = _watermark.PlaceWithinSourceContent - ? SourceContentMargins(squarePixelFrameSize, frameSize) - : NormalMargins(frameSize); - - var location = CalculatePosition( + await LoadImage( + squarePixelFrameSize, + frameSize, + _imagePath, _watermark.Location, - frameSize.Width, - frameSize.Height, - scaledWidth, - scaledHeight, - horizontalMargin, - verticalMargin); - _location = new SKPointI(location.X, location.Y); - - _animatedDurationMs = 0; - - var scaledImageInfo = new SKImageInfo(scaledWidth, scaledHeight, SKColorType.Bgra8888, SKAlphaType.Premul); - - if (_sourceCodec.FrameCount == 0) - { - // static image - using var frameBitmap = SKBitmap.Decode(_sourceCodec); - if (frameBitmap != null) - { - var scaledBitmap = new SKBitmap(scaledImageInfo); - frameBitmap.ScalePixels(scaledBitmap, SKSamplingOptions.Default); - _scaledFrames.Add(scaledBitmap); - } - } - else - { - // animated image - for (var i = 0; i < _sourceCodec.FrameCount; i++) - { - _sourceCodec.GetFrameInfo(i, out var frameInfo); - int frameDuration = frameInfo.Duration; - if (frameDuration == 0) - { - frameDuration = 100; - } - - using var frameBitmap = new SKBitmap(_sourceCodec.Info); - var pointer = frameBitmap.GetPixels(); - _sourceCodec.GetPixels(_sourceCodec.Info, pointer, new SKCodecOptions(i)); - - var scaledBitmap = new SKBitmap(scaledImageInfo); - frameBitmap.ScalePixels(scaledBitmap, SKSamplingOptions.Default); - _scaledFrames.Add(scaledBitmap); - - _animatedDurationMs += frameDuration; - _frameDelays.Add(frameDuration); - } - } - - if (_sourceCodec.FrameCount > 0 && _animatedDurationMs == 0) - { - _animatedDurationMs = int.MaxValue; - } + _watermark.Size == WatermarkSize.Scaled, + _watermark.WidthPercent, + _watermark.HorizontalMarginPercent, + _watermark.VerticalMarginPercent, + _watermark.PlaceWithinSourceContent, + cancellationToken); } catch (Exception ex) { @@ -190,61 +114,6 @@ public class WatermarkElement : GraphicsElement, IDisposable } SKBitmap frameForTimestamp = GetFrameForTimestamp(contentTime); - return ValueTask.FromResult(Optional(new PreparedElementImage(frameForTimestamp, _location, opacity, false))); - } - - private SKBitmap GetFrameForTimestamp(TimeSpan timestamp) - { - if (_scaledFrames.Count <= 1) - { - return _scaledFrames[0]; - } - - long currentTimeMs = (long)timestamp.TotalMilliseconds % _animatedDurationMs; - - long frameTime = 0; - for (var i = 0; i < _sourceCodec.FrameCount; i++) - { - frameTime += _frameDelays[i]; - if (currentTimeMs <= frameTime) - { - return _scaledFrames[i]; - } - } - - return _scaledFrames.Last(); - } - - private WatermarkMargins NormalMargins(Resolution frameSize) - { - double horizontalMargin = Math.Round(_watermark.HorizontalMarginPercent / 100.0 * frameSize.Width); - double verticalMargin = Math.Round(_watermark.VerticalMarginPercent / 100.0 * frameSize.Height); - - return new WatermarkMargins((int)Math.Round(horizontalMargin), (int)Math.Round(verticalMargin)); - } - - private WatermarkMargins SourceContentMargins(Resolution squarePixelFrameSize, Resolution frameSize) - { - int horizontalPadding = frameSize.Width - squarePixelFrameSize.Width; - int verticalPadding = frameSize.Height - squarePixelFrameSize.Height; - - double horizontalMargin = Math.Round( - _watermark.HorizontalMarginPercent / 100.0 * squarePixelFrameSize.Width - + horizontalPadding / 2.0); - double verticalMargin = Math.Round( - _watermark.VerticalMarginPercent / 100.0 * squarePixelFrameSize.Height - + verticalPadding / 2.0); - - return new WatermarkMargins((int)Math.Round(horizontalMargin), (int)Math.Round(verticalMargin)); - } - - private sealed record WatermarkMargins(int HorizontalMargin, int VerticalMargin); - - public void Dispose() - { - GC.SuppressFinalize(this); - - _sourceCodec?.Dispose(); - _scaledFrames?.ForEach(f => f.Dispose()); + return ValueTask.FromResult(Optional(new PreparedElementImage(frameForTimestamp, Location, opacity, false))); } }