From df0801f2c6a9572f713a326bf1c7620bfea0eb19 Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Sat, 9 Aug 2025 17:42:23 +0000 Subject: [PATCH] add image graphics element (#2288) --- CHANGELOG.md | 6 +- .../RefreshGraphicsElementsHandler.cs | 24 ++- ErsatzTV.Application/Graphics/Mapper.cs | 1 + .../FFmpeg/FFmpegLibraryProcessService.cs | 31 +++- ErsatzTV.Core/FileSystemLayout.cs | 2 + .../Graphics/ImageGraphicsElement.cs | 56 ++++++ .../Streaming/GraphicsEngineContext.cs | 2 + .../Streaming/GraphicsEngine.cs | 3 + .../Streaming/ImageElement.cs | 160 ++++++++++++++++++ ErsatzTV/Startup.cs | 1 + 10 files changed, 281 insertions(+), 5 deletions(-) create mode 100644 ErsatzTV.Core/Graphics/ImageGraphicsElement.cs create mode 100644 ErsatzTV.Infrastructure/Streaming/ImageElement.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index d4f24019..77695b92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,11 +21,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - The graphics engine will order by z-index when overlaying watermarks - Add *experimental* `Graphics Element` template system - Graphics elements are defined in YAML files inside ETV config folder / templates / graphics-elements subfolder - - Add `Text` graphics element type + - Add `text` graphics element type - Supported in playback troubleshooting and YAML playouts - Displays multi-line text in a specified font, color, location, z-index - Supports constant opacity and opacity expression - Supports variable replacement for music videos + - Add `image` graphics element type + - Supported in playback troubleshooting and YAML playouts + - Displays an image, similar to a watermark + - Supports constant opacity and opacity expression - YAML playout: add `graphics_on` and `graphics_off` instructions to control graphics elements - `graphics_on` requires the name of a graphics element template, e.g. `text/cool_element.yml` - The `variables` property can be used to dynamically replace text from the template diff --git a/ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElementsHandler.cs b/ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElementsHandler.cs index e4475fb7..7e6ebe5e 100644 --- a/ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElementsHandler.cs +++ b/ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElementsHandler.cs @@ -30,12 +30,12 @@ public class RefreshGraphicsElementsHandler( dbContext.GraphicsElements.Remove(existing); } - // add new elements - var newPaths = localFileSystem.ListFiles(FileSystemLayout.GraphicsElementsTextTemplatesFolder) + // add new text elements + var newTextPaths = localFileSystem.ListFiles(FileSystemLayout.GraphicsElementsTextTemplatesFolder) .Where(f => allExisting.All(e => e.Path != f)) .ToList(); - foreach (var path in newPaths) + foreach (var path in newTextPaths) { logger.LogDebug("Adding new graphics element from file {File}", path); @@ -48,6 +48,24 @@ public class RefreshGraphicsElementsHandler( await dbContext.AddAsync(graphicsElement, cancellationToken); } + // add new image elements + var newImagePaths = localFileSystem.ListFiles(FileSystemLayout.GraphicsElementsImageTemplatesFolder) + .Where(f => allExisting.All(e => e.Path != f)) + .ToList(); + + foreach (var path in newImagePaths) + { + logger.LogDebug("Adding new graphics element from file {File}", path); + + var graphicsElement = new GraphicsElement + { + Path = path, + Kind = GraphicsElementKind.Image + }; + + await dbContext.AddAsync(graphicsElement, cancellationToken); + } + await dbContext.SaveChangesAsync(cancellationToken); } } \ No newline at end of file diff --git a/ErsatzTV.Application/Graphics/Mapper.cs b/ErsatzTV.Application/Graphics/Mapper.cs index 33fafa6d..c7efed3e 100644 --- a/ErsatzTV.Application/Graphics/Mapper.cs +++ b/ErsatzTV.Application/Graphics/Mapper.cs @@ -10,6 +10,7 @@ public static class Mapper return graphicsElement.Kind switch { GraphicsElementKind.Text => new GraphicsElementViewModel(graphicsElement.Id, $"text/{fileName}"), + GraphicsElementKind.Image => new GraphicsElementViewModel(graphicsElement.Id, $"image/{fileName}"), _ => new GraphicsElementViewModel(graphicsElement.Id, graphicsElement.Path) }; } diff --git a/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs b/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs index 8431b5c7..b6e2810b 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs @@ -380,7 +380,9 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService switch (playoutItemGraphicsElement.GraphicsElement.Kind) { case GraphicsElementKind.Text: - var maybeElement = await TextGraphicsElement.FromFile(playoutItemGraphicsElement.GraphicsElement.Path); + { + var maybeElement = + await TextGraphicsElement.FromFile(playoutItemGraphicsElement.GraphicsElement.Path); if (maybeElement.IsNone) { _logger.LogWarning( @@ -399,7 +401,34 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService graphicsElementContexts.Add(new TextElementContext(element, variables)); } + + break; + } + case GraphicsElementKind.Image: + { + var maybeElement = + await ImageGraphicsElement.FromFile(playoutItemGraphicsElement.GraphicsElement.Path); + if (maybeElement.IsNone) + { + _logger.LogWarning( + "Failed to load image graphics element from file {Path}; ignoring", + playoutItemGraphicsElement.GraphicsElement.Path); + } + + foreach (var element in maybeElement) + { + // var variables = new Dictionary(); + // if (!string.IsNullOrWhiteSpace(playoutItemGraphicsElement.Variables)) + // { + // variables = JsonConvert.DeserializeObject>( + // playoutItemGraphicsElement.Variables); + // } + + graphicsElementContexts.Add(new ImageElementContext(element)); + } + break; + } default: _logger.LogInformation( "Ignoring unsupported graphics element kind {Kind}", diff --git a/ErsatzTV.Core/FileSystemLayout.cs b/ErsatzTV.Core/FileSystemLayout.cs index dcf1fcc9..535e7f70 100644 --- a/ErsatzTV.Core/FileSystemLayout.cs +++ b/ErsatzTV.Core/FileSystemLayout.cs @@ -49,6 +49,7 @@ public static class FileSystemLayout public static readonly string GraphicsElementsTemplatesFolder; public static readonly string GraphicsElementsTextTemplatesFolder; + public static readonly string GraphicsElementsImageTemplatesFolder; public static readonly string ScriptsFolder; @@ -167,6 +168,7 @@ public static class FileSystemLayout GraphicsElementsTemplatesFolder = Path.Combine(TemplatesFolder, "graphics-elements"); GraphicsElementsTextTemplatesFolder = Path.Combine(GraphicsElementsTemplatesFolder, "text"); + GraphicsElementsImageTemplatesFolder = Path.Combine(GraphicsElementsTemplatesFolder, "image"); ScriptsFolder = Path.Combine(AppDataFolder, "scripts"); diff --git a/ErsatzTV.Core/Graphics/ImageGraphicsElement.cs b/ErsatzTV.Core/Graphics/ImageGraphicsElement.cs new file mode 100644 index 00000000..6f716d2a --- /dev/null +++ b/ErsatzTV.Core/Graphics/ImageGraphicsElement.cs @@ -0,0 +1,56 @@ +using ErsatzTV.FFmpeg.State; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace ErsatzTV.Core.Graphics; + +public class ImageGraphicsElement +{ + public string Image { get; set; } + + public int? Opacity { get; set; } + + [YamlMember(Alias = "opacity_expression", ApplyNamingConventions = false)] + public string OpacityExpression { get; set; } + + public WatermarkLocation Location { get; set; } + [YamlMember(Alias = "horizontal_margin_percent", ApplyNamingConventions = false)] + public double? HorizontalMarginPercent { get; set; } + [YamlMember(Alias = "vertical_margin_percent", ApplyNamingConventions = false)] + public double? VerticalMarginPercent { get; set; } + + [YamlMember(Alias = "location_x", ApplyNamingConventions = false)] + public double? LocationX { get; set; } + [YamlMember(Alias = "location_y", ApplyNamingConventions = false)] + public double? LocationY { get; set; } + [YamlMember(Alias = "z_index", ApplyNamingConventions = false)] + public int? ZIndex { get; set; } + + public bool Scale { get; set; } + [YamlMember(Alias = "scale_width_percent", ApplyNamingConventions = false)] + public double? ScaleWidthPercent { get; set; } + + public static async Task> FromFile(string fileName) + { + try + { + string yaml = await File.ReadAllTextAsync(fileName); + + // TODO: validate schema + // if (await yamlScheduleValidator.ValidateSchedule(yaml, isImport) == false) + // { + // return Option.None; + // } + + IDeserializer deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); + + return deserializer.Deserialize(yaml); + } + catch (Exception) + { + return Option.None; + } + } +} \ No newline at end of file diff --git a/ErsatzTV.Core/Interfaces/Streaming/GraphicsEngineContext.cs b/ErsatzTV.Core/Interfaces/Streaming/GraphicsEngineContext.cs index 843a5489..55a53366 100644 --- a/ErsatzTV.Core/Interfaces/Streaming/GraphicsEngineContext.cs +++ b/ErsatzTV.Core/Interfaces/Streaming/GraphicsEngineContext.cs @@ -20,3 +20,5 @@ public record WatermarkElementContext(WatermarkOptions Options) : GraphicsElemen public record TextElementContext(TextGraphicsElement TextElement, Dictionary Variables) : GraphicsElementContext; + +public record ImageElementContext(ImageGraphicsElement ImageElement) : GraphicsElementContext; \ No newline at end of file diff --git a/ErsatzTV.Infrastructure/Streaming/GraphicsEngine.cs b/ErsatzTV.Infrastructure/Streaming/GraphicsEngine.cs index 9a1c9396..201e5efa 100644 --- a/ErsatzTV.Infrastructure/Streaming/GraphicsEngine.cs +++ b/ErsatzTV.Infrastructure/Streaming/GraphicsEngine.cs @@ -28,6 +28,9 @@ public class GraphicsEngine(ITemplateDataRepository templateDataRepository, ILog elements.Add(watermark); } + break; + case ImageElementContext imageElementContext: + elements.Add(new ImageElement(imageElementContext.ImageElement, logger)); break; case TextElementContext textElementContext: var variables = new Dictionary(); diff --git a/ErsatzTV.Infrastructure/Streaming/ImageElement.cs b/ErsatzTV.Infrastructure/Streaming/ImageElement.cs new file mode 100644 index 00000000..714b585b --- /dev/null +++ b/ErsatzTV.Infrastructure/Streaming/ImageElement.cs @@ -0,0 +1,160 @@ +using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Graphics; +using Microsoft.Extensions.Logging; +using NCalc; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.Formats.Gif; +using SixLabors.ImageSharp.Processing; +using Image = SixLabors.ImageSharp.Image; + +namespace ErsatzTV.Infrastructure.Streaming; + +public class ImageElement(ImageGraphicsElement imageGraphicsElement, ILogger logger) : IGraphicsElement, IDisposable +{ + private readonly List _scaledFrames = []; + private readonly List _frameDelays = []; + + private Option _maybeOpacityExpression; + private float _opacity; + private double _animatedDurationSeconds; + private Image _sourceImage; + private Point _location; + + public int ZIndex { get; private set; } + + public bool IsFailed { get; set; } + + public async Task InitializeAsync(Resolution frameSize, int frameRate, CancellationToken cancellationToken) + { + try + { + if (!string.IsNullOrWhiteSpace(imageGraphicsElement.OpacityExpression)) + { + var expression = new Expression(imageGraphicsElement.OpacityExpression); + expression.EvaluateFunction += OpacityExpressionHelper.EvaluateFunction; + _maybeOpacityExpression = expression; + } + else + { + _opacity = (imageGraphicsElement.Opacity ?? 100) / 100.0f; + } + + ZIndex = imageGraphicsElement.ZIndex ?? 0; + + foreach (var expression in _maybeOpacityExpression) + { + expression.EvaluateFunction += OpacityExpressionHelper.EvaluateFunction; + } + + bool isRemoteUri = Uri.TryCreate(imageGraphicsElement.Image, UriKind.Absolute, out var uriResult) + && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps); + + if (isRemoteUri) + { + using var client = new HttpClient(); + await using Stream imageStream = await client.GetStreamAsync(uriResult, cancellationToken); + _sourceImage = await Image.LoadAsync(imageStream, cancellationToken); + } + else + { + _sourceImage = await Image.LoadAsync(imageGraphicsElement.Image!, cancellationToken); + } + + int scaledWidth = _sourceImage.Width; + int scaledHeight = _sourceImage.Height; + if (imageGraphicsElement.Scale) + { + scaledWidth = (int)Math.Round((imageGraphicsElement.ScaleWidthPercent ?? 100) / 100.0 * frameSize.Width); + double aspectRatio = (double)_sourceImage.Height / _sourceImage.Width; + scaledHeight = (int)(scaledWidth * aspectRatio); + } + + int horizontalMargin = (int)Math.Round((imageGraphicsElement.HorizontalMarginPercent ?? 0) / 100.0 * frameSize.Width); + int verticalMargin = (int)Math.Round((imageGraphicsElement.VerticalMarginPercent ?? 0) / 100.0 * frameSize.Height); + + _location = WatermarkElement.CalculatePosition( + imageGraphicsElement.Location, + frameSize.Width, + frameSize.Height, + scaledWidth, + scaledHeight, + horizontalMargin, + verticalMargin); + + _animatedDurationSeconds = 0; + + for (int i = 0; i < _sourceImage.Frames.Count; i++) + { + var frame = _sourceImage.Frames.CloneFrame(i); + frame.Mutate(ctx => ctx.Resize(scaledWidth, scaledHeight)); + _scaledFrames.Add(frame); + + var frameDelay = _sourceImage.Frames[i].Metadata.GetFormatMetadata(GifFormat.Instance).FrameDelay / 100.0; + _animatedDurationSeconds += frameDelay; + _frameDelays.Add(frameDelay); + } + } + catch (Exception ex) + { + IsFailed = true; + logger.LogWarning(ex, "Failed to initialize image element; will disable for this content"); + } + } + + public ValueTask> PrepareImage( + TimeSpan timeOfDay, + TimeSpan contentTime, + TimeSpan contentTotalTime, + TimeSpan channelTime, + CancellationToken cancellationToken) + { + float opacity = _opacity; + foreach (var expression in _maybeOpacityExpression) + { + opacity = OpacityExpressionHelper.GetOpacity( + expression, + timeOfDay, + contentTime, + contentTotalTime, + channelTime); + } + + if (opacity == 0) + { + return ValueTask.FromResult(Option.None); + } + + Image frameForTimestamp = GetFrameForTimestamp(contentTime); + return ValueTask.FromResult(Optional(new PreparedElementImage(frameForTimestamp, _location, opacity, false))); + } + + private Image GetFrameForTimestamp(TimeSpan timestamp) + { + if (_scaledFrames.Count <= 1) + { + return _scaledFrames[0]; + } + + double currentTime = timestamp.TotalSeconds % _animatedDurationSeconds; + + double frameTime = 0; + for (int i = 0; i < _sourceImage.Frames.Count; i++) + { + frameTime += _frameDelays[i]; + if (currentTime <= frameTime) + { + return _scaledFrames[i]; + } + } + + return _scaledFrames.Last(); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + + _sourceImage?.Dispose(); + _scaledFrames?.ForEach(f => f.Dispose()); + } +} \ No newline at end of file diff --git a/ErsatzTV/Startup.cs b/ErsatzTV/Startup.cs index 223c247a..04b000cf 100644 --- a/ErsatzTV/Startup.cs +++ b/ErsatzTV/Startup.cs @@ -338,6 +338,7 @@ public class Startup FileSystemLayout.ChannelGuideTemplatesFolder, FileSystemLayout.GraphicsElementsTemplatesFolder, FileSystemLayout.GraphicsElementsTextTemplatesFolder, + FileSystemLayout.GraphicsElementsImageTemplatesFolder, FileSystemLayout.ScriptsFolder, FileSystemLayout.MultiEpisodeShuffleTemplatesFolder, FileSystemLayout.AudioStreamSelectorScriptsFolder