add image graphics element (#2288)

CHANGELOG.md

@@ -21,11 +21,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
   - The graphics engine will order by z-index when overlaying watermarks
 - Add *experimental* `Graphics Element` template system
   - Graphics elements are defined in YAML files inside ETV config folder / templates / graphics-elements subfolder
-  - Add `Text` graphics element type
+  - Add `text` graphics element type
     - Supported in playback troubleshooting and YAML playouts
     - Displays multi-line text in a specified font, color, location, z-index
     - Supports constant opacity and opacity expression
     - Supports variable replacement for music videos
+  - Add `image` graphics element type
+      - Supported in playback troubleshooting and YAML playouts
+      - Displays an image, similar to a watermark
+      - Supports constant opacity and opacity expression
 - YAML playout: add `graphics_on` and `graphics_off` instructions to control graphics elements
   - `graphics_on` requires the name of a graphics element template, e.g. `text/cool_element.yml`
     - The `variables` property can be used to dynamically replace text from the template

ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElementsHandler.cs

@@ -30,12 +30,12 @@ public class RefreshGraphicsElementsHandler(
             dbContext.GraphicsElements.Remove(existing);
         }
 
-        // add new elements
-        var newPaths = localFileSystem.ListFiles(FileSystemLayout.GraphicsElementsTextTemplatesFolder)
+        // add new text elements
+        var newTextPaths = localFileSystem.ListFiles(FileSystemLayout.GraphicsElementsTextTemplatesFolder)
             .Where(f => allExisting.All(e => e.Path != f))
             .ToList();
 
-        foreach (var path in newPaths)
+        foreach (var path in newTextPaths)
         {
             logger.LogDebug("Adding new graphics element from file {File}", path);
 
@@ -48,6 +48,24 @@ public class RefreshGraphicsElementsHandler(
             await dbContext.AddAsync(graphicsElement, cancellationToken);
         }
 
+        // add new image elements
+        var newImagePaths = localFileSystem.ListFiles(FileSystemLayout.GraphicsElementsImageTemplatesFolder)
+            .Where(f => allExisting.All(e => e.Path != f))
+            .ToList();
+
+        foreach (var path in newImagePaths)
+        {
+            logger.LogDebug("Adding new graphics element from file {File}", path);
+
+            var graphicsElement = new GraphicsElement
+            {
+                Path = path,
+                Kind = GraphicsElementKind.Image
+            };
+
+            await dbContext.AddAsync(graphicsElement, cancellationToken);
+        }
+
         await dbContext.SaveChangesAsync(cancellationToken);
     }
 }

ErsatzTV.Application/Graphics/Mapper.cs

@@ -10,6 +10,7 @@ public static class Mapper
         return graphicsElement.Kind switch
         {
             GraphicsElementKind.Text => new GraphicsElementViewModel(graphicsElement.Id, $"text/{fileName}"),
+            GraphicsElementKind.Image => new GraphicsElementViewModel(graphicsElement.Id, $"image/{fileName}"),
             _ => new GraphicsElementViewModel(graphicsElement.Id, graphicsElement.Path)
         };
     }

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -380,7 +380,9 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             switch (playoutItemGraphicsElement.GraphicsElement.Kind)
             {
                 case GraphicsElementKind.Text:
-                    var maybeElement = await TextGraphicsElement.FromFile(playoutItemGraphicsElement.GraphicsElement.Path);
+                {
+                    var maybeElement =
+                        await TextGraphicsElement.FromFile(playoutItemGraphicsElement.GraphicsElement.Path);
                     if (maybeElement.IsNone)
                     {
                         _logger.LogWarning(
@@ -399,7 +401,34 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
                         graphicsElementContexts.Add(new TextElementContext(element, variables));
                     }
+
                     break;
+                }
+                case GraphicsElementKind.Image:
+                {
+                    var maybeElement =
+                        await ImageGraphicsElement.FromFile(playoutItemGraphicsElement.GraphicsElement.Path);
+                    if (maybeElement.IsNone)
+                    {
+                        _logger.LogWarning(
+                            "Failed to load image graphics element from file {Path}; ignoring",
+                            playoutItemGraphicsElement.GraphicsElement.Path);
+                    }
+
+                    foreach (var element in maybeElement)
+                    {
+                        // var variables = new Dictionary<string, string>();
+                        // if (!string.IsNullOrWhiteSpace(playoutItemGraphicsElement.Variables))
+                        // {
+                        //     variables = JsonConvert.DeserializeObject<Dictionary<string, string>>(
+                        //         playoutItemGraphicsElement.Variables);
+                        // }
+
+                        graphicsElementContexts.Add(new ImageElementContext(element));
+                    }
+
+                    break;
+                }
                 default:
                     _logger.LogInformation(
                         "Ignoring unsupported graphics element kind {Kind}",

ErsatzTV.Core/FileSystemLayout.cs

@@ -49,6 +49,7 @@ public static class FileSystemLayout
 
     public static readonly string GraphicsElementsTemplatesFolder;
     public static readonly string GraphicsElementsTextTemplatesFolder;
+    public static readonly string GraphicsElementsImageTemplatesFolder;
 
     public static readonly string ScriptsFolder;
 
@@ -167,6 +168,7 @@ public static class FileSystemLayout
 
         GraphicsElementsTemplatesFolder = Path.Combine(TemplatesFolder, "graphics-elements");
         GraphicsElementsTextTemplatesFolder = Path.Combine(GraphicsElementsTemplatesFolder, "text");
+        GraphicsElementsImageTemplatesFolder = Path.Combine(GraphicsElementsTemplatesFolder, "image");
 
         ScriptsFolder = Path.Combine(AppDataFolder, "scripts");
 

ErsatzTV.Core/Graphics/ImageGraphicsElement.cs

@@ -0,0 +1,56 @@
+using ErsatzTV.FFmpeg.State;
+using YamlDotNet.Serialization;
+using YamlDotNet.Serialization.NamingConventions;
+
+namespace ErsatzTV.Core.Graphics;
+
+public class ImageGraphicsElement
+{
+    public string Image { get; set; }
+
+    public int? Opacity { get; set; }
+
+    [YamlMember(Alias = "opacity_expression", ApplyNamingConventions = false)]
+    public string OpacityExpression { get; set; }
+
+    public WatermarkLocation Location { get; set; }
+    [YamlMember(Alias = "horizontal_margin_percent", ApplyNamingConventions = false)]
+    public double? HorizontalMarginPercent { get; set; }
+    [YamlMember(Alias = "vertical_margin_percent", ApplyNamingConventions = false)]
+    public double? VerticalMarginPercent { get; set; }
+
+    [YamlMember(Alias = "location_x", ApplyNamingConventions = false)]
+    public double? LocationX { get; set; }
+    [YamlMember(Alias = "location_y", ApplyNamingConventions = false)]
+    public double? LocationY { get; set; }
+    [YamlMember(Alias = "z_index", ApplyNamingConventions = false)]
+    public int? ZIndex { get; set; }
+
+    public bool Scale { get; set; }
+    [YamlMember(Alias = "scale_width_percent", ApplyNamingConventions = false)]
+    public double? ScaleWidthPercent { get; set; }
+
+    public static async Task<Option<ImageGraphicsElement>> FromFile(string fileName)
+    {
+        try
+        {
+            string yaml = await File.ReadAllTextAsync(fileName);
+
+            // TODO: validate schema
+            // if (await yamlScheduleValidator.ValidateSchedule(yaml, isImport) == false)
+            // {
+            //     return Option<YamlPlayoutDefinition>.None;
+            // }
+
+            IDeserializer deserializer = new DeserializerBuilder()
+                .WithNamingConvention(CamelCaseNamingConvention.Instance)
+                .Build();
+
+            return deserializer.Deserialize<ImageGraphicsElement>(yaml);
+        }
+        catch (Exception)
+        {
+            return Option<ImageGraphicsElement>.None;
+        }
+    }
+}

ErsatzTV.Core/Interfaces/Streaming/GraphicsEngineContext.cs

@@ -20,3 +20,5 @@ public record WatermarkElementContext(WatermarkOptions Options) : GraphicsElemen
 
 public record TextElementContext(TextGraphicsElement TextElement, Dictionary<string, string> Variables)
     : GraphicsElementContext;
+
+public record ImageElementContext(ImageGraphicsElement ImageElement) : GraphicsElementContext;

ErsatzTV.Infrastructure/Streaming/GraphicsEngine.cs

@@ -28,6 +28,9 @@ public class GraphicsEngine(ITemplateDataRepository templateDataRepository, ILog
                         elements.Add(watermark);
                     }
 
+                    break;
+                case ImageElementContext imageElementContext:
+                    elements.Add(new ImageElement(imageElementContext.ImageElement, logger));
                     break;
                 case TextElementContext textElementContext:
                     var variables = new Dictionary<string, object>();

ErsatzTV.Infrastructure/Streaming/ImageElement.cs

@@ -0,0 +1,160 @@
+using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.Graphics;
+using Microsoft.Extensions.Logging;
+using NCalc;
+using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.Formats.Gif;
+using SixLabors.ImageSharp.Processing;
+using Image = SixLabors.ImageSharp.Image;
+
+namespace ErsatzTV.Infrastructure.Streaming;
+
+public class ImageElement(ImageGraphicsElement imageGraphicsElement, ILogger logger) : IGraphicsElement, IDisposable
+{
+    private readonly List<Image> _scaledFrames = [];
+    private readonly List<double> _frameDelays = [];
+
+    private Option<Expression> _maybeOpacityExpression;
+    private float _opacity;
+    private double _animatedDurationSeconds;
+    private Image _sourceImage;
+    private Point _location;
+
+    public int ZIndex { get; private set; }
+
+    public bool IsFailed { get; set; }
+
+    public async Task InitializeAsync(Resolution frameSize, int frameRate, CancellationToken cancellationToken)
+    {
+        try
+        {
+            if (!string.IsNullOrWhiteSpace(imageGraphicsElement.OpacityExpression))
+            {
+                var expression = new Expression(imageGraphicsElement.OpacityExpression);
+                expression.EvaluateFunction += OpacityExpressionHelper.EvaluateFunction;
+                _maybeOpacityExpression = expression;
+            }
+            else
+            {
+                _opacity = (imageGraphicsElement.Opacity ?? 100) / 100.0f;
+            }
+
+            ZIndex = imageGraphicsElement.ZIndex ?? 0;
+
+            foreach (var expression in _maybeOpacityExpression)
+            {
+                expression.EvaluateFunction += OpacityExpressionHelper.EvaluateFunction;
+            }
+
+            bool isRemoteUri = Uri.TryCreate(imageGraphicsElement.Image, UriKind.Absolute, out var uriResult)
+                               && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps);
+
+            if (isRemoteUri)
+            {
+                using var client = new HttpClient();
+                await using Stream imageStream = await client.GetStreamAsync(uriResult, cancellationToken);
+                _sourceImage = await Image.LoadAsync(imageStream, cancellationToken);
+            }
+            else
+            {
+                _sourceImage = await Image.LoadAsync(imageGraphicsElement.Image!, cancellationToken);
+            }
+
+            int scaledWidth = _sourceImage.Width;
+            int scaledHeight = _sourceImage.Height;
+            if (imageGraphicsElement.Scale)
+            {
+                scaledWidth = (int)Math.Round((imageGraphicsElement.ScaleWidthPercent ?? 100) / 100.0 * frameSize.Width);
+                double aspectRatio = (double)_sourceImage.Height / _sourceImage.Width;
+                scaledHeight = (int)(scaledWidth * aspectRatio);
+            }
+
+            int horizontalMargin = (int)Math.Round((imageGraphicsElement.HorizontalMarginPercent ?? 0) / 100.0 * frameSize.Width);
+            int verticalMargin = (int)Math.Round((imageGraphicsElement.VerticalMarginPercent ?? 0) / 100.0 * frameSize.Height);
+
+            _location = WatermarkElement.CalculatePosition(
+                imageGraphicsElement.Location,
+                frameSize.Width,
+                frameSize.Height,
+                scaledWidth,
+                scaledHeight,
+                horizontalMargin,
+                verticalMargin);
+
+            _animatedDurationSeconds = 0;
+
+            for (int i = 0; i < _sourceImage.Frames.Count; i++)
+            {
+                var frame = _sourceImage.Frames.CloneFrame(i);
+                frame.Mutate(ctx => ctx.Resize(scaledWidth, scaledHeight));
+                _scaledFrames.Add(frame);
+
+                var frameDelay = _sourceImage.Frames[i].Metadata.GetFormatMetadata(GifFormat.Instance).FrameDelay / 100.0;
+                _animatedDurationSeconds += frameDelay;
+                _frameDelays.Add(frameDelay);
+            }
+        }
+        catch (Exception ex)
+        {
+            IsFailed = true;
+            logger.LogWarning(ex, "Failed to initialize image element; will disable for this content");
+        }
+    }
+
+    public ValueTask<Option<PreparedElementImage>> PrepareImage(
+        TimeSpan timeOfDay,
+        TimeSpan contentTime,
+        TimeSpan contentTotalTime,
+        TimeSpan channelTime,
+        CancellationToken cancellationToken)
+    {
+        float opacity = _opacity;
+        foreach (var expression in _maybeOpacityExpression)
+        {
+            opacity = OpacityExpressionHelper.GetOpacity(
+                expression,
+                timeOfDay,
+                contentTime,
+                contentTotalTime,
+                channelTime);
+        }
+
+        if (opacity == 0)
+        {
+            return ValueTask.FromResult(Option<PreparedElementImage>.None);
+        }
+
+        Image frameForTimestamp = GetFrameForTimestamp(contentTime);
+        return ValueTask.FromResult(Optional(new PreparedElementImage(frameForTimestamp, _location, opacity, false)));
+    }
+
+    private Image GetFrameForTimestamp(TimeSpan timestamp)
+    {
+        if (_scaledFrames.Count <= 1)
+        {
+            return _scaledFrames[0];
+        }
+
+        double currentTime = timestamp.TotalSeconds % _animatedDurationSeconds;
+
+        double frameTime = 0;
+        for (int i = 0; i < _sourceImage.Frames.Count; i++)
+        {
+            frameTime += _frameDelays[i];
+            if (currentTime <= frameTime)
+            {
+                return _scaledFrames[i];
+            }
+        }
+
+        return _scaledFrames.Last();
+    }
+
+    public void Dispose()
+    {
+        GC.SuppressFinalize(this);
+
+        _sourceImage?.Dispose();
+        _scaledFrames?.ForEach(f => f.Dispose());
+    }
+}

ErsatzTV/Startup.cs

@@ -338,6 +338,7 @@ public class Startup
             FileSystemLayout.ChannelGuideTemplatesFolder,
             FileSystemLayout.GraphicsElementsTemplatesFolder,
             FileSystemLayout.GraphicsElementsTextTemplatesFolder,
+            FileSystemLayout.GraphicsElementsImageTemplatesFolder,
             FileSystemLayout.ScriptsFolder,
             FileSystemLayout.MultiEpisodeShuffleTemplatesFolder,
             FileSystemLayout.AudioStreamSelectorScriptsFolder