motion end behavior (#2431)

* add end behavior enum and properties

* support loop end behavior

* implement end behavior hold

* update changelog

CHANGELOG.md

@@ -32,6 +32,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     - Placement (`location`, `horizontal_margin_percent`, `vertical_margin_percent`)
     - Scaling (`scale`, `scale_width_percent`)
     - Timing (`start_seconds`)
+    - End behavior (`end_behavior`)
+      - `disappear` (default) - disappear after playing once
+      - `loop` - loop forever
+      - `hold` - hold last frame forever, or `hold_seconds`
     - Draw order (`z_index`)
 
 ### Fixed

ErsatzTV.Core/Graphics/MotionEndBehavior.cs

@@ -0,0 +1,8 @@
+namespace ErsatzTV.Core.Graphics;
+
+public enum MotionEndBehavior
+{
+    Disappear = 0,
+    Loop = 1,
+    Hold = 2
+}

ErsatzTV.Core/Graphics/MotionGraphicsElement.cs

@@ -17,6 +17,12 @@ public class MotionGraphicsElement
     [YamlMember(Alias = "start_seconds", ApplyNamingConventions = false)]
     public double? StartSeconds { get; set; }
 
+    [YamlMember(Alias = "end_behavior", ApplyNamingConventions = false)]
+    public MotionEndBehavior EndBehavior { get; set; } = MotionEndBehavior.Disappear;
+
+    [YamlMember(Alias = "hold_seconds", ApplyNamingConventions = false)]
+    public double? HoldSeconds { get; set; }
+
     public WatermarkLocation Location { get; set; }
 
     [YamlMember(Alias = "horizontal_margin_percent", ApplyNamingConventions = false)]

ErsatzTV.Infrastructure/Streaming/Graphics/GraphicsElement.cs

@@ -1,4 +1,5 @@
 using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.Interfaces.Streaming;
 using ErsatzTV.FFmpeg.State;
 using SkiaSharp;
 
@@ -10,12 +11,7 @@ public abstract class GraphicsElement : IGraphicsElement
 
     public bool IsFinished { get; set; }
 
-    public abstract Task InitializeAsync(
-        Resolution squarePixelFrameSize,
-        Resolution frameSize,
-        int frameRate,
-        TimeSpan seek,
-        CancellationToken cancellationToken);
+    public abstract Task InitializeAsync(GraphicsEngineContext context, CancellationToken cancellationToken);
 
     public abstract ValueTask<Option<PreparedElementImage>> PrepareImage(
         TimeSpan timeOfDay,

ErsatzTV.Infrastructure/Streaming/Graphics/GraphicsEngine.cs

@@ -80,14 +80,7 @@ public class GraphicsEngine(
         }
 
         // initialize all elements
-        await Task.WhenAll(
-            elements.Select(e =>
-                e.InitializeAsync(
-                    context.SquarePixelFrameSize,
-                    context.FrameSize,
-                    context.FrameRate,
-                    context.Seek,
-                    cancellationToken)));
+        await Task.WhenAll(elements.Select(e => e.InitializeAsync(context, cancellationToken)));
 
         long frameCount = 0;
         var totalFrames = (long)(context.Duration.TotalSeconds * context.FrameRate);

ErsatzTV.Infrastructure/Streaming/Graphics/IGraphicsElement.cs

@@ -1,4 +1,5 @@
 using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.Interfaces.Streaming;
 
 namespace ErsatzTV.Infrastructure.Streaming.Graphics;
 
@@ -8,12 +9,7 @@ public interface IGraphicsElement
 
     bool IsFinished { get; set; }
 
-    Task InitializeAsync(
-        Resolution squarePixelFrameSize,
-        Resolution frameSize,
-        int frameRate,
-        TimeSpan seek,
-        CancellationToken cancellationToken);
+    Task InitializeAsync(GraphicsEngineContext context, CancellationToken cancellationToken);
 
     ValueTask<Option<PreparedElementImage>> PrepareImage(
         TimeSpan timeOfDay,

ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElement.cs

@@ -1,5 +1,5 @@
-using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Graphics;
+using ErsatzTV.Core.Interfaces.Streaming;
 using Microsoft.Extensions.Logging;
 using NCalc;
 using SkiaSharp;
@@ -11,12 +11,7 @@ public class ImageElement(ImageGraphicsElement imageGraphicsElement, ILogger log
     private Option<Expression> _maybeOpacityExpression;
     private float _opacity;
 
-    public override async Task InitializeAsync(
-        Resolution squarePixelFrameSize,
-        Resolution frameSize,
-        int frameRate,
-        TimeSpan seek,
-        CancellationToken cancellationToken)
+    public override async Task InitializeAsync(GraphicsEngineContext context, CancellationToken cancellationToken)
     {
         try
         {
@@ -39,8 +34,8 @@ public class ImageElement(ImageGraphicsElement imageGraphicsElement, ILogger log
             }
 
             await LoadImage(
-                squarePixelFrameSize,
-                frameSize,
+                context.SquarePixelFrameSize,
+                context.FrameSize,
                 imageGraphicsElement.Image,
                 imageGraphicsElement.Location,
                 imageGraphicsElement.Scale,

ErsatzTV.Infrastructure/Streaming/Graphics/Image/WatermarkElement.cs

@@ -1,5 +1,6 @@
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.FFmpeg;
+using ErsatzTV.Core.Interfaces.Streaming;
 using ErsatzTV.FFmpeg.State;
 using Microsoft.Extensions.Logging;
 using NCalc;
@@ -28,12 +29,7 @@ public class WatermarkElement : ImageElementBase
 
     public bool IsValid => _imagePath != null && _watermark != null;
 
-    public override async Task InitializeAsync(
-        Resolution squarePixelFrameSize,
-        Resolution frameSize,
-        int frameRate,
-        TimeSpan seek,
-        CancellationToken cancellationToken)
+    public override async Task InitializeAsync(GraphicsEngineContext context, CancellationToken cancellationToken)
     {
         try
         {
@@ -68,8 +64,8 @@ public class WatermarkElement : ImageElementBase
             }
 
             await LoadImage(
-                squarePixelFrameSize,
-                frameSize,
+                context.SquarePixelFrameSize,
+                context.FrameSize,
                 _imagePath,
                 _watermark.Location,
                 _watermark.Size == WatermarkSize.Scaled,

ErsatzTV.Infrastructure/Streaming/Graphics/Motion/MotionElement.cs

@@ -6,6 +6,7 @@ using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Graphics;
 using ErsatzTV.Core.Interfaces.Metadata;
+using ErsatzTV.Core.Interfaces.Streaming;
 using Microsoft.Extensions.Logging;
 using SkiaSharp;
 
@@ -25,8 +26,9 @@ public class MotionElement(
     private SKPointI _point;
     private SKBitmap _canvasBitmap;
     private SKBitmap _motionFrameBitmap;
-    private bool _isFinished;
     private TimeSpan _startTime;
+    private TimeSpan _endTime;
+    private MotionElementState _state;
 
     public void Dispose()
     {
@@ -52,21 +54,24 @@ public class MotionElement(
         _motionFrameBitmap?.Dispose();
     }
 
-    public override async Task InitializeAsync(
-        Resolution squarePixelFrameSize,
-        Resolution frameSize,
-        int frameRate,
-        TimeSpan seek,
-        CancellationToken cancellationToken)
+    public override async Task InitializeAsync(GraphicsEngineContext context, CancellationToken cancellationToken)
     {
         try
         {
             _startTime = TimeSpan.FromSeconds(motionElement.StartSeconds ?? 0);
-            ProbeResult probeResult = await ProbeMotionElement(frameSize);
-            var overlayDuration = probeResult.Duration;
+            var holdDuration = TimeSpan.FromSeconds(motionElement.HoldSeconds ?? 0);
+            ProbeResult probeResult = await ProbeMotionElement(context.FrameSize);
+            var overlayDuration = motionElement.EndBehavior switch
+            {
+                MotionEndBehavior.Loop => context.Duration,
+                MotionEndBehavior.Hold => probeResult.Duration + holdDuration,
+                _ => probeResult.Duration
+            };
+
+            _endTime = _startTime + overlayDuration;
 
             // already past the time when this is supposed to play; don't do any more work
-            if (_startTime + overlayDuration < seek)
+            if (_startTime + overlayDuration < context.Seek)
             {
                 IsFinished = true;
                 return;
@@ -76,9 +81,9 @@ public class MotionElement(
             _pipeReader = pipe.Reader;
 
             var overlaySeekTime = TimeSpan.Zero;
-            if (_startTime < seek)
+            if (_startTime < context.Seek)
             {
-                overlaySeekTime = seek - _startTime;
+                overlaySeekTime = context.Seek - _startTime;
             }
 
             Resolution sourceSize = probeResult.Size;
@@ -88,7 +93,8 @@ public class MotionElement(
 
             if (motionElement.Scale)
             {
-                scaledWidth = (int)Math.Round((motionElement.ScaleWidthPercent ?? 100) / 100.0 * frameSize.Width);
+                scaledWidth = (int)Math.Round(
+                    (motionElement.ScaleWidthPercent ?? 100) / 100.0 * context.FrameSize.Width);
                 double aspectRatio = (double)sourceSize.Height / sourceSize.Width;
                 scaledHeight = (int)Math.Round(scaledWidth * aspectRatio);
             }
@@ -108,7 +114,11 @@ public class MotionElement(
 
             _frameSize = targetSize.Width * targetSize.Height * 4;
 
-            _canvasBitmap = new SKBitmap(frameSize.Width, frameSize.Height, SKColorType.Bgra8888, SKAlphaType.Unpremul);
+            _canvasBitmap = new SKBitmap(
+                context.FrameSize.Width,
+                context.FrameSize.Height,
+                SKColorType.Bgra8888,
+                SKAlphaType.Unpremul);
 
             _motionFrameBitmap = new SKBitmap(
                 targetSize.Width,
@@ -119,14 +129,14 @@ public class MotionElement(
             _point = SKPointI.Empty;
 
             (int horizontalMargin, int verticalMargin) = NormalMargins(
-                frameSize,
+                context.FrameSize,
                 motionElement.HorizontalMarginPercent ?? 0,
                 motionElement.VerticalMarginPercent ?? 0);
 
             _point = CalculatePosition(
                 motionElement.Location,
-                frameSize.Width,
-                frameSize.Height,
+                context.FrameSize.Width,
+                context.FrameSize.Height,
                 targetSize.Width,
                 targetSize.Height,
                 horizontalMargin,
@@ -134,6 +144,11 @@ public class MotionElement(
 
             List<string> arguments = ["-nostdin", "-hide_banner", "-nostats", "-loglevel", "error"];
 
+            if (motionElement.EndBehavior is MotionEndBehavior.Loop)
+            {
+                arguments.AddRange(["-stream_loop", "-1"]);
+            }
+
             foreach (string decoder in probeResult.Decoder)
             {
                 arguments.AddRange(["-c:v", decoder]);
@@ -149,26 +164,38 @@ public class MotionElement(
                 "-i", motionElement.VideoPath,
             ]);
 
-            var videoFilter = $"fps={frameRate}";
+            var videoFilter = $"fps={context.FrameRate}";
             if (motionElement.Scale)
             {
                 videoFilter += $",scale={targetSize.Width}:{targetSize.Height}";
             }
 
+            arguments.AddRange(["-vf", videoFilter]);
+
+            if (motionElement.EndBehavior is MotionEndBehavior.Loop)
+            {
+                arguments.AddRange(
+                    "-t",
+                    $"{(int)context.Duration.TotalHours:00}:{context.Duration:mm}:{context.Duration:ss\\.fffffff}");
+            }
+
             arguments.AddRange(
             [
-                "-vf", videoFilter,
                 "-f", "image2pipe",
                 "-pix_fmt", "bgra",
                 "-vcodec", "rawvideo",
                 "-"
             ]);
 
+            _state = MotionElementState.PlayingIn;
+
             Command command = Cli.Wrap("ffmpeg")
                 .WithArguments(arguments)
                 .WithWorkingDirectory(FileSystemLayout.TempFilePoolFolder)
                 .WithStandardOutputPipe(PipeTarget.ToStream(pipe.Writer.AsStream()));
 
+            //logger.LogDebug("ffmpeg motion element arguments {FFmpegArguments}", command.Arguments);
+
             _cancellationTokenSource = new CancellationTokenSource();
             var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(
                 cancellationToken,
@@ -192,60 +219,87 @@ public class MotionElement(
         TimeSpan channelTime,
         CancellationToken cancellationToken)
     {
-        if (_isFinished || contentTime < _startTime)
-        {
-            return Option<PreparedElementImage>.None;
-        }
-
-        while (true)
+        try
         {
-            ReadResult readResult = await _pipeReader.ReadAsync(cancellationToken);
-            ReadOnlySequence<byte> buffer = readResult.Buffer;
-            SequencePosition consumed = buffer.Start;
-            SequencePosition examined = buffer.End;
+            if (_state is MotionElementState.Finished || contentTime < _startTime)
+            {
+                return Option<PreparedElementImage>.None;
+            }
 
-            try
+            if (_state is MotionElementState.Holding)
             {
-                if (buffer.Length >= _frameSize)
+                if (contentTime <= _endTime)
                 {
-                    ReadOnlySequence<byte> sequence = buffer.Slice(0, _frameSize);
+                    return new PreparedElementImage(_canvasBitmap, SKPointI.Empty, 1.0f, false);
+                }
 
-                    using (SKPixmap pixmap = _motionFrameBitmap.PeekPixels())
-                    {
-                        sequence.CopyTo(pixmap.GetPixelSpan());
-                    }
+                _state = MotionElementState.Finished;
+                return Option<PreparedElementImage>.None;
+            }
 
-                    _canvasBitmap.Erase(SKColors.Transparent);
+            while (true)
+            {
+                ReadResult readResult = await _pipeReader.ReadAsync(cancellationToken);
+                ReadOnlySequence<byte> buffer = readResult.Buffer;
+                SequencePosition consumed = buffer.Start;
+                SequencePosition examined = buffer.End;
 
-                    using (var canvas = new SKCanvas(_canvasBitmap))
+                try
+                {
+                    if (buffer.Length >= _frameSize)
                     {
-                        canvas.DrawBitmap(_motionFrameBitmap, _point);
-                    }
+                        ReadOnlySequence<byte> sequence = buffer.Slice(0, _frameSize);
 
-                    // mark this frame as consumed
-                    consumed = sequence.End;
+                        using (SKPixmap pixmap = _motionFrameBitmap.PeekPixels())
+                        {
+                            sequence.CopyTo(pixmap.GetPixelSpan());
+                        }
 
-                    // we are done, return the frame
-                    return new PreparedElementImage(_canvasBitmap, SKPointI.Empty, 1.0f, false);
-                }
+                        _canvasBitmap.Erase(SKColors.Transparent);
 
-                if (readResult.IsCompleted)
-                {
-                    _isFinished = true;
+                        using (var canvas = new SKCanvas(_canvasBitmap))
+                        {
+                            canvas.DrawBitmap(_motionFrameBitmap, _point);
+                        }
+
+                        // mark this frame as consumed
+                        consumed = sequence.End;
 
-                    await _pipeReader.CompleteAsync();
-                    return Option<PreparedElementImage>.None;
+                        // we are done, return the frame
+                        return new PreparedElementImage(_canvasBitmap, SKPointI.Empty, 1.0f, false);
+                    }
+
+                    if (readResult.IsCompleted)
+                    {
+                        await _pipeReader.CompleteAsync();
+
+                        if (motionElement.EndBehavior is MotionEndBehavior.Hold)
+                        {
+                            _state = MotionElementState.Holding;
+                            return new PreparedElementImage(_canvasBitmap, SKPointI.Empty, 1.0f, false);
+                        }
+                        else
+                        {
+                            _state = MotionElementState.Finished;
+                        }
+
+                        return Option<PreparedElementImage>.None;
+                    }
                 }
-            }
-            finally
-            {
-                if (!_isFinished)
+                finally
                 {
-                    // advance the reader, consuming the processed frame and examining the entire buffer
-                    _pipeReader.AdvanceTo(consumed, examined);
+                    if (_state is not (MotionElementState.Finished or MotionElementState.Holding))
+                    {
+                        // advance the reader, consuming the processed frame and examining the entire buffer
+                        _pipeReader.AdvanceTo(consumed, examined);
+                    }
                 }
             }
         }
+        catch (TaskCanceledException)
+        {
+            return Option<PreparedElementImage>.None;
+        }
     }
 
     private async Task<ProbeResult> ProbeMotionElement(Resolution frameSize)

ErsatzTV.Infrastructure/Streaming/Graphics/Motion/MotionElementState.cs

@@ -0,0 +1,10 @@
+namespace ErsatzTV.Infrastructure.Streaming.Graphics;
+
+public enum MotionElementState
+{
+    NotStarted,
+    PlayingIn,
+    Holding,
+    // TODO: PlayingOut?
+    Finished
+}

ErsatzTV.Infrastructure/Streaming/Graphics/Subtitle/SubtitleElement.cs

@@ -2,10 +2,10 @@ using System.Buffers;
 using System.IO.Pipelines;
 using CliWrap;
 using ErsatzTV.Core;
-using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.FFmpeg;
 using ErsatzTV.Core.Graphics;
 using ErsatzTV.Core.Interfaces.FFmpeg;
+using ErsatzTV.Core.Interfaces.Streaming;
 using Microsoft.Extensions.Logging;
 using Scriban;
 using Scriban.Runtime;
@@ -52,12 +52,7 @@ public class SubtitleElement(
         _videoFrame?.Dispose();
     }
 
-    public override async Task InitializeAsync(
-        Resolution squarePixelFrameSize,
-        Resolution frameSize,
-        int frameRate,
-        TimeSpan seek,
-        CancellationToken cancellationToken)
+    public override async Task InitializeAsync(GraphicsEngineContext context, CancellationToken cancellationToken)
     {
         try
         {
@@ -65,8 +60,12 @@ public class SubtitleElement(
             _pipeReader = pipe.Reader;
 
             // video size is the same as the main frame size
-            _frameSize = frameSize.Width * frameSize.Height * 4;
-            _videoFrame = new SKBitmap(frameSize.Width, frameSize.Height, SKColorType.Bgra8888, SKAlphaType.Unpremul);
+            _frameSize = context.FrameSize.Width * context.FrameSize.Height * 4;
+            _videoFrame = new SKBitmap(
+                context.FrameSize.Width,
+                context.FrameSize.Height,
+                SKColorType.Bgra8888,
+                SKAlphaType.Unpremul);
 
             // subtitles contain their own positioning info
             _point = SKPointI.Empty;
@@ -78,10 +77,10 @@ public class SubtitleElement(
             scriptObject.Import("convert_timezone", templateFunctions.ConvertTimeZone);
             scriptObject.Import("format_datetime", templateFunctions.FormatDateTime);
 
-            var context = new TemplateContext { MemberRenamer = member => member.Name };
-            context.PushGlobal(scriptObject);
+            var templateContext = new TemplateContext { MemberRenamer = member => member.Name };
+            templateContext.PushGlobal(scriptObject);
             string inputText = await File.ReadAllTextAsync(subtitleElement.Template, cancellationToken);
-            string textToRender = await Template.Parse(inputText).RenderAsync(context);
+            string textToRender = await Template.Parse(inputText).RenderAsync(templateContext);
             await File.WriteAllTextAsync(subtitleTemplateFile, textToRender, cancellationToken);
 
             string subtitleFile = Path.GetFileName(subtitleTemplateFile);
@@ -90,7 +89,7 @@ public class SubtitleElement(
                 "-nostdin", "-hide_banner", "-nostats", "-loglevel", "error",
                 "-f", "lavfi",
                 "-i",
-                $"color=c=black@0.0:s={frameSize.Width}x{frameSize.Height}:r={frameRate},format=bgra,subtitles='{subtitleFile}':alpha=1",
+                $"color=c=black@0.0:s={context.FrameSize.Width}x{context.FrameSize.Height}:r={context.FrameRate},format=bgra,subtitles='{subtitleFile}':alpha=1",
                 "-f", "image2pipe",
                 "-pix_fmt", "bgra",
                 "-vcodec", "rawvideo",

ErsatzTV.Infrastructure/Streaming/Graphics/Text/TextElement.cs

@@ -1,6 +1,6 @@
 using System.Text.RegularExpressions;
-using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Graphics;
+using ErsatzTV.Core.Interfaces.Streaming;
 using Microsoft.Extensions.Logging;
 using NCalc;
 using SkiaSharp;
@@ -29,12 +29,7 @@ public partial class TextElement(
         _image = null;
     }
 
-    public override Task InitializeAsync(
-        Resolution squarePixelFrameSize,
-        Resolution frameSize,
-        int frameRate,
-        TimeSpan seek,
-        CancellationToken cancellationToken)
+    public override Task InitializeAsync(GraphicsEngineContext context, CancellationToken cancellationToken)
     {
         try
         {
@@ -77,13 +72,14 @@ public partial class TextElement(
             }
 
             var horizontalMargin =
-                (int)Math.Round((textElement.HorizontalMarginPercent ?? 0) / 100.0 * frameSize.Width);
-            var verticalMargin = (int)Math.Round((textElement.VerticalMarginPercent ?? 0) / 100.0 * frameSize.Height);
+                (int)Math.Round((textElement.HorizontalMarginPercent ?? 0) / 100.0 * context.FrameSize.Width);
+            var verticalMargin =
+                (int)Math.Round((textElement.VerticalMarginPercent ?? 0) / 100.0 * context.FrameSize.Height);
 
             _location = CalculatePosition(
                 textElement.Location,
-                frameSize.Width,
-                frameSize.Height,
+                context.FrameSize.Width,
+                context.FrameSize.Height,
                 _image.Width,
                 _image.Height,
                 horizontalMargin,