diff --git a/CHANGELOG.md b/CHANGELOG.md index 118b0b124..6353a4b0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Optimize QSV pipelines by merging consecutive vpp_qsv filters as much as possible (e.g. tonemap, scale and format using a single filter) - Show error messages over black/silence fallback streams by default - To disable, create the file `next/channel-config-overlays/default.json` in ETV's config folder with the contents `{"fallback":{"show_error":false}}` +- Graphics engine: + - Use `Seek Seconds` in playback troubleshooter directly as `MediaItem_SeekSeconds` in templates + - Previously, `MediaItem_SeekSeconds` was always zero, so graphics elements often started wherever troubleshooting playback started instead of being anchored ### Fixed - Next engine: @@ -36,6 +39,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Save and restore sequential schedule mid-roll, post-roll, and graphics state between builds - Fix subtitle playback with Plex other video libraries - Fix Plex other video tag generation when Plex server and ETV server use different path separators (i.e. Windows and Linux) +- Graphics engine: + - Fix motion element timing, including hold behavior when seeking into hold ## [26.9.0] - 2026-09-06 ### Fixed diff --git a/ErsatzTV.Application/Troubleshooting/Commands/PrepareTroubleshootingPlaybackHandler.cs b/ErsatzTV.Application/Troubleshooting/Commands/PrepareTroubleshootingPlaybackHandler.cs index 344ffe4c3..01632f1a2 100644 --- a/ErsatzTV.Application/Troubleshooting/Commands/PrepareTroubleshootingPlaybackHandler.cs +++ b/ErsatzTV.Application/Troubleshooting/Commands/PrepareTroubleshootingPlaybackHandler.cs @@ -228,24 +228,21 @@ public class PrepareTroubleshootingPlaybackHandler( // we cannot burst live input bool hlsRealtime = mediaItem is RemoteStream { IsLive: true }; - TimeSpan inPoint = TimeSpan.Zero; - TimeSpan outPoint = duration; + TimeSpan seek = TimeSpan.Zero; if (!hlsRealtime) { foreach (int seekSeconds in request.SeekSeconds) { - inPoint = TimeSpan.FromSeconds(seekSeconds); - if (inPoint > version.Duration) + seek = TimeSpan.FromSeconds(seekSeconds); + if (seek > version.Duration) { - inPoint = version.Duration - duration; + seek = version.Duration - duration; } - if (inPoint + duration > version.Duration) + if (seek + duration > version.Duration) { - duration = version.Duration - inPoint; + duration = version.Duration - seek; } - - outPoint = inPoint + duration; } } @@ -284,8 +281,8 @@ public class PrepareTroubleshootingPlaybackHandler( mediaItem, ffmpegProfile, channel, - inPoint, - outPoint, + seek, + duration, watermarks, graphicsElements, cancellationToken); @@ -299,7 +296,8 @@ public class PrepareTroubleshootingPlaybackHandler( ffprobePath, ffmpegProfile, channel, - inPoint, + seek, + duration, watermarks, graphicsElements, cancellationToken); @@ -311,8 +309,8 @@ public class PrepareTroubleshootingPlaybackHandler( MediaItem mediaItem, FFmpegProfile ffmpegProfile, Channel channel, - TimeSpan inPoint, - TimeSpan outPoint, + TimeSpan seek, + TimeSpan duration, List watermarks, List graphicsElements, CancellationToken cancellationToken) @@ -325,7 +323,7 @@ public class PrepareTroubleshootingPlaybackHandler( string channelBinary = channelBinaryResult.SuccessToSeq().Head(); - // ignore fractional seconds so virtual start and playout item start always match + // ignore fractional seconds so the virtual playback position has an exact seek DateTimeOffset start = DateTimeOffset.FromUnixTimeSeconds(DateTimeOffset.Now.ToUnixTimeSeconds()); ChannelConfig config = await channelConfigConverter.ToNext( @@ -343,8 +341,9 @@ public class PrepareTroubleshootingPlaybackHandler( { MediaItem = mediaItem, MediaItemId = mediaItem.Id, - Start = start.UtcDateTime, - Finish = start.UtcDateTime.Add(outPoint - inPoint), + // model joining an already-running item so canvas offsets include the seek + Start = start.UtcDateTime.Subtract(seek), + Finish = start.UtcDateTime.Add(duration), GuideStart = null, GuideFinish = null, CustomTitle = null, @@ -352,8 +351,8 @@ public class PrepareTroubleshootingPlaybackHandler( FillerKind = FillerKind.None, Playout = null, PlayoutId = 0, - InPoint = inPoint, - OutPoint = outPoint, + InPoint = TimeSpan.Zero, + OutPoint = seek + duration, ChapterTitle = null, Watermarks = [.. watermarks.Map(wm => wm.Watermark)], DisableWatermarks = request.WatermarkIds.Count == 0, @@ -378,7 +377,7 @@ public class PrepareTroubleshootingPlaybackHandler( [], TimeSpan.Zero, playoutItem, - await GetNextSubtitles(mediaItem, channel, request, inPoint), + await GetNextSubtitles(mediaItem, channel, request, playoutItem.InPoint), shouldLogMessages: true, cancellationToken); @@ -433,7 +432,8 @@ public class PrepareTroubleshootingPlaybackHandler( string ffprobePath, FFmpegProfile ffmpegProfile, Channel channel, - TimeSpan inPoint, + TimeSpan seek, + TimeSpan duration, List watermarks, List graphicsElements, CancellationToken cancellationToken) @@ -485,12 +485,6 @@ public class PrepareTroubleshootingPlaybackHandler( DateTimeOffset now = DateTimeOffset.Now; - var duration = TimeSpan.FromSeconds(Math.Min(version.Duration.TotalSeconds, 30)); - if (duration <= TimeSpan.Zero) - { - duration = TimeSpan.FromSeconds(30); - } - // we cannot burst live input bool hlsRealtime = mediaItem is RemoteStream { IsLive: true }; @@ -508,7 +502,8 @@ public class PrepareTroubleshootingPlaybackHandler( string.Empty, string.Empty, SubtitleMode, - now, + // graphics use elapsed item time; an in-point alone only seeks the media + now - seek, now + duration, now, duration, @@ -521,8 +516,8 @@ public class PrepareTroubleshootingPlaybackHandler( hlsRealtime, mediaItem is RemoteStream { IsLive: true } ? StreamInputKind.Live : StreamInputKind.Vod, FillerKind.None, - inPoint, - channelStartTime: DateTimeOffset.Now, + inPoint: TimeSpan.Zero, + channelStartTime: now, TimeSpan.Zero, Option.None, FileSystemLayout.TranscodeTroubleshootingFolder, diff --git a/ErsatzTV.Infrastructure.Tests/Streaming/MotionElementTests.cs b/ErsatzTV.Infrastructure.Tests/Streaming/MotionElementTests.cs new file mode 100644 index 000000000..319d7db3e --- /dev/null +++ b/ErsatzTV.Infrastructure.Tests/Streaming/MotionElementTests.cs @@ -0,0 +1,181 @@ +using System.ComponentModel; +using CliWrap; +using ErsatzTV.Core; +using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Graphics; +using ErsatzTV.Core.Interfaces.Metadata; +using ErsatzTV.Core.Interfaces.Streaming; +using ErsatzTV.FFmpeg; +using ErsatzTV.Infrastructure.Streaming.Graphics; +using LanguageExt; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using NUnit.Framework; +using Shouldly; +using SkiaSharp; + +namespace ErsatzTV.Infrastructure.Tests.Streaming; + +[TestFixture] +public class MotionElementTests +{ + private string _folder = null!; + private string _videoPath = null!; + + [OneTimeSetUp] + public async Task CreateAnimation() + { + _folder = Path.Combine(Path.GetTempPath(), $"etv-motion-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_folder); + Directory.CreateDirectory(FileSystemLayout.TempFilePoolFolder); + _videoPath = Path.Combine(_folder, "animation.mov"); + try + { + // One second red, then one second blue, with an alpha-capable codec. + await Cli.Wrap("ffmpeg").WithArguments(new[] + { + "-nostdin", "-v", "error", "-f", "lavfi", "-i", "color=red:s=16x16:r=10:d=1", + "-f", "lavfi", "-i", "color=blue:s=16x16:r=10:d=1", + "-filter_complex", "[0:v][1:v]concat=n=2:v=1:a=0", + "-c:v", "prores_ks", "-profile:v", "4", "-pix_fmt", "yuva444p10le", _videoPath + }).ExecuteAsync(); + } + catch (Win32Exception) + { + Assert.Ignore("Motion integration tests require ffmpeg on PATH."); + } + } + + [OneTimeTearDown] + public void RemoveAnimation() => Directory.Delete(_folder, true); + + [TestCase(0, 5, false)] + [TestCase(5, 5, false)] + [TestCase(5.5, 5.5, false)] + [TestCase(6.5, 6.5, true)] + [TestCase(6.95, 6.95, true)] + [TestCase(6.99, 6.99, true)] + [TestCase(7, 7, true)] + [TestCase(600, 600, true)] + public async Task Hold_should_show_the_correct_frame_when_joining(double seek, double time, bool blue) + { + using var element = await CreateElement(MotionEndBehavior.Hold, seek); + await AssertFrame(element, time, blue); + (await Prepare(element, 3607)).IsNone.ShouldBeTrue(); + element.IsFinished.ShouldBeTrue(); + } + + [Test] + public async Task Hold_should_resume_mid_clip_then_retain_the_final_frame() + { + using var element = await CreateElement(MotionEndBehavior.Hold, 5.5); + for (var frame = 0; frame < 20; frame++) + { + await AssertFrame(element, 5.5 + frame / 10.0, frame >= 5); + } + + await AssertFrame(element, 3606.9, true); + (await Prepare(element, 3607)).IsNone.ShouldBeTrue(); + } + + [Test] + public async Task Hold_should_play_then_retain_the_final_frame() + { + using var element = await CreateElement(MotionEndBehavior.Hold, 0); + (await Prepare(element, 4.9)).IsNone.ShouldBeTrue(); + for (var frame = 0; frame < 25; frame++) + { + await AssertFrame(element, 5 + frame / 10.0, frame >= 10); + } + } + + [TestCase(MotionEndBehavior.Hold, 3607)] + [TestCase(MotionEndBehavior.Hold, 3608)] + [TestCase(MotionEndBehavior.Disappear, 7)] + public async Task Should_be_finished_when_joining_at_or_after_end(MotionEndBehavior behavior, double seek) + { + using var element = await CreateElement(behavior, seek); + element.IsFinished.ShouldBeTrue(); + (await Prepare(element, seek)).IsNone.ShouldBeTrue(); + } + + [TestCase(MotionEndBehavior.Hold)] + [TestCase(MotionEndBehavior.Disappear)] + public async Task Should_enforce_end_time_even_while_playing(MotionEndBehavior behavior) + { + using var element = await CreateElement(behavior, 5, holdSeconds: 0); + await AssertFrame(element, 5, false); + (await Prepare(element, 7)).IsNone.ShouldBeTrue(); + element.IsFinished.ShouldBeTrue(); + } + + [TestCase(600, true)] + [TestCase(601, false)] + public async Task Loop_should_wrap_seek_and_continue_across_the_source_end(double seek, bool blue) + { + using var element = await CreateElement(MotionEndBehavior.Loop, seek); + for (var frame = 0; frame < 25; frame++) + { + bool expectedBlue = ((blue ? 10 : 0) + frame) % 20 >= 10; + await AssertFrame(element, seek + frame / 10.0, expectedBlue); + } + } + + [Test] + public async Task Loop_should_end_at_the_stream_end_without_adding_start_seconds() + { + using var element = await CreateElement(MotionEndBehavior.Loop, 0, duration: 8); + await AssertFrame(element, 5, false); + (await Prepare(element, 8)).IsNone.ShouldBeTrue(); + element.IsFinished.ShouldBeTrue(); + } + + [Test] + public async Task Should_skip_elements_starting_after_the_stream() + { + using var element = await CreateElement(MotionEndBehavior.Loop, 0, duration: 5); + element.IsFinished.ShouldBeTrue(); + } + + private async Task CreateElement( + MotionEndBehavior behavior, double seek, double holdSeconds = 3600, double duration = 4000) + { + var statistics = Substitute.For(); + statistics.GetStatistics(Arg.Any(), _videoPath).Returns( + Task.FromResult>(new MediaVersion + { + Width = 16, Height = 16, Duration = TimeSpan.FromSeconds(2), Streams = [] + })); + var element = new MotionElement(new MotionGraphicsElement + { + VideoPath = _videoPath, StartSeconds = 5, EndBehavior = behavior, HoldSeconds = holdSeconds + }, "ffprobe", "ffmpeg", statistics, NullLogger.Instance); + var size = new Resolution { Width = 16, Height = 16 }; + await element.InitializeAsync(new GraphicsEngineContext( + "1", new Movie(), [], [], size, size, new FrameRate("10"), + DateTimeOffset.Now, DateTimeOffset.Now, DateTimeOffset.Now, + TimeSpan.FromSeconds(seek), TimeSpan.FromSeconds(duration), TimeSpan.FromSeconds(seek + duration)), + CancellationToken.None); + return element; + } + + private static async Task AssertFrame(MotionElement element, double time, bool blue) + { + Option image = await Prepare(element, time); + image.IsSome.ShouldBeTrue(); + foreach (PreparedElementImage frame in image) + { + SKColor pixel = frame.Image.GetPixel(8, 8); + (blue ? pixel.Blue : pixel.Red).ShouldBeGreaterThan((byte)200); + (blue ? pixel.Red : pixel.Blue).ShouldBeLessThan((byte)30); + pixel.Alpha.ShouldBe((byte)255); + } + } + + private static async Task> Prepare(MotionElement element, double time) + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + return await element.PrepareImage(TimeSpan.Zero, TimeSpan.FromSeconds(time), TimeSpan.Zero, + TimeSpan.Zero, timeout.Token); + } +} diff --git a/ErsatzTV.Infrastructure/Streaming/Graphics/Motion/MotionElement.cs b/ErsatzTV.Infrastructure/Streaming/Graphics/Motion/MotionElement.cs index 681a3120c..e7c5e3c5e 100644 --- a/ErsatzTV.Infrastructure/Streaming/Graphics/Motion/MotionElement.cs +++ b/ErsatzTV.Infrastructure/Streaming/Graphics/Motion/MotionElement.cs @@ -23,6 +23,8 @@ public class MotionElement( private CancellationTokenSource _cancellationTokenSource; private CommandTask _commandTask; private int _frameSize; + private long _framesToSkip; + private bool _hasFrame; private PipeReader _pipeReader; private SKPointI _point; private SKBitmap _motionFrameBitmap; @@ -66,7 +68,7 @@ public class MotionElement( ProbeResult probeResult = await ProbeMotionElement(context.FrameSize); var overlayDuration = motionElement.EndBehavior switch { - MotionEndBehavior.Loop => context.Seek + context.Duration, + MotionEndBehavior.Loop => context.Seek + context.Duration - _startTime, MotionEndBehavior.Hold => probeResult.Duration + holdDuration, _ => probeResult.Duration }; @@ -74,7 +76,7 @@ public class MotionElement( _endTime = _startTime + overlayDuration; // already past the time when this is supposed to play; don't do any more work - if (_startTime + overlayDuration < context.Seek) + if (_endTime <= context.Seek || _startTime >= context.Seek + context.Duration) { IsFinished = true; return; @@ -89,6 +91,23 @@ public class MotionElement( overlaySeekTime = context.Seek - _startTime; } + if (motionElement.EndBehavior is MotionEndBehavior.Hold) + { + // seek over the prefix, but leave a second at EOF to recover the final + // frame reliably. seeking too close to EOF can produce no frames. + TimeSpan latestSeek = probeResult.Duration > TimeSpan.FromSeconds(1) + ? probeResult.Duration - TimeSpan.FromSeconds(1) + : TimeSpan.Zero; + TimeSpan inputSeek = overlaySeekTime < latestSeek ? overlaySeekTime : latestSeek; + _framesToSkip = (long)((overlaySeekTime - inputSeek).TotalSeconds * + context.FrameRate.ParsedFrameRate); + overlaySeekTime = inputSeek; + } + else if (motionElement.EndBehavior is MotionEndBehavior.Loop && probeResult.Duration > TimeSpan.Zero) + { + overlaySeekTime = TimeSpan.FromTicks(overlaySeekTime.Ticks % probeResult.Duration.Ticks); + } + Resolution sourceSize = probeResult.Size; int scaledWidth = sourceSize.Width; @@ -151,7 +170,7 @@ public class MotionElement( arguments.AddRange(["-c:v", decoder]); } - if (overlaySeekTime > TimeSpan.Zero) + if (overlaySeekTime > TimeSpan.Zero && motionElement.EndBehavior is not MotionEndBehavior.Loop) { arguments.AddRange(["-ss", FFmpegFormatter.Milliseconds(overlaySeekTime)]); } @@ -161,6 +180,12 @@ public class MotionElement( "-i", motionElement.VideoPath, ]); + // output seeking preserves the complete source on subsequent loop iterations. + if (overlaySeekTime > TimeSpan.Zero && motionElement.EndBehavior is MotionEndBehavior.Loop) + { + arguments.AddRange(["-ss", FFmpegFormatter.Milliseconds(overlaySeekTime)]); + } + var videoFilter = $"fps={context.FrameRate.RFrameRate}"; if (motionElement.Scale) { @@ -171,7 +196,8 @@ public class MotionElement( if (motionElement.EndBehavior is MotionEndBehavior.Loop) { - arguments.AddRange(["-t", FFmpegFormatter.Milliseconds(context.Duration)]); + TimeSpan playbackStart = context.Seek > _startTime ? context.Seek : _startTime; + arguments.AddRange(["-t", FFmpegFormatter.Milliseconds(_endTime - playbackStart)]); } arguments.AddRange( @@ -189,7 +215,7 @@ public class MotionElement( .WithWorkingDirectory(FileSystemLayout.TempFilePoolFolder) .WithStandardOutputPipe(PipeTarget.ToStream(pipe.Writer.AsStream())); - //logger.LogDebug("ffmpeg motion element arguments {FFmpegArguments}", command.Arguments); + logger.LogDebug("ffmpeg motion element arguments {FFmpegArguments}", command.Arguments); _cancellationTokenSource = new CancellationTokenSource(); var linkedToken = CancellationTokenSource.CreateLinkedTokenSource( @@ -216,22 +242,23 @@ public class MotionElement( { try { - if (_state is MotionElementState.Finished || contentTime < _startTime) + if (IsFinished || _state is MotionElementState.Finished || contentTime < _startTime) { return Option.None; } - if (_state is MotionElementState.Holding) + if (contentTime >= _endTime) { - if (contentTime <= _endTime) - { - return new PreparedElementImage(_motionFrameBitmap, _point, 1.0f, ZIndex, false); - } - + IsFinished = true; _state = MotionElementState.Finished; return Option.None; } + if (_state is MotionElementState.Holding) + { + return new PreparedElementImage(_motionFrameBitmap, _point, 1.0f, ZIndex, false); + } + while (true) { ReadResult readResult = await _pipeReader.ReadAsync(cancellationToken); @@ -252,6 +279,14 @@ public class MotionElement( // mark this frame as consumed consumed = sequence.End; + examined = consumed; + _hasFrame = true; + + if (_framesToSkip > 0) + { + _framesToSkip--; + continue; + } // we are done, return the frame return new PreparedElementImage(_motionFrameBitmap, _point, 1.0f, ZIndex, false); @@ -261,13 +296,14 @@ public class MotionElement( { await _pipeReader.CompleteAsync(); - if (motionElement.EndBehavior is MotionEndBehavior.Hold) + if (motionElement.EndBehavior is MotionEndBehavior.Hold && _hasFrame) { _state = MotionElementState.Holding; return new PreparedElementImage(_motionFrameBitmap, _point, 1.0f, ZIndex, false); } else { + IsFinished = true; _state = MotionElementState.Finished; } @@ -278,7 +314,7 @@ public class MotionElement( { if (_state is not (MotionElementState.Finished or MotionElementState.Holding)) { - // advance the reader, consuming the processed frame and examining the entire buffer + // leave unread frames available without waiting for more data _pipeReader.AdvanceTo(consumed, examined); } }