Browse Source

fix: graphics engine motion (#3019)

main
Jason Dove 7 hours ago committed by GitHub
parent
commit
908fa96178
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 5
      CHANGELOG.md
  2. 55
      ErsatzTV.Application/Troubleshooting/Commands/PrepareTroubleshootingPlaybackHandler.cs
  3. 181
      ErsatzTV.Infrastructure.Tests/Streaming/MotionElementTests.cs
  4. 64
      ErsatzTV.Infrastructure/Streaming/Graphics/Motion/MotionElement.cs

5
CHANGELOG.md

@ -25,6 +25,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). @@ -25,6 +25,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Optimize QSV pipelines by merging consecutive vpp_qsv filters as much as possible (e.g. tonemap, scale and format using a single filter)
- Show error messages over black/silence fallback streams by default
- To disable, create the file `next/channel-config-overlays/default.json` in ETV's config folder with the contents `{"fallback":{"show_error":false}}`
- Graphics engine:
- Use `Seek Seconds` in playback troubleshooter directly as `MediaItem_SeekSeconds` in templates
- Previously, `MediaItem_SeekSeconds` was always zero, so graphics elements often started wherever troubleshooting playback started instead of being anchored
### Fixed
- Next engine:
@ -36,6 +39,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). @@ -36,6 +39,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Save and restore sequential schedule mid-roll, post-roll, and graphics state between builds
- Fix subtitle playback with Plex other video libraries
- Fix Plex other video tag generation when Plex server and ETV server use different path separators (i.e. Windows and Linux)
- Graphics engine:
- Fix motion element timing, including hold behavior when seeking into hold
## [26.9.0] - 2026-09-06
### Fixed

55
ErsatzTV.Application/Troubleshooting/Commands/PrepareTroubleshootingPlaybackHandler.cs

@ -228,24 +228,21 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -228,24 +228,21 @@ public class PrepareTroubleshootingPlaybackHandler(
// we cannot burst live input
bool hlsRealtime = mediaItem is RemoteStream { IsLive: true };
TimeSpan inPoint = TimeSpan.Zero;
TimeSpan outPoint = duration;
TimeSpan seek = TimeSpan.Zero;
if (!hlsRealtime)
{
foreach (int seekSeconds in request.SeekSeconds)
{
inPoint = TimeSpan.FromSeconds(seekSeconds);
if (inPoint > version.Duration)
seek = TimeSpan.FromSeconds(seekSeconds);
if (seek > version.Duration)
{
inPoint = version.Duration - duration;
seek = version.Duration - duration;
}
if (inPoint + duration > version.Duration)
if (seek + duration > version.Duration)
{
duration = version.Duration - inPoint;
duration = version.Duration - seek;
}
outPoint = inPoint + duration;
}
}
@ -284,8 +281,8 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -284,8 +281,8 @@ public class PrepareTroubleshootingPlaybackHandler(
mediaItem,
ffmpegProfile,
channel,
inPoint,
outPoint,
seek,
duration,
watermarks,
graphicsElements,
cancellationToken);
@ -299,7 +296,8 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -299,7 +296,8 @@ public class PrepareTroubleshootingPlaybackHandler(
ffprobePath,
ffmpegProfile,
channel,
inPoint,
seek,
duration,
watermarks,
graphicsElements,
cancellationToken);
@ -311,8 +309,8 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -311,8 +309,8 @@ public class PrepareTroubleshootingPlaybackHandler(
MediaItem mediaItem,
FFmpegProfile ffmpegProfile,
Channel channel,
TimeSpan inPoint,
TimeSpan outPoint,
TimeSpan seek,
TimeSpan duration,
List<WatermarkOptions> watermarks,
List<GraphicsElement> graphicsElements,
CancellationToken cancellationToken)
@ -325,7 +323,7 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -325,7 +323,7 @@ public class PrepareTroubleshootingPlaybackHandler(
string channelBinary = channelBinaryResult.SuccessToSeq().Head();
// ignore fractional seconds so virtual start and playout item start always match
// ignore fractional seconds so the virtual playback position has an exact seek
DateTimeOffset start = DateTimeOffset.FromUnixTimeSeconds(DateTimeOffset.Now.ToUnixTimeSeconds());
ChannelConfig config = await channelConfigConverter.ToNext(
@ -343,8 +341,9 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -343,8 +341,9 @@ public class PrepareTroubleshootingPlaybackHandler(
{
MediaItem = mediaItem,
MediaItemId = mediaItem.Id,
Start = start.UtcDateTime,
Finish = start.UtcDateTime.Add(outPoint - inPoint),
// model joining an already-running item so canvas offsets include the seek
Start = start.UtcDateTime.Subtract(seek),
Finish = start.UtcDateTime.Add(duration),
GuideStart = null,
GuideFinish = null,
CustomTitle = null,
@ -352,8 +351,8 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -352,8 +351,8 @@ public class PrepareTroubleshootingPlaybackHandler(
FillerKind = FillerKind.None,
Playout = null,
PlayoutId = 0,
InPoint = inPoint,
OutPoint = outPoint,
InPoint = TimeSpan.Zero,
OutPoint = seek + duration,
ChapterTitle = null,
Watermarks = [.. watermarks.Map(wm => wm.Watermark)],
DisableWatermarks = request.WatermarkIds.Count == 0,
@ -378,7 +377,7 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -378,7 +377,7 @@ public class PrepareTroubleshootingPlaybackHandler(
[],
TimeSpan.Zero,
playoutItem,
await GetNextSubtitles(mediaItem, channel, request, inPoint),
await GetNextSubtitles(mediaItem, channel, request, playoutItem.InPoint),
shouldLogMessages: true,
cancellationToken);
@ -433,7 +432,8 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -433,7 +432,8 @@ public class PrepareTroubleshootingPlaybackHandler(
string ffprobePath,
FFmpegProfile ffmpegProfile,
Channel channel,
TimeSpan inPoint,
TimeSpan seek,
TimeSpan duration,
List<WatermarkOptions> watermarks,
List<GraphicsElement> graphicsElements,
CancellationToken cancellationToken)
@ -485,12 +485,6 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -485,12 +485,6 @@ public class PrepareTroubleshootingPlaybackHandler(
DateTimeOffset now = DateTimeOffset.Now;
var duration = TimeSpan.FromSeconds(Math.Min(version.Duration.TotalSeconds, 30));
if (duration <= TimeSpan.Zero)
{
duration = TimeSpan.FromSeconds(30);
}
// we cannot burst live input
bool hlsRealtime = mediaItem is RemoteStream { IsLive: true };
@ -508,7 +502,8 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -508,7 +502,8 @@ public class PrepareTroubleshootingPlaybackHandler(
string.Empty,
string.Empty,
SubtitleMode,
now,
// graphics use elapsed item time; an in-point alone only seeks the media
now - seek,
now + duration,
now,
duration,
@ -521,8 +516,8 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -521,8 +516,8 @@ public class PrepareTroubleshootingPlaybackHandler(
hlsRealtime,
mediaItem is RemoteStream { IsLive: true } ? StreamInputKind.Live : StreamInputKind.Vod,
FillerKind.None,
inPoint,
channelStartTime: DateTimeOffset.Now,
inPoint: TimeSpan.Zero,
channelStartTime: now,
TimeSpan.Zero,
Option<FrameRate>.None,
FileSystemLayout.TranscodeTroubleshootingFolder,

181
ErsatzTV.Infrastructure.Tests/Streaming/MotionElementTests.cs

@ -0,0 +1,181 @@ @@ -0,0 +1,181 @@
using System.ComponentModel;
using CliWrap;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Graphics;
using ErsatzTV.Core.Interfaces.Metadata;
using ErsatzTV.Core.Interfaces.Streaming;
using ErsatzTV.FFmpeg;
using ErsatzTV.Infrastructure.Streaming.Graphics;
using LanguageExt;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using NUnit.Framework;
using Shouldly;
using SkiaSharp;
namespace ErsatzTV.Infrastructure.Tests.Streaming;
[TestFixture]
public class MotionElementTests
{
private string _folder = null!;
private string _videoPath = null!;
[OneTimeSetUp]
public async Task CreateAnimation()
{
_folder = Path.Combine(Path.GetTempPath(), $"etv-motion-tests-{Guid.NewGuid():N}");
Directory.CreateDirectory(_folder);
Directory.CreateDirectory(FileSystemLayout.TempFilePoolFolder);
_videoPath = Path.Combine(_folder, "animation.mov");
try
{
// One second red, then one second blue, with an alpha-capable codec.
await Cli.Wrap("ffmpeg").WithArguments(new[]
{
"-nostdin", "-v", "error", "-f", "lavfi", "-i", "color=red:s=16x16:r=10:d=1",
"-f", "lavfi", "-i", "color=blue:s=16x16:r=10:d=1",
"-filter_complex", "[0:v][1:v]concat=n=2:v=1:a=0",
"-c:v", "prores_ks", "-profile:v", "4", "-pix_fmt", "yuva444p10le", _videoPath
}).ExecuteAsync();
}
catch (Win32Exception)
{
Assert.Ignore("Motion integration tests require ffmpeg on PATH.");
}
}
[OneTimeTearDown]
public void RemoveAnimation() => Directory.Delete(_folder, true);
[TestCase(0, 5, false)]
[TestCase(5, 5, false)]
[TestCase(5.5, 5.5, false)]
[TestCase(6.5, 6.5, true)]
[TestCase(6.95, 6.95, true)]
[TestCase(6.99, 6.99, true)]
[TestCase(7, 7, true)]
[TestCase(600, 600, true)]
public async Task Hold_should_show_the_correct_frame_when_joining(double seek, double time, bool blue)
{
using var element = await CreateElement(MotionEndBehavior.Hold, seek);
await AssertFrame(element, time, blue);
(await Prepare(element, 3607)).IsNone.ShouldBeTrue();
element.IsFinished.ShouldBeTrue();
}
[Test]
public async Task Hold_should_resume_mid_clip_then_retain_the_final_frame()
{
using var element = await CreateElement(MotionEndBehavior.Hold, 5.5);
for (var frame = 0; frame < 20; frame++)
{
await AssertFrame(element, 5.5 + frame / 10.0, frame >= 5);
}
await AssertFrame(element, 3606.9, true);
(await Prepare(element, 3607)).IsNone.ShouldBeTrue();
}
[Test]
public async Task Hold_should_play_then_retain_the_final_frame()
{
using var element = await CreateElement(MotionEndBehavior.Hold, 0);
(await Prepare(element, 4.9)).IsNone.ShouldBeTrue();
for (var frame = 0; frame < 25; frame++)
{
await AssertFrame(element, 5 + frame / 10.0, frame >= 10);
}
}
[TestCase(MotionEndBehavior.Hold, 3607)]
[TestCase(MotionEndBehavior.Hold, 3608)]
[TestCase(MotionEndBehavior.Disappear, 7)]
public async Task Should_be_finished_when_joining_at_or_after_end(MotionEndBehavior behavior, double seek)
{
using var element = await CreateElement(behavior, seek);
element.IsFinished.ShouldBeTrue();
(await Prepare(element, seek)).IsNone.ShouldBeTrue();
}
[TestCase(MotionEndBehavior.Hold)]
[TestCase(MotionEndBehavior.Disappear)]
public async Task Should_enforce_end_time_even_while_playing(MotionEndBehavior behavior)
{
using var element = await CreateElement(behavior, 5, holdSeconds: 0);
await AssertFrame(element, 5, false);
(await Prepare(element, 7)).IsNone.ShouldBeTrue();
element.IsFinished.ShouldBeTrue();
}
[TestCase(600, true)]
[TestCase(601, false)]
public async Task Loop_should_wrap_seek_and_continue_across_the_source_end(double seek, bool blue)
{
using var element = await CreateElement(MotionEndBehavior.Loop, seek);
for (var frame = 0; frame < 25; frame++)
{
bool expectedBlue = ((blue ? 10 : 0) + frame) % 20 >= 10;
await AssertFrame(element, seek + frame / 10.0, expectedBlue);
}
}
[Test]
public async Task Loop_should_end_at_the_stream_end_without_adding_start_seconds()
{
using var element = await CreateElement(MotionEndBehavior.Loop, 0, duration: 8);
await AssertFrame(element, 5, false);
(await Prepare(element, 8)).IsNone.ShouldBeTrue();
element.IsFinished.ShouldBeTrue();
}
[Test]
public async Task Should_skip_elements_starting_after_the_stream()
{
using var element = await CreateElement(MotionEndBehavior.Loop, 0, duration: 5);
element.IsFinished.ShouldBeTrue();
}
private async Task<MotionElement> CreateElement(
MotionEndBehavior behavior, double seek, double holdSeconds = 3600, double duration = 4000)
{
var statistics = Substitute.For<ILocalStatisticsProvider>();
statistics.GetStatistics(Arg.Any<string>(), _videoPath).Returns(
Task.FromResult<Either<BaseError, MediaVersion>>(new MediaVersion
{
Width = 16, Height = 16, Duration = TimeSpan.FromSeconds(2), Streams = []
}));
var element = new MotionElement(new MotionGraphicsElement
{
VideoPath = _videoPath, StartSeconds = 5, EndBehavior = behavior, HoldSeconds = holdSeconds
}, "ffprobe", "ffmpeg", statistics, NullLogger.Instance);
var size = new Resolution { Width = 16, Height = 16 };
await element.InitializeAsync(new GraphicsEngineContext(
"1", new Movie(), [], [], size, size, new FrameRate("10"),
DateTimeOffset.Now, DateTimeOffset.Now, DateTimeOffset.Now,
TimeSpan.FromSeconds(seek), TimeSpan.FromSeconds(duration), TimeSpan.FromSeconds(seek + duration)),
CancellationToken.None);
return element;
}
private static async Task AssertFrame(MotionElement element, double time, bool blue)
{
Option<PreparedElementImage> image = await Prepare(element, time);
image.IsSome.ShouldBeTrue();
foreach (PreparedElementImage frame in image)
{
SKColor pixel = frame.Image.GetPixel(8, 8);
(blue ? pixel.Blue : pixel.Red).ShouldBeGreaterThan((byte)200);
(blue ? pixel.Red : pixel.Blue).ShouldBeLessThan((byte)30);
pixel.Alpha.ShouldBe((byte)255);
}
}
private static async Task<Option<PreparedElementImage>> Prepare(MotionElement element, double time)
{
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10));
return await element.PrepareImage(TimeSpan.Zero, TimeSpan.FromSeconds(time), TimeSpan.Zero,
TimeSpan.Zero, timeout.Token);
}
}

64
ErsatzTV.Infrastructure/Streaming/Graphics/Motion/MotionElement.cs

@ -23,6 +23,8 @@ public class MotionElement( @@ -23,6 +23,8 @@ public class MotionElement(
private CancellationTokenSource _cancellationTokenSource;
private CommandTask<CommandResult> _commandTask;
private int _frameSize;
private long _framesToSkip;
private bool _hasFrame;
private PipeReader _pipeReader;
private SKPointI _point;
private SKBitmap _motionFrameBitmap;
@ -66,7 +68,7 @@ public class MotionElement( @@ -66,7 +68,7 @@ public class MotionElement(
ProbeResult probeResult = await ProbeMotionElement(context.FrameSize);
var overlayDuration = motionElement.EndBehavior switch
{
MotionEndBehavior.Loop => context.Seek + context.Duration,
MotionEndBehavior.Loop => context.Seek + context.Duration - _startTime,
MotionEndBehavior.Hold => probeResult.Duration + holdDuration,
_ => probeResult.Duration
};
@ -74,7 +76,7 @@ public class MotionElement( @@ -74,7 +76,7 @@ public class MotionElement(
_endTime = _startTime + overlayDuration;
// already past the time when this is supposed to play; don't do any more work
if (_startTime + overlayDuration < context.Seek)
if (_endTime <= context.Seek || _startTime >= context.Seek + context.Duration)
{
IsFinished = true;
return;
@ -89,6 +91,23 @@ public class MotionElement( @@ -89,6 +91,23 @@ public class MotionElement(
overlaySeekTime = context.Seek - _startTime;
}
if (motionElement.EndBehavior is MotionEndBehavior.Hold)
{
// seek over the prefix, but leave a second at EOF to recover the final
// frame reliably. seeking too close to EOF can produce no frames.
TimeSpan latestSeek = probeResult.Duration > TimeSpan.FromSeconds(1)
? probeResult.Duration - TimeSpan.FromSeconds(1)
: TimeSpan.Zero;
TimeSpan inputSeek = overlaySeekTime < latestSeek ? overlaySeekTime : latestSeek;
_framesToSkip = (long)((overlaySeekTime - inputSeek).TotalSeconds *
context.FrameRate.ParsedFrameRate);
overlaySeekTime = inputSeek;
}
else if (motionElement.EndBehavior is MotionEndBehavior.Loop && probeResult.Duration > TimeSpan.Zero)
{
overlaySeekTime = TimeSpan.FromTicks(overlaySeekTime.Ticks % probeResult.Duration.Ticks);
}
Resolution sourceSize = probeResult.Size;
int scaledWidth = sourceSize.Width;
@ -151,7 +170,7 @@ public class MotionElement( @@ -151,7 +170,7 @@ public class MotionElement(
arguments.AddRange(["-c:v", decoder]);
}
if (overlaySeekTime > TimeSpan.Zero)
if (overlaySeekTime > TimeSpan.Zero && motionElement.EndBehavior is not MotionEndBehavior.Loop)
{
arguments.AddRange(["-ss", FFmpegFormatter.Milliseconds(overlaySeekTime)]);
}
@ -161,6 +180,12 @@ public class MotionElement( @@ -161,6 +180,12 @@ public class MotionElement(
"-i", motionElement.VideoPath,
]);
// output seeking preserves the complete source on subsequent loop iterations.
if (overlaySeekTime > TimeSpan.Zero && motionElement.EndBehavior is MotionEndBehavior.Loop)
{
arguments.AddRange(["-ss", FFmpegFormatter.Milliseconds(overlaySeekTime)]);
}
var videoFilter = $"fps={context.FrameRate.RFrameRate}";
if (motionElement.Scale)
{
@ -171,7 +196,8 @@ public class MotionElement( @@ -171,7 +196,8 @@ public class MotionElement(
if (motionElement.EndBehavior is MotionEndBehavior.Loop)
{
arguments.AddRange(["-t", FFmpegFormatter.Milliseconds(context.Duration)]);
TimeSpan playbackStart = context.Seek > _startTime ? context.Seek : _startTime;
arguments.AddRange(["-t", FFmpegFormatter.Milliseconds(_endTime - playbackStart)]);
}
arguments.AddRange(
@ -189,7 +215,7 @@ public class MotionElement( @@ -189,7 +215,7 @@ public class MotionElement(
.WithWorkingDirectory(FileSystemLayout.TempFilePoolFolder)
.WithStandardOutputPipe(PipeTarget.ToStream(pipe.Writer.AsStream()));
//logger.LogDebug("ffmpeg motion element arguments {FFmpegArguments}", command.Arguments);
logger.LogDebug("ffmpeg motion element arguments {FFmpegArguments}", command.Arguments);
_cancellationTokenSource = new CancellationTokenSource();
var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(
@ -216,22 +242,23 @@ public class MotionElement( @@ -216,22 +242,23 @@ public class MotionElement(
{
try
{
if (_state is MotionElementState.Finished || contentTime < _startTime)
if (IsFinished || _state is MotionElementState.Finished || contentTime < _startTime)
{
return Option<PreparedElementImage>.None;
}
if (_state is MotionElementState.Holding)
if (contentTime >= _endTime)
{
if (contentTime <= _endTime)
{
return new PreparedElementImage(_motionFrameBitmap, _point, 1.0f, ZIndex, false);
}
IsFinished = true;
_state = MotionElementState.Finished;
return Option<PreparedElementImage>.None;
}
if (_state is MotionElementState.Holding)
{
return new PreparedElementImage(_motionFrameBitmap, _point, 1.0f, ZIndex, false);
}
while (true)
{
ReadResult readResult = await _pipeReader.ReadAsync(cancellationToken);
@ -252,6 +279,14 @@ public class MotionElement( @@ -252,6 +279,14 @@ public class MotionElement(
// mark this frame as consumed
consumed = sequence.End;
examined = consumed;
_hasFrame = true;
if (_framesToSkip > 0)
{
_framesToSkip--;
continue;
}
// we are done, return the frame
return new PreparedElementImage(_motionFrameBitmap, _point, 1.0f, ZIndex, false);
@ -261,13 +296,14 @@ public class MotionElement( @@ -261,13 +296,14 @@ public class MotionElement(
{
await _pipeReader.CompleteAsync();
if (motionElement.EndBehavior is MotionEndBehavior.Hold)
if (motionElement.EndBehavior is MotionEndBehavior.Hold && _hasFrame)
{
_state = MotionElementState.Holding;
return new PreparedElementImage(_motionFrameBitmap, _point, 1.0f, ZIndex, false);
}
else
{
IsFinished = true;
_state = MotionElementState.Finished;
}
@ -278,7 +314,7 @@ public class MotionElement( @@ -278,7 +314,7 @@ public class MotionElement(
{
if (_state is not (MotionElementState.Finished or MotionElementState.Holding))
{
// advance the reader, consuming the processed frame and examining the entire buffer
// leave unread frames available without waiting for more data
_pipeReader.AdvanceTo(consumed, examined);
}
}

Loading…
Cancel
Save