Browse Source

feat: graphics canvas

pull/3012/head
Jason Dove 6 days ago
parent
commit
8c51ecc256
No known key found for this signature in database
  1. 1
      CHANGELOG.md
  2. 92
      ErsatzTV.Application/Playouts/Commands/SyncNextPlayoutHandler.cs
  3. 3
      ErsatzTV.Application/Streaming/HlsSessionWorker.cs
  4. 9
      ErsatzTV.Application/Streaming/NextSessionWorker.cs
  5. 10
      ErsatzTV.Application/Streaming/Queries/GetGraphicsCanvasStream.cs
  6. 261
      ErsatzTV.Application/Streaming/Queries/GetGraphicsCanvasStreamHandler.cs
  7. 7
      ErsatzTV.Application/Troubleshooting/Commands/PrepareTroubleshootingPlaybackHandler.cs
  8. 2
      ErsatzTV.Application/Troubleshooting/Commands/StartTroubleshootingPlaybackHandler.cs
  9. 34
      ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs
  10. 3
      ErsatzTV.Core/FFmpeg/HlsSessionModel.cs
  11. 1
      ErsatzTV.Core/Interfaces/Streaming/GraphicsEngineContext.cs
  12. 26
      ErsatzTV.Core/Interfaces/Streaming/IGraphicsEngineContextFactory.cs
  13. 16
      ErsatzTV.Core/Interfaces/Troubleshooting/ITroubleshootingPlayoutItemStore.cs
  14. 54
      ErsatzTV.Core/Next/Playout.cs
  15. 15
      ErsatzTV.Core/Troubleshooting/TroubleshootingPlayoutItemStore.cs
  16. 49
      ErsatzTV.Infrastructure.Tests/Streaming/GraphicsCanvasContextTests.cs
  17. 86
      ErsatzTV.Infrastructure/Extensions/PlayoutItemQueryableExtensions.cs
  18. 231
      ErsatzTV.Infrastructure/Scheduling/PlayoutItemConverter.cs
  19. 2
      ErsatzTV.Infrastructure/Streaming/Graphics/GraphicsElementLoader.cs
  20. 80
      ErsatzTV.Infrastructure/Streaming/Graphics/GraphicsEngineContextFactory.cs
  21. 7
      ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs
  22. 51
      ErsatzTV.Tests/Controllers/GraphicsCanvasHeaderTests.cs
  23. 84
      ErsatzTV/Controllers/InternalController.cs
  24. 2
      ErsatzTV/Startup.cs

1
CHANGELOG.md

@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Added
- Next engine:
- Add full graphics engine support
- Add hardware-accelerated padding for QSV on Windows, Linux and Docker (note that this requires the latest ETV custom ffmpeg build)
- Enable `av1`, `vc1`, `vp8`, `vp9` hardware decoding using QSV when supported by GPU
- Add hardware-accelerated HDR10 tonemapping using QSV when supported by GPU

92
ErsatzTV.Application/Playouts/Commands/SyncNextPlayoutHandler.cs

@ -118,95 +118,7 @@ public partial class SyncNextPlayoutHandler( @@ -118,95 +118,7 @@ public partial class SyncNextPlayoutHandler(
List<PlayoutItem> playoutItems = await dbContext.PlayoutItems
.AsNoTracking()
.Where(i => i.Playout.Channel.Number == (mirrorChannelNumber ?? channelNumber))
// get playout deco
.Include(i => i.Playout)
.ThenInclude(p => p.Deco)
.ThenInclude(d => d.DecoWatermarks)
.ThenInclude(d => d.Watermark)
.Include(i => i.Playout)
.ThenInclude(p => p.Deco)
.ThenInclude(d => d.DecoGraphicsElements)
.ThenInclude(d => d.GraphicsElement)
// get watermarks
.Include(i => i.Watermarks)
// get graphics elements
.Include(i => i.PlayoutItemGraphicsElements)
.ThenInclude(pige => pige.GraphicsElement)
// get playout templates (and deco templates/decos)
.Include(i => i.Playout)
.ThenInclude(p => p.Templates)
.ThenInclude(t => t.DecoTemplate)
.ThenInclude(t => t.Items)
.ThenInclude(i => i.Deco)
.ThenInclude(d => d.DecoWatermarks)
.ThenInclude(d => d.Watermark)
// get playout templates (and deco templates/decos)
.Include(i => i.Playout)
.ThenInclude(p => p.Templates)
.ThenInclude(t => t.DecoTemplate)
.ThenInclude(t => t.Items)
.ThenInclude(i => i.Deco)
.ThenInclude(d => d.DecoGraphicsElements)
.ThenInclude(d => d.GraphicsElement)
.Include(i => i.MediaItem)
.ThenInclude(mi => mi.LibraryPath)
.ThenInclude(lp => lp.Library)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Episode).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Episode).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Episode).EpisodeMetadata)
.ThenInclude(em => em.Subtitles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Image).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Image).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Image).ImageMetadata)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Movie).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Movie).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Movie).MovieMetadata)
.ThenInclude(mm => mm.Subtitles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as OtherVideo).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as OtherVideo).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as OtherVideo).OtherVideoMetadata)
.ThenInclude(ovm => ovm.Subtitles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as MusicVideo).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as MusicVideo).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as RemoteStream).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as RemoteStream).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as RemoteStream).RemoteStreamMetadata)
.ThenInclude(em => em.Subtitles)
.IncludeForNextPlayout()
.AsSplitQuery()
.ToListAsync(cancellationToken);
@ -261,7 +173,7 @@ public partial class SyncNextPlayoutHandler( @@ -261,7 +173,7 @@ public partial class SyncNextPlayoutHandler(
targetFolder,
$"{first.StartOffset.ToUnixTimeMilliseconds()}_{last.FinishOffset.ToUnixTimeMilliseconds()}.json");
var playout = new Core.Next.Playout { Version = "https://ersatztv.org/playout/version/0.0.3", Items = [] };
var playout = new Core.Next.Playout { Version = "https://ersatztv.org/playout/version/0.0.4", Items = [] };
foreach (PlayoutItem playoutItem in group)
{
Option<Core.Next.PlayoutItem> maybeNextPlayoutItem = await playoutItemConverter.ToNext(

3
ErsatzTV.Application/Streaming/HlsSessionWorker.cs

@ -166,7 +166,8 @@ public class HlsSessionWorker : IHlsSessionWorker @@ -166,7 +166,8 @@ public class HlsSessionWorker : IHlsSessionWorker
public void PlayoutUpdated() => _state = HlsSessionState.PlayoutUpdated;
public HlsSessionModel GetModel() => new(_channelNumber, _state.ToString(), _transcodedUntil, _lastAccess);
public HlsSessionModel GetModel() =>
new(_channelNumber, _state.ToString(), _transcodedUntil, _lastAccess, _channelStart);
void IDisposable.Dispose()
{

9
ErsatzTV.Application/Streaming/NextSessionWorker.cs

@ -31,6 +31,7 @@ public class NextSessionWorker( @@ -31,6 +31,7 @@ public class NextSessionWorker(
private string _workingDirectory;
private string _heartbeatFileName;
private DateTimeOffset _lastTouch;
private DateTimeOffset _sessionStart;
private DateTimeOffset _lastCheckpoint;
private ChannelPlayoutMode _channelPlayoutMode = ChannelPlayoutMode.Continuous;
@ -101,7 +102,7 @@ public class NextSessionWorker( @@ -101,7 +102,7 @@ public class NextSessionWorker(
// nothing to do here; channel binary should detect that by itself
}
public HlsSessionModel GetModel() => new(_channelNumber, "next", null, _lastTouch);
public HlsSessionModel GetModel() => new(_channelNumber, "next", null, _lastTouch, _sessionStart);
public async Task Run(
string channelNumber,
@ -112,8 +113,8 @@ public class NextSessionWorker( @@ -112,8 +113,8 @@ public class NextSessionWorker(
using var checkpointCts = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token);
Task checkpointLoop = Task.CompletedTask;
DateTimeOffset sessionStart = DateTimeOffset.Now;
_lastTouch = sessionStart;
_sessionStart = DateTimeOffset.Now;
_lastTouch = _sessionStart;
_lastCheckpoint = _lastTouch;
try
@ -135,7 +136,7 @@ public class NextSessionWorker( @@ -135,7 +136,7 @@ public class NextSessionWorker(
checkpointLoop = CheckpointLoop(checkpointCts.Token);
await Mediator.Send(
new TimeShiftOnDemandPlayout(playout.PlayoutId, sessionStart, true),
new TimeShiftOnDemandPlayout(playout.PlayoutId, _sessionStart, true),
_cancellationTokenSource.Token);
// next reads serialized playout files rather than the database, so ensure it

10
ErsatzTV.Application/Streaming/Queries/GetGraphicsCanvasStream.cs

@ -0,0 +1,10 @@ @@ -0,0 +1,10 @@
using ErsatzTV.FFmpeg;
namespace ErsatzTV.Application.Streaming;
public record GetGraphicsCanvasStream(
string ChannelNumber,
int PlayoutItemId,
TimeSpan Offset,
TimeSpan Duration,
FrameRate FrameRate) : IRequest<Option<Stream>>;

261
ErsatzTV.Application/Streaming/Queries/GetGraphicsCanvasStreamHandler.cs

@ -0,0 +1,261 @@ @@ -0,0 +1,261 @@
using System.IO.Pipelines;
using CliWrap;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Extensions;
using ErsatzTV.Core.FFmpeg;
using ErsatzTV.Core.Interfaces.FFmpeg;
using ErsatzTV.Core.Interfaces.Streaming;
using ErsatzTV.Core.Interfaces.Troubleshooting;
using ErsatzTV.FFmpeg;
using ErsatzTV.Infrastructure.Data;
using ErsatzTV.Infrastructure.Extensions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace ErsatzTV.Application.Streaming;
public class GetGraphicsCanvasStreamHandler(
IDbContextFactory<TvContext> dbContextFactory,
IWatermarkSelector watermarkSelector,
IGraphicsElementSelector graphicsElementSelector,
IGraphicsEngineContextFactory graphicsEngineContextFactory,
IGraphicsEngine graphicsEngine,
IFFmpegSegmenterService ffmpegSegmenterService,
ITroubleshootingPlayoutItemStore troubleshootingPlayoutItemStore,
ILogger<GetGraphicsCanvasStreamHandler> logger)
: IRequestHandler<GetGraphicsCanvasStream, Option<Stream>>
{
public async Task<Option<Stream>> Handle(GetGraphicsCanvasStream request, CancellationToken cancellationToken)
{
await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);
Option<string> maybeFFmpegPath = await dbContext.ConfigElements
.GetValue<string>(ConfigElementKey.FFmpegPath, cancellationToken);
if (maybeFFmpegPath.IsNone)
{
logger.LogWarning("Unable to stream graphics canvas; ffmpeg path is not configured");
return None;
}
Option<CanvasItem> maybeCanvasItem = request.ChannelNumber == FileSystemLayout.TranscodeTroubleshootingChannel
? troubleshootingPlayoutItemStore.Current().Map(t => new CanvasItem(t.Channel, t.PlayoutItem, None))
: await LoadCanvasItem(dbContext, request, cancellationToken);
foreach ((Channel channel, PlayoutItem playoutItem, Option<ChannelWatermark> maybeGlobalWatermark) in maybeCanvasItem)
{
DateTimeOffset contentStart = playoutItem.StartOffset;
DateTimeOffset now = contentStart + request.Offset;
DateTimeOffset finish = playoutItem.FinishOffset < now + request.Duration
? playoutItem.FinishOffset
: now + request.Duration;
// a stale offset past the item's finish still gets the requested duration so
// next's ffmpeg never sees an empty stream
if (finish <= now)
{
logger.LogWarning(
"Graphics canvas request for playout item {PlayoutItemId} at offset {Offset} is past item finish {Finish}",
playoutItem.Id,
request.Offset,
playoutItem.FinishOffset);
finish = now + request.Duration;
}
List<WatermarkOptions> watermarks = watermarkSelector.SelectWatermarks(
maybeGlobalWatermark,
channel,
playoutItem,
now,
shouldLogMessages: false);
List<PlayoutItemGraphicsElement> graphicsElements = graphicsElementSelector.SelectGraphicsElements(
channel,
playoutItem,
now,
shouldLogMessages: false);
DateTimeOffset channelStartTime = contentStart;
if (ffmpegSegmenterService.TryGetWorker(request.ChannelNumber, out IHlsSessionWorker worker))
{
channelStartTime = worker.GetModel().StartedAt;
}
MediaVersion headVersion = playoutItem.MediaItem.GetHeadVersion();
// the engine derives content_total_seconds as Seek + ContentTotalDuration, so this is
// the time remaining in the item (legacy passes finish - now), not the media duration
TimeSpan contentTotalDuration = playoutItem.FinishOffset > now
? playoutItem.FinishOffset - now
: finish - now;
Option<GraphicsEngineContext> maybeContext = await graphicsEngineContextFactory.Create(
channel,
playoutItem.MediaItem,
headVersion,
watermarks,
graphicsElements,
request.FrameRate,
channelStartTime,
contentStart,
playoutItem.FinishOffset,
request.Offset,
finish - now,
contentTotalDuration,
cancellationToken);
// nothing selected (channel edited since sync): stream transparent frames rather
// than an error, which would kill next's ffmpeg for the whole chunk
GraphicsEngineContext context = maybeContext.IfNone(
() => new GraphicsEngineContext(
channel.Number,
playoutItem.MediaItem,
Elements: [],
TemplateVariables: [],
channel.FFmpegProfile.Resolution,
channel.FFmpegProfile.Resolution,
request.FrameRate,
channelStartTime,
contentStart,
playoutItem.FinishOffset,
request.Offset,
finish - now,
contentTotalDuration));
logger.LogDebug(
"Streaming graphics canvas for channel {ChannelNumber} item {PlayoutItemId}: offset {Offset}, duration {Duration}, rate {FrameRate}, elements {ElementCount}",
channel.Number,
playoutItem.Id,
request.Offset,
finish - now,
request.FrameRate.RFrameRate,
context.Elements.Count);
return StartCanvas(
await maybeFFmpegPath.IfNoneAsync(string.Empty),
channel.FFmpegProfile.Resolution,
request.FrameRate,
context,
cancellationToken);
}
logger.LogWarning(
"Unable to locate playout item {PlayoutItemId} for graphics canvas on channel {ChannelNumber}",
request.PlayoutItemId,
request.ChannelNumber);
return None;
}
private sealed record CanvasItem(Channel Channel, PlayoutItem PlayoutItem, Option<ChannelWatermark> GlobalWatermark);
private static async Task<Option<CanvasItem>> LoadCanvasItem(
TvContext dbContext,
GetGraphicsCanvasStream request,
CancellationToken cancellationToken)
{
Option<Channel> maybeChannel = await dbContext.Channels
.AsNoTracking()
.Include(c => c.FFmpegProfile)
.ThenInclude(p => p.Resolution)
.Include(c => c.Watermark)
.Include(c => c.Artwork)
.Include(c => c.MirrorSourceChannel)
.SelectOneAsync(c => c.Number, c => c.Number == request.ChannelNumber, cancellationToken);
foreach (Channel channel in maybeChannel)
{
// mirror channels play the source channel's items shifted by the playout offset,
// matching what SyncNextPlayoutHandler wrote for next
TimeSpan playoutOffset = TimeSpan.Zero;
string sourceChannelNumber = channel.Number;
if (channel.PlayoutSource is ChannelPlayoutSource.Mirror && channel.MirrorSourceChannel is not null)
{
sourceChannelNumber = channel.MirrorSourceChannel.Number;
playoutOffset = channel.PlayoutOffset ?? TimeSpan.Zero;
}
Option<PlayoutItem> maybePlayoutItem = await dbContext.PlayoutItems
.AsNoTracking()
.Where(i => i.Id == request.PlayoutItemId && i.Playout.Channel.Number == sourceChannelNumber)
.IncludeForNextPlayout()
.AsSplitQuery()
.SingleOrDefaultAsync(cancellationToken)
.Map(Optional);
foreach (PlayoutItem playoutItem in maybePlayoutItem)
{
playoutItem.Start += playoutOffset;
playoutItem.Finish += playoutOffset;
Option<ChannelWatermark> maybeGlobalWatermark = await dbContext.ConfigElements
.GetValue<int>(ConfigElementKey.FFmpegGlobalWatermarkId, cancellationToken)
.BindT(watermarkId => dbContext.ChannelWatermarks
.SelectOneAsync(w => w.Id, w => w.Id == watermarkId, cancellationToken));
return new CanvasItem(channel, playoutItem, maybeGlobalWatermark);
}
}
return None;
}
private Stream StartCanvas(
string ffmpegPath,
Resolution resolution,
FrameRate frameRate,
GraphicsEngineContext context,
CancellationToken cancellationToken)
{
// for process counter
var ffmpegProcess = new FFmpegProcess();
var cts = new CancellationTokenSource();
// do not use 'using' here; the token needs to live longer than this method scope
var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, cancellationToken);
var enginePipe = new Pipe();
var outputPipe = new Pipe();
// fire and forget graphics engine task
_ = graphicsEngine.Run(context, enginePipe.Writer, linkedCts.Token);
string[] arguments =
[
"-hide_banner", "-nostats", "-loglevel", "error",
"-f", "rawvideo",
"-pix_fmt", "bgra",
"-video_size", $"{resolution.Width}x{resolution.Height}",
"-framerate", frameRate.RFrameRate,
"-i", "pipe:0",
"-c:v", "ffv1", "-level", "3", "-slices", "16", "-slicecrc", "0",
"-pix_fmt", "bgra",
"-f", "nut", "pipe:1"
];
CommandTask<CommandResult> task = Cli.Wrap(ffmpegPath)
.WithArguments(arguments)
.WithStandardInputPipe(PipeSource.FromStream(enginePipe.Reader.AsStream()))
.WithStandardOutputPipe(PipeTarget.ToStream(outputPipe.Writer.AsStream()))
.WithStandardErrorPipe(PipeTarget.ToDelegate(line => logger.LogWarning("Graphics canvas ffmpeg: {Line}", line)))
.WithValidation(CommandResultValidation.None)
.ExecuteAsync(linkedCts.Token);
// ensure cleanup happens when ffmpeg exits (either naturally or via cancellation);
// cancelling stops the engine if ffmpeg exited first
_ = task.Task.ContinueWith(
(t, _) =>
{
outputPipe.Writer.Complete(t.Exception);
cts.Cancel();
ffmpegProcess.Dispose();
linkedCts.Dispose();
cts.Dispose();
},
null,
TaskScheduler.Default);
return outputPipe.Reader.AsStream();
}
}

7
ErsatzTV.Application/Troubleshooting/Commands/PrepareTroubleshootingPlaybackHandler.cs

@ -15,6 +15,7 @@ using ErsatzTV.Core.Interfaces.Metadata; @@ -15,6 +15,7 @@ using ErsatzTV.Core.Interfaces.Metadata;
using ErsatzTV.Core.Interfaces.Plex;
using ErsatzTV.Core.Interfaces.Scheduling;
using ErsatzTV.Core.Interfaces.Streaming;
using ErsatzTV.Core.Interfaces.Troubleshooting;
using ErsatzTV.Core.Next.Config;
using ErsatzTV.Core.Notifications;
using ErsatzTV.FFmpeg;
@ -42,6 +43,7 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -42,6 +43,7 @@ public class PrepareTroubleshootingPlaybackHandler(
IEntityLocker entityLocker,
IChannelConfigConverter channelConfigConverter,
IPlayoutItemConverter playoutItemConverter,
ITroubleshootingPlayoutItemStore troubleshootingPlayoutItemStore,
IMediator mediator,
LoggingLevelSwitches loggingLevelSwitches,
ILogger<PrepareTroubleshootingPlaybackHandler> logger)
@ -360,6 +362,9 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -360,6 +362,9 @@ public class PrepareTroubleshootingPlaybackHandler(
PlayoutItemGraphicsElements = [.. graphicsElements.Map(ge => new PlayoutItemGraphicsElement { GraphicsElement = ge })]
};
// the canvas endpoint resolves this item from the store since it has no database row
troubleshootingPlayoutItemStore.Store(channel, playoutItem);
Option<Core.Next.PlayoutItem> maybeNextPlayoutItem =
await playoutItemConverter.ToNext(
Some(channel),
@ -374,7 +379,7 @@ public class PrepareTroubleshootingPlaybackHandler( @@ -374,7 +379,7 @@ public class PrepareTroubleshootingPlaybackHandler(
{
var playout = new Core.Next.Playout
{
Version = "https://ersatztv.org/playout/version/0.0.3",
Version = "https://ersatztv.org/playout/version/0.0.4",
Items = [nextPlayoutItem]
};

2
ErsatzTV.Application/Troubleshooting/Commands/StartTroubleshootingPlaybackHandler.cs

@ -22,6 +22,7 @@ namespace ErsatzTV.Application.Troubleshooting; @@ -22,6 +22,7 @@ namespace ErsatzTV.Application.Troubleshooting;
public class StartTroubleshootingPlaybackHandler(
ITroubleshootingNotifier notifier,
ITroubleshootingPlayoutItemStore troubleshootingPlayoutItemStore,
IMediator mediator,
IEntityLocker entityLocker,
IRuntimeInfo runtimeInfo,
@ -242,6 +243,7 @@ public class StartTroubleshootingPlaybackHandler( @@ -242,6 +243,7 @@ public class StartTroubleshootingPlaybackHandler(
}
finally
{
troubleshootingPlayoutItemStore.Clear();
entityLocker.UnlockTroubleshootingPlayback();
loggingLevelSwitches.StreamingLevelSwitch.MinimumLevel = currentStreamingLevel;
}

34
ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@ -26,7 +26,7 @@ namespace ErsatzTV.Core.FFmpeg; @@ -26,7 +26,7 @@ namespace ErsatzTV.Core.FFmpeg;
public class FFmpegLibraryProcessService : IFFmpegProcessService
{
private readonly IConfigElementRepository _configElementRepository;
private readonly IGraphicsElementLoader _graphicsElementLoader;
private readonly IGraphicsEngineContextFactory _graphicsEngineContextFactory;
private readonly IMemoryCache _memoryCache;
private readonly IMpegTsScriptService _mpegTsScriptService;
private readonly ILocalStatisticsProvider _localStatisticsProvider;
@ -46,7 +46,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService @@ -46,7 +46,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
ITempFilePool tempFilePool,
IPipelineBuilderFactory pipelineBuilderFactory,
IConfigElementRepository configElementRepository,
IGraphicsElementLoader graphicsElementLoader,
IGraphicsEngineContextFactory graphicsEngineContextFactory,
IMemoryCache memoryCache,
IMpegTsScriptService mpegTsScriptService,
ILocalStatisticsProvider localStatisticsProvider,
@ -60,7 +60,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService @@ -60,7 +60,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
_tempFilePool = tempFilePool;
_pipelineBuilderFactory = pipelineBuilderFactory;
_configElementRepository = configElementRepository;
_graphicsElementLoader = graphicsElementLoader;
_graphicsEngineContextFactory = graphicsEngineContextFactory;
_memoryCache = memoryCache;
_mpegTsScriptService = mpegTsScriptService;
_localStatisticsProvider = localStatisticsProvider;
@ -384,7 +384,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService @@ -384,7 +384,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
Option<WatermarkInputFile> watermarkInputFile = Option<WatermarkInputFile>.None;
Option<GraphicsEngineInput> graphicsEngineInput = Option<GraphicsEngineInput>.None;
Option<GraphicsEngineContext> graphicsEngineContext = Option<GraphicsEngineContext>.None;
List<GraphicsElementContext> graphicsElementContexts = [];
List<WatermarkOptions> engineWatermarks = [];
// use ffmpeg for single permanent watermark, graphics engine for all others
if (graphicsElements.Count == 0 && watermarks.Count == 1 && watermarks.All(wm => wm.Watermark.Mode is ChannelWatermarkMode.Permanent))
@ -422,7 +422,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService @@ -422,7 +422,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
}
else
{
graphicsElementContexts.AddRange(watermarks.Map(wm => new WatermarkElementContext(wm)));
engineWatermarks.AddRange(watermarks);
}
string videoFormat = GetVideoFormat(playbackSettings);
@ -539,33 +539,29 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService @@ -539,33 +539,29 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
playbackSettings.Deinterlace);
// only use graphics engine when we have elements, and are normalizing video
if (videoFormat != VideoFormat.Copy && (graphicsElementContexts.Count > 0 || graphicsElements.Count > 0))
if (videoFormat != VideoFormat.Copy && (engineWatermarks.Count > 0 || graphicsElements.Count > 0))
{
FrameSize targetSize = await desiredState.CroppedSize.IfNoneAsync(desiredState.ScaledSize);
FrameRate frameRate = await playbackSettings.FrameRate
.IfNoneAsync(new FrameRate(videoVersion.MediaVersion.RFrameRate));
var context = new GraphicsEngineContext(
channel.Number,
graphicsEngineContext = await _graphicsEngineContextFactory.Create(
channel,
audioVersion.MediaItem,
graphicsElementContexts,
TemplateVariables: [],
new Resolution { Width = targetSize.Width, Height = targetSize.Height },
channel.FFmpegProfile.Resolution,
videoVersion.MediaVersion,
engineWatermarks,
graphicsElements,
frameRate,
channelStartTime,
start,
now + originalContentDuration,
now > start ? now - start : TimeSpan.Zero,
finish - now,
originalContentDuration);
context = await _graphicsElementLoader.LoadAll(context, graphicsElements, cancellationToken);
originalContentDuration,
cancellationToken);
if (context?.Elements?.Count > 0)
if (graphicsEngineContext.IsSome)
{
graphicsEngineInput = new GraphicsEngineInput();
graphicsEngineContext = context;
}
}

3
ErsatzTV.Core/FFmpeg/HlsSessionModel.cs

@ -4,4 +4,5 @@ public record HlsSessionModel( @@ -4,4 +4,5 @@ public record HlsSessionModel(
string ChannelNumber,
string State,
DateTimeOffset? TranscodedUntil,
DateTimeOffset LastAccess);
DateTimeOffset LastAccess,
DateTimeOffset StartedAt);

1
ErsatzTV.Core/Interfaces/Streaming/GraphicsEngineContext.cs

@ -15,6 +15,7 @@ public record GraphicsEngineContext( @@ -15,6 +15,7 @@ public record GraphicsEngineContext(
FrameRate FrameRate,
DateTimeOffset ChannelStartTime,
DateTimeOffset ContentStartTime,
DateTimeOffset ContentFinishTime,
TimeSpan Seek,
TimeSpan Duration,
TimeSpan ContentTotalDuration);

26
ErsatzTV.Core/Interfaces/Streaming/IGraphicsEngineContextFactory.cs

@ -0,0 +1,26 @@ @@ -0,0 +1,26 @@
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.FFmpeg;
using ErsatzTV.FFmpeg;
namespace ErsatzTV.Core.Interfaces.Streaming;
public interface IGraphicsEngineContextFactory
{
/// <summary>
/// Builds a fully loaded engine context, or None when nothing would be rendered.
/// </summary>
Task<Option<GraphicsEngineContext>> Create(
Channel channel,
MediaItem mediaItem,
MediaVersion videoVersion,
List<WatermarkOptions> watermarks,
List<PlayoutItemGraphicsElement> elements,
FrameRate frameRate,
DateTimeOffset channelStartTime,
DateTimeOffset contentStartTime,
DateTimeOffset contentFinishTime,
TimeSpan seek,
TimeSpan duration,
TimeSpan contentTotalDuration,
CancellationToken cancellationToken);
}

16
ErsatzTV.Core/Interfaces/Troubleshooting/ITroubleshootingPlayoutItemStore.cs

@ -0,0 +1,16 @@ @@ -0,0 +1,16 @@
using ErsatzTV.Core.Domain;
namespace ErsatzTV.Core.Interfaces.Troubleshooting;
public record TroubleshootingPlayoutItem(Channel Channel, PlayoutItem PlayoutItem);
/// <summary>
/// Media item troubleshooting builds a synthetic playout item that never reaches the database;
/// this holds it so the graphics canvas endpoint can serve the troubleshooting channel.
/// </summary>
public interface ITroubleshootingPlayoutItemStore
{
void Store(Channel channel, PlayoutItem playoutItem);
Option<TroubleshootingPlayoutItem> Current();
void Clear();
}

54
ErsatzTV.Core/Next/Playout.cs

@ -120,6 +120,13 @@ namespace ErsatzTV.Core.Next @@ -120,6 +120,13 @@ namespace ErsatzTV.Core.Next
[JsonConverter(typeof(MinMaxValueCheckConverter))]
public double? HorizontalMarginPercent { get; set; }
/// <summary>
/// Graphics layer kind.
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[JsonPropertyName("kind")]
public GraphicsLayerKind? Kind { get; set; }
/// <summary>
/// Anchor position within the primary content frame.
/// </summary>
@ -808,6 +815,18 @@ namespace ErsatzTV.Core.Next @@ -808,6 +815,18 @@ namespace ErsatzTV.Core.Next
public long? StreamIndex { get; set; }
}
/// <summary>
/// Graphics layer kind.
///
/// canvas: a full-frame layer whose frames are already the output size and carry alpha. It
/// is composited at (0,0) and is content-locked: the channel seeks it to its own position in
/// the item. `location`, margins, `width_percent`, `within_source_content`,
/// `opacity_percent` and `timing` are ignored (a warning is logged if present). HTTP canvas
/// sources receive `x-etv-channel`, `x-etv-offset-ms`, `x-etv-duration-ms` and
/// `x-etv-frame-rate` headers.
/// </summary>
public enum GraphicsLayerKind { Canvas, Media };
/// <summary>
/// Anchor position within the primary content frame.
///
@ -846,6 +865,7 @@ namespace ErsatzTV.Core.Next @@ -846,6 +865,7 @@ namespace ErsatzTV.Core.Next
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
Converters =
{
GraphicsLayerKindConverter.Singleton,
GraphicsLocationConverter.Singleton,
SourceTypeConverter.Singleton,
PeriodicClockConverter.Singleton,
@ -884,6 +904,40 @@ namespace ErsatzTV.Core.Next @@ -884,6 +904,40 @@ namespace ErsatzTV.Core.Next
public static readonly MinMaxValueCheckConverter Singleton = new MinMaxValueCheckConverter();
}
internal class GraphicsLayerKindConverter : JsonConverter<GraphicsLayerKind>
{
public override bool CanConvert(Type t) => t == typeof(GraphicsLayerKind);
public override GraphicsLayerKind Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
var value = reader.GetString();
switch (value)
{
case "canvas":
return GraphicsLayerKind.Canvas;
case "media":
return GraphicsLayerKind.Media;
}
throw new Exception("Cannot unmarshal type GraphicsLayerKind");
}
public override void Write(Utf8JsonWriter writer, GraphicsLayerKind value, JsonSerializerOptions options)
{
switch (value)
{
case GraphicsLayerKind.Canvas:
JsonSerializer.Serialize(writer, "canvas", options);
return;
case GraphicsLayerKind.Media:
JsonSerializer.Serialize(writer, "media", options);
return;
}
throw new Exception("Cannot marshal type GraphicsLayerKind");
}
public static readonly GraphicsLayerKindConverter Singleton = new GraphicsLayerKindConverter();
}
internal class GraphicsLocationConverter : JsonConverter<GraphicsLocation>
{
public override bool CanConvert(Type t) => t == typeof(GraphicsLocation);

15
ErsatzTV.Core/Troubleshooting/TroubleshootingPlayoutItemStore.cs

@ -0,0 +1,15 @@ @@ -0,0 +1,15 @@
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.Troubleshooting;
namespace ErsatzTV.Core.Troubleshooting;
public class TroubleshootingPlayoutItemStore : ITroubleshootingPlayoutItemStore
{
private volatile TroubleshootingPlayoutItem _current;
public void Store(Channel channel, PlayoutItem playoutItem) => _current = new TroubleshootingPlayoutItem(channel, playoutItem);
public Option<TroubleshootingPlayoutItem> Current() => Optional(_current);
public void Clear() => _current = null;
}

49
ErsatzTV.Infrastructure.Tests/Streaming/GraphicsCanvasContextTests.cs

@ -0,0 +1,49 @@ @@ -0,0 +1,49 @@
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.FFmpeg;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Core.Metadata;
using ErsatzTV.FFmpeg;
using ErsatzTV.Infrastructure.Streaming.Graphics;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using NUnit.Framework;
using Shouldly;
namespace ErsatzTV.Infrastructure.Tests.Streaming;
[TestFixture]
public class GraphicsCanvasContextTests
{
[TestCase(0, 44)]
[TestCase(44, 44)]
[TestCase(1760, 40)]
public async Task Should_keep_scheduled_stop_independent_of_chunk(int offsetSeconds, int durationSeconds)
{
var loader = new GraphicsElementLoader(null!, null!, Substitute.For<ITemplateDataRepository>(),
NullLogger<GraphicsElementLoader>.Instance);
var factory = new GraphicsEngineContextFactory(loader);
var start = new DateTimeOffset(2026, 9, 10, 12, 0, 0, TimeSpan.Zero);
DateTimeOffset finish = start.AddMinutes(30);
var resolution = new Resolution { Width = 1920, Height = 1080 };
var channel = new Channel(Guid.NewGuid())
{
Number = "1",
FFmpegProfile = new FFmpegProfile { Resolution = resolution, ScalingBehavior = ScalingBehavior.Stretch }
};
var result = await factory.Create(
channel, new Movie(), new MediaVersion(),
[new WatermarkOptions(new ChannelWatermark(), "watermark.png", LanguageExt.Option<int>.None)],
[], new FrameRate("24000/1001"), start, start, finish,
TimeSpan.FromSeconds(offsetSeconds), TimeSpan.FromSeconds(durationSeconds),
TimeSpan.FromMinutes(30), CancellationToken.None);
result.IsSome.ShouldBeTrue();
foreach (var context in result)
{
context.TemplateVariables[MediaItemTemplateDataKey.Stop].ShouldBe(finish);
context.TemplateVariables[MediaItemTemplateDataKey.StreamSeek].ShouldBe(TimeSpan.FromSeconds(offsetSeconds));
context.Duration.ShouldBe(TimeSpan.FromSeconds(durationSeconds));
}
}
}

86
ErsatzTV.Infrastructure/Extensions/PlayoutItemQueryableExtensions.cs

@ -14,4 +14,90 @@ public static class PlayoutItemQueryableExtensions @@ -14,4 +14,90 @@ public static class PlayoutItemQueryableExtensions
.OrderBy(pi => pi.Start)
.FirstOrDefaultAsync()
.Map(Optional);
/// <summary>
/// Everything PlayoutItemConverter.ToNext and the graphics selectors need: playout/template
/// decos with watermarks and graphics elements, item watermarks and graphics elements, and
/// media versions/streams/metadata for each playable media item type.
/// </summary>
public static IQueryable<PlayoutItem> IncludeForNextPlayout(this IQueryable<PlayoutItem> dbSet) =>
dbSet
.Include(i => i.Playout)
.ThenInclude(p => p.Deco)
.ThenInclude(d => d.DecoWatermarks)
.ThenInclude(d => d.Watermark)
.Include(i => i.Playout)
.ThenInclude(p => p.Deco)
.ThenInclude(d => d.DecoGraphicsElements)
.ThenInclude(d => d.GraphicsElement)
.Include(i => i.Watermarks)
.Include(i => i.PlayoutItemGraphicsElements)
.ThenInclude(pige => pige.GraphicsElement)
.Include(i => i.Playout)
.ThenInclude(p => p.Templates)
.ThenInclude(t => t.DecoTemplate)
.ThenInclude(t => t.Items)
.ThenInclude(i => i.Deco)
.ThenInclude(d => d.DecoWatermarks)
.ThenInclude(d => d.Watermark)
.Include(i => i.Playout)
.ThenInclude(p => p.Templates)
.ThenInclude(t => t.DecoTemplate)
.ThenInclude(t => t.Items)
.ThenInclude(i => i.Deco)
.ThenInclude(d => d.DecoGraphicsElements)
.ThenInclude(d => d.GraphicsElement)
.Include(i => i.MediaItem)
.ThenInclude(mi => mi.LibraryPath)
.ThenInclude(lp => lp.Library)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Episode).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Episode).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Episode).EpisodeMetadata)
.ThenInclude(em => em.Subtitles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Image).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Image).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Image).ImageMetadata)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Movie).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Movie).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as Movie).MovieMetadata)
.ThenInclude(mm => mm.Subtitles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as OtherVideo).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as OtherVideo).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as OtherVideo).OtherVideoMetadata)
.ThenInclude(ovm => ovm.Subtitles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as MusicVideo).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as MusicVideo).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as RemoteStream).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as RemoteStream).MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(i => i.MediaItem)
.ThenInclude(i => (i as RemoteStream).RemoteStreamMetadata)
.ThenInclude(em => em.Subtitles);
}

231
ErsatzTV.Infrastructure/Scheduling/PlayoutItemConverter.cs

@ -11,10 +11,7 @@ using ErsatzTV.Core.Interfaces.FFmpeg; @@ -11,10 +11,7 @@ using ErsatzTV.Core.Interfaces.FFmpeg;
using ErsatzTV.Core.Interfaces.Jellyfin;
using ErsatzTV.Core.Interfaces.Plex;
using ErsatzTV.Core.Interfaces.Scheduling;
using ErsatzTV.Core.Interfaces.Streaming;
using ErsatzTV.Core.Security;
using ErsatzTV.FFmpeg;
using ErsatzTV.FFmpeg.State;
using ErsatzTV.Infrastructure.Data;
using ErsatzTV.Infrastructure.Extensions;
using Microsoft.EntityFrameworkCore;
@ -32,7 +29,6 @@ public class PlayoutItemConverter( @@ -32,7 +29,6 @@ public class PlayoutItemConverter(
IFFmpegStreamSelector ffmpegStreamSelector,
IWatermarkSelector watermarkSelector,
IGraphicsElementSelector graphicsElementSelector,
IGraphicsElementLoader graphicsElementLoader,
IDbContextFactory<TvContext> dbContextFactory) : IPlayoutItemConverter
{
public async Task<Option<Core.Next.PlayoutItem>> ToNext(
@ -101,7 +97,9 @@ public class PlayoutItemConverter( @@ -101,7 +97,9 @@ public class PlayoutItemConverter(
var nextPlayoutItem = new Core.Next.PlayoutItem
{
Id = playoutItem is DynamicPlayoutItem ? Guid.NewGuid().ToString() : playoutItem.Id.ToString(CultureInfo.InvariantCulture),
Id = playoutItem is DynamicPlayoutItem
? Guid.NewGuid().ToString()
: playoutItem.Id.ToString(CultureInfo.InvariantCulture),
Start = playoutItem.StartOffset,
Finish = playoutItem.FinishOffset
};
@ -131,7 +129,9 @@ public class PlayoutItemConverter( @@ -131,7 +129,9 @@ public class PlayoutItemConverter(
Width = headVersion.Width,
Profile = s.Profile,
FieldOrder = headVersion.VideoScanKind is VideoScanKind.Interlaced ? "tt" : "progressive",
PixFmt = string.IsNullOrWhiteSpace(s.PixelFormat) ? PixelFormatForBitDepth(s.BitsPerRawSample) : s.PixelFormat,
PixFmt = string.IsNullOrWhiteSpace(s.PixelFormat)
? PixelFormatForBitDepth(s.BitsPerRawSample)
: s.PixelFormat,
FrameRate = headVersion.RFrameRate,
SampleAspectRatio = headVersion.SampleAspectRatio,
DisplayAspectRatio = headVersion.DisplayAspectRatio,
@ -183,7 +183,8 @@ public class PlayoutItemConverter( @@ -183,7 +183,8 @@ public class PlayoutItemConverter(
Params = "anullsrc=channel_layout=stereo:sample_rate=48000",
ProbeHint = new Core.Next.ProbeHint
{
Audio = [
Audio =
[
new Core.Next.AudioHint
{
StreamIndex = 0,
@ -216,14 +217,13 @@ public class PlayoutItemConverter( @@ -216,14 +217,13 @@ public class PlayoutItemConverter(
subtitles,
shouldLogMessages,
cancellationToken);
await SelectGraphics(
SelectGraphics(
maybeGlobalWatermark,
channel,
playoutItem,
nextPlayoutItem,
headVersion.RFrameRate,
shouldLogMessages,
cancellationToken);
shouldLogMessages);
}
}
@ -252,7 +252,8 @@ public class PlayoutItemConverter( @@ -252,7 +252,8 @@ public class PlayoutItemConverter(
return new Core.Next.Source
{
SourceType = Core.Next.SourceType.Dynamic,
Uri = $"http://localhost:{Settings.StreamingPort}/internal/media/fallback?exp={exp.ToUnixTimeSeconds()}&sig={sig}"
Uri =
$"http://localhost:{Settings.StreamingPort}/internal/media/fallback?exp={exp.ToUnixTimeSeconds()}&sig={sig}"
};
}
@ -333,7 +334,8 @@ public class PlayoutItemConverter( @@ -333,7 +334,8 @@ public class PlayoutItemConverter(
return new Core.Next.Source
{
SourceType = Core.Next.SourceType.Http,
Uri = $"http://localhost:{Settings.StreamingPort}/internal/media/plex/{mediaSourceId}/{pmf.Key}?exp={exp.ToUnixTimeSeconds()}&sig={sig}",
Uri =
$"http://localhost:{Settings.StreamingPort}/internal/media/plex/{mediaSourceId}/{pmf.Key}?exp={exp.ToUnixTimeSeconds()}&sig={sig}",
KeepAlive = false,
Reconnect = true
};
@ -353,7 +355,8 @@ public class PlayoutItemConverter( @@ -353,7 +355,8 @@ public class PlayoutItemConverter(
return new Core.Next.Source
{
SourceType = Core.Next.SourceType.Http,
Uri = $"http://localhost:{Settings.StreamingPort}/internal/media/jellyfin/{itemId}?exp={exp.ToUnixTimeSeconds()}&sig={sig}",
Uri =
$"http://localhost:{Settings.StreamingPort}/internal/media/jellyfin/{itemId}?exp={exp.ToUnixTimeSeconds()}&sig={sig}",
KeepAlive = false,
Reconnect = true
};
@ -374,7 +377,8 @@ public class PlayoutItemConverter( @@ -374,7 +377,8 @@ public class PlayoutItemConverter(
return new Core.Next.Source
{
SourceType = Core.Next.SourceType.Http,
Uri = $"http://localhost:{Settings.StreamingPort}/internal/media/emby/{itemId}?exp={exp.ToUnixTimeSeconds()}&sig={sig}",
Uri =
$"http://localhost:{Settings.StreamingPort}/internal/media/emby/{itemId}?exp={exp.ToUnixTimeSeconds()}&sig={sig}",
KeepAlive = false,
Reconnect = true
};
@ -512,17 +516,15 @@ public class PlayoutItemConverter( @@ -512,17 +516,15 @@ public class PlayoutItemConverter(
}
}
private async Task SelectGraphics(
private void SelectGraphics(
Option<ChannelWatermark> maybeGlobalWatermark,
Channel channel,
PlayoutItem playoutItem,
Core.Next.PlayoutItem nextPlayoutItem,
string frameRate,
bool shouldLogMessages,
CancellationToken cancellationToken)
bool shouldLogMessages)
{
nextPlayoutItem.Graphics ??= [];
var result = new List<KeyValuePair<Core.Next.GraphicsLayer, int>>();
List<WatermarkOptions> watermarks = watermarkSelector.SelectWatermarks(
maybeGlobalWatermark,
@ -531,66 +533,16 @@ public class PlayoutItemConverter( @@ -531,66 +533,16 @@ public class PlayoutItemConverter(
playoutItem.StartOffset,
shouldLogMessages: false);
// permanent or intermittent watermarks are supported
IEnumerable<WatermarkOptions> supportedWatermarks = watermarks.Where(wm =>
wm.Watermark.Mode is ChannelWatermarkMode.Permanent or ChannelWatermarkMode.Intermittent);
foreach (WatermarkOptions watermarkOptions in supportedWatermarks)
{
var layer = new Core.Next.GraphicsLayer
{
Location = ToGraphics(watermarkOptions.Watermark.Location),
HorizontalMarginPercent = watermarkOptions.Watermark.HorizontalMarginPercent,
VerticalMarginPercent = watermarkOptions.Watermark.VerticalMarginPercent,
OpacityPercent = watermarkOptions.Watermark.Opacity,
StreamIndex = await watermarkOptions.ImageStreamIndex.IfNoneAsync(0),
WithinSourceContent = watermarkOptions.Watermark.PlaceWithinSourceContent,
};
if (watermarkOptions.Watermark.Size is WatermarkSize.Scaled)
{
layer.WidthPercent = watermarkOptions.Watermark.WidthPercent;
}
if (IsRemoteUri(watermarkOptions.ImagePath))
{
layer.Source = new Core.Next.PlayoutItemSource
{
SourceType = Core.Next.SourceType.Http,
Uri = watermarkOptions.ImagePath,
};
}
else
{
layer.Source = new Core.Next.PlayoutItemSource
{
SourceType = Core.Next.SourceType.Local,
Path = watermarkOptions.ImagePath,
};
}
if (watermarkOptions.Watermark.Mode is ChannelWatermarkMode.Intermittent)
{
layer.Timing = new Core.Next.Timing
{
TimingType = Core.Next.TimingType.Periodic,
Clock = Core.Next.PeriodicClock.Wall,
FrequencyMs = watermarkOptions.Watermark.FrequencyMinutes * 60 * 1000,
HoldMs = watermarkOptions.Watermark.DurationSeconds * 1000,
};
}
result.Add(new KeyValuePair<Core.Next.GraphicsLayer, int>(layer, watermarkOptions.Watermark.ZIndex));
}
List<PlayoutItemGraphicsElement> graphicsElements = graphicsElementSelector.SelectGraphicsElements(
channel,
playoutItem,
playoutItem.StartOffset,
shouldLogMessages);
IEnumerable<PlayoutItemGraphicsElement> supportedGraphicsElements = graphicsElements
.Where(ge => ge.GraphicsElement.Kind is GraphicsElementKind.Image);
if (watermarks.Count == 0 && graphicsElements.Count == 0)
{
return;
}
var outputFrameSize = new Resolution
{
@ -598,100 +550,41 @@ public class PlayoutItemConverter( @@ -598,100 +550,41 @@ public class PlayoutItemConverter(
Height = channel.FFmpegProfile.Resolution.Height,
};
var squarePixelFrameSize = new Resolution
{
Width = outputFrameSize.Width,
Height = outputFrameSize.Height
};
var headVersion = playoutItem.MediaItem.GetHeadVersion();
Option<VideoStream> maybeVideoStream = headVersion.Streams
.Where(s => s.MediaStreamKind is MediaStreamKind.Video)
.HeadOrNone()
.Select(v => new VideoStream(
v.Index,
v.Codec,
v.Profile,
None,
ColorParams.Unknown,
new FrameSize(headVersion.Width, headVersion.Height),
headVersion.SampleAspectRatio,
headVersion.DisplayAspectRatio,
None,
StillImage: false,
ScanKind.Progressive));
foreach (var videoStream in maybeVideoStream)
{
var frameSize =
videoStream.SquarePixelFrameSize(new FrameSize(outputFrameSize.Width, outputFrameSize.Height));
squarePixelFrameSize.Width = frameSize.Width;
squarePixelFrameSize.Height = frameSize.Height;
}
var context = new GraphicsEngineContext(
channel.Number,
playoutItem.MediaItem,
Elements: [],
TemplateVariables: [],
squarePixelFrameSize,
outputFrameSize,
new FrameRate(frameRate),
playoutItem.StartOffset,
playoutItem.StartOffset,
TimeSpan.Zero,
playoutItem.OutPoint - playoutItem.InPoint,
playoutItem.MediaItem.GetDurationForPlayout());
DateTimeOffset exp = playoutItem.FinishOffset + TimeSpan.FromHours(2);
context = await graphicsElementLoader.LoadAll(context, [.. supportedGraphicsElements], cancellationToken);
string sig = InternalUrlSigner.Sign(exp, "graphics", channel.Number, $"{playoutItem.Id}");
foreach (GraphicsElementContext element in context?.Elements ?? [])
var layer = new Core.Next.GraphicsLayer
{
switch (element)
Kind = Core.Next.GraphicsLayerKind.Canvas,
Location = Core.Next.GraphicsLocation.TopLeft,
Source = new Core.Next.PlayoutItemSource
{
case ImageElementDataContext({ } image):
// opacity expressions are not supported yet
if (!string.IsNullOrWhiteSpace(image.OpacityExpression))
{
continue;
}
var layer = new Core.Next.GraphicsLayer
{
Location = ToGraphics(image.Location),
HorizontalMarginPercent = image.HorizontalMarginPercent,
VerticalMarginPercent = image.VerticalMarginPercent,
OpacityPercent = image.OpacityPercent,
StreamIndex = 0,
WithinSourceContent = image.PlaceWithinSourceContent,
WidthPercent = image.Scale ? image.ScaleWidthPercent ?? 100 : null,
};
if (IsRemoteUri(image.Image))
{
layer.Source = new Core.Next.PlayoutItemSource
{
SourceType = Core.Next.SourceType.Http,
Uri = image.Image,
};
}
else
{
layer.Source = new Core.Next.PlayoutItemSource
SourceType = Core.Next.SourceType.Http,
Uri =
$"http://localhost:{Settings.StreamingPort}/internal/graphics/{channel.Number}/{playoutItem.Id}?exp={exp.ToUnixTimeSeconds()}&sig={sig}",
Reconnect = false,
ProbeHint = new Core.Next.ProbeHint
{
FormatName = "nut",
Video =
[
new Core.Next.VideoHint
{
SourceType = Core.Next.SourceType.Local,
Path = image.Image,
};
}
result.Add(new KeyValuePair<Core.Next.GraphicsLayer, int>(layer, image.ZIndex ?? 0));
break;
Codec = "ffv1",
Width = outputFrameSize.Width,
Height = outputFrameSize.Height,
PixFmt = "bgra",
StreamIndex = 0,
FrameRate = frameRate,
}
]
}
}
}
};
nextPlayoutItem.Graphics.Clear();
nextPlayoutItem.Graphics.AddRange(result.OrderBy(kvp => kvp.Value).Select(kvp => kvp.Key));
nextPlayoutItem.Graphics.Add(layer);
}
private static async Task<List<Subtitle>> GetSubtitles(
@ -727,7 +620,10 @@ public class PlayoutItemConverter( @@ -727,7 +620,10 @@ public class PlayoutItemConverter(
return allSubtitles;
}
private static List<Subtitle> GetMusicVideoSubtitles(Channel channel, int playoutItemId, TimeSpan playoutItemInPoint)
private static List<Subtitle> GetMusicVideoSubtitles(
Channel channel,
int playoutItemId,
TimeSpan playoutItemInPoint)
{
if (channel.MusicVideoCreditsMode is not ChannelMusicVideoCreditsMode.GenerateSubtitles)
{
@ -747,7 +643,8 @@ public class PlayoutItemConverter( @@ -747,7 +643,8 @@ public class PlayoutItemConverter(
Forced = true,
IsExtracted = false,
SubtitleKind = SubtitleKind.Generated,
Path = $"http://localhost:{Settings.StreamingPort}/internal/ffmpeg/music-video-credits/{playoutItemId}{seekToMs}",
Path =
$"http://localhost:{Settings.StreamingPort}/internal/ffmpeg/music-video-credits/{playoutItemId}{seekToMs}",
SDH = false
}
];
@ -770,20 +667,6 @@ public class PlayoutItemConverter( @@ -770,20 +667,6 @@ public class PlayoutItemConverter(
}
}
private static Core.Next.GraphicsLocation ToGraphics(WatermarkLocation watermarkLocation) =>
watermarkLocation switch
{
WatermarkLocation.TopMiddle => Core.Next.GraphicsLocation.TopCenter,
WatermarkLocation.TopRight => Core.Next.GraphicsLocation.TopRight,
WatermarkLocation.LeftMiddle => Core.Next.GraphicsLocation.CenterLeft,
WatermarkLocation.MiddleCenter => Core.Next.GraphicsLocation.Center,
WatermarkLocation.RightMiddle => Core.Next.GraphicsLocation.CenterRight,
WatermarkLocation.BottomLeft => Core.Next.GraphicsLocation.BottomLeft,
WatermarkLocation.BottomMiddle => Core.Next.GraphicsLocation.BottomCenter,
WatermarkLocation.BottomRight => Core.Next.GraphicsLocation.BottomRight,
_ => Core.Next.GraphicsLocation.TopLeft
};
private static bool IsRemoteUri(string path) =>
Uri.TryCreate(path, UriKind.Absolute, out Uri uriResult)
&& (uriResult.Scheme == Uri.UriSchemeHttp ||

2
ErsatzTV.Infrastructure/Streaming/Graphics/GraphicsElementLoader.cs

@ -289,7 +289,7 @@ public partial class GraphicsElementLoader( @@ -289,7 +289,7 @@ public partial class GraphicsElementLoader(
[ChannelTemplateDataKey.ChannelStartTime] = context.ChannelStartTime,
[MediaItemTemplateDataKey.StreamSeek] = context.Seek,
[MediaItemTemplateDataKey.Start] = context.ContentStartTime,
[MediaItemTemplateDataKey.Stop] = context.ContentStartTime + context.Duration
[MediaItemTemplateDataKey.Stop] = context.ContentFinishTime
};
// media item variables

80
ErsatzTV.Infrastructure/Streaming/Graphics/GraphicsEngineContextFactory.cs

@ -0,0 +1,80 @@ @@ -0,0 +1,80 @@
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.FFmpeg;
using ErsatzTV.Core.Interfaces.Streaming;
using ErsatzTV.FFmpeg;
namespace ErsatzTV.Infrastructure.Streaming.Graphics;
public class GraphicsEngineContextFactory(IGraphicsElementLoader graphicsElementLoader)
: IGraphicsEngineContextFactory
{
public async Task<Option<GraphicsEngineContext>> Create(
Channel channel,
MediaItem mediaItem,
MediaVersion videoVersion,
List<WatermarkOptions> watermarks,
List<PlayoutItemGraphicsElement> elements,
FrameRate frameRate,
DateTimeOffset channelStartTime,
DateTimeOffset contentStartTime,
DateTimeOffset contentFinishTime,
TimeSpan seek,
TimeSpan duration,
TimeSpan contentTotalDuration,
CancellationToken cancellationToken)
{
if (watermarks.Count == 0 && elements.Count == 0)
{
return None;
}
List<GraphicsElementContext> elementContexts = [.. watermarks.Select(wm => new WatermarkElementContext(wm))];
var context = new GraphicsEngineContext(
channel.Number,
mediaItem,
elementContexts,
TemplateVariables: [],
SquarePixelFrameSize(channel.FFmpegProfile, videoVersion),
channel.FFmpegProfile.Resolution,
frameRate,
channelStartTime,
contentStartTime,
contentFinishTime,
seek,
duration,
contentTotalDuration);
context = await graphicsElementLoader.LoadAll(context, elements, cancellationToken);
return context?.Elements?.Count > 0 ? Some(context) : None;
}
// must match the content box ffmpeg produces: stretch and crop both fill the frame,
// scale-and-pad leaves the square-pixel scaled size inside the padded frame
private static Resolution SquarePixelFrameSize(FFmpegProfile profile, MediaVersion videoVersion)
{
if (profile.ScalingBehavior is ScalingBehavior.Stretch or ScalingBehavior.Crop)
{
return new Resolution { Width = profile.Resolution.Width, Height = profile.Resolution.Height };
}
var videoStream = new VideoStream(
0,
string.Empty,
string.Empty,
None,
ColorParams.Unknown,
new FrameSize(videoVersion.Width, videoVersion.Height),
videoVersion.SampleAspectRatio,
videoVersion.DisplayAspectRatio,
None,
StillImage: false,
ScanKind.Progressive);
FrameSize scaledSize = videoStream.SquarePixelFrameSize(
new FrameSize(profile.Resolution.Width, profile.Resolution.Height));
return new Resolution { Width = scaledSize.Width, Height = scaledSize.Height };
}
}

7
ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs

@ -27,6 +27,7 @@ using ErsatzTV.FFmpeg.State; @@ -27,6 +27,7 @@ using ErsatzTV.FFmpeg.State;
using ErsatzTV.Infrastructure.Images;
using ErsatzTV.Infrastructure.Metadata;
using ErsatzTV.Infrastructure.Runtime;
using ErsatzTV.Infrastructure.Streaming.Graphics;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using NSubstitute;
@ -270,6 +271,7 @@ public class TranscodingTests @@ -270,6 +271,7 @@ public class TranscodingTests
Arg.Any<List<PlayoutItemGraphicsElement>>(),
Arg.Any<CancellationToken>())
.Returns(callInfo => Task.FromResult(callInfo.Arg<GraphicsEngineContext>()));
var graphicsEngineContextFactory = new GraphicsEngineContextFactory(graphicsElementLoader);
var oldService = new FFmpegProcessService(
new FakeStreamSelector(),
@ -289,7 +291,7 @@ public class TranscodingTests @@ -289,7 +291,7 @@ public class TranscodingTests
LoggerFactory.CreateLogger<HardwareCapabilitiesFactory>()),
LoggerFactory.CreateLogger<PipelineBuilderFactory>()),
Substitute.For<IConfigElementRepository>(),
graphicsElementLoader,
graphicsEngineContextFactory,
MemoryCache,
Substitute.For<IMpegTsScriptService>(),
Substitute.For<ILocalStatisticsProvider>(),
@ -999,6 +1001,7 @@ public class TranscodingTests @@ -999,6 +1001,7 @@ public class TranscodingTests
Arg.Any<List<PlayoutItemGraphicsElement>>(),
Arg.Any<CancellationToken>())
.Returns(callInfo => Task.FromResult(callInfo.Arg<GraphicsEngineContext>()));
var graphicsEngineContextFactory = new GraphicsEngineContextFactory(graphicsElementLoader);
var oldService = new FFmpegProcessService(
new FakeStreamSelector(),
@ -1018,7 +1021,7 @@ public class TranscodingTests @@ -1018,7 +1021,7 @@ public class TranscodingTests
LoggerFactory.CreateLogger<HardwareCapabilitiesFactory>()),
LoggerFactory.CreateLogger<PipelineBuilderFactory>()),
Substitute.For<IConfigElementRepository>(),
graphicsElementLoader,
graphicsEngineContextFactory,
MemoryCache,
Substitute.For<IMpegTsScriptService>(),
Substitute.For<ILocalStatisticsProvider>(),

51
ErsatzTV.Tests/Controllers/GraphicsCanvasHeaderTests.cs

@ -0,0 +1,51 @@ @@ -0,0 +1,51 @@
using System.Globalization;
using ErsatzTV.Controllers;
using ErsatzTV.Core.Security;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging.Abstractions;
using NUnit.Framework;
using Shouldly;
namespace ErsatzTV.Tests.Controllers;
[TestFixture]
public class GraphicsCanvasHeaderTests
{
[TestCase("x-etv-frame-rate", "garbage")]
[TestCase("x-etv-frame-rate", "25/0")]
[TestCase("x-etv-frame-rate", "0/1")]
[TestCase("x-etv-frame-rate", "-25/1")]
[TestCase("x-etv-frame-rate", "25/1/1")]
[TestCase("x-etv-frame-rate", "NaN")]
[TestCase("x-etv-frame-rate", "Infinity")]
[TestCase("x-etv-frame-rate", "0")]
[TestCase("x-etv-frame-rate", "")]
[TestCase("x-etv-offset-ms", "-1")]
[TestCase("x-etv-offset-ms", "922337203685478")]
[TestCase("x-etv-duration-ms", "922337203685478")]
[TestCase("x-etv-duration-ms", "0")]
[TestCase("x-etv-channel", "2")]
public async Task Should_reject_invalid_headers_before_dispatch(string header, string value)
{
// These requests must fail before any service is called.
var controller = new InternalController(null!, null!, null!, null!, null!,
NullLogger<InternalController>.Instance)
{
ControllerContext = new ControllerContext { HttpContext = new DefaultHttpContext() }
};
controller.Request.Headers["x-etv-offset-ms"] = "0";
controller.Request.Headers["x-etv-duration-ms"] = "44000";
controller.Request.Headers["x-etv-frame-rate"] = "24000/1001";
controller.Request.Headers["x-etv-channel"] = "1";
controller.Request.Headers[header] = value;
DateTimeOffset expires = DateTimeOffset.UtcNow.AddMinutes(5);
string signature = InternalUrlSigner.Sign(expires, "graphics", "1", "123");
IActionResult result = await controller.GetGraphicsCanvas(
"1", 123,
expires.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture), signature);
result.ShouldBeOfType<BadRequestResult>();
}
}

84
ErsatzTV/Controllers/InternalController.cs

@ -1,5 +1,6 @@ @@ -1,5 +1,6 @@
using System.CommandLine.Parsing;
using System.Diagnostics;
using System.Globalization;
using System.Text;
using CliWrap;
using ErsatzTV.Application.Emby;
@ -352,6 +353,77 @@ public class InternalController : StreamingControllerBase @@ -352,6 +353,77 @@ public class InternalController : StreamingControllerBase
return new NotFoundResult();
}
[HttpGet("graphics/{channelNumber}/{playoutItemId:int}")]
public async Task<IActionResult> GetGraphicsCanvas(
string channelNumber,
int playoutItemId,
[FromQuery] string exp,
[FromQuery] string sig)
{
if (string.IsNullOrWhiteSpace(exp) || string.IsNullOrWhiteSpace(sig) ||
!InternalUrlSigner.Verify(exp, sig, "graphics", channelNumber, $"{playoutItemId}"))
{
return NotFound();
}
const long maxMilliseconds = long.MaxValue / TimeSpan.TicksPerMillisecond;
if (!TryGetHeader("x-etv-offset-ms", out string offsetMsString) ||
!long.TryParse(offsetMsString, NumberStyles.None, CultureInfo.InvariantCulture, out long offsetMs) ||
offsetMs > maxMilliseconds)
{
return BadRequest();
}
if (!TryGetHeader("x-etv-duration-ms", out string durationMsString) ||
!long.TryParse(durationMsString, NumberStyles.None, CultureInfo.InvariantCulture, out long durationMs) ||
durationMs <= 0 || durationMs > maxMilliseconds - offsetMs)
{
return BadRequest();
}
if (!TryGetHeader("x-etv-frame-rate", out string frameRate))
{
return BadRequest();
}
string[] rateParts = frameRate.Split('/');
double parsedFrameRate;
if (rateParts.Length == 2 &&
int.TryParse(rateParts[0], NumberStyles.None, CultureInfo.InvariantCulture, out int numerator) && numerator > 0 &&
int.TryParse(rateParts[1], NumberStyles.None, CultureInfo.InvariantCulture, out int denominator) && denominator > 0)
{
parsedFrameRate = numerator / (double)denominator;
}
else if (rateParts.Length != 1 ||
!double.TryParse(frameRate, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out parsedFrameRate) ||
!double.IsFinite(parsedFrameRate) || parsedFrameRate <= 0)
{
return BadRequest();
}
if (Request.Headers.ContainsKey("x-etv-channel") &&
(!TryGetHeader("x-etv-channel", out string headerChannelNumber) || headerChannelNumber != channelNumber))
{
return BadRequest();
}
Option<Stream> maybeStream = await _mediator.Send(
new GetGraphicsCanvasStream(
channelNumber,
playoutItemId,
TimeSpan.FromTicks(offsetMs * TimeSpan.TicksPerMillisecond),
TimeSpan.FromTicks(durationMs * TimeSpan.TicksPerMillisecond),
new FrameRate(frameRate) { ParsedFrameRate = parsedFrameRate }),
HttpContext.RequestAborted);
foreach (Stream stream in maybeStream)
{
return new FileStreamResult(stream, "application/octet-stream");
}
return NotFound();
}
[HttpGet("media/fallback")]
public async Task<IActionResult> GetFallbackPlayoutJson(
[FromQuery] string exp,
@ -409,6 +481,18 @@ public class InternalController : StreamingControllerBase @@ -409,6 +481,18 @@ public class InternalController : StreamingControllerBase
return NotFound();
}
private bool TryGetHeader(string name, out string value)
{
value = null;
if (Request.Headers.TryGetValue(name, out StringValues values) && values.Count == 1)
{
value = values[0];
return value is not null;
}
return false;
}
private async Task<IActionResult> GetTsLegacyStream(string channelNumber)
{
var request = new GetPlayoutItemProcessByChannelNumber(

2
ErsatzTV/Startup.cs

@ -821,6 +821,7 @@ public class Startup @@ -821,6 +821,7 @@ public class Startup
services.AddSingleton<ISmartCollectionCache, SmartCollectionCache>();
services.AddSingleton<SearchQueryParser>();
services.AddSingleton<ITroubleshootingNotifier, TroubleshootingNotifier>();
services.AddSingleton<ITroubleshootingPlayoutItemStore, TroubleshootingPlayoutItemStore>();
services.AddSingleton<CustomFontMapper>();
services.AddSingleton<GraphicsEngineFonts>();
services.AddSingleton(Program.InMemoryLogService);
@ -935,6 +936,7 @@ public class Startup @@ -935,6 +936,7 @@ public class Startup
services.AddScoped<IGraphicsElementRepository, GraphicsElementRepository>();
services.AddScoped<ITemplateDataRepository, TemplateDataRepository>();
services.AddScoped<IGraphicsElementLoader, GraphicsElementLoader>();
services.AddScoped<IGraphicsEngineContextFactory, GraphicsEngineContextFactory>();
services.AddScoped<TemplateFunctions>();
services.AddScoped<IDecoSelector, DecoSelector>();
services.AddScoped<IWatermarkSelector, WatermarkSelector>();

Loading…
Cancel
Save