From 76a589b538030bb1d0e2a76d5b0c73f546f9dc59 Mon Sep 17 00:00:00 2001
From: Jason Dove <1695733+jasongdove@users.noreply.github.com>
Date: Fri, 8 Aug 2025 19:18:15 +0000
Subject: [PATCH] add text graphics element to playback troubleshooting (#2282)
* refactor graphics engine; async frame generation
* add text graphics element to playback troubleshooting
---
CHANGELOG.md | 6 +
.../ErsatzTV.Application.csproj.DotSettings | 2 +
.../Commands/RefreshGraphicsElements.cs | 3 +
.../RefreshGraphicsElementsHandler.cs | 53 +
.../Graphics/GraphicsElementViewModel.cs | 3 +
ErsatzTV.Application/Graphics/Mapper.cs | 16 +
.../Queries/GetAllGraphicsElements.cs | 3 +
.../Queries/GetAllGraphicsElementsHandler.cs | 19 +
...layoutItemProcessByChannelNumberHandler.cs | 4 +
.../Commands/ArchiveTroubleshootingResults.cs | 1 +
.../PrepareTroubleshootingPlayback.cs | 1 +
.../PrepareTroubleshootingPlaybackHandler.cs | 5 +
ErsatzTV.Core/Domain/GraphicsElement.cs | 10 +
ErsatzTV.Core/Domain/GraphicsElementKind.cs | 7 +
ErsatzTV.Core/Domain/PlayoutItem.cs | 2 +
.../Domain/PlayoutItemGraphicsElement.cs | 9 +
ErsatzTV.Core/Domain/PlayoutItemWatermark.cs | 2 +-
.../FFmpeg/FFmpegLibraryProcessService.cs | 55 +-
ErsatzTV.Core/FileSystemLayout.cs | 6 +
ErsatzTV.Core/Graphics/TextGraphicsElement.cs | 61 +
.../FFmpeg/IFFmpegProcessService.cs | 1 +
.../Streaming/GraphicsEngineContext.cs | 3 +
...Add_PlayoutItemGraphicsElement.Designer.cs | 6274 +++++++++++++++++
...08162048_Add_PlayoutItemGraphicsElement.cs | 71 +
.../Migrations/TvContextModelSnapshot.cs | 60 +
...Add_PlayoutItemGraphicsElement.Designer.cs | 6109 ++++++++++++++++
...08162239_Add_PlayoutItemGraphicsElement.cs | 67 +
.../Migrations/TvContextModelSnapshot.cs | 58 +
.../GraphicsElementConfiguration.cs | 10 +
.../PlayoutItemConfiguration.cs | 13 +
ErsatzTV.Infrastructure/Data/TvContext.cs | 1 +
.../Streaming/GraphicsEngine.cs | 55 +-
.../Streaming/GraphicsEngineFonts.cs | 56 +
.../Streaming/IGraphicsElement.cs | 5 +-
.../Streaming/OpacityExpressionHelper.cs | 101 +
.../Streaming/PreparedElementImage.cs | 6 +
.../Streaming/TextElement.cs | 118 +
.../Streaming/WatermarkElement.cs | 268 +-
.../Core/FFmpeg/TranscodingTests.cs | 2 +
.../Controllers/Api/TroubleshootController.cs | 8 +-
ErsatzTV/Pages/PlaybackTroubleshooting.razor | 44 +-
ErsatzTV/Services/SchedulerService.cs | 6 +
ErsatzTV/Services/WorkerService.cs | 4 +
ErsatzTV/Startup.cs | 2 +
44 files changed, 13399 insertions(+), 211 deletions(-)
create mode 100644 ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElements.cs
create mode 100644 ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElementsHandler.cs
create mode 100644 ErsatzTV.Application/Graphics/GraphicsElementViewModel.cs
create mode 100644 ErsatzTV.Application/Graphics/Mapper.cs
create mode 100644 ErsatzTV.Application/Graphics/Queries/GetAllGraphicsElements.cs
create mode 100644 ErsatzTV.Application/Graphics/Queries/GetAllGraphicsElementsHandler.cs
create mode 100644 ErsatzTV.Core/Domain/GraphicsElement.cs
create mode 100644 ErsatzTV.Core/Domain/GraphicsElementKind.cs
create mode 100644 ErsatzTV.Core/Domain/PlayoutItemGraphicsElement.cs
create mode 100644 ErsatzTV.Core/Graphics/TextGraphicsElement.cs
create mode 100644 ErsatzTV.Infrastructure.MySql/Migrations/20250808162048_Add_PlayoutItemGraphicsElement.Designer.cs
create mode 100644 ErsatzTV.Infrastructure.MySql/Migrations/20250808162048_Add_PlayoutItemGraphicsElement.cs
create mode 100644 ErsatzTV.Infrastructure.Sqlite/Migrations/20250808162239_Add_PlayoutItemGraphicsElement.Designer.cs
create mode 100644 ErsatzTV.Infrastructure.Sqlite/Migrations/20250808162239_Add_PlayoutItemGraphicsElement.cs
create mode 100644 ErsatzTV.Infrastructure/Data/Configurations/GraphicsElementConfiguration.cs
create mode 100644 ErsatzTV.Infrastructure/Streaming/GraphicsEngineFonts.cs
rename {ErsatzTV.Core/Interfaces => ErsatzTV.Infrastructure}/Streaming/IGraphicsElement.cs (79%)
create mode 100644 ErsatzTV.Infrastructure/Streaming/OpacityExpressionHelper.cs
create mode 100644 ErsatzTV.Infrastructure/Streaming/PreparedElementImage.cs
create mode 100644 ErsatzTV.Infrastructure/Streaming/TextElement.cs
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 228dd464a..87f20e3e0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- `LinearFadePoints(time, start, peakStart, peakEnd, end)`
- Add `Z-Index` to watermark editor
- The graphics engine will order by z-index when overlaying watermarks
+- Add *experimental* `Graphics Element` template system
+ - Graphics elements are defined in YAML files inside ETV config folder / templates / graphics-elements subfolder
+ - Add `Text` graphics element type
+ - Supported in playback troubleshooting
+ - Displays multi-line text in a specified font, color, location, z-index
+ - Supports constant opacity and opacity expression
### Fix
- Fix database operations that were slowing down playout builds
diff --git a/ErsatzTV.Application/ErsatzTV.Application.csproj.DotSettings b/ErsatzTV.Application/ErsatzTV.Application.csproj.DotSettings
index 2138ffc30..c80a049df 100644
--- a/ErsatzTV.Application/ErsatzTV.Application.csproj.DotSettings
+++ b/ErsatzTV.Application/ErsatzTV.Application.csproj.DotSettings
@@ -11,6 +11,8 @@
True
True
True
+ True
+ True
True
True
True
diff --git a/ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElements.cs b/ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElements.cs
new file mode 100644
index 000000000..64a84302f
--- /dev/null
+++ b/ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElements.cs
@@ -0,0 +1,3 @@
+namespace ErsatzTV.Application.Graphics;
+
+public record RefreshGraphicsElements : IRequest, IBackgroundServiceRequest;
diff --git a/ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElementsHandler.cs b/ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElementsHandler.cs
new file mode 100644
index 000000000..e4475fb7d
--- /dev/null
+++ b/ErsatzTV.Application/Graphics/Commands/RefreshGraphicsElementsHandler.cs
@@ -0,0 +1,53 @@
+using ErsatzTV.Core;
+using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.Interfaces.Metadata;
+using ErsatzTV.Infrastructure.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace ErsatzTV.Application.Graphics;
+
+public class RefreshGraphicsElementsHandler(
+ IDbContextFactory dbContextFactory,
+ ILocalFileSystem localFileSystem,
+ ILogger logger)
+ : IRequestHandler
+{
+ public async Task Handle(RefreshGraphicsElements request, CancellationToken cancellationToken)
+ {
+ await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);
+
+ // cleanup existing elements
+ var allExisting = await dbContext.GraphicsElements
+ .ToListAsync(cancellationToken);
+
+ foreach (var existing in allExisting.Where(e => !localFileSystem.FileExists(e.Path)))
+ {
+ logger.LogWarning(
+ "Removing graphics element that references non-existing file {File}",
+ existing.Path);
+
+ dbContext.GraphicsElements.Remove(existing);
+ }
+
+ // add new elements
+ var newPaths = localFileSystem.ListFiles(FileSystemLayout.GraphicsElementsTextTemplatesFolder)
+ .Where(f => allExisting.All(e => e.Path != f))
+ .ToList();
+
+ foreach (var path in newPaths)
+ {
+ logger.LogDebug("Adding new graphics element from file {File}", path);
+
+ var graphicsElement = new GraphicsElement
+ {
+ Path = path,
+ Kind = GraphicsElementKind.Text
+ };
+
+ await dbContext.AddAsync(graphicsElement, cancellationToken);
+ }
+
+ await dbContext.SaveChangesAsync(cancellationToken);
+ }
+}
\ No newline at end of file
diff --git a/ErsatzTV.Application/Graphics/GraphicsElementViewModel.cs b/ErsatzTV.Application/Graphics/GraphicsElementViewModel.cs
new file mode 100644
index 000000000..2986ecfc5
--- /dev/null
+++ b/ErsatzTV.Application/Graphics/GraphicsElementViewModel.cs
@@ -0,0 +1,3 @@
+namespace ErsatzTV.Application.Graphics;
+
+public record GraphicsElementViewModel(int Id, string Name);
\ No newline at end of file
diff --git a/ErsatzTV.Application/Graphics/Mapper.cs b/ErsatzTV.Application/Graphics/Mapper.cs
new file mode 100644
index 000000000..33fafa6dd
--- /dev/null
+++ b/ErsatzTV.Application/Graphics/Mapper.cs
@@ -0,0 +1,16 @@
+using ErsatzTV.Core.Domain;
+
+namespace ErsatzTV.Application.Graphics;
+
+public static class Mapper
+{
+ public static GraphicsElementViewModel ProjectToViewModel(GraphicsElement graphicsElement)
+ {
+ var fileName = Path.GetFileName(graphicsElement.Path);
+ return graphicsElement.Kind switch
+ {
+ GraphicsElementKind.Text => new GraphicsElementViewModel(graphicsElement.Id, $"text/{fileName}"),
+ _ => new GraphicsElementViewModel(graphicsElement.Id, graphicsElement.Path)
+ };
+ }
+}
\ No newline at end of file
diff --git a/ErsatzTV.Application/Graphics/Queries/GetAllGraphicsElements.cs b/ErsatzTV.Application/Graphics/Queries/GetAllGraphicsElements.cs
new file mode 100644
index 000000000..e5986cd4f
--- /dev/null
+++ b/ErsatzTV.Application/Graphics/Queries/GetAllGraphicsElements.cs
@@ -0,0 +1,3 @@
+namespace ErsatzTV.Application.Graphics;
+
+public record GetAllGraphicsElements : IRequest>;
\ No newline at end of file
diff --git a/ErsatzTV.Application/Graphics/Queries/GetAllGraphicsElementsHandler.cs b/ErsatzTV.Application/Graphics/Queries/GetAllGraphicsElementsHandler.cs
new file mode 100644
index 000000000..793192039
--- /dev/null
+++ b/ErsatzTV.Application/Graphics/Queries/GetAllGraphicsElementsHandler.cs
@@ -0,0 +1,19 @@
+using ErsatzTV.Infrastructure.Data;
+using Microsoft.EntityFrameworkCore;
+using static ErsatzTV.Application.Graphics.Mapper;
+
+namespace ErsatzTV.Application.Graphics;
+
+public class GetAllGraphicsElementsHandler(IDbContextFactory dbContextFactory)
+ : IRequestHandler>
+{
+ public async Task> Handle(
+ GetAllGraphicsElements request,
+ CancellationToken cancellationToken)
+ {
+ await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);
+ return await dbContext.GraphicsElements
+ .ToListAsync(cancellationToken)
+ .Map(list => list.Map(ProjectToViewModel).ToList());
+ }
+}
\ No newline at end of file
diff --git a/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs b/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs
index 1870be4f8..c3ae4daee 100644
--- a/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs
+++ b/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs
@@ -88,6 +88,9 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
.ThenInclude(p => p.Deco)
.ThenInclude(d => d.Watermark)
+ // get graphics elements
+ .Include(i => i.GraphicsElements)
+
// get playout templates (and deco templates/decos)
.Include(i => i.Playout)
.ThenInclude(p => p.Templates)
@@ -341,6 +344,7 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
effectiveNow,
playoutItemWatermarks,
maybeGlobalWatermark,
+ playoutItemWithPath.PlayoutItem.GraphicsElements,
channel.FFmpegProfile.VaapiDisplay,
channel.FFmpegProfile.VaapiDriver,
channel.FFmpegProfile.VaapiDevice,
diff --git a/ErsatzTV.Application/Troubleshooting/Commands/ArchiveTroubleshootingResults.cs b/ErsatzTV.Application/Troubleshooting/Commands/ArchiveTroubleshootingResults.cs
index 11b665437..6815991ae 100644
--- a/ErsatzTV.Application/Troubleshooting/Commands/ArchiveTroubleshootingResults.cs
+++ b/ErsatzTV.Application/Troubleshooting/Commands/ArchiveTroubleshootingResults.cs
@@ -4,5 +4,6 @@ public record ArchiveTroubleshootingResults(
int MediaItemId,
int FFmpegProfileId,
List WatermarkIds,
+ List GraphicsElementIds,
bool StartFromBeginning)
: IRequest