From 0036fc5cc81429008ef7136efc14b88ec018e561 Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Thu, 2 Jul 2026 08:55:01 -0500 Subject: [PATCH] fix: subtitles with next engine (#2938) --- CHANGELOG.md | 3 + .../ExtractEmbeddedSubtitlesHandler.cs | 6 +- .../FFmpeg/CustomStreamSelectorTests.cs | 136 +++++++++++++----- ErsatzTV.Core/FFmpeg/CustomStreamSelector.cs | 30 ++-- .../FFmpeg/FFmpegLibraryProcessService.cs | 3 +- .../FFmpeg/ICustomStreamSelector.cs | 3 +- ErsatzTV.Core/Next/Playout.cs | 31 +++- .../Scheduling/PlayoutItemConverter.cs | 11 +- 8 files changed, 169 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index deaf9b83a..cc24bdcea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - A config file named `{channel_number}.json` (e.g. `1.json`) will apply to the channel with that number - Channel overlays will override values from the default overlay which will override values from the FFmpeg Profile - Add `epg_entries` support to image graphics elements +- Add (date-range based) alternate schedule system for sequential schedules ### Fixed - Fix HLS Direct playback when JWT auth is also used @@ -26,6 +27,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Use `und` language tag with sidecar subtitles that have no language in the file name - Automatically remove already-played playout items from all on demand channels - Automatically remove already-played playout items from (continuous) scripted schedules +- Extract embedded text subtitles (when enabled in settings) on channels that use custom stream selectors +- Fix subtitle playback using Next streaming engine ### Changed - Upgrade Intel driver in docker containers to support latest Battlemage devices (e.g. B70) diff --git a/ErsatzTV.Application/Subtitles/Commands/ExtractEmbeddedSubtitlesHandler.cs b/ErsatzTV.Application/Subtitles/Commands/ExtractEmbeddedSubtitlesHandler.cs index 163bab787..4c08b0a6c 100644 --- a/ErsatzTV.Application/Subtitles/Commands/ExtractEmbeddedSubtitlesHandler.cs +++ b/ErsatzTV.Application/Subtitles/Commands/ExtractEmbeddedSubtitlesHandler.cs @@ -91,7 +91,8 @@ public class ExtractEmbeddedSubtitlesHandler : ExtractEmbeddedSubtitlesHandlerBa .AsNoTracking() .Filter(p => p.Channel.SubtitleMode != ChannelSubtitleMode.None || p.ProgramSchedule.Items.Any(psi => - psi.SubtitleMode != null && psi.SubtitleMode != ChannelSubtitleMode.None)) + psi.SubtitleMode != null && psi.SubtitleMode != ChannelSubtitleMode.None) + || p.Channel.StreamSelectorMode == ChannelStreamSelectorMode.Custom) .SelectOneAsync(p => p.Id, p => p.Id == request.PlayoutId.IfNone(-1), cancellationToken); playoutIdsToCheck.AddRange(requestedPlayout.Map(p => p.Id)); @@ -103,7 +104,8 @@ public class ExtractEmbeddedSubtitlesHandler : ExtractEmbeddedSubtitlesHandlerBa .AsNoTracking() .Filter(p => p.Channel.SubtitleMode != ChannelSubtitleMode.None || p.ProgramSchedule.Items.Any(psi => - psi.SubtitleMode != null && psi.SubtitleMode != ChannelSubtitleMode.None)) + psi.SubtitleMode != null && psi.SubtitleMode != ChannelSubtitleMode.None) + || p.Channel.StreamSelectorMode == ChannelStreamSelectorMode.Custom) .Map(p => p.Id) .ToList(); } diff --git a/ErsatzTV.Core.Tests/FFmpeg/CustomStreamSelectorTests.cs b/ErsatzTV.Core.Tests/FFmpeg/CustomStreamSelectorTests.cs index 4632a2f75..5361674ee 100644 --- a/ErsatzTV.Core.Tests/FFmpeg/CustomStreamSelectorTests.cs +++ b/ErsatzTV.Core.Tests/FFmpeg/CustomStreamSelectorTests.cs @@ -104,7 +104,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -134,7 +135,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -164,7 +166,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -200,7 +203,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -231,7 +235,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -263,7 +268,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -295,7 +301,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.Subtitle.IsSome.ShouldBeFalse(); } @@ -322,7 +329,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.Subtitle.IsSome.ShouldBeTrue(); @@ -355,7 +363,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.Subtitle.IsSome.ShouldBeTrue(); @@ -397,7 +406,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.Subtitle.IsSome.ShouldBeTrue(); @@ -433,7 +443,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -471,7 +482,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -514,7 +526,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now.LocalDateTime.Date.AddHours(11).AddMinutes(59), // 11:59 AM _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -551,7 +564,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now.LocalDateTime.Date.AddHours(11).AddMinutes(59), // 11:59 AM _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -594,7 +608,12 @@ public class CustomStreamSelectorTests var start = new DateTime(2026, 1, 11, 0, 0, 0, DateTimeKind.Unspecified); // sunday var dto = new DateTimeOffset(start, tz.GetUtcOffset(start)); - StreamSelectorResult result = await streamSelector.SelectStreams(_channel, dto, _audioVersion, _subtitles); + StreamSelectorResult result = await streamSelector.SelectStreams( + _channel, + dto, + _audioVersion, + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -631,7 +650,12 @@ public class CustomStreamSelectorTests var start = new DateTime(2026, 1, 11, 0, 0, 0, DateTimeKind.Unspecified); // sunday var dto = new DateTimeOffset(start, tz.GetUtcOffset(start)); - StreamSelectorResult result = await streamSelector.SelectStreams(_channel, dto, _audioVersion, _subtitles); + StreamSelectorResult result = await streamSelector.SelectStreams( + _channel, + dto, + _audioVersion, + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -651,7 +675,8 @@ public class CustomStreamSelectorTests } [Test] - public async Task Should_Select_English_Audio_No_Subtitles_Day_Of_Week_Time_Of_Day_Content_Condition_Fail_Before() + public async Task + Should_Select_English_Audio_No_Subtitles_Day_Of_Week_Time_Of_Day_Content_Condition_Fail_Before() { // saturday from 9pm-11pm const string YAML = @@ -675,7 +700,12 @@ public class CustomStreamSelectorTests var start = new DateTime(2026, 1, 10, 20, 59, 59, DateTimeKind.Unspecified); // saturday at 8:59:59pm var dto = new DateTimeOffset(start, tz.GetUtcOffset(start)); - StreamSelectorResult result = await streamSelector.SelectStreams(_channel, dto, _audioVersion, _subtitles); + StreamSelectorResult result = await streamSelector.SelectStreams( + _channel, + dto, + _audioVersion, + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -689,7 +719,8 @@ public class CustomStreamSelectorTests } [Test] - public async Task Should_Select_English_Audio_No_Subtitles_Day_Of_Week_Time_Of_Day_Content_Condition_Fail_After() + public async Task + Should_Select_English_Audio_No_Subtitles_Day_Of_Week_Time_Of_Day_Content_Condition_Fail_After() { // saturday from 9pm-11pm const string YAML = @@ -713,7 +744,12 @@ public class CustomStreamSelectorTests var start = new DateTime(2026, 1, 10, 23, 0, 0, DateTimeKind.Unspecified); // saturday at 11:00pm var dto = new DateTimeOffset(start, tz.GetUtcOffset(start)); - StreamSelectorResult result = await streamSelector.SelectStreams(_channel, dto, _audioVersion, _subtitles); + StreamSelectorResult result = await streamSelector.SelectStreams( + _channel, + dto, + _audioVersion, + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -727,7 +763,8 @@ public class CustomStreamSelectorTests } [Test] - public async Task Should_Select_English_Audio_No_Subtitles_Day_Of_Week_Time_Of_Day_Content_Condition_Fail_Wrong_Day() + public async Task + Should_Select_English_Audio_No_Subtitles_Day_Of_Week_Time_Of_Day_Content_Condition_Fail_Wrong_Day() { // saturday from 9pm-11pm const string YAML = @@ -751,7 +788,12 @@ public class CustomStreamSelectorTests var start = new DateTime(2026, 1, 11, 22, 0, 0, DateTimeKind.Unspecified); // sunday at 10:00pm var dto = new DateTimeOffset(start, tz.GetUtcOffset(start)); - StreamSelectorResult result = await streamSelector.SelectStreams(_channel, dto, _audioVersion, _subtitles); + StreamSelectorResult result = await streamSelector.SelectStreams( + _channel, + dto, + _audioVersion, + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -789,7 +831,12 @@ public class CustomStreamSelectorTests var start = new DateTime(2026, 1, 10, 22, 0, 0, DateTimeKind.Unspecified); // saturday at 10:00pm var dto = new DateTimeOffset(start, tz.GetUtcOffset(start)); - StreamSelectorResult result = await streamSelector.SelectStreams(_channel, dto, _audioVersion, _subtitles); + StreamSelectorResult result = await streamSelector.SelectStreams( + _channel, + dto, + _audioVersion, + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -810,7 +857,8 @@ public class CustomStreamSelectorTests [Test] [SetCulture("fr-FR")] - public async Task Should_Select_English_Audio_No_Subtitles_Day_Of_Week_Time_Of_Day_Content_Condition_Match_France() + public async Task + Should_Select_English_Audio_No_Subtitles_Day_Of_Week_Time_Of_Day_Content_Condition_Match_France() { // saturday from 9pm-11pm const string YAML = @@ -834,7 +882,12 @@ public class CustomStreamSelectorTests var start = new DateTime(2026, 1, 10, 22, 0, 0, DateTimeKind.Unspecified); // saturday at 10:00pm var dto = new DateTimeOffset(start, tz.GetUtcOffset(start)); - StreamSelectorResult result = await streamSelector.SelectStreams(_channel, dto, _audioVersion, _subtitles); + StreamSelectorResult result = await streamSelector.SelectStreams( + _channel, + dto, + _audioVersion, + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -875,7 +928,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -908,7 +962,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -943,7 +998,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.Subtitle.IsSome.ShouldBeTrue(); @@ -978,7 +1034,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.Subtitle.IsSome.ShouldBeTrue(); @@ -1010,7 +1067,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.Subtitle.IsSome.ShouldBeTrue(); @@ -1042,7 +1100,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.Subtitle.IsSome.ShouldBeTrue(); @@ -1074,7 +1133,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -1106,7 +1166,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -1137,7 +1198,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeTrue(); @@ -1169,7 +1231,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.Subtitle.IsSome.ShouldBeTrue(); @@ -1202,7 +1265,8 @@ public class CustomStreamSelectorTests _channel, DateTimeOffset.Now, _audioVersion, - _subtitles); + _subtitles, + shouldLogMessages: true); result.AudioStream.IsSome.ShouldBeFalse(); result.Subtitle.IsSome.ShouldBeFalse(); diff --git a/ErsatzTV.Core/FFmpeg/CustomStreamSelector.cs b/ErsatzTV.Core/FFmpeg/CustomStreamSelector.cs index 41201bbda..a85aa43c6 100644 --- a/ErsatzTV.Core/FFmpeg/CustomStreamSelector.cs +++ b/ErsatzTV.Core/FFmpeg/CustomStreamSelector.cs @@ -5,6 +5,7 @@ using ErsatzTV.Core.Domain; using ErsatzTV.Core.FFmpeg.Selector; using ErsatzTV.Core.Interfaces.FFmpeg; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using NCalc; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; @@ -18,8 +19,11 @@ public class CustomStreamSelector(IFileSystem fileSystem, ILogger allSubtitles) + List allSubtitles, + bool shouldLogMessages) { + ILogger log = shouldLogMessages ? logger : NullLogger.Instance; + try { string streamSelectorFile = Path.Combine( @@ -28,11 +32,11 @@ public class CustomStreamSelector(IFileSystem fileSystem, ILogger s.MediaStreamKind == MediaStreamKind.Audio) @@ -44,7 +48,7 @@ public class CustomStreamSelector(IFileSystem fileSystem, ILogger LoadStreamSelector(string streamSelectorFile) + private async Task LoadStreamSelector(string streamSelectorFile, ILogger log) { try { @@ -339,7 +345,7 @@ public class CustomStreamSelector(IFileSystem fileSystem, ILogger allSubtitles); + List allSubtitles, + bool shouldLogMessages); } diff --git a/ErsatzTV.Core/Next/Playout.cs b/ErsatzTV.Core/Next/Playout.cs index 89bade4b8..5014a83b4 100644 --- a/ErsatzTV.Core/Next/Playout.cs +++ b/ErsatzTV.Core/Next/Playout.cs @@ -248,7 +248,8 @@ namespace ErsatzTV.Core.Next /// /// The hint fully replaces probing: values are not validated up front, so incorrect metadata /// surfaces as an ffmpeg error during playback rather than a probe failure. Provide an entry - /// for every stream the pipeline needs to select — typically one video and one audio. + /// for every stream the pipeline needs to select — typically one video and one audio, plus + /// any subtitle stream a track selects. /// public partial class ProbeHint { @@ -272,6 +273,14 @@ namespace ErsatzTV.Core.Next [JsonProperty("format_name")] public string FormatName { get; set; } + /// + /// Subtitle streams in the source. Omit (or use `[]`) for sources without subtitles; + /// defaults to empty. Provide an entry for any subtitle stream a track selects, otherwise + /// the requested subtitle stream cannot be located and subtitles are dropped. + /// + [JsonProperty("subtitle", NullValueHandling = NullValueHandling.Ignore)] + public List Subtitle { get; set; } + /// /// Video (and still-image) streams in the source. Omit (or use `[]`) for audio-only sources; /// defaults to empty. @@ -304,6 +313,26 @@ namespace ErsatzTV.Core.Next public long StreamIndex { get; set; } } + /// + /// Probe metadata for a single subtitle stream. + /// + public partial class SubtitleHint + { + /// + /// Codec name as reported by ffprobe (e.g. "subrip", "ass", "hdmv_pgs_subtitle", + /// "dvd_subtitle"). Compared case-insensitively; image-based codecs are handled differently + /// from text-based ones. + /// + [JsonProperty("codec")] + public string Codec { get; set; } + + /// + /// Zero-based index of this stream within the source. + /// + [JsonProperty("stream_index")] + public long StreamIndex { get; set; } + } + /// /// Probe metadata for a single video (or still-image) stream. Optional fields left out /// assume progressive, square-pixel, SDR content at 24 fps — the same fallbacks used when diff --git a/ErsatzTV.Infrastructure/Scheduling/PlayoutItemConverter.cs b/ErsatzTV.Infrastructure/Scheduling/PlayoutItemConverter.cs index 98b17433c..9d07bded9 100644 --- a/ErsatzTV.Infrastructure/Scheduling/PlayoutItemConverter.cs +++ b/ErsatzTV.Infrastructure/Scheduling/PlayoutItemConverter.cs @@ -150,11 +150,19 @@ public class PlayoutItemConverter( Codec = s.Codec, Channels = s.Channels }).ToList(); + var sourceSubtitleHints = headVersion.Streams + .Where(s => s.MediaStreamKind is MediaStreamKind.Subtitle) + .Select(s => new Core.Next.SubtitleHint + { + StreamIndex = s.Index, + Codec = s.Codec + }).ToList(); nextPlayoutItem.Source.ProbeHint = new Core.Next.ProbeHint { Audio = sourceAudioHints, Video = sourceVideoHints, + Subtitle = sourceSubtitleHints, DurationMs = (long)headVersion.Duration.TotalMilliseconds }; @@ -378,7 +386,8 @@ public class PlayoutItemConverter( channel, nextPlayoutItem.Start, audioVersion, - allSubtitles); + allSubtitles, + shouldLogMessages: false); maybeAudioStream = result.AudioStream; maybeSubtitle = result.Subtitle; }