feat: select embedded image subtitle streams for next engine during playout build (#2868)

ErsatzTV.Application/Playouts/Commands/SyncNextPlayoutHandler.cs

@@ -1,3 +1,4 @@
+using System.Collections.Immutable;
 using System.Globalization;
 using System.IO.Abstractions;
 using System.Runtime.InteropServices;
@@ -134,18 +135,27 @@ public partial class SyncNextPlayoutHandler(
             .ThenInclude(i => (i as Episode).MediaVersions)
             .ThenInclude(mv => mv.Streams)
             .Include(i => i.MediaItem)
+            .ThenInclude(i => (i as Episode).EpisodeMetadata)
+            .ThenInclude(em => em.Subtitles)
+            .Include(i => i.MediaItem)
             .ThenInclude(i => (i as Movie).MediaVersions)
             .ThenInclude(mv => mv.MediaFiles)
             .Include(i => i.MediaItem)
             .ThenInclude(i => (i as Movie).MediaVersions)
             .ThenInclude(mv => mv.Streams)
             .Include(i => i.MediaItem)
+            .ThenInclude(i => (i as Movie).MovieMetadata)
+            .ThenInclude(mm => mm.Subtitles)
+            .Include(i => i.MediaItem)
             .ThenInclude(i => (i as OtherVideo).MediaVersions)
             .ThenInclude(mv => mv.MediaFiles)
             .Include(i => i.MediaItem)
             .ThenInclude(i => (i as OtherVideo).MediaVersions)
             .ThenInclude(mv => mv.Streams)
             .Include(i => i.MediaItem)
+            .ThenInclude(i => (i as OtherVideo).OtherVideoMetadata)
+            .ThenInclude(ovm => ovm.Subtitles)
+            .Include(i => i.MediaItem)
             .ThenInclude(i => (i as MusicVideo).MediaVersions)
             .ThenInclude(mv => mv.MediaFiles)
             .Include(i => i.MediaItem)
@@ -238,6 +248,8 @@ public partial class SyncNextPlayoutHandler(
                         nextPlayoutItem,
                         playoutItem.PreferredAudioLanguageCode ?? channel.PreferredAudioLanguageCode,
                         playoutItem.PreferredAudioTitle ?? channel.PreferredAudioTitle,
+                        playoutItem.PreferredSubtitleLanguageCode ?? channel.PreferredSubtitleLanguageCode,
+                        playoutItem.SubtitleMode ?? channel.SubtitleMode,
                         cancellationToken);
                 }
 
@@ -254,13 +266,14 @@ public partial class SyncNextPlayoutHandler(
         Core.Next.PlayoutItem nextPlayoutItem,
         string preferredAudioLanguage,
         string preferredAudioTitle,
+        string preferredSubtitleLanguage,
+        ChannelSubtitleMode subtitleMode,
         CancellationToken cancellationToken)
     {
-        // TODO: NEXT: support subtitles
-        List<Subtitle> allSubtitles = [];
+        List<Subtitle> allSubtitles = await GetSubtitles(audioVersion.MediaItem);
 
         Option<MediaStream> maybeAudioStream = Option<MediaStream>.None;
-        //Option<Subtitle> maybeSubtitle = Option<Subtitle>.None;
+        Option<Subtitle> maybeSubtitle = Option<Subtitle>.None;
 
         if (channel.StreamSelectorMode is ChannelStreamSelectorMode.Custom)
         {
@@ -270,7 +283,7 @@ public partial class SyncNextPlayoutHandler(
                 audioVersion,
                 allSubtitles);
             maybeAudioStream = result.AudioStream;
-            //maybeSubtitle = result.Subtitle;
+            maybeSubtitle = result.Subtitle;
         }
 
         if (channel.StreamSelectorMode is ChannelStreamSelectorMode.Default || maybeAudioStream.IsNone)
@@ -285,13 +298,14 @@ public partial class SyncNextPlayoutHandler(
                     shouldLogMessages: false,
                     cancellationToken);
 
-            // maybeSubtitle =
-            //     await ffmpegStreamSelector.SelectSubtitleStream(
-            //         allSubtitles.ToImmutableList(),
-            //         channel,
-            //         preferredSubtitleLanguage,
-            //         subtitleMode,
-            //         cancellationToken);
+            maybeSubtitle =
+                await ffmpegStreamSelector.SelectSubtitleStream(
+                    allSubtitles.ToImmutableList(),
+                    channel,
+                    preferredSubtitleLanguage,
+                    subtitleMode,
+                    shouldLogMessages: false,
+                    cancellationToken);
         }
 
         foreach (MediaStream audioStream in maybeAudioStream)
@@ -303,6 +317,16 @@ public partial class SyncNextPlayoutHandler(
                 nextPlayoutItem.Tracks.Audio.StreamIndex = audioStream.Index;
             }
         }
+
+        foreach (Subtitle subtitle in maybeSubtitle)
+        {
+            if (nextPlayoutItem.Tracks?.Subtitle?.StreamIndex is null)
+            {
+                nextPlayoutItem.Tracks ??= new Core.Next.PlayoutItemTracks();
+                nextPlayoutItem.Tracks.Subtitle ??= new Core.Next.TrackSelection();
+                nextPlayoutItem.Tracks.Subtitle.StreamIndex = subtitle.StreamIndex;
+            }
+        }
     }
 
     private async Task<Option<Core.Next.Source>> SourceForItem(
@@ -431,4 +455,38 @@ public partial class SyncNextPlayoutHandler(
             }
         }
     }
+
+    private static async Task<List<Subtitle>> GetSubtitles(MediaItem mediaItem)
+    {
+        List<Subtitle> allSubtitles = mediaItem switch
+        {
+            Episode episode => await Optional(episode.EpisodeMetadata).Flatten().HeadOrNone()
+                .Map(mm => mm.Subtitles ?? [])
+                .IfNoneAsync([]),
+            Movie movie => await Optional(movie.MovieMetadata).Flatten().HeadOrNone()
+                .Map(mm => mm.Subtitles ?? [])
+                .IfNoneAsync([]),
+            //MusicVideo musicVideo => await GetMusicVideoSubtitles(musicVideo, channel, settings),
+            OtherVideo otherVideo => await Optional(otherVideo.OtherVideoMetadata).Flatten().HeadOrNone()
+                .Map(mm => mm.Subtitles ?? [])
+                .IfNoneAsync([]),
+            _ => []
+        };
+
+        bool isMediaServer = mediaItem is PlexMovie or PlexEpisode or
+            JellyfinMovie or JellyfinEpisode or EmbyMovie or EmbyEpisode;
+
+        if (isMediaServer)
+        {
+            return [];
+
+            // closed captions are currently unsupported
+            //allSubtitles.RemoveAll(s => s.Codec == "eia_608");
+        }
+
+        // TODO: support text subtitles; external image subtitles
+        allSubtitles.RemoveAll(s => !s.IsImage || s.SubtitleKind is not SubtitleKind.Embedded);
+
+        return allSubtitles;
+    }
 }

ErsatzTV.Core.Tests/FFmpeg/FFmpegStreamSelectorTests.cs

@@ -185,6 +185,7 @@ public class FFmpegStreamSelectorTests
                 channel,
                 "heb",
                 ChannelSubtitleMode.Any,
+                shouldLogMessages: true,
                 cancellationToken);
             selectedStream.IsSome.ShouldBeTrue();
             foreach (Subtitle stream in selectedStream)

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -162,6 +162,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
                     channel,
                     preferredSubtitleLanguage,
                     subtitleMode,
+                    shouldLogMessages: true,
                     cancellationToken);
         }
 

ErsatzTV.Core/FFmpeg/FFmpegStreamSelector.cs

@@ -155,12 +155,17 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
         Channel channel,
         string preferredSubtitleLanguage,
         ChannelSubtitleMode subtitleMode,
+        bool shouldLogMessages,
         CancellationToken cancellationToken)
     {
         if (channel.MusicVideoCreditsMode is ChannelMusicVideoCreditsMode.GenerateSubtitles &&
             subtitles.FirstOrDefault(s => s.SubtitleKind == SubtitleKind.Generated) is { } generatedSubtitle)
+        {
+            if (shouldLogMessages)
             {
                 _logger.LogDebug("Selecting generated subtitle for channel {Number}", channel.Number);
+            }
+
             return Optional(generatedSubtitle);
         }
 
@@ -176,8 +181,12 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
             .IfNoneAsync(true);
 
         if (!useEmbeddedSubtitles)
+        {
+            if (shouldLogMessages)
             {
                 _logger.LogDebug("Ignoring embedded subtitles for channel {Number}", channel.Number);
+            }
+
             candidateSubtitles = candidateSubtitles.Filter(s => s.SubtitleKind is not SubtitleKind.Embedded).ToList();
         }
 
@@ -188,18 +197,24 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
                        .ToList())
             {
                 if (!subtitle.IsExtracted)
+                {
+                    if (shouldLogMessages)
                     {
                         _logger.LogDebug(
                             "Ignoring embedded subtitle with index {Index} that has not been extracted",
                             subtitle.StreamIndex);
+                    }
 
                     candidateSubtitles.Remove(subtitle);
                 }
                 else if (string.IsNullOrWhiteSpace(subtitle.Path))
+                {
+                    if (shouldLogMessages)
                     {
                         _logger.LogDebug(
                             "BUG: ignoring embedded subtitle with index {Index} that is missing a path",
                             subtitle.StreamIndex);
+                    }
 
                     candidateSubtitles.Remove(subtitle);
                 }
@@ -209,17 +224,23 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
         var allCodes = new List<string>();
         string language = (preferredSubtitleLanguage ?? string.Empty).ToLowerInvariant();
         if (string.IsNullOrWhiteSpace(language))
+        {
+            if (shouldLogMessages)
             {
                 _logger.LogDebug("Channel {Number} has no preferred subtitle language code", channel.Number);
             }
+        }
         else
         {
             // filter to preferred language
             allCodes = GetTwoAndThreeLetterLanguageCodes(_languageCodeService.GetAllLanguageCodes([language]));
             if (allCodes.Count > 1)
+            {
+                if (shouldLogMessages)
                 {
                     _logger.LogDebug("Preferred subtitle language has multiple codes {Codes}", allCodes);
                 }
+            }
 
             candidateSubtitles = candidateSubtitles
                 .Filter(s => allCodes.Any(c => string.Equals(s.Language, c, StringComparison.OrdinalIgnoreCase)))
@@ -248,17 +269,24 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
             };
 
             foreach (Subtitle subtitle in maybeSelectedSubtitle)
+            {
+                if (shouldLogMessages)
                 {
                     _logger.LogDebug("Selecting subtitle {@Subtitle}", subtitle);
+                }
+
                 return subtitle;
             }
         }
 
+        if (shouldLogMessages)
+        {
             _logger.LogDebug(
                 "Found no subtitles for channel {ChannelNumber} with mode {Mode} matching language {Language}",
                 channel.Number,
                 subtitleMode,
                 allCodes);
+        }
 
         return None;
     }

ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegStreamSelector.cs

@@ -22,5 +22,6 @@ public interface IFFmpegStreamSelector
         Channel channel,
         string preferredSubtitleLanguage,
         ChannelSubtitleMode subtitleMode,
+        bool shouldLogMessages,
         CancellationToken cancellationToken);
 }

ErsatzTV.Core/Next/Playout.cs

@@ -171,6 +171,12 @@ namespace ErsatzTV.Core.Next
         [JsonProperty("audio")]
         public TrackSelection Audio { get; set; }
 
+        /// <summary>
+        /// Subtitle track selection.
+        /// </summary>
+        [JsonProperty("subtitle")]
+        public TrackSelection Subtitle { get; set; }
+
         /// <summary>
         /// Video track selection.
         /// </summary>

ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs

@@ -1196,6 +1196,7 @@ public class TranscodingTests
             Channel channel,
             string preferredSubtitleLanguage,
             ChannelSubtitleMode subtitleMode,
+            bool shouldLogMessages,
             CancellationToken cancellationToken) =>
             subtitles.HeadOrNone().AsTask();
     }