add subtitle selection to playback troubleshooting (#2215)

CHANGELOG.md

@@ -7,6 +7,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Added
 - Add `Troubleshoot Playback` to overflow menu on all media cards
   - This should eliminate the need to lookup media ids for content
+- Add subtitle selection to playback troubleshooting. This is limited to:
+  - Sidecar text subtitles (e.g. `srt` files)
+  - Embedded image subtitles
+  - Embedded text subtitles that have already been extracted by ETV
 
 ### Fixed
 - Fix app startup with MySql/MariaDB

ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs

@@ -81,6 +81,8 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
         DateTimeOffset now = request.Now;
 
         Either<BaseError, PlayoutItemWithPath> maybePlayoutItem = await dbContext.PlayoutItems
+            .AsNoTracking()
+
             // get playout deco
             .Include(i => i.Playout)
             .ThenInclude(p => p.Deco)

ErsatzTV.Application/Troubleshooting/Commands/PrepareTroubleshootingPlayback.cs

@@ -7,5 +7,6 @@ public record PrepareTroubleshootingPlayback(
     int MediaItemId,
     int FFmpegProfileId,
     int WatermarkId,
+    int? SubtitleId,
     bool StartFromBeginning)
     : IRequest<Either<BaseError, Command>>;

ErsatzTV.Application/Troubleshooting/Commands/PrepareTroubleshootingPlaybackHandler.cs

@@ -67,7 +67,7 @@ public class PrepareTroubleshootingPlaybackHandler(
         localFileSystem.EnsureFolderExists(FileSystemLayout.TranscodeTroubleshootingFolder);
         localFileSystem.EmptyFolder(FileSystemLayout.TranscodeTroubleshootingFolder);
 
-        ChannelSubtitleMode subtitleMode = ChannelSubtitleMode.None;
+        const ChannelSubtitleMode SUBTITLE_MODE = ChannelSubtitleMode.Any;
 
         MediaVersion version = mediaItem.GetHeadVersion();
 
@@ -117,17 +117,18 @@ public class PrepareTroubleshootingPlaybackHandler(
                 Number = ".troubleshooting",
                 FFmpegProfile = ffmpegProfile,
                 StreamingMode = StreamingMode.HttpLiveStreamingSegmenter,
-                SubtitleMode = subtitleMode
+                StreamSelectorMode = ChannelStreamSelectorMode.Troubleshooting,
+                SubtitleMode = SUBTITLE_MODE
             },
             version,
             new MediaItemAudioVersion(mediaItem, version),
             mediaPath,
             mediaPath,
-            _ => Task.FromResult(new List<Subtitle>()),
+            _ => GetSelectedSubtitle(mediaItem, request),
             string.Empty,
             string.Empty,
             string.Empty,
-            subtitleMode,
+            SUBTITLE_MODE,
             now,
             now + duration,
             now,
@@ -151,6 +152,46 @@ public class PrepareTroubleshootingPlaybackHandler(
         return process;
     }
 
+    private static async Task<List<Subtitle>> GetSelectedSubtitle(MediaItem mediaItem, PrepareTroubleshootingPlayback request)
+    {
+        if (request.SubtitleId is not null)
+        {
+            List<Subtitle> allSubtitles = mediaItem switch
+            {
+                Episode episode => await Optional(episode.EpisodeMetadata).Flatten().HeadOrNone()
+                    .Map(mm => mm.Subtitles ?? [])
+                    .IfNoneAsync([]),
+                Movie movie => await Optional(movie.MovieMetadata).Flatten().HeadOrNone()
+                    .Map(mm => mm.Subtitles ?? [])
+                    .IfNoneAsync([]),
+                OtherVideo otherVideo => await Optional(otherVideo.OtherVideoMetadata).Flatten().HeadOrNone()
+                    .Map(mm => mm.Subtitles ?? [])
+                    .IfNoneAsync([]),
+                _ => []
+            };
+
+            bool isMediaServer = mediaItem is PlexMovie or PlexEpisode or
+                JellyfinMovie or JellyfinEpisode or EmbyMovie or EmbyEpisode;
+
+            if (isMediaServer)
+            {
+                // closed captions are currently unsupported
+                allSubtitles.RemoveAll(s => s.Codec == "eia_608");
+            }
+
+            allSubtitles.RemoveAll(s => s.Id != request.SubtitleId.Value);
+
+            foreach (Subtitle subtitle in allSubtitles)
+            {
+                // pretend subtitle is forced
+                subtitle.Forced = true;
+                return [subtitle];
+            }
+        }
+
+        return [];
+    }
+
     private static async Task<Validation<BaseError, Tuple<MediaItem, string, string, FFmpegProfile>>> Validate(
         TvContext dbContext,
         PrepareTroubleshootingPlayback request) =>
@@ -166,6 +207,7 @@ public class PrepareTroubleshootingPlaybackHandler(
         PrepareTroubleshootingPlayback request)
     {
         return await dbContext.MediaItems
+            .AsNoTracking()
             .Include(mi => (mi as Episode).EpisodeMetadata)
             .ThenInclude(em => em.Subtitles)
             .Include(mi => (mi as Episode).MediaVersions)

ErsatzTV.Application/Troubleshooting/Queries/GetTroubleshootingSubtitles.cs

@@ -0,0 +1,3 @@
+namespace ErsatzTV.Application.Troubleshooting.Queries;
+
+public record GetTroubleshootingSubtitles(int MediaItemId) : IRequest<List<SubtitleViewModel>>;

ErsatzTV.Application/Troubleshooting/Queries/GetTroubleshootingSubtitlesHandler.cs

@@ -0,0 +1,50 @@
+using ErsatzTV.Core.Domain;
+using ErsatzTV.Infrastructure.Data;
+using ErsatzTV.Infrastructure.Extensions;
+using Microsoft.EntityFrameworkCore;
+
+namespace ErsatzTV.Application.Troubleshooting.Queries;
+
+public class GetTroubleshootingSubtitlesHandler(IDbContextFactory<TvContext> dbContextFactory)
+    : IRequestHandler<GetTroubleshootingSubtitles, List<SubtitleViewModel>>
+{
+    public async Task<List<SubtitleViewModel>> Handle(
+        GetTroubleshootingSubtitles request,
+        CancellationToken cancellationToken)
+    {
+        await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);
+        Option<MediaItem> maybeMediaItem = await dbContext.MediaItems
+            .AsNoTracking()
+            .Include(mi => (mi as Movie).MovieMetadata)
+            .ThenInclude(mm => mm.Subtitles)
+            .Include(mi => (mi as Episode).EpisodeMetadata)
+            .ThenInclude(mm => mm.Subtitles)
+            .Include(mi => (mi as OtherVideo).OtherVideoMetadata)
+            .ThenInclude(mm => mm.Subtitles)
+            .SelectOneAsync(mi => mi.Id, mi => mi.Id == request.MediaItemId);
+
+        foreach (MediaItem mediaItem in maybeMediaItem)
+        {
+            List<Subtitle> subtitles = GetSubtitles(mediaItem);
+
+            // remove text subtitles that are embedded but have not been extracted
+            subtitles.RemoveAll(s => s.SubtitleKind is SubtitleKind.Embedded && !s.IsImage && !s.IsExtracted);
+
+            return subtitles.Map(ProjectToViewModel).ToList();
+        }
+
+        return [];
+    }
+
+    private static List<Subtitle> GetSubtitles(MediaItem mediaItem) =>
+        mediaItem switch
+        {
+            Episode e => e.EpisodeMetadata.Head().Subtitles,
+            Movie m => m.MovieMetadata.Head().Subtitles,
+            OtherVideo ov => ov.OtherVideoMetadata.Head().Subtitles,
+            _ => []
+        };
+
+    private static SubtitleViewModel ProjectToViewModel(Subtitle subtitle) =>
+        new(subtitle.Id, subtitle.Language, subtitle.Title, subtitle.Codec);
+}

ErsatzTV.Application/Troubleshooting/SubtitleViewModel.cs

@@ -0,0 +1,3 @@
+namespace ErsatzTV.Application.Troubleshooting;
+
+public record SubtitleViewModel(int Id, string Language, string Title, string Codec);

ErsatzTV.Core.Tests/FFmpeg/FFmpegStreamSelectorTests.cs

@@ -1,3 +1,4 @@
+using System.Collections.Immutable;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.FFmpeg;
 using ErsatzTV.Core.Interfaces.Metadata;
@@ -172,7 +173,7 @@ public class FFmpegStreamSelectorTests
                 Substitute.For<ILogger<FFmpegStreamSelector>>());
 
             Option<Subtitle> selectedStream = await selector.SelectSubtitleStream(
-                subtitles,
+                subtitles.ToImmutableList(),
                 channel,
                 "heb",
                 ChannelSubtitleMode.Any);

ErsatzTV.Core/Domain/ChannelStreamSelectorMode.cs

@@ -3,5 +3,7 @@
 public enum ChannelStreamSelectorMode
 {
     Default = 0,
-    Custom = 1
+    Custom = 1,
+
+    Troubleshooting = 100
 }

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -1,4 +1,5 @@
-using CliWrap;
+using System.Collections.Immutable;
+using CliWrap;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Domain.Filler;
 using ErsatzTV.Core.Interfaces.FFmpeg;
@@ -129,12 +130,17 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
             maybeSubtitle =
                 await _ffmpegStreamSelector.SelectSubtitleStream(
-                    allSubtitles,
+                    allSubtitles.ToImmutableList(),
                     channel,
                     preferredSubtitleLanguage,
                     subtitleMode);
         }
 
+        if (channel.StreamSelectorMode is ChannelStreamSelectorMode.Troubleshooting && maybeSubtitle.IsNone)
+        {
+            maybeSubtitle = allSubtitles.HeadOrNone();
+        }
+
         foreach (Subtitle subtitle in maybeSubtitle)
         {
             if (subtitle.SubtitleKind == SubtitleKind.Sidecar)

ErsatzTV.Core/FFmpeg/FFmpegStreamSelector.cs

@@ -1,4 +1,5 @@
-using System.Diagnostics;
+using System.Collections.Immutable;
+using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using ErsatzTV.Core.Domain;
@@ -123,7 +124,7 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
     }
 
     public async Task<Option<Subtitle>> SelectSubtitleStream(
-        List<Subtitle> subtitles,
+        ImmutableList<Subtitle> subtitles,
         Channel channel,
         string preferredSubtitleLanguage,
         ChannelSubtitleMode subtitleMode)
@@ -140,6 +141,8 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
             return None;
         }
 
+        var candidateSubtitles = subtitles.ToList();
+
         bool useEmbeddedSubtitles = await _configElementRepository
             .GetValue<bool>(ConfigElementKey.FFmpegUseEmbeddedSubtitles)
             .IfNoneAsync(true);
@@ -147,10 +150,10 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
         if (!useEmbeddedSubtitles)
         {
             _logger.LogDebug("Ignoring embedded subtitles for channel {Number}", channel.Number);
-            subtitles = subtitles.Filter(s => s.SubtitleKind is not SubtitleKind.Embedded).ToList();
+            candidateSubtitles = candidateSubtitles.Filter(s => s.SubtitleKind is not SubtitleKind.Embedded).ToList();
         }
 
-        foreach (Subtitle subtitle in subtitles.Filter(s => s.SubtitleKind is SubtitleKind.Embedded && !s.IsImage)
+        foreach (Subtitle subtitle in candidateSubtitles.Filter(s => s.SubtitleKind is SubtitleKind.Embedded && !s.IsImage)
                      .ToList())
         {
             if (subtitle.IsExtracted == false)
@@ -159,7 +162,7 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
                     "Ignoring embedded subtitle with index {Index} that has not been extracted",
                     subtitle.StreamIndex);
 
-                subtitles.Remove(subtitle);
+                candidateSubtitles.Remove(subtitle);
             }
             else if (string.IsNullOrWhiteSpace(subtitle.Path))
             {
@@ -167,7 +170,7 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
                     "BUG: ignoring embedded subtitle with index {Index} that is missing a path",
                     subtitle.StreamIndex);
 
-                subtitles.Remove(subtitle);
+                candidateSubtitles.Remove(subtitle);
             }
         }
 
@@ -187,26 +190,26 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
                 _logger.LogDebug("Preferred subtitle language has multiple codes {Codes}", allCodes);
             }
 
-            subtitles = subtitles
+            candidateSubtitles = candidateSubtitles
                 .Filter(s => allCodes.Any(c => string.Equals(s.Language, c, StringComparison.OrdinalIgnoreCase)))
                 .ToList();
         }
 
-        if (subtitles.Count > 0)
+        if (candidateSubtitles.Count > 0)
         {
             Option<Subtitle> maybeSelectedSubtitle = subtitleMode switch
             {
-                ChannelSubtitleMode.Forced => subtitles
+                ChannelSubtitleMode.Forced => candidateSubtitles
                     .OrderBy(s => s.StreamIndex)
                     .Find(s => s.Forced)
                     .HeadOrNone(),
 
-                ChannelSubtitleMode.Default => subtitles
+                ChannelSubtitleMode.Default => candidateSubtitles
                     .OrderBy(s => s.Default ? 0 : 1)
                     .ThenBy(s => s.StreamIndex)
                     .HeadOrNone(),
 
-                ChannelSubtitleMode.Any => subtitles
+                ChannelSubtitleMode.Any => candidateSubtitles
                     .OrderBy(s => s.StreamIndex)
                     .HeadOrNone(),
 

ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegStreamSelector.cs

@@ -1,4 +1,5 @@
-using ErsatzTV.Core.Domain;
+using System.Collections.Immutable;
+using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.FFmpeg;
 
 namespace ErsatzTV.Core.Interfaces.FFmpeg;
@@ -15,7 +16,7 @@ public interface IFFmpegStreamSelector
         string preferredAudioTitle);
 
     Task<Option<Subtitle>> SelectSubtitleStream(
-        List<Subtitle> subtitles,
+        ImmutableList<Subtitle> subtitles,
         Channel channel,
         string preferredSubtitleLanguage,
         ChannelSubtitleMode subtitleMode);

ErsatzTV.FFmpeg/InputOption/CopyTimestampInputOption.cs

@@ -4,13 +4,13 @@ namespace ErsatzTV.FFmpeg.InputOption;
 
 public class CopyTimestampInputOption : IInputOption
 {
-    public EnvironmentVariable[] EnvironmentVariables => Array.Empty<EnvironmentVariable>();
-    public string[] GlobalOptions => Array.Empty<string>();
+    public EnvironmentVariable[] EnvironmentVariables => [];
+    public string[] GlobalOptions => [];
 
-    public string[] InputOptions(InputFile inputFile) => []; //new[] { "-copyts" };
+    public string[] InputOptions(InputFile inputFile) => ["-copyts"];
 
-    public string[] FilterOptions => Array.Empty<string>();
-    public string[] OutputOptions => Array.Empty<string>();
+    public string[] FilterOptions => [];
+    public string[] OutputOptions => [];
     public FrameState NextState(FrameState currentState) => currentState;
 
     public bool AppliesTo(AudioInputFile audioInputFile) => false;

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs

@@ -227,7 +227,7 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
 
         SetSceneDetect(videoStream, ffmpegState, desiredState, pipelineSteps);
         SetFFReport(ffmpegState, pipelineSteps);
-        SetStreamSeek(ffmpegState, videoInputFile, context, pipelineSteps);
+        SetStreamSeek(ffmpegState, videoInputFile);
         SetTimeLimit(ffmpegState, pipelineSteps);
 
         (FilterChain filterChain, ffmpegState) = BuildVideoPipeline(
@@ -834,23 +834,13 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
         }
     }
 
-    private void SetStreamSeek(
-        FFmpegState ffmpegState,
-        VideoInputFile videoInputFile,
-        PipelineContext context,
-        List<IPipelineStep> pipelineSteps)
+    private void SetStreamSeek(FFmpegState ffmpegState, VideoInputFile videoInputFile)
     {
         foreach (TimeSpan desiredStart in ffmpegState.Start.Filter(s => s > TimeSpan.Zero))
         {
             var option = new StreamSeekInputOption(desiredStart);
             _audioInputFile.Iter(a => a.AddOption(option));
             videoInputFile.AddOption(option);
-
-            // need to seek text subtitle files
-            // if (context.HasSubtitleText)
-            // {
-            //     pipelineSteps.Add(new StreamSeekFilterOption(desiredStart));
-            // }
         }
     }
 

ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs

@@ -1,4 +1,5 @@
-using System.Diagnostics;
+using System.Collections.Immutable;
+using System.Diagnostics;
 using System.Security.Cryptography;
 using System.Text;
 using Bugsnag;
@@ -1075,7 +1076,7 @@ public class TranscodingTests
                 .AsTask();
 
         public Task<Option<ErsatzTV.Core.Domain.Subtitle>> SelectSubtitleStream(
-            List<ErsatzTV.Core.Domain.Subtitle> subtitles,
+            ImmutableList<ErsatzTV.Core.Domain.Subtitle> subtitles,
             Channel channel,
             string preferredSubtitleLanguage,
             ChannelSubtitleMode subtitleMode) =>

ErsatzTV/Controllers/Api/TroubleshootController.cs

@@ -27,11 +27,13 @@ public class TroubleshootController(
         [FromQuery]
         int watermark,
         [FromQuery]
+        int? subtitleId,
+        [FromQuery]
         bool startFromBeginning,
         CancellationToken cancellationToken)
     {
         Either<BaseError, Command> result = await mediator.Send(
-            new PrepareTroubleshootingPlayback(mediaItem, ffmpegProfile, watermark, startFromBeginning),
+            new PrepareTroubleshootingPlayback(mediaItem, ffmpegProfile, watermark, subtitleId, startFromBeginning),
             cancellationToken);
 
         return await result.MatchAsync<IActionResult>(

ErsatzTV/Pages/PlaybackTroubleshooting.razor

@@ -1,6 +1,8 @@
 @page "/system/troubleshooting/playback"
 @using ErsatzTV.Application.FFmpegProfiles
 @using ErsatzTV.Application.MediaItems
+@using ErsatzTV.Application.Troubleshooting
+@using ErsatzTV.Application.Troubleshooting.Queries
 @using ErsatzTV.Application.Watermarks
 @using ErsatzTV.Core.Notifications
 @using MediatR.Courier
@@ -64,6 +66,18 @@
                     }
                 </MudSelect>
             </MudStack>
+            <MudStack Row="true" Breakpoint="Breakpoint.SmAndDown" Class="form-field-stack gap-md-8 mb-5">
+                <div class="d-flex">
+                    <MudText>Subtitle</MudText>
+                </div>
+                <MudSelect @bind-Value="_subtitleId" For="@(() => _subtitleId)" Clearable="true">
+                    <MudSelectItem T="int?" Value="@((int?)null)">(none)</MudSelectItem>
+                    @foreach (SubtitleViewModel subtitleStream in _subtitleStreams)
+                    {
+                        <MudSelectItem T="int?" Value="@subtitleStream.Id">@($"{subtitleStream.Id}: {subtitleStream.Language} - {subtitleStream.Title} ({subtitleStream.Codec})")</MudSelectItem>
+                    }
+                </MudSelect>
+            </MudStack>
             <MudStack Row="true" Breakpoint="Breakpoint.SmAndDown" Class="form-field-stack gap-md-8 mb-5">
                 <div class="d-flex">
                     <MudText>Start From Beginning</MudText>
@@ -107,9 +121,11 @@
 
     private List<FFmpegProfileViewModel> _ffmpegProfiles = [];
     private List<WatermarkViewModel> _watermarks = [];
+    private List<SubtitleViewModel> _subtitleStreams = [];
     private MediaItemInfo _info;
     private int _ffmpegProfileId;
     private int? _watermarkId;
+    private int? _subtitleId;
     private bool _startFromBeginning;
     private bool _hasPlayed;
 
@@ -151,6 +167,10 @@
         var uri = new UriBuilder(NavigationManager.ToAbsoluteUri(NavigationManager.Uri));
         uri.Path = uri.Path.Replace("/system/troubleshooting/playback", "/api/troubleshoot/playback.m3u8");
         uri.Query = $"?mediaItem={MediaItemId}&ffmpegProfile={_ffmpegProfileId}&watermark={_watermarkId ?? 0}&startFromBeginning={_startFromBeginning}";
+        if (_subtitleId is not null)
+        {
+            uri.Query += $"&subtitleId={_subtitleId.Value}";
+        }
         await JsRuntime.InvokeVoidAsync("previewChannel", uri.ToString());
 
         await Task.Delay(TimeSpan.FromSeconds(1));
@@ -170,6 +190,10 @@
             {
                 _info = info;
                 _startFromBeginning = string.Equals(info.Kind, "RemoteStream", StringComparison.OrdinalIgnoreCase);
+
+                _subtitleId = null;
+                _subtitleStreams.Clear();
+                _subtitleStreams.AddRange(await Mediator.Send(new GetTroubleshootingSubtitles(id)));
             }
 
             if (maybeInfo.IsLeft)