fix seeking with text subtitles (#2214)

CHANGELOG.md

@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Fixed
 - Fix app startup with MySql/MariaDB
 - YAML playout: fix `pad_to_next` always running over time
+- Fix playback with text subtitles when seeking into content, i.e. when first joining a channel
 
 ### Changed
 - Always tell ffmpeg to stop encoding with a specific duration

ErsatzTV.Application/Streaming/Queries/GetSeekTextSubtitleProcess.cs

@@ -0,0 +1,6 @@
+using ErsatzTV.Core;
+
+namespace ErsatzTV.Application.Streaming;
+
+public record GetSeekTextSubtitleProcess(string SubtitlePath, TimeSpan Seek)
+    : IRequest<Either<BaseError, SeekTextSubtitleProcess>>;

ErsatzTV.Application/Streaming/Queries/GetSeekTextSubtitleProcessHandler.cs

@@ -0,0 +1,47 @@
+using CliWrap;
+using ErsatzTV.Core;
+using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.Interfaces.FFmpeg;
+using ErsatzTV.Infrastructure.Data;
+using ErsatzTV.Infrastructure.Extensions;
+using Microsoft.EntityFrameworkCore;
+
+namespace ErsatzTV.Application.Streaming;
+
+public class GetSeekTextSubtitleProcessHandler(
+    IDbContextFactory<TvContext> dbContextFactory,
+    IFFmpegProcessService ffmpegProcessService)
+    : IRequestHandler<GetSeekTextSubtitleProcess,
+        Either<BaseError, SeekTextSubtitleProcess>>
+{
+    public async Task<Either<BaseError, SeekTextSubtitleProcess>> Handle(
+        GetSeekTextSubtitleProcess request,
+        CancellationToken cancellationToken)
+    {
+        await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);
+        Validation<BaseError, string> validation = await Validate(dbContext);
+        return await validation.Match(
+            ffmpegPath => GetProcess(request, ffmpegPath),
+            error => Task.FromResult<Either<BaseError, SeekTextSubtitleProcess>>(error.Join()));
+    }
+
+    private async Task<Either<BaseError, SeekTextSubtitleProcess>> GetProcess(
+        GetSeekTextSubtitleProcess request,
+        string ffmpegPath)
+    {
+        Command process = await ffmpegProcessService.SeekTextSubtitle(
+            ffmpegPath,
+            request.SubtitlePath,
+            request.Seek);
+
+        return new SeekTextSubtitleProcess(process);
+    }
+
+    private static async Task<Validation<BaseError, string>> Validate(TvContext dbContext) =>
+        await FFmpegPathMustExist(dbContext);
+
+    private static Task<Validation<BaseError, string>> FFmpegPathMustExist(TvContext dbContext) =>
+        dbContext.ConfigElements.GetValue<string>(ConfigElementKey.FFmpegPath)
+            .FilterT(File.Exists)
+            .Map(maybePath => maybePath.ToValidation<BaseError>("FFmpeg path does not exist on filesystem"));
+}

ErsatzTV.Application/Streaming/SeekTextSubtitleProcess.cs

@@ -0,0 +1,5 @@
+using CliWrap;
+
+namespace ErsatzTV.Application.Streaming;
+
+public record SeekTextSubtitleProcess(Command Process);

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -141,6 +141,11 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             {
                 // proxy to avoid dealing with escaping
                 subtitle.Path = $"http://localhost:{Settings.StreamingPort}/media/subtitle/{subtitle.Id}";
+
+                foreach (TimeSpan seek in playbackSettings.StreamSeek)
+                {
+                    subtitle.Path += $"?seekToMs={(int)seek.TotalMilliseconds}";
+                }
             }
         }
 
@@ -920,6 +925,45 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             watermarkWidthPercent,
             cancellationToken);
 
+    public async Task<Command> SeekTextSubtitle(string ffmpegPath, string inputFile, TimeSpan seek)
+    {
+        var videoInputFile = new VideoInputFile(
+            inputFile,
+            new List<VideoStream>
+            {
+                new(
+                    0,
+                    string.Empty,
+                    string.Empty,
+                    None,
+                    ColorParams.Default,
+                    FrameSize.Unknown,
+                    string.Empty,
+                    string.Empty,
+                    None,
+                    true,
+                    ScanKind.Progressive)
+            });
+
+        IPipelineBuilder pipelineBuilder = await _pipelineBuilderFactory.GetBuilder(
+            HardwareAccelerationMode.None,
+            videoInputFile,
+            None,
+            None,
+            None,
+            Option<ConcatInputFile>.None,
+            Option<string>.None,
+            None,
+            None,
+            FileSystemLayout.FFmpegReportsFolder,
+            FileSystemLayout.FontsCacheFolder,
+            ffmpegPath);
+
+        FFmpegPipeline pipeline = pipelineBuilder.Seek(inputFile, seek);
+
+        return GetCommand(ffmpegPath, videoInputFile, None, None, None, pipeline, false);
+    }
+
     private static Option<WatermarkInputFile> GetWatermarkInputFile(
         Option<WatermarkOptions> watermarkOptions,
         Option<List<FadePoint>> maybeFadePoints)

ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs

@@ -90,4 +90,6 @@ public interface IFFmpegProcessService
         int verticalMarginPercent,
         int watermarkWidthPercent,
         CancellationToken cancellationToken);
+
+    Task<Command> SeekTextSubtitle(string ffmpegPath, string inputFile, TimeSpan seek);
 }

ErsatzTV.FFmpeg/InputOption/CopyTimestampInputOption.cs

@@ -7,7 +7,7 @@ public class CopyTimestampInputOption : IInputOption
     public EnvironmentVariable[] EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public string[] GlobalOptions => Array.Empty<string>();
 
-    public string[] InputOptions(InputFile inputFile) => new[] { "-copyts" };
+    public string[] InputOptions(InputFile inputFile) => []; //new[] { "-copyts" };
 
     public string[] FilterOptions => Array.Empty<string>();
     public string[] OutputOptions => Array.Empty<string>();

ErsatzTV.FFmpeg/OutputFormat/OutputFormatAss.cs

@@ -0,0 +1,13 @@
+using ErsatzTV.FFmpeg.Environment;
+
+namespace ErsatzTV.FFmpeg.OutputFormat;
+
+public class OutputFormatAss : IPipelineStep
+{
+    public EnvironmentVariable[] EnvironmentVariables => [];
+    public string[] GlobalOptions => [];
+    public string[] InputOptions(InputFile inputFile) => [];
+    public string[] FilterOptions => [];
+    public string[] OutputOptions => ["-f", "ass"];
+    public FrameState NextState(FrameState currentState) => currentState;
+}

ErsatzTV.FFmpeg/OutputFormat/OutputFormatSrt.cs

@@ -0,0 +1,13 @@
+using ErsatzTV.FFmpeg.Environment;
+
+namespace ErsatzTV.FFmpeg.OutputFormat;
+
+public class OutputFormatSrt : IPipelineStep
+{
+    public EnvironmentVariable[] EnvironmentVariables => [];
+    public string[] GlobalOptions => [];
+    public string[] InputOptions(InputFile inputFile) => [];
+    public string[] FilterOptions => [];
+    public string[] OutputOptions => ["-f", "srt"];
+    public FrameState NextState(FrameState currentState) => currentState;
+}

ErsatzTV.FFmpeg/OutputFormat/OutputFormatWebVtt.cs

@@ -0,0 +1,13 @@
+using ErsatzTV.FFmpeg.Environment;
+
+namespace ErsatzTV.FFmpeg.OutputFormat;
+
+public class OutputFormatWebVtt : IPipelineStep
+{
+    public EnvironmentVariable[] EnvironmentVariables => [];
+    public string[] GlobalOptions => [];
+    public string[] InputOptions(InputFile inputFile) => [];
+    public string[] FilterOptions => [];
+    public string[] OutputOptions => ["-f", "webvtt"];
+    public FrameState NextState(FrameState currentState) => currentState;
+}

ErsatzTV.FFmpeg/Pipeline/IPipelineBuilder.cs

@@ -3,6 +3,7 @@ namespace ErsatzTV.FFmpeg.Pipeline;
 public interface IPipelineBuilder
 {
     FFmpegPipeline Resize(string outputFile, FrameSize scaledSize);
+    FFmpegPipeline Seek(string inputFile, TimeSpan seek);
     FFmpegPipeline Concat(ConcatInputFile concatInputFile, FFmpegState ffmpegState);
     FFmpegPipeline WrapSegmenter(ConcatInputFile concatInputFile, FFmpegState ffmpegState);
     FFmpegPipeline Build(FFmpegState ffmpegState, FrameState desiredState);

ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs

@@ -542,8 +542,6 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
         {
             if (context.HasSubtitleText)
             {
-                videoInputFile.AddOption(new CopyTimestampInputOption());
-
                 if (videoInputFile.FilterSteps.Count == 0 && videoInputFile.InputOptions.OfType<CuvidDecoder>().Any())
                 {
                     // change the hw accel output to software so the explicit download isn't needed

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs

@@ -65,13 +65,37 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
         IPipelineFilterStep scaleStep = new ScaleImageFilter(scaledSize);
         _videoInputFile.Iter(f => f.FilterSteps.Add(scaleStep));
 
-        pipelineSteps.Add(new VideoFilter(new[] { scaleStep }));
+        pipelineSteps.Add(new VideoFilter([scaleStep]));
         pipelineSteps.Add(scaleStep);
         pipelineSteps.Add(new FileNameOutputOption(outputFile));
 
         return new FFmpegPipeline(pipelineSteps, false);
     }
 
+    public FFmpegPipeline Seek(string inputFile, TimeSpan seek)
+    {
+        IPipelineStep outputFormat = Path.GetExtension(inputFile).ToLowerInvariant() switch
+        {
+            "ass" or "ssa" => new OutputFormatAss(),
+            "vtt" => new OutputFormatWebVtt(),
+            _ => new OutputFormatSrt()
+        };
+
+        var pipelineSteps = new List<IPipelineStep>
+        {
+            new NoStandardInputOption(),
+            new HideBannerOption(),
+            new NoStatsOption(),
+            new LoglevelErrorOption(),
+            new StreamSeekFilterOption(seek),
+            new EncoderCopySubtitle(),
+            outputFormat,
+            new PipeProtocol(),
+        };
+
+        return new FFmpegPipeline(pipelineSteps, false);
+    }
+
     public FFmpegPipeline Concat(ConcatInputFile concatInputFile, FFmpegState ffmpegState)
     {
         var pipelineSteps = new List<IPipelineStep>
@@ -823,10 +847,10 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
             videoInputFile.AddOption(option);
 
             // need to seek text subtitle files
-            if (context.HasSubtitleText)
-            {
-                pipelineSteps.Add(new StreamSeekFilterOption(desiredStart));
-            }
+            // if (context.HasSubtitleText)
+            // {
+            //     pipelineSteps.Add(new StreamSeekFilterOption(desiredStart));
+            // }
         }
     }
 

ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs

@@ -508,8 +508,6 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
         {
             if (context.HasSubtitleText)
             {
-                videoInputFile.AddOption(new CopyTimestampInputOption());
-
                 var downloadFilter = new HardwareDownloadFilter(currentState);
                 currentState = downloadFilter.NextState(currentState);
                 videoInputFile.FilterSteps.Add(downloadFilter);

ErsatzTV.FFmpeg/Pipeline/SoftwarePipelineBuilder.cs

@@ -270,8 +270,6 @@ public class SoftwarePipelineBuilder : PipelineBuilderBase
         {
             if (context.HasSubtitleText)
             {
-                videoInputFile.AddOption(new CopyTimestampInputOption());
-
                 var subtitlesFilter = new SubtitlesFilter(fontsFolder, subtitle);
                 videoInputFile.FilterSteps.Add(subtitlesFilter);
             }

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -449,8 +449,6 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
         {
             if (context.HasSubtitleText)
             {
-                videoInputFile.AddOption(new CopyTimestampInputOption());
-
                 // if (videoInputFile.FilterSteps.Count == 0 && videoInputFile.InputOptions.OfType<CuvidDecoder>().Any())
                 // {
                 //     // change the hw accel output to software so the explicit download isn't needed

ErsatzTV/Controllers/InternalController.cs

@@ -190,27 +190,65 @@ public class InternalController : ControllerBase
     }
 
     [HttpGet("/media/subtitle/{id:int}")]
-    public async Task<IActionResult> GetSubtitle(int id)
+    public async Task<IActionResult> GetSubtitle(int id, [FromQuery] long? seekToMs)
     {
-        Either<BaseError, string> path = await _mediator.Send(new GetSubtitlePathById(id));
-        return path.Match<IActionResult>(
-            Left: _ => new NotFoundResult(),
-            Right: r =>
+        Either<BaseError, string> maybePath = await _mediator.Send(new GetSubtitlePathById(id));
+
+        foreach (string path in maybePath.RightToSeq())
+        {
+            string mimeType = Path.GetExtension(path).ToLowerInvariant() switch
+            {
+                "ass" or "ssa" => "text/x-ssa",
+                "vtt" => "text/vtt",
+                _ => "application/x-subrip"
+            };
+
+            if (seekToMs is > 0)
             {
-                if (r.StartsWith("http", StringComparison.OrdinalIgnoreCase))
+                Either<BaseError, SeekTextSubtitleProcess> maybeProcess = await _mediator.Send(
+                    new GetSeekTextSubtitleProcess(path, TimeSpan.FromMilliseconds(seekToMs.Value)));
+                foreach (SeekTextSubtitleProcess processModel in maybeProcess.RightToSeq())
                 {
-                    return new RedirectResult(r);
+                    Command command = processModel.Process;
+
+                    _logger.LogDebug("ffmpeg text subtitle arguments {FFmpegArguments}", command.Arguments);
+
+                    var process = new FFmpegProcess
+                    {
+                        StartInfo = new ProcessStartInfo
+                        {
+                            FileName = command.TargetFilePath,
+                            Arguments = command.Arguments,
+                            RedirectStandardOutput = true,
+                            RedirectStandardError = false,
+                            UseShellExecute = false,
+                            CreateNoWindow = true
+                        }
+                    };
+
+                    HttpContext.Response.RegisterForDispose(process);
+
+                    foreach ((string key, string value) in command.EnvironmentVariables)
+                    {
+                        process.StartInfo.Environment[key] = value;
+                    }
+
+                    process.Start();
+                    return new FileStreamResult(process.StandardOutput.BaseStream, mimeType);
                 }
 
-                string mimeType = Path.GetExtension(r).ToLowerInvariant() switch
-                {
-                    "ass" or "ssa" => "text/x-ssa",
-                    "vtt" => "text/vtt",
-                    _ => "application/x-subrip"
-                };
+                return new NotFoundResult();
+            }
 
-                return new PhysicalFileResult(r, mimeType);
-            });
+            if (path.StartsWith("http", StringComparison.OrdinalIgnoreCase))
+            {
+                return new RedirectResult(path);
+            }
+
+            return new PhysicalFileResult(path, mimeType);
+        }
+
+        return new NotFoundResult();
     }
 
     private async Task<IActionResult> GetSegmenterV2Stream(string channelNumber)