subtitle improvements with hls direct (#1290)

* wip: hls direct subtitles

* convert picture subtitles with hls direct

* use mp4 for hls direct to support more codecs

* disable subtitle conversion in hls direct

* fix tests

* update changelog

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Synchronize actor metadata from Jellyfin and Emby television libraries
   - New libraries and new episodes will get actor data automatically
   - Existing libraries can deep scan (one time) to retrieve actor data for existing episodes
+- `HLS Direct` streaming mode: stream copy dvd subtitles 
 
 ### Fixed
 - Fix extracting embedded text subtitles that had been incompletely extracted in the past 
@@ -21,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 - Timeout playout builds after 2 minutes; this should prevent playout bugs from blocking other functionality
+- `HLS Direct` streaming mode: Use MP4 container instead MPEG-TS container to improve codec compatibility (e.g. FLAC audio)
 
 ## [0.7.8-beta] - 2023-04-29
 ### Added

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -91,9 +91,11 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             hlsRealtime,
             targetFramerate);
 
+        var allSubtitles = await getSubtitles(playbackSettings);
+        
         Option<Subtitle> maybeSubtitle =
             await _ffmpegStreamSelector.SelectSubtitleStream(
-                await getSubtitles(playbackSettings),
+                allSubtitles,
                 channel,
                 preferredSubtitleLanguage,
                 subtitleMode);
@@ -213,13 +215,31 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
                     _ => Path.Combine(FileSystemLayout.SubtitleCacheFolder, subtitle.Path)
                 };
 
+                SubtitleMethod method = SubtitleMethod.Burn;
+                if (channel.StreamingMode == StreamingMode.HttpLiveStreamingDirect)
+                {
+                    method = subtitle.Codec switch
+                    {
+                        // MP4 supports vobsub
+                        "dvdsub" or "dvd_subtitle" or "vobsub" => SubtitleMethod.Copy,
+
+                        // MP4 does not support PGS
+                        "pgs" or "pgssub" or "hdmv_pgs_subtitle" => SubtitleMethod.None,
+
+                        // ignore text subtitles for now
+                        _ => SubtitleMethod.None
+                    };
+
+                    if (method == SubtitleMethod.None)
+                    {
+                        return None;
+                    }
+                }
+
                 return new SubtitleInputFile(
                     path,
                     new List<ErsatzTV.FFmpeg.MediaStream> { ffmpegSubtitleStream },
-                    false);
-
-                // TODO: figure out HLS direct
-                // channel.StreamingMode == StreamingMode.HttpLiveStreamingDirect);
+                    method);
             }).Flatten();
 
         Option<WatermarkInputFile> watermarkInputFile = GetWatermarkInputFile(watermarkOptions, maybeFadePoints);
@@ -228,9 +248,12 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
         HardwareAccelerationMode hwAccel = GetHardwareAccelerationMode(playbackSettings, fillerKind);
 
-        OutputFormatKind outputFormat = channel.StreamingMode == StreamingMode.HttpLiveStreamingSegmenter
-            ? OutputFormatKind.Hls
-            : OutputFormatKind.MpegTs;
+        OutputFormatKind outputFormat = channel.StreamingMode switch
+        {
+            StreamingMode.HttpLiveStreamingSegmenter => OutputFormatKind.Hls,
+            StreamingMode.HttpLiveStreamingDirect => OutputFormatKind.Mp4,
+            _ => OutputFormatKind.MpegTs
+        };
 
         Option<string> hlsPlaylistPath = outputFormat == OutputFormatKind.Hls
             ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live.m3u8")
@@ -415,7 +438,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         var subtitleInputFile = new SubtitleInputFile(
             subtitleFile,
             new List<ErsatzTV.FFmpeg.MediaStream> { ffmpegSubtitleStream },
-            false);
+            SubtitleMethod.Burn);
 
         _logger.LogDebug("FFmpeg desired error state {FrameState}", desiredState);
 

ErsatzTV.Core/FFmpeg/FFmpegStreamSelector.cs

@@ -138,15 +138,6 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
             return None;
         }
 
-        if (channel.StreamingMode == StreamingMode.HttpLiveStreamingDirect &&
-            string.IsNullOrWhiteSpace(preferredSubtitleLanguage))
-        {
-            // _logger.LogDebug(
-            //     "Channel {Number} is HLS Direct with no preferred subtitle language; using all subtitle streams",
-            //     channel.Number);
-            return None;
-        }
-
         string language = (preferredSubtitleLanguage ?? string.Empty).ToLowerInvariant();
         if (string.IsNullOrWhiteSpace(language))
         {

ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs

@@ -306,7 +306,7 @@ public class PipelineBuilderBaseTests
             Option<string>.None,
             Option<string>.None,
             Option<string>.None,
-            OutputFormatKind.MpegTs,
+            OutputFormatKind.Mp4,
             Option<string>.None,
             Option<string>.None,
             0,
@@ -334,7 +334,7 @@ public class PipelineBuilderBaseTests
 
         // 0.4.0 reference: "-nostdin -threads 1 -hide_banner -loglevel error -nostats -fflags +genpts+discardcorrupt+igndts -re -ss 00:14:33.6195516 -i /tmp/whatever.mkv -map 0:0 -map 0:a -c:v copy -flags cgop -sc_threshold 0 -c:a copy -movflags +faststart -muxdelay 0 -muxpreload 0 -metadata service_provider="ErsatzTV" -metadata service_name="ErsatzTV" -t 00:06:39.6934484 -f mpegts -mpegts_flags +initial_discontinuity pipe:1"
         command.Should().Be(
-            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -re -i /tmp/whatever.mkv -map 0:1 -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -sc_threshold 0 -c:v copy -c:a copy -f mpegts -mpegts_flags +initial_discontinuity pipe:1");
+            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -re -i /tmp/whatever.mkv -map 0:1 -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart+frag_keyframe+delay_moov -flags cgop -sc_threshold 0 -c:v copy -c:a copy -f mp4 pipe:1");
     }
 
     [Test]
@@ -386,7 +386,7 @@ public class PipelineBuilderBaseTests
             Option<string>.None,
             Option<string>.None,
             Option<string>.None,
-            OutputFormatKind.MpegTs,
+            OutputFormatKind.Mp4,
             Option<string>.None,
             Option<string>.None,
             0,
@@ -412,7 +412,7 @@ public class PipelineBuilderBaseTests
         string command = PrintCommand(videoInputFile, audioInputFile, None, None, result);
 
         command.Should().Be(
-            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -re -i /tmp/whatever.mkv -map 0:a -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -sc_threshold 0 -c:v copy -c:a copy -f mpegts -mpegts_flags +initial_discontinuity pipe:1");
+            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -re -i /tmp/whatever.mkv -map 0:a -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart+frag_keyframe+delay_moov -flags cgop -sc_threshold 0 -c:v copy -c:a copy -f mp4 pipe:1");
     }
 
     [Test]

ErsatzTV.FFmpeg/Encoder/EncoderDvdSubtitle.cs

@@ -0,0 +1,8 @@
+namespace ErsatzTV.FFmpeg.Encoder;
+
+public class EncoderDvdSubtitle : EncoderBase
+{
+    public override string Name => "dvdsub";
+    public override StreamKind Kind => StreamKind.Subtitle;
+    public override FrameState NextState(FrameState currentState) => currentState;
+}

ErsatzTV.FFmpeg/Filter/ComplexFilter.cs

@@ -138,7 +138,8 @@ public class ComplexFilter : IPipelineStep
             }
         }
 
-        foreach (SubtitleInputFile subtitleInputFile in _maybeSubtitleInputFile.Filter(s => !s.Copy))
+        foreach (SubtitleInputFile subtitleInputFile in _maybeSubtitleInputFile.Filter(
+                     s => s.Method == SubtitleMethod.Burn))
         {
             int inputIndex = distinctPaths.IndexOf(subtitleInputFile.Path);
             foreach ((int index, _, _) in subtitleInputFile.Streams)
@@ -235,13 +236,18 @@ public class ComplexFilter : IPipelineStep
 
         result.AddRange(new[] { "-map", audioLabel, "-map", videoLabel });
 
-        foreach (SubtitleInputFile subtitleInputFile in _maybeSubtitleInputFile.Filter(s => s.Copy))
+        foreach (SubtitleInputFile subtitleInputFile in _maybeSubtitleInputFile.Filter(
+                     s => s.Method == SubtitleMethod.Copy ||
+                          s is { IsImageBased: true, Method: SubtitleMethod.Convert })) // TODO: support converting text subtitles?
         {
-            int inputIndex = distinctPaths.IndexOf(subtitleInputFile.Path);
-            foreach ((int index, _, _) in subtitleInputFile.Streams)
+            if (subtitleInputFile.Streams.Any())
             {
-                subtitleLabel = $"{inputIndex}:{index}";
-                result.AddRange(new[] { "-map", subtitleLabel });
+                int inputIndex = distinctPaths.IndexOf(subtitleInputFile.Path);
+                foreach ((int index, _, _) in subtitleInputFile.Streams)
+                {
+                    subtitleLabel = $"{inputIndex}:{index}";
+                    result.AddRange(new[] { "-map", subtitleLabel });
+                }
             }
         }
 

ErsatzTV.FFmpeg/InputFile.cs

@@ -76,7 +76,7 @@ public record VideoInputFile(string Path, IList<VideoStream> VideoStreams) : Inp
 public record WatermarkInputFile
     (string Path, IList<VideoStream> VideoStreams, WatermarkState DesiredState) : VideoInputFile(Path, VideoStreams);
 
-public record SubtitleInputFile(string Path, IList<MediaStream> SubtitleStreams, bool Copy) : InputFile(
+public record SubtitleInputFile(string Path, IList<MediaStream> SubtitleStreams, SubtitleMethod Method) : InputFile(
     Path,
     SubtitleStreams)
 {

ErsatzTV.FFmpeg/Option/Mp4OutputOptions.cs

@@ -0,0 +1,6 @@
+namespace ErsatzTV.FFmpeg.Option;
+
+public class Mp4OutputOptions : OutputOption
+{
+    public override IList<string> OutputOptions => new List<string> { "-movflags", "+faststart+frag_keyframe+delay_moov" };
+}

ErsatzTV.FFmpeg/OutputFormat/OutputFormatKind.cs

@@ -4,5 +4,6 @@ public enum OutputFormatKind
 {
     None,
     MpegTs,
+    Mp4,
     Hls
 }

ErsatzTV.FFmpeg/OutputFormat/OutputFormatMp4.cs

@@ -0,0 +1,13 @@
+using ErsatzTV.FFmpeg.Environment;
+
+namespace ErsatzTV.FFmpeg.OutputFormat;
+
+public class OutputFormatMp4 : IPipelineStep
+{
+    public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
+    public IList<string> GlobalOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
+    public IList<string> FilterOptions => Array.Empty<string>();
+    public IList<string> OutputOptions => new List<string> { "-f", "mp4" };
+    public FrameState NextState(FrameState currentState) => currentState;
+}

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs

@@ -141,6 +141,12 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
 
     public FFmpegPipeline Build(FFmpegState ffmpegState, FrameState desiredState)
     {
+        OutputOption outputOption = new FastStartOutputOption();
+        if (ffmpegState.OutputFormat == OutputFormatKind.Mp4)
+        {
+            outputOption = new Mp4OutputOptions();
+        }
+        
         var pipelineSteps = new List<IPipelineStep>
         {
             new NoStandardInputOption(),
@@ -149,7 +155,7 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
             new LoglevelErrorOption(),
             new StandardFormatFlags(),
             new NoDemuxDecodeDelayOutputOption(),
-            new FastStartOutputOption(),
+            outputOption,
             new ClosedGopOutputOption()
         };
 
@@ -163,8 +169,8 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
         var context = new PipelineContext(
             _hardwareAccelerationMode,
             _watermarkInputFile.IsSome,
-            _subtitleInputFile.Map(s => s is { IsImageBased: true, Copy: false }).IfNone(false),
-            _subtitleInputFile.Map(s => s is { IsImageBased: false }).IfNone(false),
+            _subtitleInputFile.Map(s => s is { IsImageBased: true, Method: SubtitleMethod.Burn }).IfNone(false),
+            _subtitleInputFile.Map(s => s is { IsImageBased: false, Method: SubtitleMethod.Burn }).IfNone(false),
             desiredState.Deinterlaced,
             desiredState.PixelFormat.Map(pf => pf.BitDepth).IfNone(8) == 10);
 
@@ -244,6 +250,10 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
                 pipelineSteps.Add(new OutputFormatMpegTs());
                 pipelineSteps.Add(new PipeProtocol());
                 break;
+            case OutputFormatKind.Mp4:
+                pipelineSteps.Add(new OutputFormatMp4());
+                pipelineSteps.Add(new PipeProtocol());
+                break;
             case OutputFormatKind.Hls:
                 foreach (string playlistPath in ffmpegState.HlsPlaylistPath)
                 {
@@ -380,9 +390,19 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
         PipelineContext context,
         ICollection<IPipelineStep> pipelineSteps)
     {
-        if (_subtitleInputFile.Map(s => s.Copy) == Some(true))
+        foreach (SubtitleInputFile subtitleInputFile in _subtitleInputFile)
         {
-            pipelineSteps.Add(new EncoderCopySubtitle());
+            if (subtitleInputFile.Method == SubtitleMethod.Copy)
+            {
+                pipelineSteps.Add(new EncoderCopySubtitle());
+            }
+            else if (subtitleInputFile.Method == SubtitleMethod.Convert)
+            {
+                if (subtitleInputFile.IsImageBased)
+                {
+                    pipelineSteps.Add(new EncoderDvdSubtitle());
+                }
+            }
         }
 
         ffmpegState = SetAccelState(videoStream, ffmpegState, desiredState, context, pipelineSteps);

ErsatzTV.FFmpeg/SubtitleMethod.cs

@@ -0,0 +1,9 @@
+namespace ErsatzTV.FFmpeg;
+
+public enum SubtitleMethod
+{
+    None,
+    Burn,
+    Convert,
+    Copy
+}

ErsatzTV/Controllers/InternalController.cs

@@ -73,8 +73,14 @@ public class InternalController : ControllerBase
                                 process.StartInfo.Environment[key] = value;
                             }
 
+                            var contentType = "video/mp2t";
+                            if (mode.ToLowerInvariant() == "hls-direct")
+                            {
+                                contentType = "video/mp4";
+                            }
+                            
                             process.Start();
-                            return new FileStreamResult(process.StandardOutput.BaseStream, "video/mp2t");
+                            return new FileStreamResult(process.StandardOutput.BaseStream, contentType);
                         },
                         error =>
                         {