hls direct mkv container (#1292)

* use mkv container for hls direct

* add setting for mp4/mkv container with hls direct

* cleanup

* update changelog

CHANGELOG.md

@@ -8,7 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Synchronize actor metadata from Jellyfin and Emby television libraries
   - New libraries and new episodes will get actor data automatically
   - Existing libraries can deep scan (one time) to retrieve actor data for existing episodes
-- `HLS Direct` streaming mode: stream copy dvd subtitles 
+- `HLS Direct` streaming mode
+    - Use `MP4` container/output format by default, with new global option to use `MKV` container/output format
+    - `MP4` output format: stream copy dvd subtitles
+    - `MKV` output format: stream copy all embedded subtitles
 
 ### Fixed
 - Fix extracting embedded text subtitles that had been incompletely extracted in the past 
@@ -22,7 +25,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 - Timeout playout builds after 2 minutes; this should prevent playout bugs from blocking other functionality
-- `HLS Direct` streaming mode: Use MP4 container instead MPEG-TS container to improve codec compatibility (e.g. FLAC audio)
 
 ## [0.7.8-beta] - 2023-04-29
 ### Added

ErsatzTV.Application/FFmpegProfiles/Commands/UpdateFFmpegSettingsHandler.cs

@@ -75,6 +75,9 @@ public class UpdateFFmpegSettingsHandler : IRequestHandler<UpdateFFmpegSettings,
         await _configElementRepository.Upsert(
             ConfigElementKey.FFmpegSaveReports,
             request.Settings.SaveReports.ToString());
+        await _configElementRepository.Upsert(
+            ConfigElementKey.FFmpegHlsDirectOutputFormat,
+            request.Settings.HlsDirectOutputFormat);
 
         if (request.Settings.SaveReports && !Directory.Exists(FileSystemLayout.FFmpegReportsFolder))
         {

ErsatzTV.Application/FFmpegProfiles/FFmpegSettingsViewModel.cs

@@ -1,4 +1,6 @@
-namespace ErsatzTV.Application.FFmpegProfiles;
+using ErsatzTV.FFmpeg.OutputFormat;
+
+namespace ErsatzTV.Application.FFmpegProfiles;
 
 public class FFmpegSettingsViewModel
 {
@@ -12,4 +14,5 @@ public class FFmpegSettingsViewModel
     public int HlsSegmenterIdleTimeout { get; set; }
     public int WorkAheadSegmenterLimit { get; set; }
     public int InitialSegmentCount { get; set; }
+    public OutputFormatKind HlsDirectOutputFormat { get; set; }
 }

ErsatzTV.Application/FFmpegProfiles/Queries/GetFFmpegSettingsHandler.cs

@@ -1,5 +1,6 @@
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Repositories;
+using ErsatzTV.FFmpeg.OutputFormat;
 
 namespace ErsatzTV.Application.FFmpegProfiles;
 
@@ -32,6 +33,8 @@ public class GetFFmpegSettingsHandler : IRequestHandler<GetFFmpegSettings, FFmpe
             await _configElementRepository.GetValue<int>(ConfigElementKey.FFmpegWorkAheadSegmenters);
         Option<int> initialSegmentCount =
             await _configElementRepository.GetValue<int>(ConfigElementKey.FFmpegInitialSegmentCount);
+        Option<OutputFormatKind> outputFormatKind =
+            await _configElementRepository.GetValue<OutputFormatKind>(ConfigElementKey.FFmpegHlsDirectOutputFormat);
 
         var result = new FFmpegSettingsViewModel
         {
@@ -42,7 +45,8 @@ public class GetFFmpegSettingsHandler : IRequestHandler<GetFFmpegSettings, FFmpe
             PreferredAudioLanguageCode = await preferredAudioLanguageCode.IfNoneAsync("eng"),
             HlsSegmenterIdleTimeout = await hlsSegmenterIdleTimeout.IfNoneAsync(60),
             WorkAheadSegmenterLimit = await workAheadSegmenterLimit.IfNoneAsync(1),
-            InitialSegmentCount = await initialSegmentCount.IfNoneAsync(1)
+            InitialSegmentCount = await initialSegmentCount.IfNoneAsync(1),
+            HlsDirectOutputFormat = await outputFormatKind.IfNoneAsync(OutputFormatKind.Mp4)
         };
 
         foreach (int watermarkId in watermark)

ErsatzTV.Core/Domain/ConfigElementKey.cs

@@ -18,6 +18,7 @@ public class ConfigElementKey
     public static ConfigElementKey FFmpegSegmenterTimeout => new("ffmpeg.segmenter.timeout_seconds");
     public static ConfigElementKey FFmpegWorkAheadSegmenters => new("ffmpeg.segmenter.work_ahead_limit");
     public static ConfigElementKey FFmpegInitialSegmentCount => new("ffmpeg.segmenter.initial_segment_count");
+    public static ConfigElementKey FFmpegHlsDirectOutputFormat => new("ffmpeg.hls_direct.output_format");
     public static ConfigElementKey SearchIndexVersion => new("search_index.version");
     public static ConfigElementKey HDHRTunerCount => new("hdhr.tuner_count");
     public static ConfigElementKey ChannelsPageSize => new("pages.channels.page_size");

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -2,6 +2,7 @@
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Domain.Filler;
 using ErsatzTV.Core.Interfaces.FFmpeg;
+using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.FFmpeg;
 using ErsatzTV.FFmpeg.Environment;
 using ErsatzTV.FFmpeg.Format;
@@ -19,6 +20,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
     private readonly IFFmpegStreamSelector _ffmpegStreamSelector;
     private readonly ILogger<FFmpegLibraryProcessService> _logger;
     private readonly IPipelineBuilderFactory _pipelineBuilderFactory;
+    private readonly IConfigElementRepository _configElementRepository;
     private readonly FFmpegPlaybackSettingsCalculator _playbackSettingsCalculator;
     private readonly ITempFilePool _tempFilePool;
 
@@ -28,6 +30,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         IFFmpegStreamSelector ffmpegStreamSelector,
         ITempFilePool tempFilePool,
         IPipelineBuilderFactory pipelineBuilderFactory,
+        IConfigElementRepository configElementRepository,
         ILogger<FFmpegLibraryProcessService> logger)
     {
         _ffmpegProcessService = ffmpegProcessService;
@@ -35,6 +38,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         _ffmpegStreamSelector = ffmpegStreamSelector;
         _tempFilePool = tempFilePool;
         _pipelineBuilderFactory = pipelineBuilderFactory;
+        _configElementRepository = configElementRepository;
         _logger = logger;
     }
 
@@ -194,6 +198,28 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
                 return new AudioInputFile(audioPath, new List<AudioStream> { ffmpegAudioStream }, audioState);
             });
 
+        OutputFormatKind outputFormat = OutputFormatKind.MpegTs;
+        if (channel.StreamingMode == StreamingMode.HttpLiveStreamingSegmenter)
+        {
+            outputFormat = OutputFormatKind.Hls;
+        }
+        else if (channel.StreamingMode == StreamingMode.HttpLiveStreamingDirect)
+        {
+            // use mp4 by default
+            outputFormat = OutputFormatKind.Mp4;
+            
+            // override with setting if applicable
+            Option<OutputFormatKind> maybeOutputFormat = await _configElementRepository
+                .GetValue<OutputFormatKind>(ConfigElementKey.FFmpegHlsDirectOutputFormat);
+            foreach (OutputFormatKind of in maybeOutputFormat)
+            {
+                outputFormat = of;
+            }
+        }
+
+        Option<string> subtitleLanguage = Option<string>.None;
+        Option<string> subtitleTitle = Option<string>.None;
+
         Option<SubtitleInputFile> subtitleInputFile = maybeSubtitle.Map<Option<SubtitleInputFile>>(
             subtitle =>
             {
@@ -218,13 +244,18 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
                 SubtitleMethod method = SubtitleMethod.Burn;
                 if (channel.StreamingMode == StreamingMode.HttpLiveStreamingDirect)
                 {
-                    method = subtitle.Codec switch
+                    method = (outputFormat, subtitle.SubtitleKind, subtitle.Codec) switch
                     {
+                        // mkv supports all subtitle codecs, maybe?
+                        (OutputFormatKind.Mkv, SubtitleKind.Embedded, _) => SubtitleMethod.Copy,
+
                         // MP4 supports vobsub
-                        "dvdsub" or "dvd_subtitle" or "vobsub" => SubtitleMethod.Copy,
+                        (OutputFormatKind.Mp4, SubtitleKind.Embedded, "dvdsub" or "dvd_subtitle" or "vobsub") =>
+                            SubtitleMethod.Copy,
 
                         // MP4 does not support PGS
-                        "pgs" or "pgssub" or "hdmv_pgs_subtitle" => SubtitleMethod.None,
+                        (OutputFormatKind.Mp4, SubtitleKind.Embedded, "pgs" or "pgssub" or "hdmv_pgs_subtitle") =>
+                            SubtitleMethod.None,
 
                         // ignore text subtitles for now
                         _ => SubtitleMethod.None
@@ -234,6 +265,19 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
                     {
                         return None;
                     }
+                    
+                    // hls direct won't use extracted embedded subtitles
+                    if (subtitle.SubtitleKind == SubtitleKind.Embedded)
+                    {
+                        path = videoPath;
+                        ffmpegSubtitleStream = ffmpegSubtitleStream with { Index = subtitle.StreamIndex };
+                    }
+                }
+
+                if (method == SubtitleMethod.Copy)
+                {
+                    subtitleLanguage = Optional(subtitle.Language);
+                    subtitleTitle = Optional(subtitle.Title);
                 }
 
                 return new SubtitleInputFile(
@@ -248,13 +292,6 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
         HardwareAccelerationMode hwAccel = GetHardwareAccelerationMode(playbackSettings, fillerKind);
 
-        OutputFormatKind outputFormat = channel.StreamingMode switch
-        {
-            StreamingMode.HttpLiveStreamingSegmenter => OutputFormatKind.Hls,
-            StreamingMode.HttpLiveStreamingDirect => OutputFormatKind.Mp4,
-            _ => OutputFormatKind.MpegTs
-        };
-
         Option<string> hlsPlaylistPath = outputFormat == OutputFormatKind.Hls
             ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live.m3u8")
             : Option<string>.None;
@@ -291,6 +328,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             "ErsatzTV",
             channel.Name,
             maybeAudioStream.Map(s => Optional(s.Language)).Flatten(),
+            subtitleLanguage,
+            subtitleTitle,
             outputFormat,
             hlsPlaylistPath,
             hlsSegmentTemplate,
@@ -424,6 +463,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             "ErsatzTV",
             channel.Name,
             None,
+            None,
+            None,
             outputFormat,
             hlsPlaylistPath,
             hlsSegmentTemplate,

ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs

@@ -80,6 +80,8 @@ public class PipelineBuilderBaseTests
             Option<string>.None,
             Option<string>.None,
             Option<string>.None,
+            Option<string>.None,
+            Option<string>.None,
             OutputFormatKind.MpegTs,
             Option<string>.None,
             Option<string>.None,
@@ -166,6 +168,8 @@ public class PipelineBuilderBaseTests
             Option<string>.None,
             Option<string>.None,
             Option<string>.None,
+            Option<string>.None,
+            Option<string>.None,
             OutputFormatKind.MpegTs,
             Option<string>.None,
             Option<string>.None,
@@ -306,6 +310,8 @@ public class PipelineBuilderBaseTests
             Option<string>.None,
             Option<string>.None,
             Option<string>.None,
+            Option<string>.None,
+            Option<string>.None,
             OutputFormatKind.Mp4,
             Option<string>.None,
             Option<string>.None,
@@ -386,6 +392,8 @@ public class PipelineBuilderBaseTests
             Option<string>.None,
             Option<string>.None,
             Option<string>.None,
+            Option<string>.None,
+            Option<string>.None,
             OutputFormatKind.Mp4,
             Option<string>.None,
             Option<string>.None,

ErsatzTV.FFmpeg/FFmpegState.cs

@@ -14,6 +14,8 @@ public record FFmpegState(
     Option<string> MetadataServiceProvider,
     Option<string> MetadataServiceName,
     Option<string> MetadataAudioLanguage,
+    Option<string> MetadataSubtitleLanguage,
+    Option<string> MetadataSubtitleTitle,
     OutputFormatKind OutputFormat,
     Option<string> HlsPlaylistPath,
     Option<string> HlsSegmentTemplate,
@@ -36,6 +38,8 @@ public record FFmpegState(
             "ErsatzTV",
             channelName,
             Option<string>.None,
+            Option<string>.None,
+            Option<string>.None,
             OutputFormatKind.MpegTs,
             Option<string>.None,
             Option<string>.None,

ErsatzTV.FFmpeg/Option/Metadata/MetadataSubtitleLanguageOutputOption.cs

@@ -0,0 +1,11 @@
+namespace ErsatzTV.FFmpeg.Option.Metadata;
+
+public class MetadataSubtitleLanguageOutputOption : OutputOption
+{
+    private readonly string _subtitleLanguage;
+
+    public MetadataSubtitleLanguageOutputOption(string subtitleLanguage) => _subtitleLanguage = subtitleLanguage;
+
+    public override IList<string> OutputOptions => new List<string>
+        { "-metadata:s:s:0", $"language={_subtitleLanguage}" };
+}

ErsatzTV.FFmpeg/Option/Metadata/MetadataSubtitleTitleOutputOption.cs

@@ -0,0 +1,11 @@
+namespace ErsatzTV.FFmpeg.Option.Metadata;
+
+public class MetadataSubtitleTitleOutputOption : OutputOption
+{
+    private readonly string _subtitleTitle;
+
+    public MetadataSubtitleTitleOutputOption(string subtitleTitle) => _subtitleTitle = subtitleTitle;
+
+    public override IList<string> OutputOptions => new List<string>
+        { "-metadata:s:s:0", $"title={_subtitleTitle}" };
+}

ErsatzTV.FFmpeg/OutputFormat/OutputFormatKind.cs

@@ -3,6 +3,7 @@
 public enum OutputFormatKind
 {
     None,
+    Mkv,
     MpegTs,
     Mp4,
     Hls

ErsatzTV.FFmpeg/OutputFormat/OutputFormatMkv.cs

@@ -0,0 +1,13 @@
+using ErsatzTV.FFmpeg.Environment;
+
+namespace ErsatzTV.FFmpeg.OutputFormat;
+
+public class OutputFormatMkv : IPipelineStep
+{
+    public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
+    public IList<string> GlobalOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
+    public IList<string> FilterOptions => Array.Empty<string>();
+    public IList<string> OutputOptions => new List<string> { "-f", "matroska" };
+    public FrameState NextState(FrameState currentState) => currentState;
+}

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs

@@ -204,6 +204,7 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
         SetMetadataServiceProvider(ffmpegState, pipelineSteps);
         SetMetadataServiceName(ffmpegState, pipelineSteps);
         SetMetadataAudioLanguage(ffmpegState, pipelineSteps);
+        SetMetadataSubtitle(ffmpegState, pipelineSteps);
         SetOutputFormat(ffmpegState, desiredState, pipelineSteps, videoStream);
 
         var complexFilter = new ComplexFilter(
@@ -246,6 +247,10 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
     {
         switch (ffmpegState.OutputFormat)
         {
+            case OutputFormatKind.Mkv:
+                pipelineSteps.Add(new OutputFormatMkv());
+                pipelineSteps.Add(new PipeProtocol());
+                break;
             case OutputFormatKind.MpegTs:
                 pipelineSteps.Add(new OutputFormatMpegTs());
                 pipelineSteps.Add(new PipeProtocol());
@@ -280,6 +285,19 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
         }
     }
 
+    private static void SetMetadataSubtitle(FFmpegState ffmpegState, List<IPipelineStep> pipelineSteps)
+    {
+        foreach (string desiredSubtitleLanguage in ffmpegState.MetadataSubtitleLanguage)
+        {
+            pipelineSteps.Add(new MetadataSubtitleLanguageOutputOption(desiredSubtitleLanguage));
+        }
+
+        foreach (string desiredSubtitleTitle in ffmpegState.MetadataSubtitleTitle)
+        {
+            pipelineSteps.Add(new MetadataSubtitleTitleOutputOption(desiredSubtitleTitle));
+        }
+    }
+
     private static void SetMetadataServiceName(FFmpegState ffmpegState, List<IPipelineStep> pipelineSteps)
     {
         foreach (string desiredServiceName in ffmpegState.MetadataServiceName)

ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs

@@ -21,6 +21,7 @@ using ErsatzTV.FFmpeg.Filter.Vaapi;
 using ErsatzTV.FFmpeg.Format;
 using ErsatzTV.FFmpeg.Pipeline;
 using ErsatzTV.FFmpeg.State;
+using ErsatzTV.Infrastructure.Data.Repositories;
 using ErsatzTV.Infrastructure.Images;
 using ErsatzTV.Infrastructure.Runtime;
 using ErsatzTV.Scanner.Core.Interfaces.Metadata;
@@ -250,6 +251,7 @@ public class TranscodingTests
                     MemoryCache,
                     LoggerFactory.CreateLogger<HardwareCapabilitiesFactory>()),
                 LoggerFactory.CreateLogger<PipelineBuilderFactory>()),
+            new Mock<ConfigElementRepository>().Object,
             LoggerFactory.CreateLogger<FFmpegLibraryProcessService>());
 
         var songVideoGenerator = new SongVideoGenerator(tempFilePool, mockImageCache.Object, service);
@@ -869,6 +871,7 @@ public class TranscodingTests
                     MemoryCache,
                     LoggerFactory.CreateLogger<HardwareCapabilitiesFactory>()),
                 LoggerFactory.CreateLogger<PipelineBuilderFactory>()),
+            new Mock<ConfigElementRepository>().Object,
             LoggerFactory.CreateLogger<FFmpegLibraryProcessService>());
 
         return service;

ErsatzTV/Pages/Settings.razor

@@ -7,6 +7,7 @@
 @using System.Globalization
 @using ErsatzTV.Application.Configuration
 @using ErsatzTV.Core.Domain.Filler
+@using ErsatzTV.FFmpeg.OutputFormat
 @using Serilog.Events
 @implements IDisposable
 @inject IMediator _mediator
@@ -95,6 +96,13 @@
                                       Required="true"
                                       RequiredError="HLS Segmenter initial segment count is required!"/>
                     </MudElement>
+                    <MudSelect Class="mt-3"
+                               Label="HLS Direct Output Format"
+                               @bind-Value="_ffmpegSettings.HlsDirectOutputFormat"
+                               For="@(() => _ffmpegSettings.HlsDirectOutputFormat)">
+                        <MudSelectItem T="OutputFormatKind" Value="@OutputFormatKind.Mp4">MP4</MudSelectItem>
+                        <MudSelectItem T="OutputFormatKind" Value="@OutputFormatKind.Mkv">MKV</MudSelectItem>
+                    </MudSelect>
                 </MudForm>
             </MudCardContent>
             <MudCardActions>