add new subtitle settings (#1590)

CHANGELOG.md

@@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [Unreleased]
 ### Added
 - Add `show_studio` to search index for seasons and episodes
+- Add two new global subtitle settings:
+  - `Use embedded subtitles`
+    - Default value: `true`
+    - When disabled, embedded subtitles will not be considered for extraction (text subtitles), or playback (all embedded subtitles)
+  - `Extract and use embedded (text) subtitles`
+    - Default value: `false`
+    - When enabled, embedded text subtitles will be periodically extracted, and considered for playback
 
 ### Fixed
 - Fix antiforgery error caused by reusing existing browser tabs across docker container restarts

ErsatzTV.Application/FFmpegProfiles/Commands/UpdateFFmpegSettingsHandler.cs

@@ -1,5 +1,7 @@
 using System.Diagnostics;
 using System.Globalization;
+using System.Threading.Channels;
+using ErsatzTV.Application.Subtitles;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Metadata;
@@ -11,13 +13,16 @@ public class UpdateFFmpegSettingsHandler : IRequestHandler<UpdateFFmpegSettings,
 {
     private readonly IConfigElementRepository _configElementRepository;
     private readonly ILocalFileSystem _localFileSystem;
+    private readonly ChannelWriter<IBackgroundServiceRequest> _workerChannel;
 
     public UpdateFFmpegSettingsHandler(
         IConfigElementRepository configElementRepository,
-        ILocalFileSystem localFileSystem)
+        ILocalFileSystem localFileSystem,
+        ChannelWriter<IBackgroundServiceRequest> workerChannel)
     {
         _configElementRepository = configElementRepository;
         _localFileSystem = localFileSystem;
+        _workerChannel = workerChannel;
     }
 
     public Task<Either<BaseError, Unit>> Handle(
@@ -87,6 +92,26 @@ public class UpdateFFmpegSettingsHandler : IRequestHandler<UpdateFFmpegSettings,
             ConfigElementKey.FFmpegPreferredLanguageCode,
             request.Settings.PreferredAudioLanguageCode);
 
+        await _configElementRepository.Upsert(
+            ConfigElementKey.FFmpegUseEmbeddedSubtitles,
+            request.Settings.UseEmbeddedSubtitles);
+
+        // do not extract when subtitles are not used
+        if (request.Settings.UseEmbeddedSubtitles == false)
+        {
+            request.Settings.ExtractEmbeddedSubtitles = false;
+        }
+        
+        await _configElementRepository.Upsert(
+            ConfigElementKey.FFmpegExtractEmbeddedSubtitles,
+            request.Settings.ExtractEmbeddedSubtitles);
+        
+        // queue extracting all embedded subtitles
+        if (request.Settings.ExtractEmbeddedSubtitles)
+        {
+            await _workerChannel.WriteAsync(new ExtractEmbeddedSubtitles(Option<int>.None));
+        }
+
         if (request.Settings.GlobalWatermarkId is not null)
         {
             await _configElementRepository.Upsert(

ErsatzTV.Application/FFmpegProfiles/FFmpegSettingsViewModel.cs

@@ -8,6 +8,8 @@ public class FFmpegSettingsViewModel
     public string FFprobePath { get; set; }
     public int DefaultFFmpegProfileId { get; set; }
     public string PreferredAudioLanguageCode { get; set; }
+    public bool UseEmbeddedSubtitles { get; set; }
+    public bool ExtractEmbeddedSubtitles { get; set; }
     public bool SaveReports { get; set; }
     public int? GlobalWatermarkId { get; set; }
     public int? GlobalFallbackFillerId { get; set; }

ErsatzTV.Application/FFmpegProfiles/Queries/GetFFmpegSettingsHandler.cs

@@ -23,6 +23,10 @@ public class GetFFmpegSettingsHandler : IRequestHandler<GetFFmpegSettings, FFmpe
             await _configElementRepository.GetValue<bool>(ConfigElementKey.FFmpegSaveReports);
         Option<string> preferredAudioLanguageCode =
             await _configElementRepository.GetValue<string>(ConfigElementKey.FFmpegPreferredLanguageCode);
+        Option<bool> useEmbeddedSubtitles =
+            await _configElementRepository.GetValue<bool>(ConfigElementKey.FFmpegUseEmbeddedSubtitles);
+        Option<bool> extractEmbeddedSubtitles =
+            await _configElementRepository.GetValue<bool>(ConfigElementKey.FFmpegExtractEmbeddedSubtitles);
         Option<int> watermark =
             await _configElementRepository.GetValue<int>(ConfigElementKey.FFmpegGlobalWatermarkId);
         Option<int> fallbackFiller =
@@ -42,6 +46,8 @@ public class GetFFmpegSettingsHandler : IRequestHandler<GetFFmpegSettings, FFmpe
             FFprobePath = await ffprobePath.IfNoneAsync(string.Empty),
             DefaultFFmpegProfileId = await defaultFFmpegProfileId.IfNoneAsync(0),
             SaveReports = await saveReports.IfNoneAsync(false),
+            UseEmbeddedSubtitles = await useEmbeddedSubtitles.IfNoneAsync(true),
+            ExtractEmbeddedSubtitles = await extractEmbeddedSubtitles.IfNoneAsync(false),
             PreferredAudioLanguageCode = await preferredAudioLanguageCode.IfNoneAsync("eng"),
             HlsSegmenterIdleTimeout = await hlsSegmenterIdleTimeout.IfNoneAsync(60),
             WorkAheadSegmenterLimit = await workAheadSegmenterLimit.IfNoneAsync(1),

ErsatzTV.Application/Subtitles/Commands/ExtractEmbeddedSubtitles.cs

@@ -2,5 +2,4 @@
 
 namespace ErsatzTV.Application.Subtitles;
 
-public record ExtractEmbeddedSubtitles(Option<int> PlayoutId) : IRequest<Either<BaseError, Unit>>,
-    IBackgroundServiceRequest;
+public record ExtractEmbeddedSubtitles(Option<int> PlayoutId) : IRequest<Option<BaseError>>, IBackgroundServiceRequest;

ErsatzTV.Application/Subtitles/Commands/ExtractEmbeddedSubtitlesHandler.cs

@@ -12,6 +12,7 @@ using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Extensions;
 using ErsatzTV.Core.Interfaces.Locking;
 using ErsatzTV.Core.Interfaces.Metadata;
+using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Infrastructure.Data;
 using ErsatzTV.Infrastructure.Extensions;
 using Microsoft.EntityFrameworkCore;
@@ -20,10 +21,11 @@ using Microsoft.Extensions.Logging;
 namespace ErsatzTV.Application.Subtitles;
 
 [SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms")]
-public class ExtractEmbeddedSubtitlesHandler : IRequestHandler<ExtractEmbeddedSubtitles, Either<BaseError, Unit>>
+public class ExtractEmbeddedSubtitlesHandler : IRequestHandler<ExtractEmbeddedSubtitles, Option<BaseError>>
 {
     private readonly IDbContextFactory<TvContext> _dbContextFactory;
     private readonly IEntityLocker _entityLocker;
+    private readonly IConfigElementRepository _configElementRepository;
     private readonly ILocalFileSystem _localFileSystem;
     private readonly ILogger<ExtractEmbeddedSubtitlesHandler> _logger;
     private readonly ChannelWriter<IBackgroundServiceRequest> _workerChannel;
@@ -32,17 +34,19 @@ public class ExtractEmbeddedSubtitlesHandler : IRequestHandler<ExtractEmbeddedSu
         IDbContextFactory<TvContext> dbContextFactory,
         ILocalFileSystem localFileSystem,
         IEntityLocker entityLocker,
+        IConfigElementRepository configElementRepository,
         ChannelWriter<IBackgroundServiceRequest> workerChannel,
         ILogger<ExtractEmbeddedSubtitlesHandler> logger)
     {
         _dbContextFactory = dbContextFactory;
         _localFileSystem = localFileSystem;
         _entityLocker = entityLocker;
+        _configElementRepository = configElementRepository;
         _workerChannel = workerChannel;
         _logger = logger;
     }
 
-    public async Task<Either<BaseError, Unit>> Handle(
+    public async Task<Option<BaseError>> Handle(
         ExtractEmbeddedSubtitles request,
         CancellationToken cancellationToken)
     {
@@ -51,14 +55,14 @@ public class ExtractEmbeddedSubtitlesHandler : IRequestHandler<ExtractEmbeddedSu
         return await validation.Match(
             async ffmpegPath =>
             {
-                Either<BaseError, Unit> result = await ExtractAll(dbContext, request, ffmpegPath, cancellationToken);
+                Option<BaseError> result = await ExtractAll(dbContext, request, ffmpegPath, cancellationToken);
                 await _workerChannel.WriteAsync(new ReleaseMemory(false), cancellationToken);
                 return result;
             },
-            error => Task.FromResult<Either<BaseError, Unit>>(error.Join()));
+            error => Task.FromResult<Option<BaseError>>(error.Join()));
     }
 
-    private async Task<Either<BaseError, Unit>> ExtractAll(
+    private async Task<Option<BaseError>> ExtractAll(
         TvContext dbContext,
         ExtractEmbeddedSubtitles request,
         string ffmpegPath,
@@ -66,6 +70,26 @@ public class ExtractEmbeddedSubtitlesHandler : IRequestHandler<ExtractEmbeddedSu
     {
         try
         {
+            bool useEmbeddedSubtitles = await _configElementRepository
+                .GetValue<bool>(ConfigElementKey.FFmpegUseEmbeddedSubtitles)
+                .IfNoneAsync(true);
+
+            if (!useEmbeddedSubtitles)
+            {
+                _logger.LogDebug("Embedded subtitles are NOT enabled; nothing to extract");
+                return Option<BaseError>.None;
+            }
+            
+            bool extractEmbeddedSubtitles = await _configElementRepository
+                .GetValue<bool>(ConfigElementKey.FFmpegExtractEmbeddedSubtitles)
+                .IfNoneAsync(false);
+
+            if (!extractEmbeddedSubtitles)
+            {
+                _logger.LogDebug("Embedded subtitle extraction is NOT enabled");
+                return Option<BaseError>.None;
+            }
+            
             DateTime now = DateTime.UtcNow;
             DateTime until = now.AddHours(1);
 
@@ -102,11 +126,11 @@ public class ExtractEmbeddedSubtitlesHandler : IRequestHandler<ExtractEmbeddedSu
                     _logger.LogDebug(
                         "Playout {PlayoutId} does not have subtitles enabled; nothing to extract",
                         playoutId);
-                    return Unit.Default;
+                    return Option<BaseError>.None;
                 }
 
                 _logger.LogDebug("No playouts have subtitles enabled; nothing to extract");
-                return Unit.Default;
+                return Option<BaseError>.None;
             }
 
             foreach (int playoutId in playoutIdsToCheck)
@@ -157,7 +181,7 @@ public class ExtractEmbeddedSubtitlesHandler : IRequestHandler<ExtractEmbeddedSu
             {
                 if (cancellationToken.IsCancellationRequested)
                 {
-                    return Unit.Default;
+                    return Option<BaseError>.None;
                 }
 
                 // extract subtitles and fonts for each item and update db
@@ -171,13 +195,13 @@ public class ExtractEmbeddedSubtitlesHandler : IRequestHandler<ExtractEmbeddedSu
             {
                 _entityLocker.UnlockPlayout(playoutId);
             }
-
-            return Unit.Default;
         }
         catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
         {
-            return Unit.Default;
+            // do nothing
         }
+
+        return Option<BaseError>.None;
     }
 
     private static async Task<List<int>> GetMediaItemIdsWithTextSubtitles(

ErsatzTV.Application/Troubleshooting/Queries/GetTroubleshootingInfoHandler.cs

@@ -8,6 +8,7 @@ using ErsatzTV.Core.Health;
 using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.FFmpeg.Capabilities;
 using ErsatzTV.FFmpeg.Capabilities.Qsv;
+using ErsatzTV.FFmpeg.OutputFormat;
 using ErsatzTV.FFmpeg.Runtime;
 using ErsatzTV.Infrastructure.Data;
 using Microsoft.EntityFrameworkCore;
@@ -149,6 +150,10 @@ public class GetTroubleshootingInfoHandler : IRequestHandler<GetTroubleshootingI
             await _configElementRepository.GetValue<bool>(ConfigElementKey.FFmpegSaveReports);
         Option<string> preferredAudioLanguageCode =
             await _configElementRepository.GetValue<string>(ConfigElementKey.FFmpegPreferredLanguageCode);
+        Option<bool> useEmbeddedSubtitles =
+            await _configElementRepository.GetValue<bool>(ConfigElementKey.FFmpegUseEmbeddedSubtitles);
+        Option<bool> extractEmbeddedSubtitles =
+            await _configElementRepository.GetValue<bool>(ConfigElementKey.FFmpegExtractEmbeddedSubtitles);
         Option<int> watermark =
             await _configElementRepository.GetValue<int>(ConfigElementKey.FFmpegGlobalWatermarkId);
         Option<int> fallbackFiller =
@@ -159,6 +164,8 @@ public class GetTroubleshootingInfoHandler : IRequestHandler<GetTroubleshootingI
             await _configElementRepository.GetValue<int>(ConfigElementKey.FFmpegWorkAheadSegmenters);
         Option<int> initialSegmentCount =
             await _configElementRepository.GetValue<int>(ConfigElementKey.FFmpegInitialSegmentCount);
+        Option<OutputFormatKind> outputFormatKind =
+            await _configElementRepository.GetValue<OutputFormatKind>(ConfigElementKey.FFmpegHlsDirectOutputFormat);
 
         var result = new FFmpegSettingsViewModel
         {
@@ -166,10 +173,13 @@ public class GetTroubleshootingInfoHandler : IRequestHandler<GetTroubleshootingI
             FFprobePath = await ffprobePath.IfNoneAsync(string.Empty),
             DefaultFFmpegProfileId = await defaultFFmpegProfileId.IfNoneAsync(0),
             SaveReports = await saveReports.IfNoneAsync(false),
+            UseEmbeddedSubtitles = await useEmbeddedSubtitles.IfNoneAsync(true),
+            ExtractEmbeddedSubtitles = await extractEmbeddedSubtitles.IfNoneAsync(false),
             PreferredAudioLanguageCode = await preferredAudioLanguageCode.IfNoneAsync("eng"),
             HlsSegmenterIdleTimeout = await hlsSegmenterIdleTimeout.IfNoneAsync(60),
             WorkAheadSegmenterLimit = await workAheadSegmenterLimit.IfNoneAsync(1),
-            InitialSegmentCount = await initialSegmentCount.IfNoneAsync(1)
+            InitialSegmentCount = await initialSegmentCount.IfNoneAsync(1),
+            HlsDirectOutputFormat = await outputFormatKind.IfNoneAsync(OutputFormatKind.MpegTs)
         };
 
         foreach (int watermarkId in watermark)

ErsatzTV.Core/Domain/ConfigElementKey.cs

@@ -15,6 +15,8 @@ public class ConfigElementKey
     public static ConfigElementKey FFmpegDefaultProfileId => new("ffmpeg.default_profile_id");
     public static ConfigElementKey FFmpegDefaultResolutionId => new("ffmpeg.default_resolution_id");
     public static ConfigElementKey FFmpegSaveReports => new("ffmpeg.save_reports");
+    public static ConfigElementKey FFmpegUseEmbeddedSubtitles => new("ffmpeg.use_embedded_subtitles");
+    public static ConfigElementKey FFmpegExtractEmbeddedSubtitles => new("ffmpeg.extract_embedded_subtitles");
     public static ConfigElementKey FFmpegPreferredLanguageCode => new("ffmpeg.preferred_language_code");
     public static ConfigElementKey FFmpegGlobalWatermarkId => new("ffmpeg.global_watermark_id");
     public static ConfigElementKey FFmpegGlobalFallbackFillerId => new("ffmpeg.global_fallback_filler_id");

ErsatzTV.Core/FFmpeg/FFmpegStreamSelector.cs

@@ -138,6 +138,36 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
             return None;
         }
 
+        bool useEmbeddedSubtitles = await _configElementRepository
+            .GetValue<bool>(ConfigElementKey.FFmpegUseEmbeddedSubtitles)
+            .IfNoneAsync(true);
+
+        if (!useEmbeddedSubtitles)
+        {
+            _logger.LogDebug("Ignoring embedded subtitles for channel {Number}", channel.Number);
+            subtitles = subtitles.Filter(s => s.SubtitleKind is not SubtitleKind.Embedded).ToList();
+        }
+
+        foreach (Subtitle subtitle in subtitles.Filter(s => s.SubtitleKind is SubtitleKind.Embedded && !s.IsImage).ToList())
+        {
+            if (subtitle.IsExtracted == false)
+            {
+                _logger.LogDebug(
+                    "Ignoring embedded subtitle with index {Index} that has not been extracted",
+                    subtitle.StreamIndex);
+
+                subtitles.Remove(subtitle);
+            }
+            else if (string.IsNullOrWhiteSpace(subtitle.Path))
+            {
+                _logger.LogDebug(
+                    "BUG: ignoring embedded subtitle with index {Index} that is missing a path",
+                    subtitle.StreamIndex);
+
+                subtitles.Remove(subtitle);
+            }
+        }
+        
         var allCodes = new List<string>();
         string language = (preferredSubtitleLanguage ?? string.Empty).ToLowerInvariant();
         if (string.IsNullOrWhiteSpace(language))
@@ -160,29 +190,29 @@ public class FFmpegStreamSelector : IFFmpegStreamSelector
 
         if (subtitles.Count > 0)
         {
-            switch (subtitleMode)
+            Option<Subtitle> maybeSelectedSubtitle = subtitleMode switch
             {
-                case ChannelSubtitleMode.Forced:
-                    foreach (Subtitle subtitle in subtitles.OrderBy(s => s.StreamIndex).Find(s => s.Forced))
-                    {
-                        return subtitle;
-                    }
+                ChannelSubtitleMode.Forced => subtitles
+                    .OrderBy(s => s.StreamIndex)
+                    .Find(s => s.Forced)
+                    .HeadOrNone(),
 
-                    break;
-                case ChannelSubtitleMode.Default:
-                    foreach (Subtitle subtitle in subtitles.OrderBy(s => s.Default ? 0 : 1).ThenBy(s => s.StreamIndex))
-                    {
-                        return subtitle;
-                    }
+                ChannelSubtitleMode.Default => subtitles
+                    .OrderBy(s => s.Default ? 0 : 1)
+                    .ThenBy(s => s.StreamIndex)
+                    .HeadOrNone(),
 
-                    break;
-                case ChannelSubtitleMode.Any:
-                    foreach (Subtitle subtitle in subtitles.OrderBy(s => s.StreamIndex).HeadOrNone())
-                    {
-                        return subtitle;
-                    }
+                ChannelSubtitleMode.Any => subtitles
+                    .OrderBy(s => s.StreamIndex)
+                    .HeadOrNone(),
 
-                    break;
+                _ => Option<Subtitle>.None
+            };
+
+            foreach (Subtitle subtitle in maybeSelectedSubtitle)
+            {
+                _logger.LogDebug("Selecting subtitle {@Subtitle}", subtitle);
+                return subtitle;
             }
         }
 

ErsatzTV/Pages/Settings.razor

@@ -43,6 +43,19 @@
                     <MudSelectItem Value="@culture.ThreeLetterISOLanguageName">@culture.EnglishName</MudSelectItem>
                 }
             </MudSelect>
+            <MudElement HtmlTag="div" Class="mt-3">
+                <MudSwitch T="bool"
+                           Label="Use embedded subtitles"
+                           Color="Color.Primary"
+                           @bind-Checked="@_ffmpegSettings.UseEmbeddedSubtitles"/>
+            </MudElement>
+            <MudElement HtmlTag="div" Class="mt-3">
+                <MudSwitch T="bool"
+                           Label="Extract and use embedded (text) subtitles"
+                           Color="Color.Primary"
+                           @bind-Checked="@_ffmpegSettings.ExtractEmbeddedSubtitles"
+                           Disabled="@(_ffmpegSettings.UseEmbeddedSubtitles == false)"/>
+            </MudElement>
             <MudElement HtmlTag="div" Class="mt-3">
                 <MudSwitch T="bool"
                            Label="Save troubleshooting reports to disk"