add hls segmenter settings to optimize performance (#418)

* add hls segmenter settings to optimize performance

* use consistent setting defaults

CHANGELOG.md

@@ -7,6 +7,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Added
 - Include more cuda (nvidia) filters in docker image
 - Enable deinterlacing with nvidia using new `yadif_cuda` filter
+- Add two HLS Segmenter settings: idle timeout and work-ahead limit
+  - `HLS Segmenter Idle Timeout` - the number of seconds to keep transcoding a channel while no requests have been received from any client
+    - This setting must be greater than or equal to 30 (seconds)
+  - `Work-Ahead HLS Segmenter Limit` - the number of segmenters (channels) that will work-ahead simultaneously (if multiple channels are being watched)
+    - "working ahead" means transcoding at full speed, which can take a lot of resources
+    - This setting must be greater than or equal to 0
 
 ### Changed
 - Upgrade ffmpeg from 4.3 to 4.4 in all docker images

ErsatzTV.Application/FFmpegProfiles/Commands/UpdateFFmpegSettingsHandler.cs

@@ -119,6 +119,14 @@ namespace ErsatzTV.Application.FFmpegProfiles.Commands
                 ConfigElementKey.FFmpegVaapiDriver,
                 (int)request.Settings.VaapiDriver);
 
+            await _configElementRepository.Upsert(
+                ConfigElementKey.FFmpegSegmenterTimeout,
+                request.Settings.HlsSegmenterIdleTimeout);
+
+            await _configElementRepository.Upsert(
+                ConfigElementKey.FFmpegWorkAheadSegmenters,
+                request.Settings.WorkAheadSegmenterLimit);
+
             return Unit.Default;
         }
     }

ErsatzTV.Application/FFmpegProfiles/FFmpegSettingsViewModel.cs

@@ -11,5 +11,7 @@ namespace ErsatzTV.Application.FFmpegProfiles
         public bool SaveReports { get; set; }
         public int? GlobalWatermarkId { get; set; }
         public VaapiDriver VaapiDriver { get; set; }
+        public int HlsSegmenterIdleTimeout { get; set; }
+        public int WorkAheadSegmenterLimit { get; set; }
     }
 }

ErsatzTV.Application/FFmpegProfiles/Queries/GetFFmpegSettingsHandler.cs

@@ -31,6 +31,10 @@ namespace ErsatzTV.Application.FFmpegProfiles.Queries
                 await _configElementRepository.GetValue<int>(ConfigElementKey.FFmpegGlobalWatermarkId);
             Option<int> vaapiDriver =
                 await _configElementRepository.GetValue<int>(ConfigElementKey.FFmpegVaapiDriver);
+            Option<int> hlsSegmenterIdleTimeout =
+                await _configElementRepository.GetValue<int>(ConfigElementKey.FFmpegSegmenterTimeout);
+            Option<int> workAheadSegmenterLimit =
+                await _configElementRepository.GetValue<int>(ConfigElementKey.FFmpegWorkAheadSegmenters);
 
             var result = new FFmpegSettingsViewModel
             {
@@ -39,7 +43,9 @@ namespace ErsatzTV.Application.FFmpegProfiles.Queries
                 DefaultFFmpegProfileId = await defaultFFmpegProfileId.IfNoneAsync(0),
                 SaveReports = await saveReports.IfNoneAsync(false),
                 PreferredLanguageCode = await preferredLanguageCode.IfNoneAsync("eng"),
-                VaapiDriver = (VaapiDriver)await vaapiDriver.IfNoneAsync(0)
+                VaapiDriver = (VaapiDriver)await vaapiDriver.IfNoneAsync(0),
+                HlsSegmenterIdleTimeout = await hlsSegmenterIdleTimeout.IfNoneAsync(60),
+                WorkAheadSegmenterLimit = await workAheadSegmenterLimit.IfNoneAsync(1),
             };
 
             foreach (int watermarkId in watermark)

ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs

@@ -3,9 +3,11 @@ using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 using ErsatzTV.Core;
+using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Errors;
 using ErsatzTV.Core.Interfaces.FFmpeg;
 using ErsatzTV.Core.Interfaces.Metadata;
+using ErsatzTV.Core.Interfaces.Repositories;
 using LanguageExt;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -18,18 +20,21 @@ namespace ErsatzTV.Application.Streaming.Commands
         private readonly ILogger<StartFFmpegSessionHandler> _logger;
         private readonly IServiceScopeFactory _serviceScopeFactory;
         private readonly IFFmpegSegmenterService _ffmpegSegmenterService;
+        private readonly IConfigElementRepository _configElementRepository;
         private readonly ILocalFileSystem _localFileSystem;
 
         public StartFFmpegSessionHandler(
             ILocalFileSystem localFileSystem,
             ILogger<StartFFmpegSessionHandler> logger,
             IServiceScopeFactory serviceScopeFactory,
-            IFFmpegSegmenterService ffmpegSegmenterService)
+            IFFmpegSegmenterService ffmpegSegmenterService,
+            IConfigElementRepository configElementRepository)
         {
             _localFileSystem = localFileSystem;
             _logger = logger;
             _serviceScopeFactory = serviceScopeFactory;
             _ffmpegSegmenterService = ffmpegSegmenterService;
+            _configElementRepository = configElementRepository;
         }
 
         public Task<Either<BaseError, Unit>> Handle(StartFFmpegSession request, CancellationToken cancellationToken) =>
@@ -42,12 +47,16 @@ namespace ErsatzTV.Application.Streaming.Commands
 
         private async Task<Unit> StartProcess(StartFFmpegSession request)
         {
+            TimeSpan idleTimeout = await _configElementRepository
+                .GetValue<int>(ConfigElementKey.FFmpegSegmenterTimeout)
+                .Map(maybeTimeout => maybeTimeout.Match(i => TimeSpan.FromSeconds(i), () => TimeSpan.FromMinutes(1)));
+            
             using IServiceScope scope = _serviceScopeFactory.CreateScope();
             HlsSessionWorker worker = scope.ServiceProvider.GetRequiredService<HlsSessionWorker>();
             _ffmpegSegmenterService.SessionWorkers.AddOrUpdate(request.ChannelNumber, _ => worker, (_, _) => worker);
 
             // fire and forget worker
-            _ = worker.Run(request.ChannelNumber)
+            _ = worker.Run(request.ChannelNumber, idleTimeout)
                 .ContinueWith(
                     _ => _ffmpegSegmenterService.SessionWorkers.TryRemove(
                         request.ChannelNumber,

ErsatzTV.Application/Streaming/HlsSessionWorker.cs

@@ -6,23 +6,27 @@ using System.Threading.Tasks;
 using System.Timers;
 using ErsatzTV.Application.Streaming.Queries;
 using ErsatzTV.Core;
+using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.FFmpeg;
 using ErsatzTV.Core.Interfaces.FFmpeg;
+using ErsatzTV.Core.Interfaces.Repositories;
 using LanguageExt;
 using MediatR;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Timer = System.Timers.Timer;
+using static LanguageExt.Prelude;
 
 namespace ErsatzTV.Application.Streaming
 {
     public class HlsSessionWorker : IHlsSessionWorker
     {
+        private static int _workAheadCount;
         private readonly IServiceScopeFactory _serviceScopeFactory;
         private readonly ILogger<HlsSessionWorker> _logger;
         private DateTimeOffset _lastAccess;
         private DateTimeOffset _transcodedUntil;
-        private readonly Timer _timer = new(TimeSpan.FromMinutes(2).TotalMilliseconds) { AutoReset = false };
+        private Timer _timer;
         private readonly object _sync = new();
         private DateTimeOffset _playlistStart;
 
@@ -40,19 +44,23 @@ namespace ErsatzTV.Application.Streaming
             {
                 _lastAccess = DateTimeOffset.Now;
 
-                _timer.Stop();
-                _timer.Start();
+                _timer?.Stop();
+                _timer?.Start();
             }
         }
 
-        public async Task Run(string channelNumber)
+        public async Task Run(string channelNumber, TimeSpan idleTimeout)
         {
             var cts = new CancellationTokenSource();
             void Cancel(object o, ElapsedEventArgs e) => cts.Cancel();
 
             try
             {
-                _timer.Elapsed += Cancel;
+                lock (_sync)
+                {
+                    _timer = new Timer(idleTimeout.TotalMilliseconds) { AutoReset = false };
+                    _timer.Elapsed += Cancel;
+                }
 
                 CancellationToken cancellationToken = cts.Token;
 
@@ -62,16 +70,15 @@ namespace ErsatzTV.Application.Streaming
                 _transcodedUntil = DateTimeOffset.Now;
                 _playlistStart = _transcodedUntil;
 
-                // start initial transcode WITHOUT realtime throttle
-                if (!await Transcode(channelNumber, true, false, cancellationToken))
+                bool initialWorkAhead = Volatile.Read(ref _workAheadCount) < await GetWorkAheadLimit();
+                if (!await Transcode(channelNumber, true, !initialWorkAhead, cancellationToken))
                 {
                     return;
                 }
 
                 while (!cancellationToken.IsCancellationRequested)
                 {
-                    // TODO: configurable? 5 minutes?
-                    if (DateTimeOffset.Now - _lastAccess > TimeSpan.FromMinutes(2))
+                    if (DateTimeOffset.Now - _lastAccess > idleTimeout)
                     {
                         _logger.LogInformation("Stopping idle HLS session for channel {Channel}", channelNumber);
                         return;
@@ -83,7 +90,9 @@ namespace ErsatzTV.Application.Streaming
                     {
                         // only use realtime encoding when we're at least 30 seconds ahead
                         bool realtime = transcodedBuffer >= TimeSpan.FromSeconds(30);
-                        if (!await Transcode(channelNumber, false, realtime, cancellationToken))
+                        bool subsequentWorkAhead =
+                            !realtime && Volatile.Read(ref _workAheadCount) < await GetWorkAheadLimit();
+                        if (!await Transcode(channelNumber, false, !subsequentWorkAhead, cancellationToken))
                         {
                             return;
                         }
@@ -97,7 +106,10 @@ namespace ErsatzTV.Application.Streaming
             }
             finally
             {
-                _timer.Elapsed -= Cancel;
+                lock (_sync)
+                {
+                    _timer.Elapsed -= Cancel;
+                }
             }
         }
 
@@ -105,6 +117,18 @@ namespace ErsatzTV.Application.Streaming
         {
             try
             {
+                if (!realtime)
+                {
+                    Interlocked.Increment(ref _workAheadCount);
+                    _logger.LogInformation("HLS segmenter will work ahead for channel {Channel}", channelNumber);
+                }
+                else
+                {
+                    _logger.LogInformation(
+                        "HLS segmenter will NOT work ahead for channel {Channel}",
+                        channelNumber);
+                }
+
                 using IServiceScope scope = _serviceScopeFactory.CreateScope();
                 IMediator mediator = scope.ServiceProvider.GetRequiredService<IMediator>();
 
@@ -134,7 +158,7 @@ namespace ErsatzTV.Application.Streaming
                 foreach (PlayoutItemProcessModel processModel in result.RightAsEnumerable())
                 {
                     await TrimAndDelete(channelNumber, cancellationToken);
-                    
+
                     Process process = processModel.Process;
 
                     _logger.LogDebug(
@@ -166,6 +190,10 @@ namespace ErsatzTV.Application.Streaming
                 _logger.LogError(ex, "Error transcoding channel {Channel}", channelNumber);
                 return false;
             }
+            finally
+            {
+                Interlocked.Decrement(ref _workAheadCount);
+            }
 
             return true;
         }
@@ -203,5 +231,13 @@ namespace ErsatzTV.Application.Streaming
                 _playlistStart = trimResult.PlaylistStart;
             }
         }
+
+        private async Task<int> GetWorkAheadLimit()
+        {
+            using IServiceScope scope = _serviceScopeFactory.CreateScope();
+            IConfigElementRepository repo = scope.ServiceProvider.GetRequiredService<IConfigElementRepository>();
+            return await repo.GetValue<int>(ConfigElementKey.FFmpegWorkAheadSegmenters)
+                .Map(maybeCount => maybeCount.Match(identity, () => 1));
+        }
     }
 }

ErsatzTV.Core/Domain/ConfigElementKey.cs

@@ -14,6 +14,8 @@
         public static ConfigElementKey FFmpegPreferredLanguageCode => new("ffmpeg.preferred_language_code");
         public static ConfigElementKey FFmpegGlobalWatermarkId => new("ffmpeg.global_watermark_id");
         public static ConfigElementKey FFmpegVaapiDriver => new("ffmpeg.vaapi_driver");
+        public static ConfigElementKey FFmpegSegmenterTimeout => new("ffmpeg.segmenter.timeout_seconds");
+        public static ConfigElementKey FFmpegWorkAheadSegmenters => new("ffmpeg.segmenter.work_ahead_limit");
         public static ConfigElementKey SearchIndexVersion => new("search_index.version");
         public static ConfigElementKey HDHRTunerCount => new("hdhr.tuner_count");
         public static ConfigElementKey ChannelsPageSize => new("pages.channels.page_size");

ErsatzTV/Pages/Settings.razor

@@ -64,6 +64,24 @@
                             <MudSelectItem Value="@driver">@driver</MudSelectItem>
                         }
                     </MudSelect>
+                    <MudElement HtmlTag="div" Class="mt-3">
+                        <MudTextField T="int"
+                                      Label="HLS Segmenter Idle Timeout"
+                                      @bind-Value="_ffmpegSettings.HlsSegmenterIdleTimeout"
+                                      Validation="@(new Func<int, string>(ValidateHlsSegmenterIdleTimeout))"
+                                      Required="true"
+                                      RequiredError="HLS Segmenter idle timeout is required!"
+                                      Adornment="Adornment.End"
+                                      AdornmentText="Seconds"/>
+                    </MudElement>
+                    <MudElement HtmlTag="div" Class="mt-3">
+                        <MudTextField T="int"
+                                      Label="Work-Ahead HLS Segmenter Limit"
+                                      @bind-Value="_ffmpegSettings.WorkAheadSegmenterLimit"
+                                      Validation="@(new Func<int, string>(ValidateWorkAheadSegmenterLimit))"
+                                      Required="true"
+                                      RequiredError="Work-ahead HLS Segmenter limit is required!"/>
+                    </MudElement>
                 </MudForm>
             </MudCardContent>
             <MudCardActions>
@@ -169,6 +187,10 @@
     
     private static string ValidatePlayoutDaysToBuild(int daysToBuild) => daysToBuild <= 0 ? "Days to build must be greater than zero" : null;
 
+    private static string ValidateHlsSegmenterIdleTimeout(int idleTimeout) => idleTimeout < 30 ? "HLS Segmenter idle timeout must be greater than or equal to 30" : null;
+
+    private static string ValidateWorkAheadSegmenterLimit(int limit) => limit < 0 ? "Work-Ahead HLS Segmenter limit must be greater than or equal to 0" : null;
+
     private async Task LoadFFmpegProfilesAsync() =>
         _ffmpegProfiles = await _mediator.Send(new GetAllFFmpegProfiles());