From 04adbfeffa115bf9383f72551db055550997d1c9 Mon Sep 17 00:00:00 2001 From: Jason Dove Date: Tue, 12 Oct 2021 06:31:11 -0500 Subject: [PATCH] add hls segmenter settings to optimize performance (#418) * add hls segmenter settings to optimize performance * use consistent setting defaults --- CHANGELOG.md | 6 ++ .../Commands/UpdateFFmpegSettingsHandler.cs | 8 +++ .../FFmpegProfiles/FFmpegSettingsViewModel.cs | 2 + .../Queries/GetFFmpegSettingsHandler.cs | 8 ++- .../Commands/StartFFmpegSessionHandler.cs | 13 +++- .../Streaming/HlsSessionWorker.cs | 60 +++++++++++++++---- ErsatzTV.Core/Domain/ConfigElementKey.cs | 2 + ErsatzTV/Pages/Settings.razor | 22 +++++++ 8 files changed, 106 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9779b3ed2..541b90d59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - Include more cuda (nvidia) filters in docker image - Enable deinterlacing with nvidia using new `yadif_cuda` filter +- Add two HLS Segmenter settings: idle timeout and work-ahead limit + - `HLS Segmenter Idle Timeout` - the number of seconds to keep transcoding a channel while no requests have been received from any client + - This setting must be greater than or equal to 30 (seconds) + - `Work-Ahead HLS Segmenter Limit` - the number of segmenters (channels) that will work-ahead simultaneously (if multiple channels are being watched) + - "working ahead" means transcoding at full speed, which can take a lot of resources + - This setting must be greater than or equal to 0 ### Changed - Upgrade ffmpeg from 4.3 to 4.4 in all docker images diff --git a/ErsatzTV.Application/FFmpegProfiles/Commands/UpdateFFmpegSettingsHandler.cs b/ErsatzTV.Application/FFmpegProfiles/Commands/UpdateFFmpegSettingsHandler.cs index e690acf94..0aca8ec28 100644 --- a/ErsatzTV.Application/FFmpegProfiles/Commands/UpdateFFmpegSettingsHandler.cs +++ b/ErsatzTV.Application/FFmpegProfiles/Commands/UpdateFFmpegSettingsHandler.cs @@ -119,6 +119,14 @@ namespace ErsatzTV.Application.FFmpegProfiles.Commands ConfigElementKey.FFmpegVaapiDriver, (int)request.Settings.VaapiDriver); + await _configElementRepository.Upsert( + ConfigElementKey.FFmpegSegmenterTimeout, + request.Settings.HlsSegmenterIdleTimeout); + + await _configElementRepository.Upsert( + ConfigElementKey.FFmpegWorkAheadSegmenters, + request.Settings.WorkAheadSegmenterLimit); + return Unit.Default; } } diff --git a/ErsatzTV.Application/FFmpegProfiles/FFmpegSettingsViewModel.cs b/ErsatzTV.Application/FFmpegProfiles/FFmpegSettingsViewModel.cs index 339c759d3..a594a2d42 100644 --- a/ErsatzTV.Application/FFmpegProfiles/FFmpegSettingsViewModel.cs +++ b/ErsatzTV.Application/FFmpegProfiles/FFmpegSettingsViewModel.cs @@ -11,5 +11,7 @@ namespace ErsatzTV.Application.FFmpegProfiles public bool SaveReports { get; set; } public int? GlobalWatermarkId { get; set; } public VaapiDriver VaapiDriver { get; set; } + public int HlsSegmenterIdleTimeout { get; set; } + public int WorkAheadSegmenterLimit { get; set; } } } diff --git a/ErsatzTV.Application/FFmpegProfiles/Queries/GetFFmpegSettingsHandler.cs b/ErsatzTV.Application/FFmpegProfiles/Queries/GetFFmpegSettingsHandler.cs index 725fc00d4..8bd0da3b0 100644 --- a/ErsatzTV.Application/FFmpegProfiles/Queries/GetFFmpegSettingsHandler.cs +++ b/ErsatzTV.Application/FFmpegProfiles/Queries/GetFFmpegSettingsHandler.cs @@ -31,6 +31,10 @@ namespace ErsatzTV.Application.FFmpegProfiles.Queries await _configElementRepository.GetValue(ConfigElementKey.FFmpegGlobalWatermarkId); Option vaapiDriver = await _configElementRepository.GetValue(ConfigElementKey.FFmpegVaapiDriver); + Option hlsSegmenterIdleTimeout = + await _configElementRepository.GetValue(ConfigElementKey.FFmpegSegmenterTimeout); + Option workAheadSegmenterLimit = + await _configElementRepository.GetValue(ConfigElementKey.FFmpegWorkAheadSegmenters); var result = new FFmpegSettingsViewModel { @@ -39,7 +43,9 @@ namespace ErsatzTV.Application.FFmpegProfiles.Queries DefaultFFmpegProfileId = await defaultFFmpegProfileId.IfNoneAsync(0), SaveReports = await saveReports.IfNoneAsync(false), PreferredLanguageCode = await preferredLanguageCode.IfNoneAsync("eng"), - VaapiDriver = (VaapiDriver)await vaapiDriver.IfNoneAsync(0) + VaapiDriver = (VaapiDriver)await vaapiDriver.IfNoneAsync(0), + HlsSegmenterIdleTimeout = await hlsSegmenterIdleTimeout.IfNoneAsync(60), + WorkAheadSegmenterLimit = await workAheadSegmenterLimit.IfNoneAsync(1), }; foreach (int watermarkId in watermark) diff --git a/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs b/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs index 61637325b..c398d7aca 100644 --- a/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs +++ b/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs @@ -3,9 +3,11 @@ using System.IO; using System.Threading; using System.Threading.Tasks; using ErsatzTV.Core; +using ErsatzTV.Core.Domain; using ErsatzTV.Core.Errors; using ErsatzTV.Core.Interfaces.FFmpeg; using ErsatzTV.Core.Interfaces.Metadata; +using ErsatzTV.Core.Interfaces.Repositories; using LanguageExt; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -18,18 +20,21 @@ namespace ErsatzTV.Application.Streaming.Commands private readonly ILogger _logger; private readonly IServiceScopeFactory _serviceScopeFactory; private readonly IFFmpegSegmenterService _ffmpegSegmenterService; + private readonly IConfigElementRepository _configElementRepository; private readonly ILocalFileSystem _localFileSystem; public StartFFmpegSessionHandler( ILocalFileSystem localFileSystem, ILogger logger, IServiceScopeFactory serviceScopeFactory, - IFFmpegSegmenterService ffmpegSegmenterService) + IFFmpegSegmenterService ffmpegSegmenterService, + IConfigElementRepository configElementRepository) { _localFileSystem = localFileSystem; _logger = logger; _serviceScopeFactory = serviceScopeFactory; _ffmpegSegmenterService = ffmpegSegmenterService; + _configElementRepository = configElementRepository; } public Task> Handle(StartFFmpegSession request, CancellationToken cancellationToken) => @@ -42,12 +47,16 @@ namespace ErsatzTV.Application.Streaming.Commands private async Task StartProcess(StartFFmpegSession request) { + TimeSpan idleTimeout = await _configElementRepository + .GetValue(ConfigElementKey.FFmpegSegmenterTimeout) + .Map(maybeTimeout => maybeTimeout.Match(i => TimeSpan.FromSeconds(i), () => TimeSpan.FromMinutes(1))); + using IServiceScope scope = _serviceScopeFactory.CreateScope(); HlsSessionWorker worker = scope.ServiceProvider.GetRequiredService(); _ffmpegSegmenterService.SessionWorkers.AddOrUpdate(request.ChannelNumber, _ => worker, (_, _) => worker); // fire and forget worker - _ = worker.Run(request.ChannelNumber) + _ = worker.Run(request.ChannelNumber, idleTimeout) .ContinueWith( _ => _ffmpegSegmenterService.SessionWorkers.TryRemove( request.ChannelNumber, diff --git a/ErsatzTV.Application/Streaming/HlsSessionWorker.cs b/ErsatzTV.Application/Streaming/HlsSessionWorker.cs index e9a380931..41aefe5a3 100644 --- a/ErsatzTV.Application/Streaming/HlsSessionWorker.cs +++ b/ErsatzTV.Application/Streaming/HlsSessionWorker.cs @@ -6,23 +6,27 @@ using System.Threading.Tasks; using System.Timers; using ErsatzTV.Application.Streaming.Queries; using ErsatzTV.Core; +using ErsatzTV.Core.Domain; using ErsatzTV.Core.FFmpeg; using ErsatzTV.Core.Interfaces.FFmpeg; +using ErsatzTV.Core.Interfaces.Repositories; using LanguageExt; using MediatR; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Timer = System.Timers.Timer; +using static LanguageExt.Prelude; namespace ErsatzTV.Application.Streaming { public class HlsSessionWorker : IHlsSessionWorker { + private static int _workAheadCount; private readonly IServiceScopeFactory _serviceScopeFactory; private readonly ILogger _logger; private DateTimeOffset _lastAccess; private DateTimeOffset _transcodedUntil; - private readonly Timer _timer = new(TimeSpan.FromMinutes(2).TotalMilliseconds) { AutoReset = false }; + private Timer _timer; private readonly object _sync = new(); private DateTimeOffset _playlistStart; @@ -40,19 +44,23 @@ namespace ErsatzTV.Application.Streaming { _lastAccess = DateTimeOffset.Now; - _timer.Stop(); - _timer.Start(); + _timer?.Stop(); + _timer?.Start(); } } - public async Task Run(string channelNumber) + public async Task Run(string channelNumber, TimeSpan idleTimeout) { var cts = new CancellationTokenSource(); void Cancel(object o, ElapsedEventArgs e) => cts.Cancel(); try { - _timer.Elapsed += Cancel; + lock (_sync) + { + _timer = new Timer(idleTimeout.TotalMilliseconds) { AutoReset = false }; + _timer.Elapsed += Cancel; + } CancellationToken cancellationToken = cts.Token; @@ -62,16 +70,15 @@ namespace ErsatzTV.Application.Streaming _transcodedUntil = DateTimeOffset.Now; _playlistStart = _transcodedUntil; - // start initial transcode WITHOUT realtime throttle - if (!await Transcode(channelNumber, true, false, cancellationToken)) + bool initialWorkAhead = Volatile.Read(ref _workAheadCount) < await GetWorkAheadLimit(); + if (!await Transcode(channelNumber, true, !initialWorkAhead, cancellationToken)) { return; } while (!cancellationToken.IsCancellationRequested) { - // TODO: configurable? 5 minutes? - if (DateTimeOffset.Now - _lastAccess > TimeSpan.FromMinutes(2)) + if (DateTimeOffset.Now - _lastAccess > idleTimeout) { _logger.LogInformation("Stopping idle HLS session for channel {Channel}", channelNumber); return; @@ -83,7 +90,9 @@ namespace ErsatzTV.Application.Streaming { // only use realtime encoding when we're at least 30 seconds ahead bool realtime = transcodedBuffer >= TimeSpan.FromSeconds(30); - if (!await Transcode(channelNumber, false, realtime, cancellationToken)) + bool subsequentWorkAhead = + !realtime && Volatile.Read(ref _workAheadCount) < await GetWorkAheadLimit(); + if (!await Transcode(channelNumber, false, !subsequentWorkAhead, cancellationToken)) { return; } @@ -97,7 +106,10 @@ namespace ErsatzTV.Application.Streaming } finally { - _timer.Elapsed -= Cancel; + lock (_sync) + { + _timer.Elapsed -= Cancel; + } } } @@ -105,6 +117,18 @@ namespace ErsatzTV.Application.Streaming { try { + if (!realtime) + { + Interlocked.Increment(ref _workAheadCount); + _logger.LogInformation("HLS segmenter will work ahead for channel {Channel}", channelNumber); + } + else + { + _logger.LogInformation( + "HLS segmenter will NOT work ahead for channel {Channel}", + channelNumber); + } + using IServiceScope scope = _serviceScopeFactory.CreateScope(); IMediator mediator = scope.ServiceProvider.GetRequiredService(); @@ -134,7 +158,7 @@ namespace ErsatzTV.Application.Streaming foreach (PlayoutItemProcessModel processModel in result.RightAsEnumerable()) { await TrimAndDelete(channelNumber, cancellationToken); - + Process process = processModel.Process; _logger.LogDebug( @@ -166,6 +190,10 @@ namespace ErsatzTV.Application.Streaming _logger.LogError(ex, "Error transcoding channel {Channel}", channelNumber); return false; } + finally + { + Interlocked.Decrement(ref _workAheadCount); + } return true; } @@ -203,5 +231,13 @@ namespace ErsatzTV.Application.Streaming _playlistStart = trimResult.PlaylistStart; } } + + private async Task GetWorkAheadLimit() + { + using IServiceScope scope = _serviceScopeFactory.CreateScope(); + IConfigElementRepository repo = scope.ServiceProvider.GetRequiredService(); + return await repo.GetValue(ConfigElementKey.FFmpegWorkAheadSegmenters) + .Map(maybeCount => maybeCount.Match(identity, () => 1)); + } } } diff --git a/ErsatzTV.Core/Domain/ConfigElementKey.cs b/ErsatzTV.Core/Domain/ConfigElementKey.cs index 9c34d0526..da0f45ee2 100644 --- a/ErsatzTV.Core/Domain/ConfigElementKey.cs +++ b/ErsatzTV.Core/Domain/ConfigElementKey.cs @@ -14,6 +14,8 @@ public static ConfigElementKey FFmpegPreferredLanguageCode => new("ffmpeg.preferred_language_code"); public static ConfigElementKey FFmpegGlobalWatermarkId => new("ffmpeg.global_watermark_id"); public static ConfigElementKey FFmpegVaapiDriver => new("ffmpeg.vaapi_driver"); + public static ConfigElementKey FFmpegSegmenterTimeout => new("ffmpeg.segmenter.timeout_seconds"); + public static ConfigElementKey FFmpegWorkAheadSegmenters => new("ffmpeg.segmenter.work_ahead_limit"); public static ConfigElementKey SearchIndexVersion => new("search_index.version"); public static ConfigElementKey HDHRTunerCount => new("hdhr.tuner_count"); public static ConfigElementKey ChannelsPageSize => new("pages.channels.page_size"); diff --git a/ErsatzTV/Pages/Settings.razor b/ErsatzTV/Pages/Settings.razor index 87401618b..8b7b19c02 100644 --- a/ErsatzTV/Pages/Settings.razor +++ b/ErsatzTV/Pages/Settings.razor @@ -64,6 +64,24 @@ @driver } + + + + + + @@ -169,6 +187,10 @@ private static string ValidatePlayoutDaysToBuild(int daysToBuild) => daysToBuild <= 0 ? "Days to build must be greater than zero" : null; + private static string ValidateHlsSegmenterIdleTimeout(int idleTimeout) => idleTimeout < 30 ? "HLS Segmenter idle timeout must be greater than or equal to 30" : null; + + private static string ValidateWorkAheadSegmenterLimit(int limit) => limit < 0 ? "Work-Ahead HLS Segmenter limit must be greater than or equal to 0" : null; + private async Task LoadFFmpegProfilesAsync() => _ffmpegProfiles = await _mediator.Send(new GetAllFFmpegProfiles());