memory improvements (#1127)

* regularly release memory

* don't aggressively GC while legacy streaming

* update changelog

CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
+### Added
+- Attempt to release memory periodically
+
 ### Fixed
 - Fix schedule editor crashing due to bad music video artist data
 - Fix bug where playouts would not maintain smart collection progress on schedules that use multiple smart collections

ErsatzTV.Application/FFmpegProfiles/Commands/UpdateFFmpegSettingsHandler.cs

@@ -52,7 +52,7 @@ public class UpdateFFmpegSettingsHandler : IRequestHandler<UpdateFFmpegSettings,
             UseShellExecute = false
         };
 
-        var test = new Process
+        using var test = new Process
         {
             StartInfo = startInfo
         };

ErsatzTV.Application/Maintenance/Commands/ReleaseMemory.cs

@@ -0,0 +1,3 @@
+namespace ErsatzTV.Application.Maintenance;
+
+public record ReleaseMemory(bool ForceAggressive) : IRequest<Unit>, IBackgroundServiceRequest;

ErsatzTV.Application/Maintenance/Commands/ReleaseMemoryHandler.cs

@@ -0,0 +1,41 @@
+using ErsatzTV.Core.FFmpeg;
+using ErsatzTV.Core.Interfaces.FFmpeg;
+using Microsoft.Extensions.Logging;
+
+namespace ErsatzTV.Application.Maintenance;
+
+public class ReleaseMemoryHandler : IRequestHandler<ReleaseMemory, Unit>
+{
+    private readonly IFFmpegSegmenterService _ffmpegSegmenterService;
+    private readonly ILogger<ReleaseMemoryHandler> _logger;
+
+    public ReleaseMemoryHandler(
+        IFFmpegSegmenterService ffmpegSegmenterService,
+        ILogger<ReleaseMemoryHandler> logger)
+    {
+        _ffmpegSegmenterService = ffmpegSegmenterService;
+        _logger = logger;
+    }
+
+    public Task<Unit> Handle(ReleaseMemory request, CancellationToken cancellationToken)
+    {
+        bool hasActiveWorkers = _ffmpegSegmenterService.SessionWorkers.Any() || FFmpegProcess.ProcessCount > 0;
+        if (request.ForceAggressive || !hasActiveWorkers)
+        {
+            _logger.LogDebug("Starting aggressive garbage collection");
+            GC.Collect(2, GCCollectionMode.Aggressive, blocking: true, compacting: true);
+        }
+        else
+        {
+            _logger.LogDebug("Starting garbage collection");
+            GC.Collect(2, GCCollectionMode.Forced, blocking: false);
+        }
+
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+
+        _logger.LogDebug("Completed garbage collection");
+
+        return Task.FromResult(Unit.Default);
+    }
+}

ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs

@@ -1,4 +1,6 @@
 using System.Diagnostics;
+using System.Threading.Channels;
+using ErsatzTV.Application.Maintenance;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Errors;
@@ -14,6 +16,7 @@ namespace ErsatzTV.Application.Streaming;
 public class StartFFmpegSessionHandler : IRequestHandler<StartFFmpegSession, Either<BaseError, Unit>>
 {
     private readonly IConfigElementRepository _configElementRepository;
+    private readonly ChannelWriter<IBackgroundServiceRequest> _workerChannel;
     private readonly IFFmpegSegmenterService _ffmpegSegmenterService;
     private readonly ILocalFileSystem _localFileSystem;
     private readonly ILogger<StartFFmpegSessionHandler> _logger;
@@ -24,13 +27,15 @@ public class StartFFmpegSessionHandler : IRequestHandler<StartFFmpegSession, Eit
         ILogger<StartFFmpegSessionHandler> logger,
         IServiceScopeFactory serviceScopeFactory,
         IFFmpegSegmenterService ffmpegSegmenterService,
-        IConfigElementRepository configElementRepository)
+        IConfigElementRepository configElementRepository,
+        ChannelWriter<IBackgroundServiceRequest> workerChannel)
     {
         _localFileSystem = localFileSystem;
         _logger = logger;
         _serviceScopeFactory = serviceScopeFactory;
         _ffmpegSegmenterService = ffmpegSegmenterService;
         _configElementRepository = configElementRepository;
+        _workerChannel = workerChannel;
     }
 
     public Task<Either<BaseError, Unit>> Handle(StartFFmpegSession request, CancellationToken cancellationToken) =>
@@ -54,9 +59,14 @@ public class StartFFmpegSessionHandler : IRequestHandler<StartFFmpegSession, Eit
         // fire and forget worker
         _ = worker.Run(request.ChannelNumber, idleTimeout, cancellationToken)
             .ContinueWith(
-                _ => _ffmpegSegmenterService.SessionWorkers.TryRemove(
+                _ =>
-                    request.ChannelNumber,
+                {
-                    out IHlsSessionWorker _),
+                    _ffmpegSegmenterService.SessionWorkers.TryRemove(
+                        request.ChannelNumber,
+                        out IHlsSessionWorker _);
+
+                    _workerChannel.TryWrite(new ReleaseMemory(false));
+                },
                 TaskScheduler.Default);
 
         string playlistFileName = Path.Combine(

ErsatzTV.Core/FFmpeg/FFmpegProcess.cs

@@ -0,0 +1,20 @@
+using System.Diagnostics;
+
+namespace ErsatzTV.Core.FFmpeg;
+
+public class FFmpegProcess : Process
+{
+    public static int ProcessCount;
+
+    public FFmpegProcess()
+    {
+        Interlocked.Increment(ref ProcessCount);
+    }
+
+    protected override void Dispose(bool disposing)
+    {
+        base.Dispose(disposing);
+
+        Interlocked.Decrement(ref ProcessCount);
+    }
+}

ErsatzTV.Scanner/Core/Metadata/LocalStatisticsProvider.cs

@@ -287,7 +287,7 @@ public class LocalStatisticsProvider : ILocalStatisticsProvider
             startInfo.ArgumentList.Add("null");
             startInfo.ArgumentList.Add("-");
 
-            var probe = new Process
+            using var probe = new Process
             {
                 StartInfo = startInfo
             };

ErsatzTV/Controllers/Api/MaintenanceController.cs

@@ -0,0 +1,20 @@
+using ErsatzTV.Application.Maintenance;
+using MediatR;
+using Microsoft.AspNetCore.Mvc;
+
+namespace ErsatzTV.Controllers.Api;
+
+[ApiController]
+public class MaintenanceController
+{
+    private readonly IMediator _mediator;
+
+    public MaintenanceController(IMediator mediator) => _mediator = mediator;
+
+    [HttpGet("/api/maintenance/gc")]
+    public async Task<IActionResult> GarbageCollection([FromQuery] bool force = false)
+    {
+        await _mediator.Send(new ReleaseMemory(force));
+        return new OkResult();
+    }
+}

ErsatzTV/Controllers/ArtworkController.cs

@@ -145,6 +145,7 @@ public class ArtworkController : ControllerBase
             Right: async r =>
             {
                 HttpClient client = _httpClientFactory.CreateClient();
+                HttpContext.Response.RegisterForDispose(client);
                 client.DefaultRequestHeaders.Add("X-Plex-Token", r.AuthToken);
 
                 var fullPath = new Uri(r.Uri, transcodePath);
@@ -152,6 +153,8 @@ public class ArtworkController : ControllerBase
                     fullPath,
                     HttpCompletionOption.ResponseHeadersRead,
                     cancellationToken);
+                HttpContext.Response.RegisterForDispose(response);
+
                 Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken);
 
                 return new FileStreamResult(
@@ -170,12 +173,15 @@ public class ArtworkController : ControllerBase
             Right: async vm =>
             {
                 HttpClient client = _httpClientFactory.CreateClient();
+                HttpContext.Response.RegisterForDispose(client);
 
                 Url fullPath = JellyfinUrl.ForArtwork(vm.Address, path);
                 HttpResponseMessage response = await client.GetAsync(
                     fullPath,
                     HttpCompletionOption.ResponseHeadersRead,
                     cancellationToken);
+                HttpContext.Response.RegisterForDispose(response);
+
                 Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken);
 
                 return new FileStreamResult(
@@ -194,12 +200,15 @@ public class ArtworkController : ControllerBase
             Right: async vm =>
             {
                 HttpClient client = _httpClientFactory.CreateClient();
+                HttpContext.Response.RegisterForDispose(client);
 
                 Url fullPath = EmbyUrl.ForArtwork(vm.Address, path);
                 HttpResponseMessage response = await client.GetAsync(
                     fullPath,
                     HttpCompletionOption.ResponseHeadersRead,
                     cancellationToken);
+                HttpContext.Response.RegisterForDispose(response);
+
                 Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken);
 
                 return new FileStreamResult(

ErsatzTV/Controllers/InternalController.cs

@@ -1,6 +1,7 @@
 using System.Diagnostics;
 using CliWrap;
 using ErsatzTV.Application.Streaming;
+using ErsatzTV.Core.FFmpeg;
 using ErsatzTV.Extensions;
 using MediatR;
 using Microsoft.AspNetCore.Mvc;
@@ -47,7 +48,7 @@ public class InternalController : ControllerBase
                             Command command = processModel.Process;
 
                             _logger.LogInformation("ffmpeg arguments {FFmpegArguments}", command.Arguments);
-                            var process = new Process
+                            var process = new FFmpegProcess
                             {
                                 StartInfo = new ProcessStartInfo
                                 {
@@ -59,6 +60,7 @@ public class InternalController : ControllerBase
                                     CreateNoWindow = true
                                 }
                             };
+                            HttpContext.Response.RegisterForDispose(process);
 
                             foreach ((string key, string value) in command.EnvironmentVariables)
                             {

ErsatzTV/Controllers/IptvController.cs

@@ -97,7 +97,7 @@ public class IptvController : ControllerBase
 
                         _logger.LogInformation("Starting ts stream for channel {ChannelNumber}", channelNumber);
                         _logger.LogInformation("ffmpeg arguments {FFmpegArguments}", command.Arguments);
-                        var process = new Process
+                        var process = new FFmpegProcess
                         {
                             StartInfo = new ProcessStartInfo
                             {
@@ -109,6 +109,7 @@ public class IptvController : ControllerBase
                                 CreateNoWindow = true
                             }
                         };
+                        HttpContext.Response.RegisterForDispose(process);
 
                         foreach ((string key, string value) in command.EnvironmentVariables)
                         {

ErsatzTV/Services/SchedulerService.cs

@@ -100,6 +100,8 @@ public class SchedulerService : BackgroundService
             await ScanEmbyMediaSources(cancellationToken);
 #endif
             await MatchTraktLists(cancellationToken);
+
+            await ReleaseMemory(cancellationToken);
         }
         catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
         {
@@ -270,4 +272,7 @@ public class SchedulerService : BackgroundService
 
     private ValueTask DeleteOrphanedArtwork(CancellationToken cancellationToken) =>
         _workerChannel.WriteAsync(new DeleteOrphanedArtwork(), cancellationToken);
+
+    private ValueTask ReleaseMemory(CancellationToken cancellationToken) =>
+        _workerChannel.WriteAsync(new ReleaseMemory(false), cancellationToken);
 }

ErsatzTV/Services/WorkerService.cs

@@ -113,6 +113,9 @@ public class WorkerService : BackgroundService
                         case ExtractEmbeddedSubtitles extractEmbeddedSubtitles:
                             await mediator.Send(extractEmbeddedSubtitles, cancellationToken);
                             break;
+                        case ReleaseMemory aggressivelyReleaseMemory:
+                            await mediator.Send(aggressivelyReleaseMemory, cancellationToken);
+                            break;
                     }
                 }
                 catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested)