From 91676151152446c1cbb13bd7ebf4b65d171dd58f Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Mon, 7 Sep 2026 08:45:36 -0500 Subject: [PATCH] fix: health check improvements --- CHANGELOG.md | 4 + .../Queries/GetAllHealthCheckResults.cs | 2 +- .../GetAllHealthCheckResultsHandler.cs | 16 +- .../Commands/DeleteOrphanedArtworkHandler.cs | 10 +- ErsatzTV.Core/Health/IHealthCheckService.cs | 1 + .../Health/HealthCheckService.cs | 159 ++++++++++++------ ErsatzTV/Pages/Index.razor | 31 +++- ErsatzTV/Startup.cs | 2 +- 8 files changed, 150 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da1b88097..f0956247d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Fixed +- Fix health checks causing a flood of (harmless) logged errors when quickly navigating away from home page + - Health check results will now be cached for 5 minutes by default; a refresh button has been added to immediately re-run all checks + ## [26.9.0] - 2026-09-06 ### Fixed - Fix text subtitle playback (regression from `v26.8.1`) diff --git a/ErsatzTV.Application/Health/Queries/GetAllHealthCheckResults.cs b/ErsatzTV.Application/Health/Queries/GetAllHealthCheckResults.cs index dd51aa2a1..25ca9222b 100644 --- a/ErsatzTV.Application/Health/Queries/GetAllHealthCheckResults.cs +++ b/ErsatzTV.Application/Health/Queries/GetAllHealthCheckResults.cs @@ -2,4 +2,4 @@ namespace ErsatzTV.Application.Health; -public record GetAllHealthCheckResults : IRequest>; +public record GetAllHealthCheckResults(bool Refresh) : IRequest>; diff --git a/ErsatzTV.Application/Health/Queries/GetAllHealthCheckResultsHandler.cs b/ErsatzTV.Application/Health/Queries/GetAllHealthCheckResultsHandler.cs index 489872a67..295cf4a13 100644 --- a/ErsatzTV.Application/Health/Queries/GetAllHealthCheckResultsHandler.cs +++ b/ErsatzTV.Application/Health/Queries/GetAllHealthCheckResultsHandler.cs @@ -2,25 +2,23 @@ namespace ErsatzTV.Application.Health; -public class GetAllHealthCheckResultsHandler : IRequestHandler> +public class GetAllHealthCheckResultsHandler(IHealthCheckService healthCheckService) + : IRequestHandler> { - private readonly IHealthCheckService _healthCheckService; - - public GetAllHealthCheckResultsHandler(IHealthCheckService healthCheckService) => - _healthCheckService = healthCheckService; - public async Task> Handle( GetAllHealthCheckResults request, CancellationToken cancellationToken) { try { - List results = await _healthCheckService.PerformHealthChecks(cancellationToken); + List results = request.Refresh + ? await healthCheckService.PerformHealthChecks(cancellationToken) + : await healthCheckService.GetCachedHealthChecks(cancellationToken); return results.Filter(r => r.Status != HealthCheckStatus.NotApplicable).ToList(); } - catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException) + catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException) { - return new List(); + return []; } } } diff --git a/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs b/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs index 02cc4f6a7..5f65af364 100644 --- a/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs +++ b/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs @@ -43,24 +43,24 @@ public class DeleteOrphanedArtworkHandler( int deletedActors = await artworkRepository.DeleteOrphanedActors(request.MaxToDelete, cancellationToken); if (deletedActors > 0) { - logger.LogInformation("Deleted {Count} orphaned actors", deletedActors); + logger.LogDebug("Deleted {Count} orphaned actors", deletedActors); } else { - logger.LogInformation("No orphaned actors to delete"); + logger.LogDebug("No orphaned actors to delete"); } int deletedArtwork = await artworkRepository.DeleteOrphanedArtwork(request.MaxToDelete, cancellationToken); if (deletedArtwork > 0) { - logger.LogInformation("Deleted {Count} orphaned artwork", deletedArtwork); + logger.LogDebug("Deleted {Count} orphaned artwork", deletedArtwork); } else { - logger.LogInformation("No orphaned artwork to delete"); + logger.LogDebug("No orphaned artwork to delete"); } - logger.LogInformation("Done cleaning!"); + logger.LogDebug("Done cleaning!"); } private async Task CleanUpFileSystem(CancellationToken cancellationToken) diff --git a/ErsatzTV.Core/Health/IHealthCheckService.cs b/ErsatzTV.Core/Health/IHealthCheckService.cs index c9d3dac2a..4ed0453c2 100644 --- a/ErsatzTV.Core/Health/IHealthCheckService.cs +++ b/ErsatzTV.Core/Health/IHealthCheckService.cs @@ -3,5 +3,6 @@ public interface IHealthCheckService { Task> PerformHealthChecks(CancellationToken cancellationToken); + Task> GetCachedHealthChecks(CancellationToken cancellationToken); HealthCheckSummary GetHealthCheckSummary(); } diff --git a/ErsatzTV.Infrastructure/Health/HealthCheckService.cs b/ErsatzTV.Infrastructure/Health/HealthCheckService.cs index 3bda2544b..a1fb1b703 100644 --- a/ErsatzTV.Infrastructure/Health/HealthCheckService.cs +++ b/ErsatzTV.Infrastructure/Health/HealthCheckService.cs @@ -2,63 +2,91 @@ using ErsatzTV.Core.Health.Checks; using MediatR; using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace ErsatzTV.Infrastructure.Health; -public class HealthCheckService : IHealthCheckService +public class HealthCheckService( + IServiceScopeFactory serviceScopeFactory, + IMemoryCache memoryCache, + ILogger logger) : IHealthCheckService { private const string CacheKey = "healthcheck.summary"; - private readonly List _checks; // ReSharper disable SuggestBaseTypeForParameterInConstructor - private readonly IMemoryCache _memoryCache; - private readonly IMediator _mediator; - private readonly ILogger _logger; - - public HealthCheckService( - IMacOsConfigFolderHealthCheck macOsConfigFolderHealthCheck, - IFFmpegVersionHealthCheck ffmpegVersionHealthCheck, - IFFmpegCapabilitiesHealthCheck ffmpegCapabilitiesHealthCheck, - IFFmpegReportsHealthCheck ffmpegReportsHealthCheck, - IHardwareAccelerationHealthCheck hardwareAccelerationHealthCheck, - IMovieMetadataHealthCheck movieMetadataHealthCheck, - IEpisodeMetadataHealthCheck episodeMetadataHealthCheck, - IZeroDurationHealthCheck zeroDurationHealthCheck, - IFileNotFoundHealthCheck fileNotFoundHealthCheck, - IUnavailableHealthCheck unavailableHealthCheck, - IVaapiDriverHealthCheck vaapiDriverHealthCheck, - IUnifiedDockerHealthCheck unifiedDockerHealthCheck, - IDowngradeHealthCheck downgradeHealthCheck, - IEmptyScheduleHealthCheck emptyScheduleHealthCheck, - IMemoryCache memoryCache, - IMediator mediator, - ILogger logger) + private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(5); + private static readonly TimeSpan RunTimeout = TimeSpan.FromMinutes(15); + + // this order is also the display order + private static readonly Type[] CheckTypes = + [ + typeof(IDowngradeHealthCheck), + typeof(IMacOsConfigFolderHealthCheck), + typeof(IUnifiedDockerHealthCheck), + typeof(IFFmpegVersionHealthCheck), + typeof(IFFmpegCapabilitiesHealthCheck), + typeof(IFFmpegReportsHealthCheck), + typeof(IHardwareAccelerationHealthCheck), + typeof(IMovieMetadataHealthCheck), + typeof(IEpisodeMetadataHealthCheck), + typeof(IZeroDurationHealthCheck), + typeof(IFileNotFoundHealthCheck), + typeof(IUnavailableHealthCheck), + typeof(IEmptyScheduleHealthCheck), + typeof(IVaapiDriverHealthCheck) + ]; + + private readonly Lock _sync = new(); + + private Task> _inFlight; + private List _results; + private DateTimeOffset _resultsExpireAt; + + public Task> GetCachedHealthChecks(CancellationToken cancellationToken) => + Run(bypassCache: false, cancellationToken); + + public Task> PerformHealthChecks(CancellationToken cancellationToken) => + Run(bypassCache: true, cancellationToken); + + public HealthCheckSummary GetHealthCheckSummary() => + memoryCache.Get(CacheKey) ?? new HealthCheckSummary(0, 0); + + private async Task> Run(bool bypassCache, CancellationToken cancellationToken) { - _memoryCache = memoryCache; - _mediator = mediator; - _logger = logger; - _checks = - [ - downgradeHealthCheck, - macOsConfigFolderHealthCheck, - unifiedDockerHealthCheck, - ffmpegVersionHealthCheck, - ffmpegCapabilitiesHealthCheck, - ffmpegReportsHealthCheck, - hardwareAccelerationHealthCheck, - movieMetadataHealthCheck, - episodeMetadataHealthCheck, - zeroDurationHealthCheck, - fileNotFoundHealthCheck, - unavailableHealthCheck, - emptyScheduleHealthCheck, - vaapiDriverHealthCheck - ]; + Task> run; + + lock (_sync) + { + if (!bypassCache && _results is not null && DateTimeOffset.UtcNow < _resultsExpireAt) + { + return _results; + } + + // only one run at a time; two runs at once start ffmpeg twice, and can finish + // out of order and store stale results + if (_inFlight is null || _inFlight.IsCompleted) + { + _inFlight = RunChecks(); + } + + run = _inFlight; + } + + return await run.WaitAsync(cancellationToken); } - public async Task> PerformHealthChecks(CancellationToken cancellationToken) + private async Task> RunChecks() { - List result = await _checks.Map(c => + // this token is not the caller's; the checks start ffmpeg and read the whole media + // library, so a closed page must not stop a run that other callers share + using var cts = new CancellationTokenSource(RunTimeout); + CancellationToken cancellationToken = cts.Token; + + using IServiceScope scope = serviceScopeFactory.CreateScope(); + + List results = await CheckTypes + .Map(t => (IHealthCheck)scope.ServiceProvider.GetRequiredService(t)) + .Map(c => { var failedResult = new HealthCheckResult( c.Title, @@ -69,27 +97,46 @@ public class HealthCheckService : IHealthCheckService return TryAsync(() => c.Check(cancellationToken)).IfFail(ex => LogAndReturn(ex, failedResult)); }) .SequenceParallel() - .Map(results => results.ToList()); + .Map(r => r.ToList()); + + if (cancellationToken.IsCancellationRequested) + { + // after a timeout each unfinished check reports a failure that is not real, so do not + // cache or publish it; an exception here stops the host through RunHealthChecksService + logger.LogWarning("Health checks did not complete within {Timeout}", RunTimeout); + return results; + } var summary = new HealthCheckSummary( - result.Count(x => x.Status is HealthCheckStatus.Warning), - result.Count(x => x.Status is HealthCheckStatus.Fail)); + results.Count(x => x.Status is HealthCheckStatus.Warning), + results.Count(x => x.Status is HealthCheckStatus.Fail)); - _memoryCache.Set(CacheKey, summary); + lock (_sync) + { + _results = results; + _resultsExpireAt = DateTimeOffset.UtcNow.Add(CacheDuration); + } - await _mediator.Publish(summary, cancellationToken); + memoryCache.Set(CacheKey, summary); - return result; - } + try + { + await scope.ServiceProvider.GetRequiredService().Publish(summary, cancellationToken); + } + catch (Exception ex) + { + // an exception from a notification handler must not stop the host + logger.LogWarning(ex, "Failed to publish health check summary"); + } - public HealthCheckSummary GetHealthCheckSummary() => - _memoryCache.Get(CacheKey) ?? new HealthCheckSummary(0, 0); + return results; + } private HealthCheckResult LogAndReturn(Exception ex, HealthCheckResult failedResult) { if (ex is not OperationCanceledException) { - _logger.LogWarning(ex, "Failed to run health check {Title}", failedResult.Title); + logger.LogWarning(ex, "Failed to run health check {Title}", failedResult.Title); } return failedResult; diff --git a/ErsatzTV/Pages/Index.razor b/ErsatzTV/Pages/Index.razor index abb3697b4..80863b40b 100644 --- a/ErsatzTV/Pages/Index.razor +++ b/ErsatzTV/Pages/Index.razor @@ -50,12 +50,20 @@ } else { - Health Checks +
+ Health Checks + +
+ ServerData="@(ServerReload)" + @ref="_healthCheckTable"> Check Message @@ -145,6 +153,10 @@ @code { private CancellationTokenSource _cts; + private MudTable _healthCheckTable; + + private bool _refreshHealthChecks; + private string _releaseNotes; protected override void OnInitialized() @@ -234,9 +246,22 @@ } } + private async Task RefreshHealthChecks() + { + _refreshHealthChecks = true; + + if (_healthCheckTable != null) + { + await _healthCheckTable.ReloadServerData(); + } + } + private async Task> ServerReload(TableState state, CancellationToken cancellationToken) { - List healthCheckResults = await Mediator.Send(new GetAllHealthCheckResults(), cancellationToken); + bool refresh = _refreshHealthChecks; + _refreshHealthChecks = false; + + List healthCheckResults = await Mediator.Send(new GetAllHealthCheckResults(refresh), cancellationToken); return new TableData { diff --git a/ErsatzTV/Startup.cs b/ErsatzTV/Startup.cs index 1a7d76bcc..bd37413a5 100644 --- a/ErsatzTV/Startup.cs +++ b/ErsatzTV/Startup.cs @@ -847,6 +847,7 @@ public class Startup services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); AddChannel(services); AddChannel(services); AddChannel(services); @@ -869,7 +870,6 @@ public class Startup services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); services.AddScoped(); services.AddScoped();