@ -2,63 +2,91 @@
@@ -2,63 +2,91 @@
using ErsatzTV.Core.Health.Checks ;
using MediatR ;
using Microsoft.Extensions.Caching.Memory ;
using Microsoft.Extensions.DependencyInjection ;
using Microsoft.Extensions.Logging ;
namespace ErsatzTV.Infrastructure.Health ;
public class HealthCheckService : IHealthCheckService
public class HealthCheckService (
IServiceScopeFactory serviceScopeFactory ,
IMemoryCache memoryCache ,
ILogger < HealthCheckService > logger ) : IHealthCheckService
{
private const string CacheKey = "healthcheck.summary" ;
private readonly List < IHealthCheck > _ checks ; // ReSharper disable SuggestBaseTypeForParameterInConstructor
private readonly IMemoryCache _ memoryCache ;
private readonly IMediator _ mediator ;
private readonly ILogger < HealthCheckService > _l ogger ;
public HealthCheckService (
IMacOsConfigFolderHealthCheck macOsConfigFolderHealthCheck ,
IFFmpegVersionHealthCheck ffmpegVersionHealthCheck ,
IFFmpegCapabilitiesHealthCheck ffmpegCapabilitiesHealthCheck ,
IFFmpegReportsHealthCheck ffmpegReportsHealthCheck ,
IHardwareAccelerationHealthCheck hardwareAccelerationHealthCheck ,
IMovieMetadataHealthCheck movieMetadataHealthCheck ,
IEpisodeMetadataHealthCheck episodeMetadataHealthCheck ,
IZeroDurationHealthCheck zeroDurationHealthCheck ,
IFileNotFoundHealthCheck fileNotFoundHealthCheck ,
IUnavailableHealthCheck unavailableHealthCheck ,
IVaapiDriverHealthCheck vaapiDriverHealthCheck ,
IUnifiedDockerHealthCheck unifiedDockerHealthCheck ,
IDowngradeHealthCheck downgradeHealthCheck ,
IEmptyScheduleHealthCheck emptyScheduleHealthCheck ,
IMemoryCache memoryCache ,
IMediator mediator ,
ILogger < HealthCheckService > logger )
private static readonly TimeSpan CacheDuration = TimeSpan . FromMinutes ( 5 ) ;
private static readonly TimeSpan RunTimeout = TimeSpan . FromMinutes ( 1 5 ) ;
// this order is also the display order
private static readonly Type [ ] CheckTypes =
[
typeof ( IDowngradeHealthCheck ) ,
typeof ( IMacOsConfigFolderHealthCheck ) ,
typeof ( IUnifiedDockerHealthCheck ) ,
typeof ( IFFmpegVersionHealthCheck ) ,
typeof ( IFFmpegCapabilitiesHealthCheck ) ,
typeof ( IFFmpegReportsHealthCheck ) ,
typeof ( IHardwareAccelerationHealthCheck ) ,
typeof ( IMovieMetadataHealthCheck ) ,
typeof ( IEpisodeMetadataHealthCheck ) ,
typeof ( IZeroDurationHealthCheck ) ,
typeof ( IFileNotFoundHealthCheck ) ,
typeof ( IUnavailableHealthCheck ) ,
typeof ( IEmptyScheduleHealthCheck ) ,
typeof ( IVaapiDriverHealthCheck )
] ;
private readonly Lock _ sync = new ( ) ;
private Task < List < HealthCheckResult > > _ inFlight ;
private List < HealthCheckResult > _ results ;
private DateTimeOffset _ resultsExpireAt ;
public Task < List < HealthCheckResult > > GetCachedHealthChecks ( CancellationToken cancellationToken ) = >
Run ( bypassCache : false , cancellationToken ) ;
public Task < List < HealthCheckResult > > PerformHealthChecks ( CancellationToken cancellationToken ) = >
Run ( bypassCache : true , cancellationToken ) ;
public HealthCheckSummary GetHealthCheckSummary ( ) = >
memoryCache . Get < HealthCheckSummary > ( CacheKey ) ? ? new HealthCheckSummary ( 0 , 0 ) ;
private async Task < List < HealthCheckResult > > Run ( bool bypassCache , CancellationToken cancellationToken )
{
_ memoryCache = memoryCache ;
_ mediator = mediator ;
_l ogger = logger ;
_ checks =
[
downgradeHealthCheck ,
macOsConfigFolderHealthCheck ,
unifiedDockerHealthCheck ,
ffmpegVersionHealthCheck ,
ffmpegCapabilitiesHealthCheck ,
ffmpegReportsHealthCheck ,
hardwareAccelerationHealthCheck ,
movieMetadataHealthCheck ,
episodeMetadataHealthCheck ,
zeroDurationHealthCheck ,
fileNotFoundHealthCheck ,
unavailableHealthCheck ,
emptyScheduleHealthCheck ,
vaapiDriverHealthCheck
] ;
Task < List < HealthCheckResult > > run ;
lock ( _ sync )
{
if ( ! bypassCache & & _ results is not null & & DateTimeOffset . UtcNow < _ resultsExpireAt )
{
return _ results ;
}
// only one run at a time; two runs at once start ffmpeg twice, and can finish
// out of order and store stale results
if ( _ inFlight is null | | _ inFlight . IsCompleted )
{
_ inFlight = RunChecks ( ) ;
}
run = _ inFlight ;
}
return await run . WaitAsync ( cancellationToken ) ;
}
public async Task < List < HealthCheckResult > > PerformHealthChecks ( CancellationToken cancellationToken )
private async Task < List < HealthCheckResult > > RunChecks ( )
{
List < HealthCheckResult > result = await _ checks . Map ( c = >
// this token is not the caller's; the checks start ffmpeg and read the whole media
// library, so a closed page must not stop a run that other callers share
using var cts = new CancellationTokenSource ( RunTimeout ) ;
CancellationToken cancellationToken = cts . Token ;
using IServiceScope scope = serviceScopeFactory . CreateScope ( ) ;
List < HealthCheckResult > results = await CheckTypes
. Map ( t = > ( IHealthCheck ) scope . ServiceProvider . GetRequiredService ( t ) )
. Map ( c = >
{
var failedResult = new HealthCheckResult (
c . Title ,
@ -69,27 +97,46 @@ public class HealthCheckService : IHealthCheckService
@@ -69,27 +97,46 @@ public class HealthCheckService : IHealthCheckService
return TryAsync ( ( ) = > c . Check ( cancellationToken ) ) . IfFail ( ex = > LogAndReturn ( ex , failedResult ) ) ;
} )
. SequenceParallel ( )
. Map ( results = > results . ToList ( ) ) ;
. Map ( r = > r . ToList ( ) ) ;
if ( cancellationToken . IsCancellationRequested )
{
// after a timeout each unfinished check reports a failure that is not real, so do not
// cache or publish it; an exception here stops the host through RunHealthChecksService
logger . LogWarning ( "Health checks did not complete within {Timeout}" , RunTimeout ) ;
return results ;
}
var summary = new HealthCheckSummary (
result . Count ( x = > x . Status is HealthCheckStatus . Warning ) ,
result . Count ( x = > x . Status is HealthCheckStatus . Fail ) ) ;
results . Count ( x = > x . Status is HealthCheckStatus . Warning ) ,
results . Count ( x = > x . Status is HealthCheckStatus . Fail ) ) ;
_ memoryCache . Set ( CacheKey , summary ) ;
lock ( _ sync )
{
_ results = results ;
_ resultsExpireAt = DateTimeOffset . UtcNow . Add ( CacheDuration ) ;
}
await _ mediator . Publish ( summary , cancellationToken ) ;
memoryCache . Set ( CacheKey , summary ) ;
return result ;
}
try
{
await scope . ServiceProvider . GetRequiredService < IMediator > ( ) . Publish ( summary , cancellationToken ) ;
}
catch ( Exception ex )
{
// an exception from a notification handler must not stop the host
logger . LogWarning ( ex , "Failed to publish health check summary" ) ;
}
public HealthCheckSummary GetHealthCheckSummary ( ) = >
_ memoryCache . Get < HealthCheckSummary > ( CacheKey ) ? ? new HealthCheckSummary ( 0 , 0 ) ;
return results ;
}
private HealthCheckResult LogAndReturn ( Exception ex , HealthCheckResult failedResult )
{
if ( ex is not OperationCanceledException )
{
_l ogger . LogWarning ( ex , "Failed to run health check {Title}" , failedResult . Title ) ;
l ogger. LogWarning ( ex , "Failed to run health check {Title}" , failedResult . Title ) ;
}
return failedResult ;
Jason Dove
1 week ago