From 35d24ffea653f2aa8fa5dc8d5267abe44d370025 Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Fri, 16 Jan 2026 13:38:31 -0600 Subject: [PATCH] cleanup artwork cache folder (#2779) * cleanup artwork cache folder * fixes * ignore watermarks that no longer exist on the file system --- CHANGELOG.md | 7 + .../Commands/DeleteOrphanedArtworkHandler.cs | 147 +++++++++++++++++- .../FFmpeg/WatermarkSelectorTests.cs | 13 +- ErsatzTV.Core/FFmpeg/WatermarkSelector.cs | 95 ++++++++--- ErsatzTV.Infrastructure/Images/ImageCache.cs | 9 +- .../Core/FFmpeg/TranscodingTests.cs | 5 +- .../Controllers/Api/MaintenanceController.cs | 13 +- ErsatzTV/Services/SchedulerService.cs | 4 - 8 files changed, 257 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e9dd75c5..25754d485 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - `Software` - force software padding - This can be used to work around buggy GPU driver behavior where padding is green instead of black - This is most often seen with VAAPI acceleration (radeonsi or i965 drivers) +- Add API endpoint to clean artwork cache folder (on demand) + - POST `/api/maintenance/clean_artwork` + +### Changed +- Disable automatic artwork database cleanup + - This will be re-enabled at some point in the future (after more testing) + - For now, the API should be used to clean as needed ### Fixed - Use code signing on all Windows executables (`ErsatzTV-Windows.exe`, `ErsatzTV.exe`, `ErsatzTV.Scanner.exe`) diff --git a/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs b/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs index 64b58f565..627135be2 100644 --- a/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs +++ b/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs @@ -1,14 +1,147 @@ -using ErsatzTV.Core; +using System.IO.Abstractions; +using ErsatzTV.Core; using ErsatzTV.Core.Interfaces.Repositories; +using ErsatzTV.Infrastructure.Data; +using ErsatzTV.Infrastructure.Images; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; namespace ErsatzTV.Application.Maintenance; -public class DeleteOrphanedArtworkHandler(IArtworkRepository artworkRepository) +public class DeleteOrphanedArtworkHandler( + IDbContextFactory dbContextFactory, + IArtworkRepository artworkRepository, + IFileSystem fileSystem, + ILogger logger) : IRequestHandler> { - public Task> - Handle(DeleteOrphanedArtwork request, CancellationToken cancellationToken) => - artworkRepository.GetOrphanedArtworkIds() - .Bind(artworkRepository.Delete) - .Map(_ => Right(Unit.Default)); + public async Task> Handle( + DeleteOrphanedArtwork request, + CancellationToken cancellationToken) + { + try + { + await CleanUpDatabase(); + await CleanUpFileSystem(cancellationToken); + + return Unit.Default; + } + catch (Exception e) + { + return BaseError.New(e.Message); + } + } + + private async Task CleanUpDatabase() + { + List ids = await artworkRepository.GetOrphanedArtworkIds(); + if (ids.Count > 0) + { + await artworkRepository.Delete(ids); + } + } + + private async Task CleanUpFileSystem(CancellationToken cancellationToken) + { + await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken); + + System.Collections.Generic.HashSet validFiles = []; + + var lastId = 0; + while (true) + { + List result = await dbContext.Artwork + .TagWithCallSite() + .AsNoTracking() + .Where(a => a.Id > lastId) + .OrderBy(a => a.Id) + .Take(1000) + .Select(a => new MinimalArtwork(a.Id, a.Path, a.BlurHash43, a.BlurHash54, a.BlurHash64)) + .ToListAsync(cancellationToken); + + if (result.Count == 0) + { + break; + } + + foreach (MinimalArtwork artwork in result) + { + if (!string.IsNullOrWhiteSpace(artwork.Path) && !artwork.Path.Contains('/')) + { + validFiles.Add(artwork.Path); + } + + if (!string.IsNullOrWhiteSpace(artwork.BlurHash43)) + { + validFiles.Add(ImageCache.GetBlurHashFileName(artwork.BlurHash43)); + } + + if (!string.IsNullOrWhiteSpace(artwork.BlurHash54)) + { + validFiles.Add(ImageCache.GetBlurHashFileName(artwork.BlurHash54)); + } + + if (!string.IsNullOrWhiteSpace(artwork.BlurHash64)) + { + validFiles.Add(ImageCache.GetBlurHashFileName(artwork.BlurHash64)); + } + } + + lastId = result.Last().Id; + } + + logger.LogDebug("Loaded {Count} artwork hashes (valid file names)", validFiles.Count); + + var deleted = 0; + foreach (string file in fileSystem.Directory.EnumerateFiles( + FileSystemLayout.ArtworkCacheFolder, + "*.*", + SearchOption.AllDirectories)) + { + string fileName = fileSystem.Path.GetFileName(file); + if (!validFiles.Contains(fileName)) + { + try + { + fileSystem.File.Delete(file); + deleted++; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Could not delete artwork file {File}", file); + } + } + } + + logger.LogDebug("Deleted {Count} unused artwork cache files", deleted); + + DeleteEmptySubfolders(FileSystemLayout.ArtworkCacheFolder); + } + + private void DeleteEmptySubfolders(string path) + { + if (!fileSystem.Directory.Exists(path)) + { + return; + } + + foreach (string sub in fileSystem.Directory.GetDirectories(path)) + { + DeleteEmptySubfolders(sub); + } + + if (!fileSystem.Directory.EnumerateFileSystemEntries(path).Any()) + { + try + { + fileSystem.Directory.Delete(path); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Could not delete empty cache folder {Folder}", path); + } + } + } + + private sealed record MinimalArtwork(int Id, string Path, string BlurHash43, string BlurHash54, string BlurHash64); } diff --git a/ErsatzTV.Core.Tests/FFmpeg/WatermarkSelectorTests.cs b/ErsatzTV.Core.Tests/FFmpeg/WatermarkSelectorTests.cs index d647152ab..685c9192e 100644 --- a/ErsatzTV.Core.Tests/FFmpeg/WatermarkSelectorTests.cs +++ b/ErsatzTV.Core.Tests/FFmpeg/WatermarkSelectorTests.cs @@ -8,6 +8,7 @@ using NSubstitute; using NUnit.Framework; using Serilog; using Shouldly; +using Testably.Abstractions.Testing; namespace ErsatzTV.Core.Tests.FFmpeg; @@ -65,8 +66,18 @@ public class WatermarkSelectorTests var loggerFactory = new LoggerFactory().AddSerilog(Log.Logger); + // watermarks should always exist; effectively ignoring filesystem checks for now + var mockFileSystem = new MockFileSystem(); + mockFileSystem.Initialize() + .WithFile("/tmp/watermark"); + + var fakeImageCache = Substitute.For(); + fakeImageCache.GetPathForImage(Arg.Any(), Arg.Is(ArtworkKind.Watermark), Arg.Any>()) + .Returns(_ => "/tmp/watermark"); + WatermarkSelector = new WatermarkSelector( - Substitute.For(), + mockFileSystem, + fakeImageCache, new DecoSelector(loggerFactory.CreateLogger()), loggerFactory.CreateLogger()); diff --git a/ErsatzTV.Core/FFmpeg/WatermarkSelector.cs b/ErsatzTV.Core/FFmpeg/WatermarkSelector.cs index 10342a9f4..6519860c6 100644 --- a/ErsatzTV.Core/FFmpeg/WatermarkSelector.cs +++ b/ErsatzTV.Core/FFmpeg/WatermarkSelector.cs @@ -1,3 +1,4 @@ +using System.IO.Abstractions; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Domain.Filler; using ErsatzTV.Core.Domain.Scheduling; @@ -8,7 +9,11 @@ using Microsoft.Extensions.Logging; namespace ErsatzTV.Core.FFmpeg; -public class WatermarkSelector(IImageCache imageCache, IDecoSelector decoSelector, ILogger logger) +public class WatermarkSelector( + IFileSystem fileSystem, + IImageCache imageCache, + IDecoSelector decoSelector, + ILogger logger) : IWatermarkSelector { public List SelectWatermarks( @@ -170,10 +175,18 @@ public class WatermarkSelector(IImageCache imageCache, IDecoSelector decoSelecto { // used for song progress overlay case ChannelWatermarkImageSource.Resource: - return new WatermarkOptions( - watermark, - Path.Combine(FileSystemLayout.ResourcesCacheFolder, watermark.Image), - Option.None); + string resourcePath = fileSystem.Path.Combine( + FileSystemLayout.ResourcesCacheFolder, + watermark.Image); + if (fileSystem.File.Exists(resourcePath)) + { + return new WatermarkOptions(watermark, resourcePath, Option.None); + } + + logger.LogWarning( + "Watermark resource no longer exists at {Path} and will be ignored", + resourcePath); + return None; case ChannelWatermarkImageSource.Custom: // bad form validation makes this possible if (string.IsNullOrWhiteSpace(watermark.Image)) @@ -190,10 +203,16 @@ public class WatermarkSelector(IImageCache imageCache, IDecoSelector decoSelecto watermark.Image, ArtworkKind.Watermark, Option.None); - return new WatermarkOptions( - watermark, - customPath, - None); + + if (fileSystem.File.Exists(customPath)) + { + return new WatermarkOptions(watermark, customPath, None); + } + + logger.LogWarning( + "Custom watermark no longer exists at {Path} and will be ignored", + customPath); + return None; case ChannelWatermarkImageSource.ChannelLogo: logger.LogDebug("Watermark will come from playout item (channel logo)"); @@ -207,7 +226,15 @@ public class WatermarkSelector(IImageCache imageCache, IDecoSelector decoSelecto : imageCache.GetPathForImage(logoArtwork.Path, ArtworkKind.Logo, Option.None); } - return new WatermarkOptions(watermark, channelPath, None); + if (fileSystem.File.Exists(channelPath)) + { + return new WatermarkOptions(watermark, channelPath, None); + } + + logger.LogWarning( + "Channel logo no longer exists at {Path} and will be ignored", + channelPath); + return None; default: throw new NotSupportedException("Unsupported watermark image source"); } @@ -225,10 +252,16 @@ public class WatermarkSelector(IImageCache imageCache, IDecoSelector decoSelecto channel.Watermark.Image, ArtworkKind.Watermark, Option.None); - return new WatermarkOptions( - channel.Watermark, - customPath, - None); + + if (fileSystem.File.Exists(customPath)) + { + return new WatermarkOptions(channel.Watermark, customPath, None); + } + + logger.LogWarning( + "Custom watermark no longer exists at {Path} and will be ignored", + customPath); + return None; case ChannelWatermarkImageSource.ChannelLogo: logger.LogDebug("Watermark will come from channel (channel logo)"); @@ -242,7 +275,15 @@ public class WatermarkSelector(IImageCache imageCache, IDecoSelector decoSelecto : imageCache.GetPathForImage(logoArtwork.Path, ArtworkKind.Logo, Option.None); } - return new WatermarkOptions(channel.Watermark, channelPath, None); + if (fileSystem.File.Exists(channelPath)) + { + return new WatermarkOptions(channel.Watermark, channelPath, None); + } + + logger.LogWarning( + "Channel logo no longer exists at {Path} and will be ignored", + channelPath); + return None; default: throw new NotSupportedException("Unsupported watermark image source"); } @@ -260,10 +301,16 @@ public class WatermarkSelector(IImageCache imageCache, IDecoSelector decoSelecto watermark.Image, ArtworkKind.Watermark, Option.None); - return new WatermarkOptions( - watermark, - customPath, - None); + + if (fileSystem.File.Exists(customPath)) + { + return new WatermarkOptions(watermark, customPath, None); + } + + logger.LogWarning( + "Custom watermark no longer exists at {Path} and will be ignored", + customPath); + return None; case ChannelWatermarkImageSource.ChannelLogo: logger.LogDebug("Watermark will come from global (channel logo)"); @@ -277,7 +324,15 @@ public class WatermarkSelector(IImageCache imageCache, IDecoSelector decoSelecto : imageCache.GetPathForImage(logoArtwork.Path, ArtworkKind.Logo, Option.None); } - return new WatermarkOptions(watermark, channelPath, None); + if (fileSystem.File.Exists(channelPath)) + { + return new WatermarkOptions(watermark, channelPath, None); + } + + logger.LogWarning( + "Channel logo no longer exists at {Path} and will be ignored", + channelPath); + return None; default: throw new NotSupportedException("Unsupported watermark image source"); } diff --git a/ErsatzTV.Infrastructure/Images/ImageCache.cs b/ErsatzTV.Infrastructure/Images/ImageCache.cs index 1c46feb57..0688f76fb 100644 --- a/ErsatzTV.Infrastructure/Images/ImageCache.cs +++ b/ErsatzTV.Infrastructure/Images/ImageCache.cs @@ -22,6 +22,12 @@ public class ImageCache(IFileSystem fileSystem, ILocalFileSystem localFileSystem static ImageCache() => Crypto = SHA1.Create(); + public static string GetBlurHashFileName(string blurHash) + { + byte[] bytes = Encoding.UTF8.GetBytes(blurHash); + return Convert.ToBase64String(bytes).Replace("+", "_").Replace("/", "-").Replace("=", ""); + } + public async Task> SaveArtworkToCache(Stream stream, ArtworkKind artworkKind) { try @@ -149,8 +155,7 @@ public class ImageCache(IFileSystem fileSystem, ILocalFileSystem localFileSystem public Task WriteBlurHash(string blurHash, IDisplaySize targetSize) { - byte[] bytes = Encoding.UTF8.GetBytes(blurHash); - string base64 = Convert.ToBase64String(bytes).Replace("+", "_").Replace("/", "-").Replace("=", ""); + string base64 = GetBlurHashFileName(blurHash); string targetFile = GetPathForImage(base64, ArtworkKind.Poster, targetSize.Height) ?? string.Empty; if (!fileSystem.File.Exists(targetFile)) { diff --git a/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs b/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs index 7e14380ba..ff8e988a4 100644 --- a/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs +++ b/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs @@ -34,6 +34,7 @@ using NSubstitute; using NUnit.Framework; using Serilog; using Shouldly; +using Testably.Abstractions; using Testably.Abstractions.Testing; using MediaStream = ErsatzTV.Core.Domain.MediaStream; @@ -236,7 +237,7 @@ public class TranscodingTests StreamingMode streamingMode) { var localFileSystem = new LocalFileSystem( - new MockFileSystem(), + new RealFileSystem(), Substitute.For(), LoggerFactory.CreateLogger()); var fileSystem = new MockFileSystem(); @@ -367,6 +368,7 @@ public class TranscodingTests DateTimeOffset now = DateTimeOffset.Now; WatermarkSelector watermarkSelector = new WatermarkSelector( + new MockFileSystem(), mockImageCache, new DecoSelector(LoggerFactory.CreateLogger()), LoggerFactory.CreateLogger()); @@ -696,6 +698,7 @@ public class TranscodingTests .Returns(Path.Combine(TestContext.CurrentContext.TestDirectory, "Resources", "ErsatzTV.png")); WatermarkSelector watermarkSelector = new WatermarkSelector( + new RealFileSystem(), mockImageCache, new DecoSelector(LoggerFactory.CreateLogger()), LoggerFactory.CreateLogger()); diff --git a/ErsatzTV/Controllers/Api/MaintenanceController.cs b/ErsatzTV/Controllers/Api/MaintenanceController.cs index 048e65728..b6009e32a 100644 --- a/ErsatzTV/Controllers/Api/MaintenanceController.cs +++ b/ErsatzTV/Controllers/Api/MaintenanceController.cs @@ -1,3 +1,5 @@ +using System.Threading.Channels; +using ErsatzTV.Application; using ErsatzTV.Application.Maintenance; using ErsatzTV.Core; using MediatR; @@ -7,7 +9,7 @@ namespace ErsatzTV.Controllers.Api; [ApiController] [EndpointGroupName("general")] -public class MaintenanceController(IMediator mediator) +public class MaintenanceController(IMediator mediator, ChannelWriter workerChannel) { [HttpGet("/api/maintenance/gc")] [Tags("Maintenance")] @@ -36,4 +38,13 @@ public class MaintenanceController(IMediator mediator) return new OkResult(); } + + [HttpPost("/api/maintenance/clean_artwork")] + [Tags("Maintenance")] + [EndpointSummary("Clean artwork cache")] + public async Task CleanArtwork(CancellationToken cancellationToken) + { + await workerChannel.WriteAsync(new DeleteOrphanedArtwork(), cancellationToken); + return new OkResult(); + } } diff --git a/ErsatzTV/Services/SchedulerService.cs b/ErsatzTV/Services/SchedulerService.cs index b278117fe..a6765c882 100644 --- a/ErsatzTV/Services/SchedulerService.cs +++ b/ErsatzTV/Services/SchedulerService.cs @@ -119,7 +119,6 @@ public class SchedulerService : BackgroundService { try { - await DeleteOrphanedArtwork(cancellationToken); await DeleteOrphanedSubtitles(cancellationToken); await RefreshMpegTsScripts(cancellationToken); await RefreshChannelGuideChannelList(cancellationToken); @@ -390,9 +389,6 @@ public class SchedulerService : BackgroundService private ValueTask RefreshGraphicsElements(CancellationToken cancellationToken) => _workerChannel.WriteAsync(new RefreshGraphicsElements(), cancellationToken); - private ValueTask DeleteOrphanedArtwork(CancellationToken cancellationToken) => - _workerChannel.WriteAsync(new DeleteOrphanedArtwork(), cancellationToken); - private ValueTask DeleteOrphanedSubtitles(CancellationToken cancellationToken) => _workerChannel.WriteAsync(new DeleteOrphanedSubtitles(), cancellationToken);