fix: heal artwork cache (#3007)

CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+### Added
+- Periodically delete unused artwork from cache folder on disk
+
 ### Fixed
 - Fix health checks causing a flood of (harmless) logged errors when quickly navigating away from home page
   - Health check results will now be cached for 5 minutes by default; a refresh button has been added to immediately re-run all checks

ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs

@@ -1,6 +1,9 @@
-using System.Globalization;
+using System.Collections.Immutable;
+using System.Globalization;
 using System.IO.Abstractions;
 using ErsatzTV.Core;
+using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.Interfaces.Images;
 using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Infrastructure.Data;
 using ErsatzTV.Infrastructure.Images;
@@ -14,9 +17,15 @@ public class DeleteOrphanedArtworkHandler(
     IDbContextFactory<TvContext> dbContextFactory,
     IArtworkRepository artworkRepository,
     IFileSystem fileSystem,
+    IImageCache imageCache,
     ILogger<DeleteOrphanedArtworkHandler> logger)
     : IRequestHandler<DeleteOrphanedArtwork, Either<BaseError, Unit>>
 {
+    private static readonly ImmutableHashSet<string> ImageFileExtensions = new[]
+    {
+        ".jpg", ".jpeg", ".png", ".gif", ".tbn", ".webp"
+    }.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase);
+
     public async Task<Either<BaseError, Unit>> Handle(
         DeleteOrphanedArtwork request,
         CancellationToken cancellationToken)
@@ -25,8 +34,9 @@ public class DeleteOrphanedArtworkHandler(
         {
             await CleanUpDatabase(request, cancellationToken);
 
-            // temporarily disabled since this is now scheduled
-            //await CleanUpFileSystem(cancellationToken);
+            System.Collections.Generic.HashSet<string> cacheFiles = await CleanUpFileSystem(cancellationToken);
+
+            await CacheMissingArtwork(cacheFiles, cancellationToken);
 
             return Unit.Default;
         }
@@ -59,15 +69,15 @@ public class DeleteOrphanedArtworkHandler(
         {
             logger.LogDebug("No orphaned artwork to delete");
         }
-
-        logger.LogDebug("Done cleaning!");
     }
 
-    private async Task CleanUpFileSystem(CancellationToken cancellationToken)
+    private async Task<System.Collections.Generic.HashSet<string>> CleanUpFileSystem(
+        CancellationToken cancellationToken)
     {
         await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);
 
         System.Collections.Generic.HashSet<string> validFiles = [];
+        System.Collections.Generic.HashSet<string> cacheFiles = [];
 
         List<string> watermarks = await dbContext.ChannelWatermarks
             .TagWithCallSite()
@@ -147,14 +157,27 @@ public class DeleteOrphanedArtworkHandler(
                     logger.LogWarning(ex, "Could not delete artwork file {File}", file);
                 }
             }
+            else
+            {
+                cacheFiles.Add(fileName);
+            }
         }
 
-        logger.LogDebug(
-            "Deleted {Count} unused artwork cache files totaling {Size}",
-            deleted,
-            bytes.Bytes().Humanize(CultureInfo.CurrentCulture));
+        if (deleted > 0)
+        {
+            logger.LogDebug(
+                "Deleted {Count} unused artwork cache files totaling {Size}",
+                deleted,
+                bytes.Bytes().Humanize(CultureInfo.CurrentCulture));
+        }
+        else
+        {
+            logger.LogDebug("No unused artwork cache files to delete");
+        }
 
         DeleteEmptySubfolders(FileSystemLayout.ArtworkCacheFolder);
+
+        return cacheFiles;
     }
 
     private void DeleteEmptySubfolders(string path)
@@ -190,5 +213,121 @@ public class DeleteOrphanedArtworkHandler(
         }
     }
 
-    private sealed record MinimalArtwork(int Id, string Path, string BlurHash43, string BlurHash54, string BlurHash64);
+    private async Task CacheMissingArtwork(
+        System.Collections.Generic.HashSet<string> cacheFiles,
+        CancellationToken cancellationToken)
+    {
+        await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);
+
+        // artwork is deduplicated by source, so one cache file can serve many rows
+        System.Collections.Generic.HashSet<string> restoredPaths = [];
+
+        var restored = 0;
+        var unrestorable = 0;
+
+        var lastId = 0;
+        while (true)
+        {
+            List<MissingArtwork> result = await dbContext.Artwork
+                .TagWithCallSite()
+                .AsNoTracking()
+                .Where(a => a.Id > lastId)
+                .OrderBy(a => a.Id)
+                .Take(1000)
+                .Select(a => new MissingArtwork(a.Id, a.Path, a.ArtworkKind, a.SourcePath))
+                .ToListAsync(cancellationToken);
+
+            if (result.Count == 0)
+            {
+                break;
+            }
+
+            foreach (MissingArtwork artwork in result)
+            {
+                if (string.IsNullOrWhiteSpace(artwork.Path) || artwork.Path.Contains('/'))
+                {
+                    continue;
+                }
+
+                if (cacheFiles.Contains(artwork.Path) || restoredPaths.Contains(artwork.Path))
+                {
+                    continue;
+                }
+
+                // the source is a remote key for media server artwork, and the media file itself
+                // for embedded cover art; neither one is an image to copy into the cache
+                if (string.IsNullOrWhiteSpace(artwork.SourcePath) || !IsImageFile(artwork.SourcePath) ||
+                    !fileSystem.File.Exists(artwork.SourcePath))
+                {
+                    unrestorable++;
+                    continue;
+                }
+
+                logger.LogDebug(
+                    "Restoring missing artwork {Path} from source {SourcePath}",
+                    artwork.Path,
+                    artwork.SourcePath);
+
+                Either<BaseError, string> copyResult =
+                    await imageCache.CopyArtworkToCache(artwork.SourcePath, artwork.Kind);
+
+                foreach (BaseError error in copyResult.LeftToSeq())
+                {
+                    logger.LogWarning(
+                        "Failed to restore artwork {Path} from source {SourcePath}: {Error}",
+                        artwork.Path,
+                        artwork.SourcePath,
+                        error.Value);
+                }
+
+                foreach (string cacheName in copyResult.RightToSeq())
+                {
+                    restored++;
+                    restoredPaths.Add(artwork.Path);
+                    cacheFiles.Add(cacheName);
+
+                    // the cache file name comes from the source path and its write time; rows
+                    // written from an ffmpeg-converted copy of the source have a different name
+                    if (!string.Equals(cacheName, artwork.Path, StringComparison.Ordinal))
+                    {
+                        string stalePath = artwork.Path;
+                        int repointed = await dbContext.Artwork
+                            .Where(a => a.Path == stalePath)
+                            .ExecuteUpdateAsync(
+                                s => s.SetProperty(a => a.Path, cacheName),
+                                cancellationToken);
+
+                        logger.LogDebug(
+                            "Repointed {Count} artwork rows from {StalePath} to {Path}",
+                            repointed,
+                            stalePath,
+                            cacheName);
+                    }
+                }
+            }
+
+            lastId = result.Last().Id;
+        }
+
+        if (restored > 0)
+        {
+            logger.LogDebug("Restored {Count} missing artwork files to the cache", restored);
+        }
+
+        if (unrestorable > 0)
+        {
+            logger.LogDebug("{Count} missing artwork files have no local source to restore from", unrestorable);
+        }
+    }
+
+    private bool IsImageFile(string path) => ImageFileExtensions.Contains(fileSystem.Path.GetExtension(path));
+
+    private sealed record MinimalArtwork(
+        int Id,
+        string Path,
+        string BlurHash43,
+        string BlurHash54,
+        string BlurHash64);
+
+    private sealed record MissingArtwork(int Id, string Path, ArtworkKind Kind, string SourcePath);
 }

ErsatzTV.Core/Interfaces/Images/IImageCache.cs

@@ -7,6 +7,7 @@ public interface IImageCache
 {
     Task<Either<BaseError, string>> SaveArtworkToCache(Stream stream, ArtworkKind artworkKind);
     Task<Either<BaseError, string>> CopyArtworkToCache(string path, ArtworkKind artworkKind);
+    bool IsCached(string path, ArtworkKind artworkKind);
     string GetPathForImage(string fileName, ArtworkKind artworkKind, Option<int> maybeMaxHeight);
     Task<string> CalculateBlurHash(string fileName, ArtworkKind artworkKind, int x, int y);
     Task<string> WriteBlurHash(string blurHash, IDisplaySize targetSize);

ErsatzTV.Infrastructure/Images/ImageCache.cs

@@ -99,6 +99,12 @@ public class ImageCache(IFileSystem fileSystem, ILocalFileSystem localFileSystem
         }
     }
 
+    public bool IsCached(string path, ArtworkKind artworkKind)
+    {
+        string finalPath = GetPathForImage(path, artworkKind, Option<int>.None);
+        return fileSystem.File.Exists(finalPath);
+    }
+
     public virtual string GetPathForImage(string fileName, ArtworkKind artworkKind, Option<int> maybeMaxHeight)
     {
         string subfolder = maybeMaxHeight.Match(

ErsatzTV.Scanner/Core/Metadata/LocalFolderScanner.cs

@@ -139,12 +139,14 @@ public abstract class LocalFolderScanner
     {
         DateTime lastWriteTime = _fileSystem.File.GetLastWriteTime(artworkFile);
 
-        metadata.Artwork ??= new List<Artwork>();
+        metadata.Artwork ??= [];
 
         Option<Artwork> maybeArtwork = metadata.Artwork.FirstOrDefault(a => a.ArtworkKind == artworkKind);
 
+        bool cacheMissing = maybeArtwork.Match(a => !_imageCache.IsCached(a.Path, artworkKind), false);
+
         bool shouldRefresh = maybeArtwork.Match(
-            artwork => lastWriteTime.Subtract(artwork.DateUpdated) > TimeSpan.FromSeconds(1),
+            artwork => cacheMissing || lastWriteTime.Subtract(artwork.DateUpdated) > TimeSpan.FromSeconds(1),
             true);
 
         if (shouldRefresh)
@@ -154,7 +156,7 @@ public abstract class LocalFolderScanner
                 _logger.LogDebug("Refreshing {Attribute} from {Path}", artworkKind, artworkFile);
 
                 string sourcePath = artworkFile;
-                if (await _metadataRepository.CloneArtwork(
+                if (!cacheMissing && await _metadataRepository.CloneArtwork(
                         metadata,
                         maybeArtwork,
                         artworkKind,