properly restore all local library items from trash during scans (#1660)

CHANGELOG.md

@@ -55,6 +55,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix bug where replacing files in Plex would be missed by subsequent ETV library scans
   - This fix will require a one-time re-scan of each Plex library in full
   - After the initial full scan, incremental scans will behave as normal
+- Fix edge case where some local episodes, music videos, other videos, songs, images would not automatically be restored from trash
 
 ### Changed
 - Log search index updates under scanner category at debug level, to indicate a potential cause for the UI being out of date

ErsatzTV.Core/Interfaces/Repositories/ILibraryRepository.cs

@@ -18,5 +18,4 @@ public interface ILibraryRepository
     Task<LibraryFolder> GetOrAddFolder(LibraryPath libraryPath, Option<int> maybeParentFolder, string folder);
     Task UpdateLibraryFolderId(MediaFile mediaFile, int libraryFolderId);
     Task UpdatePath(LibraryPath libraryPath, string normalizedLibraryPath);
-    Task<System.Collections.Generic.HashSet<string>> FindAllMissingFiles(LibraryPath libraryPath);
 }

ErsatzTV.Infrastructure/Data/Repositories/LibraryRepository.cs

@@ -1,6 +1,5 @@
 using Dapper;
 using ErsatzTV.Core.Domain;
-using ErsatzTV.Core.Extensions;
 using ErsatzTV.Core.Interfaces.Metadata;
 using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Infrastructure.Extensions;
@@ -212,41 +211,6 @@ public class LibraryRepository : ILibraryRepository
             new { Path = normalizedLibraryPath, libraryPath.Id });
     }
 
-    public async Task<System.Collections.Generic.HashSet<string>> FindAllMissingFiles(LibraryPath libraryPath)
-    {
-        var result = new System.Collections.Generic.HashSet<string>();
-
-        await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync();
-        IAsyncEnumerable<MediaItem> items = dbContext.MediaItems
-            .AsNoTracking()
-            .Filter(mi => mi.LibraryPathId == libraryPath.Id)
-            .Filter(mi => mi.State == MediaItemState.FileNotFound)
-            .Include(mi => (mi as Movie).MediaVersions)
-            .ThenInclude(mv => mv.MediaFiles)
-            .Include(mi => (mi as Episode).MediaVersions)
-            .ThenInclude(mv => mv.MediaFiles)
-            .Include(mi => (mi as Song).MediaVersions)
-            .ThenInclude(mv => mv.MediaFiles)
-            .Include(mi => (mi as MusicVideo).MediaVersions)
-            .ThenInclude(mv => mv.MediaFiles)
-            .Include(mi => (mi as OtherVideo).MediaVersions)
-            .ThenInclude(mv => mv.MediaFiles)
-            .Include(mi => (mi as Image).MediaVersions)
-            .ThenInclude(mv => mv.MediaFiles)
-            .AsAsyncEnumerable();
-
-        await foreach (MediaItem item in items)
-        {
-            MediaVersion version = item.GetHeadVersion();
-            foreach (MediaFile file in version.MediaFiles)
-            {
-                result.Add(file.Path);
-            }
-        }
-
-        return result;
-    }
-
     private static LibraryFolder CreateNewFolder(LibraryPath libraryPath, Option<int> maybeParentFolder, string folder)
     {
         int? parentId = null;

ErsatzTV.Infrastructure/Data/Repositories/MediaItemRepository.cs

@@ -91,7 +91,7 @@ public class MediaItemRepository : IMediaItemRepository
         return await dbContext.Connection.QueryAsync<string>(
                 @"SELECT MF.Path
                 FROM MediaItem M
-                INNER JOIN MediaVersion MV on M.Id = COALESCE(MovieId, MusicVideoId, OtherVideoId, SongId, EpisodeId)
+                INNER JOIN MediaVersion MV on M.Id = COALESCE(MovieId, MusicVideoId, OtherVideoId, SongId, EpisodeId, ImageId)
                 INNER JOIN MediaFile MF on MV.Id = MF.MediaVersionId
                 WHERE M.State IN (1,2) AND M.LibraryPathId = @LibraryPathId",
                 new { LibraryPathId = libraryPath.Id })

ErsatzTV.Scanner/Core/Metadata/ImageFolderScanner.cs

@@ -1,4 +1,5 @@
-using Bugsnag;
+using System.Collections.Immutable;
+using Bugsnag;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Errors;
@@ -20,6 +21,7 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner
     private readonly IClient _client;
     private readonly IImageRepository _imageRepository;
     private readonly ILibraryRepository _libraryRepository;
+    private readonly IMediaItemRepository _mediaItemRepository;
     private readonly ILocalFileSystem _localFileSystem;
     private readonly ILocalMetadataProvider _localMetadataProvider;
     private readonly ILogger<ImageFolderScanner> _logger;
@@ -54,6 +56,7 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner
         _mediator = mediator;
         _imageRepository = imageRepository;
         _libraryRepository = libraryRepository;
+        _mediaItemRepository = mediaItemRepository;
         _client = client;
         _logger = logger;
     }
@@ -83,6 +86,8 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner
                 await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath);
             }
 
+            ImmutableHashSet<string> allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath);
+
             if (ShouldIncludeFolder(libraryPath.Path) && allFolders.Add(libraryPath.Path))
             {
                 folderQueue.Enqueue(libraryPath.Path);
@@ -139,10 +144,23 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner
                     maybeParentFolder,
                     imageFolder);
 
-                // skip folder if etag matches
-                if (allFiles.Count == 0 || knownFolder.Etag == etag)
+                if (knownFolder.Etag == etag)
+                {
+                    if (allFiles.Any(allTrashedItems.Contains))
+                    {
+                        _logger.LogDebug("Previously trashed items are now present in folder {Folder}", imageFolder);
+                    }
+                    else
+                    {
+                        // etag matches and no trashed items are now present, continue to next folder
+                        continue;
+                    }
+                }
+                else
                 {
-                    continue;
+                    _logger.LogDebug(
+                        "UPDATE: Etag has changed for folder {Folder}",
+                        imageFolder);
                 }
 
                 // walk up to get duration, if needed

ErsatzTV.Scanner/Core/Metadata/MusicVideoFolderScanner.cs

@@ -1,4 +1,5 @@
-using Bugsnag;
+using System.Collections.Immutable;
+using Bugsnag;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Errors;
@@ -20,6 +21,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
     private readonly IArtistRepository _artistRepository;
     private readonly IClient _client;
     private readonly ILibraryRepository _libraryRepository;
+    private readonly IMediaItemRepository _mediaItemRepository;
     private readonly ILocalFileSystem _localFileSystem;
     private readonly ILocalMetadataProvider _localMetadataProvider;
     private readonly ILocalSubtitlesProvider _localSubtitlesProvider;
@@ -59,6 +61,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
         _artistRepository = artistRepository;
         _musicVideoRepository = musicVideoRepository;
         _libraryRepository = libraryRepository;
+        _mediaItemRepository = mediaItemRepository;
         _mediator = mediator;
         _client = client;
         _logger = logger;
@@ -84,6 +87,8 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
                 await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath);
             }
 
+            ImmutableHashSet<string> allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath);
+
             var allArtistFolders = _localFileSystem.ListSubdirectories(libraryPath.Path)
                 .Filter(ShouldIncludeFolder)
                 .OrderBy(identity)
@@ -151,6 +156,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
                         ffprobePath,
                         result.Item,
                         artistFolder,
+                        allTrashedItems,
                         cancellationToken);
 
                     foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
@@ -312,6 +318,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
         string ffprobePath,
         Artist artist,
         string artistFolder,
+        ImmutableHashSet<string> allTrashedItems,
         CancellationToken cancellationToken)
     {
         var folderQueue = new Queue<string>();
@@ -345,10 +352,23 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
                 maybeParentFolder,
                 musicVideoFolder);
 
-            // skip folder if etag matches
             if (knownFolder.Etag == etag)
             {
-                continue;
+                if (allFiles.Any(allTrashedItems.Contains))
+                {
+                    _logger.LogDebug("Previously trashed items are now present in folder {Folder}", musicVideoFolder);
+                }
+                else
+                {
+                    // etag matches and no trashed items are now present, continue to next folder
+                    continue;
+                }
+            }
+            else
+            {
+                _logger.LogDebug(
+                    "UPDATE: Etag has changed for folder {Folder}",
+                    musicVideoFolder);
             }
 
             var hasErrors = false;

ErsatzTV.Scanner/Core/Metadata/OtherVideoFolderScanner.cs

@@ -1,4 +1,5 @@
-using Bugsnag;
+using System.Collections.Immutable;
+using Bugsnag;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Errors;
@@ -19,6 +20,7 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan
 {
     private readonly IClient _client;
     private readonly ILibraryRepository _libraryRepository;
+    private readonly IMediaItemRepository _mediaItemRepository;
     private readonly ILocalFileSystem _localFileSystem;
     private readonly ILocalMetadataProvider _localMetadataProvider;
     private readonly ILocalSubtitlesProvider _localSubtitlesProvider;
@@ -57,6 +59,7 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan
         _mediator = mediator;
         _otherVideoRepository = otherVideoRepository;
         _libraryRepository = libraryRepository;
+        _mediaItemRepository = mediaItemRepository;
         _client = client;
         _logger = logger;
     }
@@ -78,16 +81,7 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan
             var allFolders = new System.Collections.Generic.HashSet<string>();
             var folderQueue = new Queue<string>();
 
-            System.Collections.Generic.HashSet<string> allMissingFiles =
-                await _libraryRepository.FindAllMissingFiles(libraryPath);
-
-            if (allMissingFiles.Count > 0)
-            {
-                _logger.LogDebug(
-                    "Library path {Path} has {Count} missing files",
-                    libraryPath.Path,
-                    allMissingFiles.Count);
-            }
+            ImmutableHashSet<string> allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath);
 
             string normalizedLibraryPath = libraryPath.Path.TrimEnd(
                 Path.DirectorySeparatorChar,
@@ -160,35 +154,25 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan
                     maybeParentFolder,
                     otherVideoFolder);
 
-                bool hasMissingFiles = allFiles.Any(allMissingFiles.Contains);
-                bool isSameEtag = !hasMissingFiles && knownFolder.Etag == etag;
-
-                _logger.LogDebug(
-                    "Scanning other video folder {Folder}; file count: {Count}, etag: {LastEtag} => {Etag}; has missing files: {HasMissingFiles}",
-                    otherVideoFolder,
-                    allFiles.Count,
-                    etag,
-                    knownFolder.Etag,
-                    hasMissingFiles);
-
-                // skip empty folder
-                if (allFiles.Count == 0)
+                if (knownFolder.Etag == etag)
                 {
-                    _logger.LogDebug("Skipping empty other videos folder");
-                    continue;
+                    if (allFiles.Any(allTrashedItems.Contains))
+                    {
+                        _logger.LogDebug("Previously trashed items are now present in folder {Folder}", otherVideoFolder);
+                    }
+                    else
+                    {
+                        // etag matches and no trashed items are now present, continue to next folder
+                        continue;
+                    }
                 }
-                
-                // skip folder if etag matches
-                if (isSameEtag)
+                else
                 {
-                    _logger.LogDebug("Skipping unchanged other videos folder, that contains no missing items");
-                    continue;
+                    _logger.LogDebug(
+                        "UPDATE: Etag has changed for folder {Folder}",
+                        otherVideoFolder);
                 }
 
-                _logger.LogDebug(
-                    "UPDATE: Etag has changed for folder {Folder}",
-                    otherVideoFolder);
-
                 var hasErrors = false;
 
                 foreach (string file in allFiles.OrderBy(identity))

ErsatzTV.Scanner/Core/Metadata/SongFolderScanner.cs

@@ -1,4 +1,5 @@
-using Bugsnag;
+using System.Collections.Immutable;
+using Bugsnag;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Errors;
@@ -19,6 +20,7 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner
 {
     private readonly IClient _client;
     private readonly ILibraryRepository _libraryRepository;
+    private readonly IMediaItemRepository _mediaItemRepository;
     private readonly ILocalFileSystem _localFileSystem;
     private readonly ILocalMetadataProvider _localMetadataProvider;
     private readonly ILogger<SongFolderScanner> _logger;
@@ -54,6 +56,7 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner
         _mediator = mediator;
         _songRepository = songRepository;
         _libraryRepository = libraryRepository;
+        _mediaItemRepository = mediaItemRepository;
         _client = client;
         _logger = logger;
     }
@@ -82,6 +85,8 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner
                 await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath);
             }
 
+            ImmutableHashSet<string> allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath);
+
             if (ShouldIncludeFolder(libraryPath.Path))
             {
                 folderQueue.Enqueue(libraryPath.Path);
@@ -135,11 +140,24 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner
                     libraryPath,
                     maybeParentFolder,
                     songFolder);
-
-                // skip folder if etag matches
-                if (allFiles.Count == 0 || knownFolder.Etag == etag)
+                
+                if (knownFolder.Etag == etag)
+                {
+                    if (allFiles.Any(allTrashedItems.Contains))
+                    {
+                        _logger.LogDebug("Previously trashed items are now present in folder {Folder}", songFolder);
+                    }
+                    else
+                    {
+                        // etag matches and no trashed items are now present, continue to next folder
+                        continue;
+                    }
+                }
+                else
                 {
-                    continue;
+                    _logger.LogDebug(
+                        "UPDATE: Etag has changed for folder {Folder}",
+                        songFolder);
                 }
 
                 _logger.LogDebug(

ErsatzTV.Scanner/Core/Metadata/TelevisionFolderScanner.cs

@@ -1,4 +1,5 @@
-using Bugsnag;
+using System.Collections.Immutable;
+using Bugsnag;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Errors;
@@ -20,6 +21,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
     private readonly IClient _client;
     private readonly IFallbackMetadataProvider _fallbackMetadataProvider;
     private readonly ILibraryRepository _libraryRepository;
+    private readonly IMediaItemRepository _mediaItemRepository;
     private readonly ILocalFileSystem _localFileSystem;
     private readonly ILocalMetadataProvider _localMetadataProvider;
     private readonly ILocalSubtitlesProvider _localSubtitlesProvider;
@@ -60,6 +62,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
         _localSubtitlesProvider = localSubtitlesProvider;
         _metadataRepository = metadataRepository;
         _libraryRepository = libraryRepository;
+        _mediaItemRepository = mediaItemRepository;
         _mediator = mediator;
         _client = client;
         _fallbackMetadataProvider = fallbackMetadataProvider;
@@ -85,6 +88,8 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
             {
                 await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath);
             }
+            
+            ImmutableHashSet<string> allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath);
 
             var allShowFolders = _localFileSystem.ListSubdirectories(libraryPath.Path)
                 .Filter(ShouldIncludeFolder)
@@ -153,6 +158,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
                         ffprobePath,
                         result.Item,
                         showFolder,
+                        allTrashedItems,
                         cancellationToken);
 
                     foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
@@ -227,6 +233,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
         string ffprobePath,
         Show show,
         string showFolder,
+        ImmutableHashSet<string> allTrashedItems,
         CancellationToken cancellationToken)
     {
         foreach (string seasonFolder in _localFileSystem.ListSubdirectories(showFolder).Filter(ShouldIncludeFolder)
@@ -248,7 +255,15 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
             // skip folder if etag matches
             if (knownFolder.Etag == etag)
             {
-                continue;
+                if (allTrashedItems.Any(f => f.StartsWith(seasonFolder, StringComparison.OrdinalIgnoreCase)))
+                {
+                    _logger.LogDebug("Previously trashed items are now present in folder {Folder}", seasonFolder);
+                }
+                else
+                {
+                    // etag matches and no trashed items are now present, continue to next folder
+                    continue;
+                }
             }
 
             Option<int> maybeSeasonNumber = _fallbackMetadataProvider.GetSeasonNumberForFolder(seasonFolder);