optimize local library scanning by using etags (#196)

* use etags to optimize local movie scanner

* use etags to optimize local television scanner

* use etags to optimize local music video scanner

* code cleanup

ErsatzTV.Application/MediaSources/Commands/ScanLocalLibraryHandler.cs

@@ -90,7 +90,6 @@ namespace ErsatzTV.Application.MediaSources.Commands
                             await _movieFolderScanner.ScanFolder(
                                 libraryPath,
                                 ffprobePath,
-                                lastScan,
                                 progressMin,
                                 progressMax);
                             break;
@@ -98,7 +97,6 @@ namespace ErsatzTV.Application.MediaSources.Commands
                             await _televisionFolderScanner.ScanFolder(
                                 libraryPath,
                                 ffprobePath,
-                                lastScan,
                                 progressMin,
                                 progressMax);
                             break;
@@ -106,7 +104,6 @@ namespace ErsatzTV.Application.MediaSources.Commands
                             await _musicVideoFolderScanner.ScanFolder(
                                 libraryPath,
                                 ffprobePath,
-                                lastScan,
                                 progressMin,
                                 progressMax);
                             break;

ErsatzTV.Core.Tests/Metadata/MovieFolderScannerTests.cs

@@ -81,12 +81,11 @@ namespace ErsatzTV.Core.Tests.Metadata
                 MovieFolderScanner service = GetService(
                     new FakeFileEntry(Path.Combine(FakeRoot, Path.Combine("Movie (2020)", "Movie (2020).mkv")))
                 );
-                var libraryPath = new LibraryPath { Path = BadFakeRoot };
+                var libraryPath = new LibraryPath { Path = BadFakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -106,12 +105,12 @@ namespace ErsatzTV.Core.Tests.Metadata
                 MovieFolderScanner service = GetService(
                     new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now }
                 );
-                var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
+                var libraryPath = new LibraryPath
+                    { Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -147,12 +146,12 @@ namespace ErsatzTV.Core.Tests.Metadata
                     new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now },
                     new FakeFileEntry(metadataPath)
                 );
-                var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
+                var libraryPath = new LibraryPath
+                    { Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -189,12 +188,12 @@ namespace ErsatzTV.Core.Tests.Metadata
                     new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now },
                     new FakeFileEntry(metadataPath)
                 );
-                var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
+                var libraryPath = new LibraryPath
+                    { Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -235,12 +234,12 @@ namespace ErsatzTV.Core.Tests.Metadata
                     new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now },
                     new FakeFileEntry(posterPath) { LastWriteTime = DateTime.Now }
                 );
-                var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
+                var libraryPath = new LibraryPath
+                    { Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -284,12 +283,12 @@ namespace ErsatzTV.Core.Tests.Metadata
                     new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now },
                     new FakeFileEntry(posterPath) { LastWriteTime = DateTime.Now }
                 );
-                var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
+                var libraryPath = new LibraryPath
+                    { Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -333,12 +332,12 @@ namespace ErsatzTV.Core.Tests.Metadata
                     new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now },
                     new FakeFileEntry(posterPath) { LastWriteTime = DateTime.Now }
                 );
-                var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
+                var libraryPath = new LibraryPath
+                    { Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -381,12 +380,12 @@ namespace ErsatzTV.Core.Tests.Metadata
                             Path.GetDirectoryName(moviePath) ?? string.Empty,
                             $"Movie (2020)-{extraFile}{videoExtension}"))
                 );
-                var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
+                var libraryPath = new LibraryPath
+                    { Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -425,12 +424,12 @@ namespace ErsatzTV.Core.Tests.Metadata
                             Path.GetDirectoryName(moviePath) ?? string.Empty,
                             Path.Combine(extraFolder, $"Movie (2020){videoExtension}")))
                 );
-                var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
+                var libraryPath = new LibraryPath
+                    { Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -463,12 +462,12 @@ namespace ErsatzTV.Core.Tests.Metadata
                 MovieFolderScanner service = GetService(
                     new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now }
                 );
-                var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
+                var libraryPath = new LibraryPath
+                    { Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -503,12 +502,12 @@ namespace ErsatzTV.Core.Tests.Metadata
                 MovieFolderScanner service = GetService(
                     new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now }
                 );
-                var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
+                var libraryPath = new LibraryPath
+                    { Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -532,12 +531,12 @@ namespace ErsatzTV.Core.Tests.Metadata
                 MovieFolderScanner service = GetService(
                     new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now }
                 );
-                var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
+                var libraryPath = new LibraryPath
+                    { Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
 
                 Either<BaseError, Unit> result = await service.ScanFolder(
                     libraryPath,
                     FFprobePath,
-                    DateTimeOffset.MinValue,
                     0,
                     1);
 
@@ -558,6 +557,7 @@ namespace ErsatzTV.Core.Tests.Metadata
                     _imageCache.Object,
                     new Mock<ISearchIndex>().Object,
                     new Mock<ISearchRepository>().Object,
+                    new Mock<ILibraryRepository>().Object,
                     new Mock<IMediator>().Object,
                     new Mock<ILogger<MovieFolderScanner>>().Object
                 );

ErsatzTV.Core/Domain/Library/LibraryFolder.cs

@@ -0,0 +1,11 @@
+namespace ErsatzTV.Core.Domain
+{
+    public class LibraryFolder
+    {
+        public int Id { get; set; }
+        public string Path { get; set; }
+        public int LibraryPathId { get; set; }
+        public LibraryPath LibraryPath { get; set; }
+        public string Etag { get; set; }
+    }
+}

ErsatzTV.Core/Domain/Library/LibraryPath.cs

@@ -13,5 +13,6 @@ namespace ErsatzTV.Core.Domain
         public Library Library { get; set; }
 
         public List<MediaItem> MediaItems { get; set; }
+        public List<LibraryFolder> LibraryFolders { get; set; }
     }
 }

ErsatzTV.Core/Interfaces/Metadata/IMovieFolderScanner.cs

@@ -1,5 +1,4 @@
-using System;
+using System.Threading.Tasks;
-using System.Threading.Tasks;
 using ErsatzTV.Core.Domain;
 using LanguageExt;
 
@@ -10,7 +9,6 @@ namespace ErsatzTV.Core.Interfaces.Metadata
         Task<Either<BaseError, Unit>> ScanFolder(
             LibraryPath libraryPath,
             string ffprobePath,
-            DateTimeOffset lastScan,
             decimal progressMin,
             decimal progressMax);
     }

ErsatzTV.Core/Interfaces/Metadata/IMusicVideoFolderScanner.cs

@@ -1,5 +1,4 @@
-using System;
+using System.Threading.Tasks;
-using System.Threading.Tasks;
 using ErsatzTV.Core.Domain;
 using LanguageExt;
 
@@ -10,7 +9,6 @@ namespace ErsatzTV.Core.Interfaces.Metadata
         Task<Either<BaseError, Unit>> ScanFolder(
             LibraryPath libraryPath,
             string ffprobePath,
-            DateTimeOffset lastScan,
             decimal progressMin,
             decimal progressMax);
     }

ErsatzTV.Core/Interfaces/Metadata/ITelevisionFolderScanner.cs

@@ -1,5 +1,4 @@
-using System;
+using System.Threading.Tasks;
-using System.Threading.Tasks;
 using ErsatzTV.Core.Domain;
 using LanguageExt;
 
@@ -10,7 +9,6 @@ namespace ErsatzTV.Core.Interfaces.Metadata
         Task<Either<BaseError, Unit>> ScanFolder(
             LibraryPath libraryPath,
             string ffprobePath,
-            DateTimeOffset lastScan,
             decimal progressMin,
             decimal progressMax);
     }

ErsatzTV.Core/Interfaces/Repositories/ILibraryRepository.cs

@@ -18,5 +18,6 @@ namespace ErsatzTV.Core.Interfaces.Repositories
         Task<int> CountMediaItemsByPath(int libraryPathId);
         Task<List<int>> GetMediaIdsByLocalPath(int libraryPathId);
         Task DeleteLocalPath(int libraryPathId);
+        Task<Unit> SetEtag(LibraryPath libraryPath, Option<LibraryFolder> knownFolder, string path, string etag);
     }
 }

ErsatzTV.Core/Metadata/FolderEtag.cs

@@ -0,0 +1,35 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+using ErsatzTV.Core.Interfaces.Metadata;
+using static LanguageExt.Prelude;
+
+namespace ErsatzTV.Core.Metadata
+{
+    public static class FolderEtag
+    {
+        private static readonly MD5CryptoServiceProvider Crypto = new();
+
+        public static string Calculate(string folder, ILocalFileSystem localFileSystem)
+        {
+            IEnumerable<string> allFiles = localFileSystem.ListFiles(folder);
+
+            var sb = new StringBuilder();
+            foreach (string file in allFiles.OrderBy(identity))
+            {
+                sb.Append(file);
+                sb.Append(localFileSystem.GetLastWriteTime(file).Ticks);
+            }
+
+            var hash = new StringBuilder();
+            byte[] bytes = Crypto.ComputeHash(Encoding.UTF8.GetBytes(sb.ToString()));
+            foreach (byte t in bytes)
+            {
+                hash.Append(t.ToString("x2"));
+            }
+
+            return hash.ToString();
+        }
+    }
+}

ErsatzTV.Core/Metadata/LocalFolderScanner.cs

@@ -78,7 +78,7 @@ namespace ErsatzTV.Core.Metadata
 
                 string path = version.MediaFiles.Head().Path;
 
-                if (version.DateUpdated < _localFileSystem.GetLastWriteTime(path) || !version.Streams.Any())
+                if (version.DateUpdated != _localFileSystem.GetLastWriteTime(path) || !version.Streams.Any())
                 {
                     _logger.LogDebug("Refreshing {Attribute} for {Path}", "Statistics", path);
                     Either<BaseError, bool> refreshResult =

ErsatzTV.Core/Metadata/MovieFolderScanner.cs

@@ -20,6 +20,7 @@ namespace ErsatzTV.Core.Metadata
 {
     public class MovieFolderScanner : LocalFolderScanner, IMovieFolderScanner
     {
+        private readonly ILibraryRepository _libraryRepository;
         private readonly ILocalFileSystem _localFileSystem;
         private readonly ILocalMetadataProvider _localMetadataProvider;
         private readonly ILogger<MovieFolderScanner> _logger;
@@ -37,6 +38,7 @@ namespace ErsatzTV.Core.Metadata
             IImageCache imageCache,
             ISearchIndex searchIndex,
             ISearchRepository searchRepository,
+            ILibraryRepository libraryRepository,
             IMediator mediator,
             ILogger<MovieFolderScanner> logger)
             : base(localFileSystem, localStatisticsProvider, metadataRepository, imageCache, logger)
@@ -46,6 +48,7 @@ namespace ErsatzTV.Core.Metadata
             _localMetadataProvider = localMetadataProvider;
             _searchIndex = searchIndex;
             _searchRepository = searchRepository;
+            _libraryRepository = libraryRepository;
             _mediator = mediator;
             _logger = logger;
         }
@@ -53,7 +56,6 @@ namespace ErsatzTV.Core.Metadata
         public async Task<Either<BaseError, Unit>> ScanFolder(
             LibraryPath libraryPath,
             string ffprobePath,
-            DateTimeOffset lastScan,
             decimal progressMin,
             decimal progressMax)
         {
@@ -81,7 +83,9 @@ namespace ErsatzTV.Core.Metadata
                 string movieFolder = folderQueue.Dequeue();
                 foldersCompleted++;
 
-                var allFiles = _localFileSystem.ListFiles(movieFolder)
+                var filesForEtag = _localFileSystem.ListFiles(movieFolder).ToList();
+
+                var allFiles = filesForEtag
                     .Filter(f => VideoFileExtensions.Contains(Path.GetExtension(f)))
                     .Filter(
                         f => !ExtraFiles.Any(
@@ -98,11 +102,21 @@ namespace ErsatzTV.Core.Metadata
                     continue;
                 }
 
-                if (allFiles.All(file => _localFileSystem.GetLastWriteTime(file) < lastScan))
+                string etag = FolderEtag.Calculate(movieFolder, _localFileSystem);
+                Option<LibraryFolder> knownFolder = libraryPath.LibraryFolders
+                    .Filter(f => f.Path == movieFolder)
+                    .HeadOrNone();
+
+                // skip folder if etag matches
+                if (await knownFolder.Map(f => f.Etag).IfNoneAsync(string.Empty) == etag)
                 {
                     continue;
                 }
 
+                _logger.LogDebug(
+                    "UPDATE: Etag has changed for folder {Folder}",
+                    movieFolder);
+
                 foreach (string file in allFiles.OrderBy(identity))
                 {
                     // TODO: figure out how to rebuild playlists
@@ -124,6 +138,8 @@ namespace ErsatzTV.Core.Metadata
                             {
                                 await _searchIndex.UpdateItems(_searchRepository, new List<MediaItem> { result.Item });
                             }
+
+                            await _libraryRepository.SetEtag(libraryPath, knownFolder, movieFolder, etag);
                         },
                         error =>
                         {
@@ -158,7 +174,7 @@ namespace ErsatzTV.Core.Metadata
                     {
                         bool shouldUpdate = Optional(movie.MovieMetadata).Flatten().HeadOrNone().Match(
                             m => m.MetadataKind == MetadataKind.Fallback ||
-                                 m.DateUpdated < _localFileSystem.GetLastWriteTime(nfoFile),
+                                 m.DateUpdated != _localFileSystem.GetLastWriteTime(nfoFile),
                             true);
 
                         if (shouldUpdate)

ErsatzTV.Core/Metadata/MusicVideoFolderScanner.cs

@@ -20,6 +20,7 @@ namespace ErsatzTV.Core.Metadata
     public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScanner
     {
         private readonly IArtistRepository _artistRepository;
+        private readonly ILibraryRepository _libraryRepository;
         private readonly ILocalFileSystem _localFileSystem;
         private readonly ILocalMetadataProvider _localMetadataProvider;
         private readonly ILogger<MusicVideoFolderScanner> _logger;
@@ -38,6 +39,7 @@ namespace ErsatzTV.Core.Metadata
             ISearchRepository searchRepository,
             IArtistRepository artistRepository,
             IMusicVideoRepository musicVideoRepository,
+            ILibraryRepository libraryRepository,
             IMediator mediator,
             ILogger<MusicVideoFolderScanner> logger) : base(
             localFileSystem,
@@ -52,6 +54,7 @@ namespace ErsatzTV.Core.Metadata
             _searchRepository = searchRepository;
             _artistRepository = artistRepository;
             _musicVideoRepository = musicVideoRepository;
+            _libraryRepository = libraryRepository;
             _mediator = mediator;
             _logger = logger;
         }
@@ -59,7 +62,6 @@ namespace ErsatzTV.Core.Metadata
         public async Task<Either<BaseError, Unit>> ScanFolder(
             LibraryPath libraryPath,
             string ffprobePath,
-            DateTimeOffset lastScan,
             decimal progressMin,
             decimal progressMax)
         {
@@ -96,9 +98,7 @@ namespace ErsatzTV.Core.Metadata
                             libraryPath,
                             ffprobePath,
                             result.Item,
-                            artistFolder,
+                            artistFolder);
-                            // force scanning all folders if we're adding a new artist
-                            result.IsAdded ? DateTimeOffset.MinValue : lastScan);
 
                         if (result.IsAdded)
                         {
@@ -167,7 +167,7 @@ namespace ErsatzTV.Core.Metadata
                     {
                         bool shouldUpdate = Optional(artist.ArtistMetadata).Flatten().HeadOrNone().Match(
                             m => m.MetadataKind == MetadataKind.Fallback ||
-                                 m.DateUpdated < _localFileSystem.GetLastWriteTime(nfoFile),
+                                 m.DateUpdated != _localFileSystem.GetLastWriteTime(nfoFile),
                             true);
 
                         if (shouldUpdate)
@@ -222,12 +222,11 @@ namespace ErsatzTV.Core.Metadata
             }
         }
 
-        public async Task ScanMusicVideos(
+        private async Task ScanMusicVideos(
             LibraryPath libraryPath,
             string ffprobePath,
             Artist artist,
-            string artistFolder,
+            string artistFolder)
-            DateTimeOffset lastScan)
         {
             var folderQueue = new Queue<string>();
             folderQueue.Enqueue(artistFolder);
@@ -247,7 +246,13 @@ namespace ErsatzTV.Core.Metadata
                     folderQueue.Enqueue(subdirectory);
                 }
 
-                if (_localFileSystem.GetLastWriteTime(musicVideoFolder) < lastScan)
+                string etag = FolderEtag.Calculate(musicVideoFolder, _localFileSystem);
+                Option<LibraryFolder> knownFolder = libraryPath.LibraryFolders
+                    .Filter(f => f.Path == musicVideoFolder)
+                    .HeadOrNone();
+
+                // skip folder if etag matches
+                if (await knownFolder.Map(f => f.Etag).IfNoneAsync(string.Empty) == etag)
                 {
                     continue;
                 }
@@ -272,6 +277,8 @@ namespace ErsatzTV.Core.Metadata
                             {
                                 await _searchIndex.UpdateItems(_searchRepository, new List<MediaItem> { result.Item });
                             }
+
+                            await _libraryRepository.SetEtag(libraryPath, knownFolder, musicVideoFolder, etag);
                         },
                         error =>
                         {
@@ -293,7 +300,7 @@ namespace ErsatzTV.Core.Metadata
                     {
                         bool shouldUpdate = Optional(musicVideo.MusicVideoMetadata).Flatten().HeadOrNone().Match(
                             m => m.MetadataKind == MetadataKind.Fallback ||
-                                 m.DateUpdated < _localFileSystem.GetLastWriteTime(nfoFile),
+                                 m.DateUpdated != _localFileSystem.GetLastWriteTime(nfoFile),
                             true);
 
                         if (shouldUpdate)

ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs

@@ -19,6 +19,7 @@ namespace ErsatzTV.Core.Metadata
 {
     public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScanner
     {
+        private readonly ILibraryRepository _libraryRepository;
         private readonly ILocalFileSystem _localFileSystem;
         private readonly ILocalMetadataProvider _localMetadataProvider;
         private readonly ILogger<TelevisionFolderScanner> _logger;
@@ -36,6 +37,7 @@ namespace ErsatzTV.Core.Metadata
             IImageCache imageCache,
             ISearchIndex searchIndex,
             ISearchRepository searchRepository,
+            ILibraryRepository libraryRepository,
             IMediator mediator,
             ILogger<TelevisionFolderScanner> logger) : base(
             localFileSystem,
@@ -49,6 +51,7 @@ namespace ErsatzTV.Core.Metadata
             _localMetadataProvider = localMetadataProvider;
             _searchIndex = searchIndex;
             _searchRepository = searchRepository;
+            _libraryRepository = libraryRepository;
             _mediator = mediator;
             _logger = logger;
         }
@@ -56,7 +59,6 @@ namespace ErsatzTV.Core.Metadata
         public async Task<Either<BaseError, Unit>> ScanFolder(
             LibraryPath libraryPath,
             string ffprobePath,
-            DateTimeOffset lastScan,
             decimal progressMin,
             decimal progressMax)
         {
@@ -91,9 +93,7 @@ namespace ErsatzTV.Core.Metadata
                             libraryPath,
                             ffprobePath,
                             result.Item,
-                            showFolder,
+                            showFolder);
-                            // force scanning all folders if we're adding a new show
-                            result.IsAdded ? DateTimeOffset.MinValue : lastScan);
 
                         if (result.IsAdded)
                         {
@@ -146,12 +146,22 @@ namespace ErsatzTV.Core.Metadata
             LibraryPath libraryPath,
             string ffprobePath,
             Show show,
-            string showFolder,
+            string showFolder)
-            DateTimeOffset lastScan)
         {
             foreach (string seasonFolder in _localFileSystem.ListSubdirectories(showFolder).Filter(ShouldIncludeFolder)
                 .OrderBy(identity))
             {
+                string etag = FolderEtag.Calculate(seasonFolder, _localFileSystem);
+                Option<LibraryFolder> knownFolder = libraryPath.LibraryFolders
+                    .Filter(f => f.Path == seasonFolder)
+                    .HeadOrNone();
+
+                // skip folder if etag matches
+                if (await knownFolder.Map(f => f.Etag).IfNoneAsync(string.Empty) == etag)
+                {
+                    continue;
+                }
+
                 Option<int> maybeSeasonNumber = SeasonNumberForFolder(seasonFolder);
                 await maybeSeasonNumber.IfSomeAsync(
                     async seasonNumber =>
@@ -161,7 +171,11 @@ namespace ErsatzTV.Core.Metadata
                             .BindT(season => UpdatePoster(season, seasonFolder));
 
                         await maybeSeason.Match(
-                            season => ScanEpisodes(libraryPath, ffprobePath, season, seasonFolder, lastScan),
+                            async season =>
+                            {
+                                await ScanEpisodes(libraryPath, ffprobePath, season, seasonFolder);
+                                await _libraryRepository.SetEtag(libraryPath, knownFolder, seasonFolder, etag);
+                            },
                             error =>
                             {
                                 _logger.LogWarning(
@@ -180,14 +194,8 @@ namespace ErsatzTV.Core.Metadata
             LibraryPath libraryPath,
             string ffprobePath,
             Season season,
-            string seasonPath,
+            string seasonPath)
-            DateTimeOffset lastScan)
         {
-            if (_localFileSystem.GetLastWriteTime(seasonPath) < lastScan)
-            {
-                return Unit.Default;
-            }
-
             foreach (string file in _localFileSystem.ListFiles(seasonPath)
                 .Filter(f => VideoFileExtensions.Contains(Path.GetExtension(f))).OrderBy(identity))
             {
@@ -219,7 +227,7 @@ namespace ErsatzTV.Core.Metadata
                     {
                         bool shouldUpdate = Optional(show.ShowMetadata).Flatten().HeadOrNone().Match(
                             m => m.MetadataKind == MetadataKind.Fallback ||
-                                 m.DateUpdated < _localFileSystem.GetLastWriteTime(nfoFile),
+                                 m.DateUpdated != _localFileSystem.GetLastWriteTime(nfoFile),
                             true);
 
                         if (shouldUpdate)
@@ -261,7 +269,7 @@ namespace ErsatzTV.Core.Metadata
                     {
                         bool shouldUpdate = Optional(episode.EpisodeMetadata).Flatten().HeadOrNone().Match(
                             m => m.MetadataKind == MetadataKind.Fallback ||
-                                 m.DateUpdated < _localFileSystem.GetLastWriteTime(nfoFile),
+                                 m.DateUpdated != _localFileSystem.GetLastWriteTime(nfoFile),
                             true);
 
                         if (shouldUpdate)

ErsatzTV.Infrastructure/Data/Configurations/Library/LibraryFolderConfiguration.cs

@@ -0,0 +1,11 @@
+using ErsatzTV.Core.Domain;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace ErsatzTV.Infrastructure.Data.Configurations
+{
+    public class LibraryFolderConfiguration : IEntityTypeConfiguration<LibraryFolder>
+    {
+        public void Configure(EntityTypeBuilder<LibraryFolder> builder) => builder.ToTable("LibraryFolder");
+    }
+}

ErsatzTV.Infrastructure/Data/Configurations/Library/LibraryPathConfiguration.cs

@@ -14,6 +14,11 @@ namespace ErsatzTV.Infrastructure.Data.Configurations
                 .WithOne(i => i.LibraryPath)
                 .HasForeignKey(i => i.LibraryPathId)
                 .OnDelete(DeleteBehavior.Cascade);
+
+            builder.HasMany(p => p.LibraryFolders)
+                .WithOne(f => f.LibraryPath)
+                .HasForeignKey(f => f.LibraryPathId)
+                .OnDelete(DeleteBehavior.Cascade);
         }
     }
 }

ErsatzTV.Infrastructure/Data/Repositories/LibraryRepository.cs

@@ -35,6 +35,7 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
             using TvContext context = _dbContextFactory.CreateDbContext();
             return context.Libraries
                 .Include(l => l.Paths)
+                .ThenInclude(p => p.LibraryFolders)
                 .OrderBy(l => l.Id)
                 .SingleOrDefaultAsync(l => l.Id == libraryId)
                 .Map(Optional);
@@ -104,5 +105,30 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
             context.LibraryPaths.Remove(libraryPath);
             await context.SaveChangesAsync();
         }
+
+        public Task<Unit> SetEtag(
+            LibraryPath libraryPath,
+            Option<LibraryFolder> knownFolder,
+            string path,
+            string etag) =>
+            knownFolder.Match(
+                async folder =>
+                {
+                    await _dbConnection.ExecuteAsync(
+                        "UPDATE LibraryFolder SET Etag = @Etag WHERE Id = @Id",
+                        new { folder.Id, Etag = etag });
+                },
+                async () =>
+                {
+                    await using TvContext context = _dbContextFactory.CreateDbContext();
+                    await context.LibraryFolders.AddAsync(
+                        new LibraryFolder
+                        {
+                            Path = path,
+                            Etag = etag,
+                            LibraryPathId = libraryPath.Id
+                        });
+                    await context.SaveChangesAsync();
+                }).ToUnit();
     }
 }

ErsatzTV.Infrastructure/Data/TvContext.cs

@@ -21,6 +21,7 @@ namespace ErsatzTV.Infrastructure.Data
         public DbSet<Library> Libraries { get; set; }
         public DbSet<LocalLibrary> LocalLibraries { get; set; }
         public DbSet<LibraryPath> LibraryPaths { get; set; }
+        public DbSet<LibraryFolder> LibraryFolders { get; set; }
         public DbSet<PlexLibrary> PlexLibraries { get; set; }
         public DbSet<JellyfinLibrary> JellyfinLibraries { get; set; }
         public DbSet<PlexPathReplacement> PlexPathReplacements { get; set; }

ErsatzTV.Infrastructure/Migrations/20210521015037_Add_LibraryFolder.cs

@@ -0,0 +1,40 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace ErsatzTV.Infrastructure.Migrations
+{
+    public partial class Add_LibraryFolder : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                "LibraryFolder",
+                table => new
+                {
+                    Id = table.Column<int>("INTEGER", nullable: false)
+                        .Annotation("Sqlite:Autoincrement", true),
+                    Path = table.Column<string>("TEXT", nullable: true),
+                    LibraryPathId = table.Column<int>("INTEGER", nullable: false),
+                    Etag = table.Column<string>("TEXT", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_LibraryFolder", x => x.Id);
+                    table.ForeignKey(
+                        "FK_LibraryFolder_LibraryPath_LibraryPathId",
+                        x => x.LibraryPathId,
+                        "LibraryPath",
+                        "Id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateIndex(
+                "IX_LibraryFolder_LibraryPathId",
+                "LibraryFolder",
+                "LibraryPathId");
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder) =>
+            migrationBuilder.DropTable(
+                "LibraryFolder");
+    }
+}

ErsatzTV.Infrastructure/Migrations/TvContextModelSnapshot.cs

@@ -517,6 +517,30 @@ namespace ErsatzTV.Infrastructure.Migrations
                     b.ToTable("Library");
                 });
 
+            modelBuilder.Entity(
+                "ErsatzTV.Core.Domain.LibraryFolder",
+                b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER");
+
+                    b.Property<string>("Etag")
+                        .HasColumnType("TEXT");
+
+                    b.Property<int>("LibraryPathId")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<string>("Path")
+                        .HasColumnType("TEXT");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("LibraryPathId");
+
+                    b.ToTable("LibraryFolder");
+                });
+
             modelBuilder.Entity(
                 "ErsatzTV.Core.Domain.LibraryPath",
                 b =>
@@ -1811,6 +1835,19 @@ namespace ErsatzTV.Infrastructure.Migrations
                     b.Navigation("MediaSource");
                 });
 
+            modelBuilder.Entity(
+                "ErsatzTV.Core.Domain.LibraryFolder",
+                b =>
+                {
+                    b.HasOne("ErsatzTV.Core.Domain.LibraryPath", "LibraryPath")
+                        .WithMany("LibraryFolders")
+                        .HasForeignKey("LibraryPathId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.Navigation("LibraryPath");
+                });
+
             modelBuilder.Entity(
                 "ErsatzTV.Core.Domain.LibraryPath",
                 b =>
@@ -2559,7 +2596,14 @@ namespace ErsatzTV.Infrastructure.Migrations
 
             modelBuilder.Entity("ErsatzTV.Core.Domain.Library", b => { b.Navigation("Paths"); });
 
-            modelBuilder.Entity("ErsatzTV.Core.Domain.LibraryPath", b => { b.Navigation("MediaItems"); });
+            modelBuilder.Entity(
+                "ErsatzTV.Core.Domain.LibraryPath",
+                b =>
+                {
+                    b.Navigation("LibraryFolders");
+
+                    b.Navigation("MediaItems");
+                });
 
             modelBuilder.Entity("ErsatzTV.Core.Domain.MediaItem", b => { b.Navigation("CollectionItems"); });