Browse Source

optimize local library scanning by using etags (#196)

* use etags to optimize local movie scanner

* use etags to optimize local television scanner

* use etags to optimize local music video scanner

* code cleanup
pull/197/head v0.0.37-prealpha
Jason Dove 5 years ago committed by GitHub
parent
commit
0799fe25d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      ErsatzTV.Application/MediaSources/Commands/ScanLocalLibraryHandler.cs
  2. 48
      ErsatzTV.Core.Tests/Metadata/MovieFolderScannerTests.cs
  3. 11
      ErsatzTV.Core/Domain/Library/LibraryFolder.cs
  4. 1
      ErsatzTV.Core/Domain/Library/LibraryPath.cs
  5. 4
      ErsatzTV.Core/Interfaces/Metadata/IMovieFolderScanner.cs
  6. 4
      ErsatzTV.Core/Interfaces/Metadata/IMusicVideoFolderScanner.cs
  7. 4
      ErsatzTV.Core/Interfaces/Metadata/ITelevisionFolderScanner.cs
  8. 1
      ErsatzTV.Core/Interfaces/Repositories/ILibraryRepository.cs
  9. 35
      ErsatzTV.Core/Metadata/FolderEtag.cs
  10. 2
      ErsatzTV.Core/Metadata/LocalFolderScanner.cs
  11. 24
      ErsatzTV.Core/Metadata/MovieFolderScanner.cs
  12. 27
      ErsatzTV.Core/Metadata/MusicVideoFolderScanner.cs
  13. 40
      ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs
  14. 11
      ErsatzTV.Infrastructure/Data/Configurations/Library/LibraryFolderConfiguration.cs
  15. 5
      ErsatzTV.Infrastructure/Data/Configurations/Library/LibraryPathConfiguration.cs
  16. 26
      ErsatzTV.Infrastructure/Data/Repositories/LibraryRepository.cs
  17. 1
      ErsatzTV.Infrastructure/Data/TvContext.cs
  18. 2509
      ErsatzTV.Infrastructure/Migrations/20210521015037_Add_LibraryFolder.Designer.cs
  19. 40
      ErsatzTV.Infrastructure/Migrations/20210521015037_Add_LibraryFolder.cs
  20. 46
      ErsatzTV.Infrastructure/Migrations/TvContextModelSnapshot.cs

3
ErsatzTV.Application/MediaSources/Commands/ScanLocalLibraryHandler.cs

@ -90,7 +90,6 @@ namespace ErsatzTV.Application.MediaSources.Commands @@ -90,7 +90,6 @@ namespace ErsatzTV.Application.MediaSources.Commands
await _movieFolderScanner.ScanFolder(
libraryPath,
ffprobePath,
lastScan,
progressMin,
progressMax);
break;
@ -98,7 +97,6 @@ namespace ErsatzTV.Application.MediaSources.Commands @@ -98,7 +97,6 @@ namespace ErsatzTV.Application.MediaSources.Commands
await _televisionFolderScanner.ScanFolder(
libraryPath,
ffprobePath,
lastScan,
progressMin,
progressMax);
break;
@ -106,7 +104,6 @@ namespace ErsatzTV.Application.MediaSources.Commands @@ -106,7 +104,6 @@ namespace ErsatzTV.Application.MediaSources.Commands
await _musicVideoFolderScanner.ScanFolder(
libraryPath,
ffprobePath,
lastScan,
progressMin,
progressMax);
break;

48
ErsatzTV.Core.Tests/Metadata/MovieFolderScannerTests.cs

@ -81,12 +81,11 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -81,12 +81,11 @@ namespace ErsatzTV.Core.Tests.Metadata
MovieFolderScanner service = GetService(
new FakeFileEntry(Path.Combine(FakeRoot, Path.Combine("Movie (2020)", "Movie (2020).mkv")))
);
var libraryPath = new LibraryPath { Path = BadFakeRoot };
var libraryPath = new LibraryPath { Path = BadFakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -106,12 +105,12 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -106,12 +105,12 @@ namespace ErsatzTV.Core.Tests.Metadata
MovieFolderScanner service = GetService(
new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now }
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
var libraryPath = new LibraryPath
{ Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -147,12 +146,12 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -147,12 +146,12 @@ namespace ErsatzTV.Core.Tests.Metadata
new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now },
new FakeFileEntry(metadataPath)
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
var libraryPath = new LibraryPath
{ Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -189,12 +188,12 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -189,12 +188,12 @@ namespace ErsatzTV.Core.Tests.Metadata
new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now },
new FakeFileEntry(metadataPath)
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
var libraryPath = new LibraryPath
{ Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -235,12 +234,12 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -235,12 +234,12 @@ namespace ErsatzTV.Core.Tests.Metadata
new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now },
new FakeFileEntry(posterPath) { LastWriteTime = DateTime.Now }
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
var libraryPath = new LibraryPath
{ Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -284,12 +283,12 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -284,12 +283,12 @@ namespace ErsatzTV.Core.Tests.Metadata
new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now },
new FakeFileEntry(posterPath) { LastWriteTime = DateTime.Now }
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
var libraryPath = new LibraryPath
{ Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -333,12 +332,12 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -333,12 +332,12 @@ namespace ErsatzTV.Core.Tests.Metadata
new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now },
new FakeFileEntry(posterPath) { LastWriteTime = DateTime.Now }
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
var libraryPath = new LibraryPath
{ Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -381,12 +380,12 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -381,12 +380,12 @@ namespace ErsatzTV.Core.Tests.Metadata
Path.GetDirectoryName(moviePath) ?? string.Empty,
$"Movie (2020)-{extraFile}{videoExtension}"))
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
var libraryPath = new LibraryPath
{ Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -425,12 +424,12 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -425,12 +424,12 @@ namespace ErsatzTV.Core.Tests.Metadata
Path.GetDirectoryName(moviePath) ?? string.Empty,
Path.Combine(extraFolder, $"Movie (2020){videoExtension}")))
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
var libraryPath = new LibraryPath
{ Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -463,12 +462,12 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -463,12 +462,12 @@ namespace ErsatzTV.Core.Tests.Metadata
MovieFolderScanner service = GetService(
new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now }
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
var libraryPath = new LibraryPath
{ Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -503,12 +502,12 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -503,12 +502,12 @@ namespace ErsatzTV.Core.Tests.Metadata
MovieFolderScanner service = GetService(
new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now }
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
var libraryPath = new LibraryPath
{ Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -532,12 +531,12 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -532,12 +531,12 @@ namespace ErsatzTV.Core.Tests.Metadata
MovieFolderScanner service = GetService(
new FakeFileEntry(moviePath) { LastWriteTime = DateTime.Now }
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
var libraryPath = new LibraryPath
{ Id = 1, Path = FakeRoot, LibraryFolders = new List<LibraryFolder>() };
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue,
0,
1);
@ -558,6 +557,7 @@ namespace ErsatzTV.Core.Tests.Metadata @@ -558,6 +557,7 @@ namespace ErsatzTV.Core.Tests.Metadata
_imageCache.Object,
new Mock<ISearchIndex>().Object,
new Mock<ISearchRepository>().Object,
new Mock<ILibraryRepository>().Object,
new Mock<IMediator>().Object,
new Mock<ILogger<MovieFolderScanner>>().Object
);

11
ErsatzTV.Core/Domain/Library/LibraryFolder.cs

@ -0,0 +1,11 @@ @@ -0,0 +1,11 @@
namespace ErsatzTV.Core.Domain
{
public class LibraryFolder
{
public int Id { get; set; }
public string Path { get; set; }
public int LibraryPathId { get; set; }
public LibraryPath LibraryPath { get; set; }
public string Etag { get; set; }
}
}

1
ErsatzTV.Core/Domain/Library/LibraryPath.cs

@ -13,5 +13,6 @@ namespace ErsatzTV.Core.Domain @@ -13,5 +13,6 @@ namespace ErsatzTV.Core.Domain
public Library Library { get; set; }
public List<MediaItem> MediaItems { get; set; }
public List<LibraryFolder> LibraryFolders { get; set; }
}
}

4
ErsatzTV.Core/Interfaces/Metadata/IMovieFolderScanner.cs

@ -1,5 +1,4 @@ @@ -1,5 +1,4 @@
using System;
using System.Threading.Tasks;
using System.Threading.Tasks;
using ErsatzTV.Core.Domain;
using LanguageExt;
@ -10,7 +9,6 @@ namespace ErsatzTV.Core.Interfaces.Metadata @@ -10,7 +9,6 @@ namespace ErsatzTV.Core.Interfaces.Metadata
Task<Either<BaseError, Unit>> ScanFolder(
LibraryPath libraryPath,
string ffprobePath,
DateTimeOffset lastScan,
decimal progressMin,
decimal progressMax);
}

4
ErsatzTV.Core/Interfaces/Metadata/IMusicVideoFolderScanner.cs

@ -1,5 +1,4 @@ @@ -1,5 +1,4 @@
using System;
using System.Threading.Tasks;
using System.Threading.Tasks;
using ErsatzTV.Core.Domain;
using LanguageExt;
@ -10,7 +9,6 @@ namespace ErsatzTV.Core.Interfaces.Metadata @@ -10,7 +9,6 @@ namespace ErsatzTV.Core.Interfaces.Metadata
Task<Either<BaseError, Unit>> ScanFolder(
LibraryPath libraryPath,
string ffprobePath,
DateTimeOffset lastScan,
decimal progressMin,
decimal progressMax);
}

4
ErsatzTV.Core/Interfaces/Metadata/ITelevisionFolderScanner.cs

@ -1,5 +1,4 @@ @@ -1,5 +1,4 @@
using System;
using System.Threading.Tasks;
using System.Threading.Tasks;
using ErsatzTV.Core.Domain;
using LanguageExt;
@ -10,7 +9,6 @@ namespace ErsatzTV.Core.Interfaces.Metadata @@ -10,7 +9,6 @@ namespace ErsatzTV.Core.Interfaces.Metadata
Task<Either<BaseError, Unit>> ScanFolder(
LibraryPath libraryPath,
string ffprobePath,
DateTimeOffset lastScan,
decimal progressMin,
decimal progressMax);
}

1
ErsatzTV.Core/Interfaces/Repositories/ILibraryRepository.cs

@ -18,5 +18,6 @@ namespace ErsatzTV.Core.Interfaces.Repositories @@ -18,5 +18,6 @@ namespace ErsatzTV.Core.Interfaces.Repositories
Task<int> CountMediaItemsByPath(int libraryPathId);
Task<List<int>> GetMediaIdsByLocalPath(int libraryPathId);
Task DeleteLocalPath(int libraryPathId);
Task<Unit> SetEtag(LibraryPath libraryPath, Option<LibraryFolder> knownFolder, string path, string etag);
}
}

35
ErsatzTV.Core/Metadata/FolderEtag.cs

@ -0,0 +1,35 @@ @@ -0,0 +1,35 @@
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using ErsatzTV.Core.Interfaces.Metadata;
using static LanguageExt.Prelude;
namespace ErsatzTV.Core.Metadata
{
public static class FolderEtag
{
private static readonly MD5CryptoServiceProvider Crypto = new();
public static string Calculate(string folder, ILocalFileSystem localFileSystem)
{
IEnumerable<string> allFiles = localFileSystem.ListFiles(folder);
var sb = new StringBuilder();
foreach (string file in allFiles.OrderBy(identity))
{
sb.Append(file);
sb.Append(localFileSystem.GetLastWriteTime(file).Ticks);
}
var hash = new StringBuilder();
byte[] bytes = Crypto.ComputeHash(Encoding.UTF8.GetBytes(sb.ToString()));
foreach (byte t in bytes)
{
hash.Append(t.ToString("x2"));
}
return hash.ToString();
}
}
}

2
ErsatzTV.Core/Metadata/LocalFolderScanner.cs

@ -78,7 +78,7 @@ namespace ErsatzTV.Core.Metadata @@ -78,7 +78,7 @@ namespace ErsatzTV.Core.Metadata
string path = version.MediaFiles.Head().Path;
if (version.DateUpdated < _localFileSystem.GetLastWriteTime(path) || !version.Streams.Any())
if (version.DateUpdated != _localFileSystem.GetLastWriteTime(path) || !version.Streams.Any())
{
_logger.LogDebug("Refreshing {Attribute} for {Path}", "Statistics", path);
Either<BaseError, bool> refreshResult =

24
ErsatzTV.Core/Metadata/MovieFolderScanner.cs

@ -20,6 +20,7 @@ namespace ErsatzTV.Core.Metadata @@ -20,6 +20,7 @@ namespace ErsatzTV.Core.Metadata
{
public class MovieFolderScanner : LocalFolderScanner, IMovieFolderScanner
{
private readonly ILibraryRepository _libraryRepository;
private readonly ILocalFileSystem _localFileSystem;
private readonly ILocalMetadataProvider _localMetadataProvider;
private readonly ILogger<MovieFolderScanner> _logger;
@ -37,6 +38,7 @@ namespace ErsatzTV.Core.Metadata @@ -37,6 +38,7 @@ namespace ErsatzTV.Core.Metadata
IImageCache imageCache,
ISearchIndex searchIndex,
ISearchRepository searchRepository,
ILibraryRepository libraryRepository,
IMediator mediator,
ILogger<MovieFolderScanner> logger)
: base(localFileSystem, localStatisticsProvider, metadataRepository, imageCache, logger)
@ -46,6 +48,7 @@ namespace ErsatzTV.Core.Metadata @@ -46,6 +48,7 @@ namespace ErsatzTV.Core.Metadata
_localMetadataProvider = localMetadataProvider;
_searchIndex = searchIndex;
_searchRepository = searchRepository;
_libraryRepository = libraryRepository;
_mediator = mediator;
_logger = logger;
}
@ -53,7 +56,6 @@ namespace ErsatzTV.Core.Metadata @@ -53,7 +56,6 @@ namespace ErsatzTV.Core.Metadata
public async Task<Either<BaseError, Unit>> ScanFolder(
LibraryPath libraryPath,
string ffprobePath,
DateTimeOffset lastScan,
decimal progressMin,
decimal progressMax)
{
@ -81,7 +83,9 @@ namespace ErsatzTV.Core.Metadata @@ -81,7 +83,9 @@ namespace ErsatzTV.Core.Metadata
string movieFolder = folderQueue.Dequeue();
foldersCompleted++;
var allFiles = _localFileSystem.ListFiles(movieFolder)
var filesForEtag = _localFileSystem.ListFiles(movieFolder).ToList();
var allFiles = filesForEtag
.Filter(f => VideoFileExtensions.Contains(Path.GetExtension(f)))
.Filter(
f => !ExtraFiles.Any(
@ -98,11 +102,21 @@ namespace ErsatzTV.Core.Metadata @@ -98,11 +102,21 @@ namespace ErsatzTV.Core.Metadata
continue;
}
if (allFiles.All(file => _localFileSystem.GetLastWriteTime(file) < lastScan))
string etag = FolderEtag.Calculate(movieFolder, _localFileSystem);
Option<LibraryFolder> knownFolder = libraryPath.LibraryFolders
.Filter(f => f.Path == movieFolder)
.HeadOrNone();
// skip folder if etag matches
if (await knownFolder.Map(f => f.Etag).IfNoneAsync(string.Empty) == etag)
{
continue;
}
_logger.LogDebug(
"UPDATE: Etag has changed for folder {Folder}",
movieFolder);
foreach (string file in allFiles.OrderBy(identity))
{
// TODO: figure out how to rebuild playlists
@ -124,6 +138,8 @@ namespace ErsatzTV.Core.Metadata @@ -124,6 +138,8 @@ namespace ErsatzTV.Core.Metadata
{
await _searchIndex.UpdateItems(_searchRepository, new List<MediaItem> { result.Item });
}
await _libraryRepository.SetEtag(libraryPath, knownFolder, movieFolder, etag);
},
error =>
{
@ -158,7 +174,7 @@ namespace ErsatzTV.Core.Metadata @@ -158,7 +174,7 @@ namespace ErsatzTV.Core.Metadata
{
bool shouldUpdate = Optional(movie.MovieMetadata).Flatten().HeadOrNone().Match(
m => m.MetadataKind == MetadataKind.Fallback ||
m.DateUpdated < _localFileSystem.GetLastWriteTime(nfoFile),
m.DateUpdated != _localFileSystem.GetLastWriteTime(nfoFile),
true);
if (shouldUpdate)

27
ErsatzTV.Core/Metadata/MusicVideoFolderScanner.cs

@ -20,6 +20,7 @@ namespace ErsatzTV.Core.Metadata @@ -20,6 +20,7 @@ namespace ErsatzTV.Core.Metadata
public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScanner
{
private readonly IArtistRepository _artistRepository;
private readonly ILibraryRepository _libraryRepository;
private readonly ILocalFileSystem _localFileSystem;
private readonly ILocalMetadataProvider _localMetadataProvider;
private readonly ILogger<MusicVideoFolderScanner> _logger;
@ -38,6 +39,7 @@ namespace ErsatzTV.Core.Metadata @@ -38,6 +39,7 @@ namespace ErsatzTV.Core.Metadata
ISearchRepository searchRepository,
IArtistRepository artistRepository,
IMusicVideoRepository musicVideoRepository,
ILibraryRepository libraryRepository,
IMediator mediator,
ILogger<MusicVideoFolderScanner> logger) : base(
localFileSystem,
@ -52,6 +54,7 @@ namespace ErsatzTV.Core.Metadata @@ -52,6 +54,7 @@ namespace ErsatzTV.Core.Metadata
_searchRepository = searchRepository;
_artistRepository = artistRepository;
_musicVideoRepository = musicVideoRepository;
_libraryRepository = libraryRepository;
_mediator = mediator;
_logger = logger;
}
@ -59,7 +62,6 @@ namespace ErsatzTV.Core.Metadata @@ -59,7 +62,6 @@ namespace ErsatzTV.Core.Metadata
public async Task<Either<BaseError, Unit>> ScanFolder(
LibraryPath libraryPath,
string ffprobePath,
DateTimeOffset lastScan,
decimal progressMin,
decimal progressMax)
{
@ -96,9 +98,7 @@ namespace ErsatzTV.Core.Metadata @@ -96,9 +98,7 @@ namespace ErsatzTV.Core.Metadata
libraryPath,
ffprobePath,
result.Item,
artistFolder,
// force scanning all folders if we're adding a new artist
result.IsAdded ? DateTimeOffset.MinValue : lastScan);
artistFolder);
if (result.IsAdded)
{
@ -167,7 +167,7 @@ namespace ErsatzTV.Core.Metadata @@ -167,7 +167,7 @@ namespace ErsatzTV.Core.Metadata
{
bool shouldUpdate = Optional(artist.ArtistMetadata).Flatten().HeadOrNone().Match(
m => m.MetadataKind == MetadataKind.Fallback ||
m.DateUpdated < _localFileSystem.GetLastWriteTime(nfoFile),
m.DateUpdated != _localFileSystem.GetLastWriteTime(nfoFile),
true);
if (shouldUpdate)
@ -222,12 +222,11 @@ namespace ErsatzTV.Core.Metadata @@ -222,12 +222,11 @@ namespace ErsatzTV.Core.Metadata
}
}
public async Task ScanMusicVideos(
private async Task ScanMusicVideos(
LibraryPath libraryPath,
string ffprobePath,
Artist artist,
string artistFolder,
DateTimeOffset lastScan)
string artistFolder)
{
var folderQueue = new Queue<string>();
folderQueue.Enqueue(artistFolder);
@ -247,7 +246,13 @@ namespace ErsatzTV.Core.Metadata @@ -247,7 +246,13 @@ namespace ErsatzTV.Core.Metadata
folderQueue.Enqueue(subdirectory);
}
if (_localFileSystem.GetLastWriteTime(musicVideoFolder) < lastScan)
string etag = FolderEtag.Calculate(musicVideoFolder, _localFileSystem);
Option<LibraryFolder> knownFolder = libraryPath.LibraryFolders
.Filter(f => f.Path == musicVideoFolder)
.HeadOrNone();
// skip folder if etag matches
if (await knownFolder.Map(f => f.Etag).IfNoneAsync(string.Empty) == etag)
{
continue;
}
@ -272,6 +277,8 @@ namespace ErsatzTV.Core.Metadata @@ -272,6 +277,8 @@ namespace ErsatzTV.Core.Metadata
{
await _searchIndex.UpdateItems(_searchRepository, new List<MediaItem> { result.Item });
}
await _libraryRepository.SetEtag(libraryPath, knownFolder, musicVideoFolder, etag);
},
error =>
{
@ -293,7 +300,7 @@ namespace ErsatzTV.Core.Metadata @@ -293,7 +300,7 @@ namespace ErsatzTV.Core.Metadata
{
bool shouldUpdate = Optional(musicVideo.MusicVideoMetadata).Flatten().HeadOrNone().Match(
m => m.MetadataKind == MetadataKind.Fallback ||
m.DateUpdated < _localFileSystem.GetLastWriteTime(nfoFile),
m.DateUpdated != _localFileSystem.GetLastWriteTime(nfoFile),
true);
if (shouldUpdate)

40
ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs

@ -19,6 +19,7 @@ namespace ErsatzTV.Core.Metadata @@ -19,6 +19,7 @@ namespace ErsatzTV.Core.Metadata
{
public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScanner
{
private readonly ILibraryRepository _libraryRepository;
private readonly ILocalFileSystem _localFileSystem;
private readonly ILocalMetadataProvider _localMetadataProvider;
private readonly ILogger<TelevisionFolderScanner> _logger;
@ -36,6 +37,7 @@ namespace ErsatzTV.Core.Metadata @@ -36,6 +37,7 @@ namespace ErsatzTV.Core.Metadata
IImageCache imageCache,
ISearchIndex searchIndex,
ISearchRepository searchRepository,
ILibraryRepository libraryRepository,
IMediator mediator,
ILogger<TelevisionFolderScanner> logger) : base(
localFileSystem,
@ -49,6 +51,7 @@ namespace ErsatzTV.Core.Metadata @@ -49,6 +51,7 @@ namespace ErsatzTV.Core.Metadata
_localMetadataProvider = localMetadataProvider;
_searchIndex = searchIndex;
_searchRepository = searchRepository;
_libraryRepository = libraryRepository;
_mediator = mediator;
_logger = logger;
}
@ -56,7 +59,6 @@ namespace ErsatzTV.Core.Metadata @@ -56,7 +59,6 @@ namespace ErsatzTV.Core.Metadata
public async Task<Either<BaseError, Unit>> ScanFolder(
LibraryPath libraryPath,
string ffprobePath,
DateTimeOffset lastScan,
decimal progressMin,
decimal progressMax)
{
@ -91,9 +93,7 @@ namespace ErsatzTV.Core.Metadata @@ -91,9 +93,7 @@ namespace ErsatzTV.Core.Metadata
libraryPath,
ffprobePath,
result.Item,
showFolder,
// force scanning all folders if we're adding a new show
result.IsAdded ? DateTimeOffset.MinValue : lastScan);
showFolder);
if (result.IsAdded)
{
@ -146,12 +146,22 @@ namespace ErsatzTV.Core.Metadata @@ -146,12 +146,22 @@ namespace ErsatzTV.Core.Metadata
LibraryPath libraryPath,
string ffprobePath,
Show show,
string showFolder,
DateTimeOffset lastScan)
string showFolder)
{
foreach (string seasonFolder in _localFileSystem.ListSubdirectories(showFolder).Filter(ShouldIncludeFolder)
.OrderBy(identity))
{
string etag = FolderEtag.Calculate(seasonFolder, _localFileSystem);
Option<LibraryFolder> knownFolder = libraryPath.LibraryFolders
.Filter(f => f.Path == seasonFolder)
.HeadOrNone();
// skip folder if etag matches
if (await knownFolder.Map(f => f.Etag).IfNoneAsync(string.Empty) == etag)
{
continue;
}
Option<int> maybeSeasonNumber = SeasonNumberForFolder(seasonFolder);
await maybeSeasonNumber.IfSomeAsync(
async seasonNumber =>
@ -161,7 +171,11 @@ namespace ErsatzTV.Core.Metadata @@ -161,7 +171,11 @@ namespace ErsatzTV.Core.Metadata
.BindT(season => UpdatePoster(season, seasonFolder));
await maybeSeason.Match(
season => ScanEpisodes(libraryPath, ffprobePath, season, seasonFolder, lastScan),
async season =>
{
await ScanEpisodes(libraryPath, ffprobePath, season, seasonFolder);
await _libraryRepository.SetEtag(libraryPath, knownFolder, seasonFolder, etag);
},
error =>
{
_logger.LogWarning(
@ -180,14 +194,8 @@ namespace ErsatzTV.Core.Metadata @@ -180,14 +194,8 @@ namespace ErsatzTV.Core.Metadata
LibraryPath libraryPath,
string ffprobePath,
Season season,
string seasonPath,
DateTimeOffset lastScan)
string seasonPath)
{
if (_localFileSystem.GetLastWriteTime(seasonPath) < lastScan)
{
return Unit.Default;
}
foreach (string file in _localFileSystem.ListFiles(seasonPath)
.Filter(f => VideoFileExtensions.Contains(Path.GetExtension(f))).OrderBy(identity))
{
@ -219,7 +227,7 @@ namespace ErsatzTV.Core.Metadata @@ -219,7 +227,7 @@ namespace ErsatzTV.Core.Metadata
{
bool shouldUpdate = Optional(show.ShowMetadata).Flatten().HeadOrNone().Match(
m => m.MetadataKind == MetadataKind.Fallback ||
m.DateUpdated < _localFileSystem.GetLastWriteTime(nfoFile),
m.DateUpdated != _localFileSystem.GetLastWriteTime(nfoFile),
true);
if (shouldUpdate)
@ -261,7 +269,7 @@ namespace ErsatzTV.Core.Metadata @@ -261,7 +269,7 @@ namespace ErsatzTV.Core.Metadata
{
bool shouldUpdate = Optional(episode.EpisodeMetadata).Flatten().HeadOrNone().Match(
m => m.MetadataKind == MetadataKind.Fallback ||
m.DateUpdated < _localFileSystem.GetLastWriteTime(nfoFile),
m.DateUpdated != _localFileSystem.GetLastWriteTime(nfoFile),
true);
if (shouldUpdate)

11
ErsatzTV.Infrastructure/Data/Configurations/Library/LibraryFolderConfiguration.cs

@ -0,0 +1,11 @@ @@ -0,0 +1,11 @@
using ErsatzTV.Core.Domain;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
namespace ErsatzTV.Infrastructure.Data.Configurations
{
public class LibraryFolderConfiguration : IEntityTypeConfiguration<LibraryFolder>
{
public void Configure(EntityTypeBuilder<LibraryFolder> builder) => builder.ToTable("LibraryFolder");
}
}

5
ErsatzTV.Infrastructure/Data/Configurations/Library/LibraryPathConfiguration.cs

@ -14,6 +14,11 @@ namespace ErsatzTV.Infrastructure.Data.Configurations @@ -14,6 +14,11 @@ namespace ErsatzTV.Infrastructure.Data.Configurations
.WithOne(i => i.LibraryPath)
.HasForeignKey(i => i.LibraryPathId)
.OnDelete(DeleteBehavior.Cascade);
builder.HasMany(p => p.LibraryFolders)
.WithOne(f => f.LibraryPath)
.HasForeignKey(f => f.LibraryPathId)
.OnDelete(DeleteBehavior.Cascade);
}
}
}

26
ErsatzTV.Infrastructure/Data/Repositories/LibraryRepository.cs

@ -35,6 +35,7 @@ namespace ErsatzTV.Infrastructure.Data.Repositories @@ -35,6 +35,7 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
using TvContext context = _dbContextFactory.CreateDbContext();
return context.Libraries
.Include(l => l.Paths)
.ThenInclude(p => p.LibraryFolders)
.OrderBy(l => l.Id)
.SingleOrDefaultAsync(l => l.Id == libraryId)
.Map(Optional);
@ -104,5 +105,30 @@ namespace ErsatzTV.Infrastructure.Data.Repositories @@ -104,5 +105,30 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
context.LibraryPaths.Remove(libraryPath);
await context.SaveChangesAsync();
}
public Task<Unit> SetEtag(
LibraryPath libraryPath,
Option<LibraryFolder> knownFolder,
string path,
string etag) =>
knownFolder.Match(
async folder =>
{
await _dbConnection.ExecuteAsync(
"UPDATE LibraryFolder SET Etag = @Etag WHERE Id = @Id",
new { folder.Id, Etag = etag });
},
async () =>
{
await using TvContext context = _dbContextFactory.CreateDbContext();
await context.LibraryFolders.AddAsync(
new LibraryFolder
{
Path = path,
Etag = etag,
LibraryPathId = libraryPath.Id
});
await context.SaveChangesAsync();
}).ToUnit();
}
}

1
ErsatzTV.Infrastructure/Data/TvContext.cs

@ -21,6 +21,7 @@ namespace ErsatzTV.Infrastructure.Data @@ -21,6 +21,7 @@ namespace ErsatzTV.Infrastructure.Data
public DbSet<Library> Libraries { get; set; }
public DbSet<LocalLibrary> LocalLibraries { get; set; }
public DbSet<LibraryPath> LibraryPaths { get; set; }
public DbSet<LibraryFolder> LibraryFolders { get; set; }
public DbSet<PlexLibrary> PlexLibraries { get; set; }
public DbSet<JellyfinLibrary> JellyfinLibraries { get; set; }
public DbSet<PlexPathReplacement> PlexPathReplacements { get; set; }

2509
ErsatzTV.Infrastructure/Migrations/20210521015037_Add_LibraryFolder.Designer.cs generated

File diff suppressed because it is too large Load Diff

40
ErsatzTV.Infrastructure/Migrations/20210521015037_Add_LibraryFolder.cs

@ -0,0 +1,40 @@ @@ -0,0 +1,40 @@
using Microsoft.EntityFrameworkCore.Migrations;
namespace ErsatzTV.Infrastructure.Migrations
{
public partial class Add_LibraryFolder : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
"LibraryFolder",
table => new
{
Id = table.Column<int>("INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
Path = table.Column<string>("TEXT", nullable: true),
LibraryPathId = table.Column<int>("INTEGER", nullable: false),
Etag = table.Column<string>("TEXT", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_LibraryFolder", x => x.Id);
table.ForeignKey(
"FK_LibraryFolder_LibraryPath_LibraryPathId",
x => x.LibraryPathId,
"LibraryPath",
"Id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
"IX_LibraryFolder_LibraryPathId",
"LibraryFolder",
"LibraryPathId");
}
protected override void Down(MigrationBuilder migrationBuilder) =>
migrationBuilder.DropTable(
"LibraryFolder");
}
}

46
ErsatzTV.Infrastructure/Migrations/TvContextModelSnapshot.cs

@ -517,6 +517,30 @@ namespace ErsatzTV.Infrastructure.Migrations @@ -517,6 +517,30 @@ namespace ErsatzTV.Infrastructure.Migrations
b.ToTable("Library");
});
modelBuilder.Entity(
"ErsatzTV.Core.Domain.LibraryFolder",
b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("Etag")
.HasColumnType("TEXT");
b.Property<int>("LibraryPathId")
.HasColumnType("INTEGER");
b.Property<string>("Path")
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("LibraryPathId");
b.ToTable("LibraryFolder");
});
modelBuilder.Entity(
"ErsatzTV.Core.Domain.LibraryPath",
b =>
@ -1811,6 +1835,19 @@ namespace ErsatzTV.Infrastructure.Migrations @@ -1811,6 +1835,19 @@ namespace ErsatzTV.Infrastructure.Migrations
b.Navigation("MediaSource");
});
modelBuilder.Entity(
"ErsatzTV.Core.Domain.LibraryFolder",
b =>
{
b.HasOne("ErsatzTV.Core.Domain.LibraryPath", "LibraryPath")
.WithMany("LibraryFolders")
.HasForeignKey("LibraryPathId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("LibraryPath");
});
modelBuilder.Entity(
"ErsatzTV.Core.Domain.LibraryPath",
b =>
@ -2559,7 +2596,14 @@ namespace ErsatzTV.Infrastructure.Migrations @@ -2559,7 +2596,14 @@ namespace ErsatzTV.Infrastructure.Migrations
modelBuilder.Entity("ErsatzTV.Core.Domain.Library", b => { b.Navigation("Paths"); });
modelBuilder.Entity("ErsatzTV.Core.Domain.LibraryPath", b => { b.Navigation("MediaItems"); });
modelBuilder.Entity(
"ErsatzTV.Core.Domain.LibraryPath",
b =>
{
b.Navigation("LibraryFolders");
b.Navigation("MediaItems");
});
modelBuilder.Entity("ErsatzTV.Core.Domain.MediaItem", b => { b.Navigation("CollectionItems"); });

Loading…
Cancel
Save