diff --git a/CHANGELOG.md b/CHANGELOG.md index e3cba6174..9785f5489 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - Update Plex, Jellyfin and Emby movie library scanners to share a significant amount of code - This should help maintain feature parity going forward +- Optimize search-index rebuilding to complete 100x faster ### Added - Add `unavailable` state for Emby movie libraries diff --git a/ErsatzTV.Application/ErsatzTV.Application.csproj b/ErsatzTV.Application/ErsatzTV.Application.csproj index 026e7303a..917aa78b5 100644 --- a/ErsatzTV.Application/ErsatzTV.Application.csproj +++ b/ErsatzTV.Application/ErsatzTV.Application.csproj @@ -8,7 +8,8 @@ - + + diff --git a/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs b/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs index 880e72507..412449dc8 100644 --- a/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs +++ b/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs @@ -1,8 +1,10 @@ -using ErsatzTV.Core; +using System.Diagnostics; +using ErsatzTV.Core; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Interfaces.Metadata; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Interfaces.Search; +using Humanizer; using Microsoft.Extensions.Logging; namespace ErsatzTV.Application.Search; @@ -41,12 +43,13 @@ public class RebuildSearchIndexHandler : IRequestHandler itemIds = await _searchRepository.GetItemIdsToIndex(); - await _searchIndex.Rebuild(_searchRepository, itemIds); + var sw = Stopwatch.StartNew(); + await _searchIndex.Rebuild(_searchRepository); await _configElementRepository.Upsert(ConfigElementKey.SearchIndexVersion, _searchIndex.Version); + sw.Stop(); - _logger.LogInformation("Done migrating search index"); + _logger.LogInformation("Done migrating search index in {Duration}", sw.Elapsed.Humanize()); } else { diff --git a/ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj b/ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj index 07534ec9c..b7a10f6ef 100644 --- a/ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj +++ b/ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj @@ -8,7 +8,7 @@ - + @@ -24,7 +24,7 @@ - + diff --git a/ErsatzTV.Core/ErsatzTV.Core.csproj b/ErsatzTV.Core/ErsatzTV.Core.csproj index b87579a62..de909e47f 100644 --- a/ErsatzTV.Core/ErsatzTV.Core.csproj +++ b/ErsatzTV.Core/ErsatzTV.Core.csproj @@ -9,7 +9,7 @@ - + @@ -21,7 +21,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/ErsatzTV.Core/Interfaces/Repositories/ISearchRepository.cs b/ErsatzTV.Core/Interfaces/Repositories/ISearchRepository.cs index 691246c5d..50c38625b 100644 --- a/ErsatzTV.Core/Interfaces/Repositories/ISearchRepository.cs +++ b/ErsatzTV.Core/Interfaces/Repositories/ISearchRepository.cs @@ -4,10 +4,10 @@ namespace ErsatzTV.Core.Interfaces.Repositories; public interface ISearchRepository { - Task> GetItemIdsToIndex(); Task> GetItemToIndex(int id); Task> GetLanguagesForShow(Show show); Task> GetLanguagesForSeason(Season season); Task> GetLanguagesForArtist(Artist artist); Task> GetAllLanguageCodes(List mediaCodes); + IAsyncEnumerable GetAllMediaItems(); } diff --git a/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs b/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs index 0a49849fa..4eaa76b19 100644 --- a/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs +++ b/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs @@ -9,7 +9,7 @@ public interface ISearchIndex : IDisposable { public int Version { get; } Task Initialize(ILocalFileSystem localFileSystem); - Task Rebuild(ISearchRepository searchRepository, List itemIds); + Task Rebuild(ISearchRepository searchRepository); Task RebuildItems(ISearchRepository searchRepository, List itemIds); Task AddItems(ISearchRepository searchRepository, List items); Task UpdateItems(ISearchRepository searchRepository, List items); diff --git a/ErsatzTV.FFmpeg/ErsatzTV.FFmpeg.csproj b/ErsatzTV.FFmpeg/ErsatzTV.FFmpeg.csproj index 9591668be..212b64b9b 100644 --- a/ErsatzTV.FFmpeg/ErsatzTV.FFmpeg.csproj +++ b/ErsatzTV.FFmpeg/ErsatzTV.FFmpeg.csproj @@ -7,7 +7,7 @@ - + diff --git a/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs index c88e3e037..95425f096 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs @@ -12,13 +12,6 @@ public class SearchRepository : ISearchRepository public SearchRepository(IDbContextFactory dbContextFactory) => _dbContextFactory = dbContextFactory; - public async Task> GetItemIdsToIndex() - { - await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(); - return await dbContext.Connection.QueryAsync(@"SELECT Id FROM MediaItem") - .Map(result => result.ToList()); - } - public async Task> GetItemToIndex(int id) { await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(); @@ -103,9 +96,7 @@ public class SearchRepository : ISearchRepository .ThenInclude(mm => mm.Streams) .Include(mi => mi.TraktListItems) .ThenInclude(tli => tli.TraktList) - .OrderBy(mi => mi.Id) - .SingleOrDefaultAsync(mi => mi.Id == id) - .Map(Optional); + .SelectOneAsync(mi => mi.Id, mi => mi.Id == id); } public async Task> GetLanguagesForShow(Show show) @@ -151,4 +142,91 @@ public class SearchRepository : ISearchRepository await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(); return await dbContext.LanguageCodes.GetAllLanguageCodes(mediaCodes); } + + public IAsyncEnumerable GetAllMediaItems() + { + TvContext dbContext = _dbContextFactory.CreateDbContext(); + return dbContext.MediaItems + .AsNoTracking() + .Include(mi => mi.LibraryPath) + .ThenInclude(lp => lp.Library) + .Include(mi => (mi as Movie).MovieMetadata) + .ThenInclude(mm => mm.Genres) + .Include(mi => (mi as Movie).MovieMetadata) + .ThenInclude(mm => mm.Tags) + .Include(mi => (mi as Movie).MovieMetadata) + .ThenInclude(mm => mm.Studios) + .Include(mi => (mi as Movie).MovieMetadata) + .ThenInclude(mm => mm.Actors) + .Include(mi => (mi as Movie).MovieMetadata) + .ThenInclude(mm => mm.Directors) + .Include(mi => (mi as Movie).MovieMetadata) + .ThenInclude(mm => mm.Writers) + .Include(mi => (mi as Movie).MediaVersions) + .ThenInclude(mm => mm.Streams) + .Include(mi => (mi as Episode).EpisodeMetadata) + .ThenInclude(em => em.Genres) + .Include(mi => (mi as Episode).EpisodeMetadata) + .ThenInclude(em => em.Tags) + .Include(mi => (mi as Episode).EpisodeMetadata) + .ThenInclude(em => em.Studios) + .Include(mi => (mi as Episode).EpisodeMetadata) + .ThenInclude(em => em.Actors) + .Include(mi => (mi as Episode).EpisodeMetadata) + .ThenInclude(em => em.Directors) + .Include(mi => (mi as Episode).EpisodeMetadata) + .ThenInclude(em => em.Writers) + .Include(mi => (mi as Episode).EpisodeMetadata) + .ThenInclude(em => em.Guids) + .Include(mi => (mi as Episode).MediaVersions) + .ThenInclude(em => em.Streams) + .Include(mi => (mi as Episode).Season) + .Include(mi => (mi as Season).SeasonMetadata) + .ThenInclude(sm => sm.Genres) + .Include(mi => (mi as Season).SeasonMetadata) + .ThenInclude(sm => sm.Tags) + .Include(mi => (mi as Season).SeasonMetadata) + .ThenInclude(sm => sm.Studios) + .Include(mi => (mi as Season).SeasonMetadata) + .ThenInclude(sm => sm.Actors) + .Include(mi => (mi as Season).Show) + .ThenInclude(sm => sm.ShowMetadata) + .Include(mi => (mi as Show).ShowMetadata) + .ThenInclude(mm => mm.Genres) + .Include(mi => (mi as Show).ShowMetadata) + .ThenInclude(mm => mm.Tags) + .Include(mi => (mi as Show).ShowMetadata) + .ThenInclude(mm => mm.Studios) + .Include(mi => (mi as Show).ShowMetadata) + .ThenInclude(mm => mm.Actors) + .Include(mi => (mi as MusicVideo).Artist) + .ThenInclude(mm => mm.ArtistMetadata) + .Include(mi => (mi as MusicVideo).MusicVideoMetadata) + .ThenInclude(mm => mm.Genres) + .Include(mi => (mi as MusicVideo).MusicVideoMetadata) + .ThenInclude(mm => mm.Tags) + .Include(mi => (mi as MusicVideo).MusicVideoMetadata) + .ThenInclude(mm => mm.Studios) + .Include(mi => (mi as MusicVideo).MediaVersions) + .ThenInclude(mm => mm.Streams) + .Include(mi => (mi as Artist).ArtistMetadata) + .ThenInclude(mm => mm.Genres) + .Include(mi => (mi as Artist).ArtistMetadata) + .ThenInclude(mm => mm.Styles) + .Include(mi => (mi as Artist).ArtistMetadata) + .ThenInclude(mm => mm.Moods) + .Include(mi => (mi as OtherVideo).OtherVideoMetadata) + .ThenInclude(mm => mm.Tags) + .Include(mi => (mi as OtherVideo).MediaVersions) + .ThenInclude(mm => mm.Streams) + .Include(mi => (mi as Song).SongMetadata) + .ThenInclude(mm => mm.Tags) + .Include(mi => (mi as Song).SongMetadata) + .ThenInclude(mm => mm.Genres) + .Include(mi => (mi as Song).MediaVersions) + .ThenInclude(mm => mm.Streams) + .Include(mi => mi.TraktListItems) + .ThenInclude(tli => tli.TraktList) + .AsAsyncEnumerable(); + } } diff --git a/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj b/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj index 4da202b8c..9e620578d 100644 --- a/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj +++ b/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj @@ -9,7 +9,7 @@ - + diff --git a/ErsatzTV.Infrastructure/Search/SearchIndex.cs b/ErsatzTV.Infrastructure/Search/SearchIndex.cs index b568cb4fa..cef971282 100644 --- a/ErsatzTV.Infrastructure/Search/SearchIndex.cs +++ b/ErsatzTV.Infrastructure/Search/SearchIndex.cs @@ -197,11 +197,15 @@ public sealed class SearchIndex : ISearchIndex _directory?.Dispose(); } - public async Task Rebuild(ISearchRepository searchRepository, List itemIds) + public async Task Rebuild(ISearchRepository searchRepository) { _writer.DeleteAll(); + _writer.Commit(); - await RebuildItems(searchRepository, itemIds); + await foreach (MediaItem mediaItem in searchRepository.GetAllMediaItems()) + { + await RebuildItem(searchRepository, mediaItem); + } _writer.Commit(); return Unit.Default; @@ -213,39 +217,44 @@ public sealed class SearchIndex : ISearchIndex { foreach (MediaItem mediaItem in await searchRepository.GetItemToIndex(id)) { - switch (mediaItem) - { - case Movie movie: - await UpdateMovie(searchRepository, movie); - break; - case Show show: - await UpdateShow(searchRepository, show); - break; - case Season season: - await UpdateSeason(searchRepository, season); - break; - case Artist artist: - await UpdateArtist(searchRepository, artist); - break; - case MusicVideo musicVideo: - await UpdateMusicVideo(searchRepository, musicVideo); - break; - case Episode episode: - await UpdateEpisode(searchRepository, episode); - break; - case OtherVideo otherVideo: - await UpdateOtherVideo(searchRepository, otherVideo); - break; - case Song song: - await UpdateSong(searchRepository, song); - break; - } + await RebuildItem(searchRepository, mediaItem); } } return Unit.Default; } + private async Task RebuildItem(ISearchRepository searchRepository, MediaItem mediaItem) + { + switch (mediaItem) + { + case Movie movie: + await UpdateMovie(searchRepository, movie); + break; + case Show show: + await UpdateShow(searchRepository, show); + break; + case Season season: + await UpdateSeason(searchRepository, season); + break; + case Artist artist: + await UpdateArtist(searchRepository, artist); + break; + case MusicVideo musicVideo: + await UpdateMusicVideo(searchRepository, musicVideo); + break; + case Episode episode: + await UpdateEpisode(searchRepository, episode); + break; + case OtherVideo otherVideo: + await UpdateOtherVideo(searchRepository, otherVideo); + break; + case Song song: + await UpdateSong(searchRepository, song); + break; + } + } + private static Option GetSearchPageMap( IndexSearcher searcher, Query query, diff --git a/ErsatzTV/ErsatzTV.csproj b/ErsatzTV/ErsatzTV.csproj index fa4bb690e..0b534ed45 100644 --- a/ErsatzTV/ErsatzTV.csproj +++ b/ErsatzTV/ErsatzTV.csproj @@ -59,7 +59,7 @@ - + @@ -76,7 +76,7 @@ - +