optimize search-index rebuilding (#765)

* update dependencies

* optimize search-index rebuilding

* cleanup logging

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Changed
 - Update Plex, Jellyfin and Emby movie library scanners to share a significant amount of code
   - This should help maintain feature parity going forward
+- Optimize search-index rebuilding to complete 100x faster
 
 ### Added
 - Add `unavailable` state for Emby movie libraries

ErsatzTV.Application/ErsatzTV.Application.csproj

@@ -8,7 +8,8 @@
 
     <ItemGroup>
       <PackageReference Include="Bugsnag" Version="3.0.1" />
-      <PackageReference Include="CliWrap" Version="3.4.3" />
+      <PackageReference Include="CliWrap" Version="3.4.4" />
+      <PackageReference Include="Humanizer.Core" Version="2.14.1" />
       <PackageReference Include="MediatR" Version="10.0.1" />
       <PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="6.0.0" />
       <PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.1.46">

ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs

@@ -1,8 +1,10 @@
-using ErsatzTV.Core;
+using System.Diagnostics;
+using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Metadata;
 using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Core.Interfaces.Search;
+using Humanizer;
 using Microsoft.Extensions.Logging;
 
 namespace ErsatzTV.Application.Search;
@@ -41,12 +43,13 @@ public class RebuildSearchIndexHandler : IRequestHandler<RebuildSearchIndex, Uni
         {
             _logger.LogInformation("Migrating search index to version {Version}", _searchIndex.Version);
 
-            List<int> itemIds = await _searchRepository.GetItemIdsToIndex();
+            var sw = Stopwatch.StartNew();
-            await _searchIndex.Rebuild(_searchRepository, itemIds);
+            await _searchIndex.Rebuild(_searchRepository);
 
             await _configElementRepository.Upsert(ConfigElementKey.SearchIndexVersion, _searchIndex.Version);
+            sw.Stop();
 
-            _logger.LogInformation("Done migrating search index");
+            _logger.LogInformation("Done migrating search index in {Duration}", sw.Elapsed.Humanize());
         }
         else
         {

ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj

@@ -8,7 +8,7 @@
 
     <ItemGroup>
       <PackageReference Include="Bugsnag" Version="3.0.1" />
-      <PackageReference Include="CliWrap" Version="3.4.3" />
+      <PackageReference Include="CliWrap" Version="3.4.4" />
       <PackageReference Include="FluentAssertions" Version="6.6.0" />
       <PackageReference Include="LanguageExt.Core" Version="4.0.4" />
       <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="6.0.1" />
@@ -24,7 +24,7 @@
       <PackageReference Include="Moq" Version="4.17.2" />
       <PackageReference Include="NUnit" Version="3.13.3" />
       <PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
-      <PackageReference Include="Serilog" Version="2.10.0" />
+      <PackageReference Include="Serilog" Version="2.11.0" />
       <PackageReference Include="Serilog.Extensions.Logging" Version="3.1.0" />
       <PackageReference Include="Serilog.Sinks.Debug" Version="2.0.0" />
     </ItemGroup>

ErsatzTV.Core/ErsatzTV.Core.csproj

@@ -9,7 +9,7 @@
     <ItemGroup>
       <PackageReference Include="Bugsnag" Version="3.0.1" />
       <PackageReference Include="Destructurama.Attributed" Version="3.0.0" />
-      <PackageReference Include="Flurl" Version="3.0.4" />
+      <PackageReference Include="Flurl" Version="3.0.5" />
       <PackageReference Include="LanguageExt.Core" Version="4.0.4" />
       <PackageReference Include="MediatR" Version="10.0.1" />
       <PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="6.0.0" />
@@ -21,7 +21,7 @@
         <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       </PackageReference>
       <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-      <PackageReference Include="Serilog" Version="2.10.0" />
+      <PackageReference Include="Serilog" Version="2.11.0" />
       <PackageReference Include="Serilog.Sinks.Console" Version="4.0.1" />
     </ItemGroup>
     

ErsatzTV.Core/Interfaces/Repositories/ISearchRepository.cs

@@ -4,10 +4,10 @@ namespace ErsatzTV.Core.Interfaces.Repositories;
 
 public interface ISearchRepository
 {
-    Task<List<int>> GetItemIdsToIndex();
     Task<Option<MediaItem>> GetItemToIndex(int id);
     Task<List<string>> GetLanguagesForShow(Show show);
     Task<List<string>> GetLanguagesForSeason(Season season);
     Task<List<string>> GetLanguagesForArtist(Artist artist);
     Task<List<string>> GetAllLanguageCodes(List<string> mediaCodes);
+    IAsyncEnumerable<MediaItem> GetAllMediaItems();
 }

ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs

@@ -9,7 +9,7 @@ public interface ISearchIndex : IDisposable
 {
     public int Version { get; }
     Task<bool> Initialize(ILocalFileSystem localFileSystem);
-    Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds);
+    Task<Unit> Rebuild(ISearchRepository searchRepository);
     Task<Unit> RebuildItems(ISearchRepository searchRepository, List<int> itemIds);
     Task<Unit> AddItems(ISearchRepository searchRepository, List<MediaItem> items);
     Task<Unit> UpdateItems(ISearchRepository searchRepository, List<MediaItem> items);

ErsatzTV.FFmpeg/ErsatzTV.FFmpeg.csproj

@@ -7,7 +7,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="CliWrap" Version="3.4.3" />
+      <PackageReference Include="CliWrap" Version="3.4.4" />
       <PackageReference Include="LanguageExt.Core" Version="4.0.4" />
       <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
     </ItemGroup>

ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs

@@ -12,13 +12,6 @@ public class SearchRepository : ISearchRepository
 
     public SearchRepository(IDbContextFactory<TvContext> dbContextFactory) => _dbContextFactory = dbContextFactory;
 
-    public async Task<List<int>> GetItemIdsToIndex()
-    {
-        await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync();
-        return await dbContext.Connection.QueryAsync<int>(@"SELECT Id FROM MediaItem")
-            .Map(result => result.ToList());
-    }
-
     public async Task<Option<MediaItem>> GetItemToIndex(int id)
     {
         await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync();
@@ -103,9 +96,7 @@ public class SearchRepository : ISearchRepository
             .ThenInclude(mm => mm.Streams)
             .Include(mi => mi.TraktListItems)
             .ThenInclude(tli => tli.TraktList)
-            .OrderBy(mi => mi.Id)
+            .SelectOneAsync(mi => mi.Id, mi => mi.Id == id);
-            .SingleOrDefaultAsync(mi => mi.Id == id)
-            .Map(Optional);
     }
 
     public async Task<List<string>> GetLanguagesForShow(Show show)
@@ -151,4 +142,91 @@ public class SearchRepository : ISearchRepository
         await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync();
         return await dbContext.LanguageCodes.GetAllLanguageCodes(mediaCodes);
     }
+
+    public IAsyncEnumerable<MediaItem> GetAllMediaItems()
+    {
+        TvContext dbContext = _dbContextFactory.CreateDbContext();
+        return dbContext.MediaItems
+            .AsNoTracking()
+            .Include(mi => mi.LibraryPath)
+            .ThenInclude(lp => lp.Library)
+            .Include(mi => (mi as Movie).MovieMetadata)
+            .ThenInclude(mm => mm.Genres)
+            .Include(mi => (mi as Movie).MovieMetadata)
+            .ThenInclude(mm => mm.Tags)
+            .Include(mi => (mi as Movie).MovieMetadata)
+            .ThenInclude(mm => mm.Studios)
+            .Include(mi => (mi as Movie).MovieMetadata)
+            .ThenInclude(mm => mm.Actors)
+            .Include(mi => (mi as Movie).MovieMetadata)
+            .ThenInclude(mm => mm.Directors)
+            .Include(mi => (mi as Movie).MovieMetadata)
+            .ThenInclude(mm => mm.Writers)
+            .Include(mi => (mi as Movie).MediaVersions)
+            .ThenInclude(mm => mm.Streams)
+            .Include(mi => (mi as Episode).EpisodeMetadata)
+            .ThenInclude(em => em.Genres)
+            .Include(mi => (mi as Episode).EpisodeMetadata)
+            .ThenInclude(em => em.Tags)
+            .Include(mi => (mi as Episode).EpisodeMetadata)
+            .ThenInclude(em => em.Studios)
+            .Include(mi => (mi as Episode).EpisodeMetadata)
+            .ThenInclude(em => em.Actors)
+            .Include(mi => (mi as Episode).EpisodeMetadata)
+            .ThenInclude(em => em.Directors)
+            .Include(mi => (mi as Episode).EpisodeMetadata)
+            .ThenInclude(em => em.Writers)
+            .Include(mi => (mi as Episode).EpisodeMetadata)
+            .ThenInclude(em => em.Guids)
+            .Include(mi => (mi as Episode).MediaVersions)
+            .ThenInclude(em => em.Streams)
+            .Include(mi => (mi as Episode).Season)
+            .Include(mi => (mi as Season).SeasonMetadata)
+            .ThenInclude(sm => sm.Genres)
+            .Include(mi => (mi as Season).SeasonMetadata)
+            .ThenInclude(sm => sm.Tags)
+            .Include(mi => (mi as Season).SeasonMetadata)
+            .ThenInclude(sm => sm.Studios)
+            .Include(mi => (mi as Season).SeasonMetadata)
+            .ThenInclude(sm => sm.Actors)
+            .Include(mi => (mi as Season).Show)
+            .ThenInclude(sm => sm.ShowMetadata)
+            .Include(mi => (mi as Show).ShowMetadata)
+            .ThenInclude(mm => mm.Genres)
+            .Include(mi => (mi as Show).ShowMetadata)
+            .ThenInclude(mm => mm.Tags)
+            .Include(mi => (mi as Show).ShowMetadata)
+            .ThenInclude(mm => mm.Studios)
+            .Include(mi => (mi as Show).ShowMetadata)
+            .ThenInclude(mm => mm.Actors)
+            .Include(mi => (mi as MusicVideo).Artist)
+            .ThenInclude(mm => mm.ArtistMetadata)
+            .Include(mi => (mi as MusicVideo).MusicVideoMetadata)
+            .ThenInclude(mm => mm.Genres)
+            .Include(mi => (mi as MusicVideo).MusicVideoMetadata)
+            .ThenInclude(mm => mm.Tags)
+            .Include(mi => (mi as MusicVideo).MusicVideoMetadata)
+            .ThenInclude(mm => mm.Studios)
+            .Include(mi => (mi as MusicVideo).MediaVersions)
+            .ThenInclude(mm => mm.Streams)
+            .Include(mi => (mi as Artist).ArtistMetadata)
+            .ThenInclude(mm => mm.Genres)
+            .Include(mi => (mi as Artist).ArtistMetadata)
+            .ThenInclude(mm => mm.Styles)
+            .Include(mi => (mi as Artist).ArtistMetadata)
+            .ThenInclude(mm => mm.Moods)
+            .Include(mi => (mi as OtherVideo).OtherVideoMetadata)
+            .ThenInclude(mm => mm.Tags)
+            .Include(mi => (mi as OtherVideo).MediaVersions)
+            .ThenInclude(mm => mm.Streams)
+            .Include(mi => (mi as Song).SongMetadata)
+            .ThenInclude(mm => mm.Tags)
+            .Include(mi => (mi as Song).SongMetadata)
+            .ThenInclude(mm => mm.Genres)
+            .Include(mi => (mi as Song).MediaVersions)
+            .ThenInclude(mm => mm.Streams)
+            .Include(mi => mi.TraktListItems)
+            .ThenInclude(tli => tli.TraktList)
+            .AsAsyncEnumerable();
+    }
 }

ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj

@@ -9,7 +9,7 @@
 
     <ItemGroup>
       <PackageReference Include="Blurhash.System.Drawing.Common" Version="2.2.0" />
-      <PackageReference Include="CliWrap" Version="3.4.3" />
+      <PackageReference Include="CliWrap" Version="3.4.4" />
       <PackageReference Include="Dapper" Version="2.0.123" />
       <PackageReference Include="Lucene.Net" Version="4.8.0-beta00016" />
       <PackageReference Include="Lucene.Net.Analysis.Common" Version="4.8.0-beta00016" />

ErsatzTV.Infrastructure/Search/SearchIndex.cs

@@ -197,11 +197,15 @@ public sealed class SearchIndex : ISearchIndex
         _directory?.Dispose();
     }
 
-    public async Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds)
+    public async Task<Unit> Rebuild(ISearchRepository searchRepository)
     {
         _writer.DeleteAll();
+        _writer.Commit();
 
-        await RebuildItems(searchRepository, itemIds);
+        await foreach (MediaItem mediaItem in searchRepository.GetAllMediaItems())
+        {
+            await RebuildItem(searchRepository, mediaItem);
+        }
 
         _writer.Commit();
         return Unit.Default;
@@ -213,39 +217,44 @@ public sealed class SearchIndex : ISearchIndex
         {
             foreach (MediaItem mediaItem in await searchRepository.GetItemToIndex(id))
             {
-                switch (mediaItem)
+                await RebuildItem(searchRepository, mediaItem);
-                {
-                    case Movie movie:
-                        await UpdateMovie(searchRepository, movie);
-                        break;
-                    case Show show:
-                        await UpdateShow(searchRepository, show);
-                        break;
-                    case Season season:
-                        await UpdateSeason(searchRepository, season);
-                        break;
-                    case Artist artist:
-                        await UpdateArtist(searchRepository, artist);
-                        break;
-                    case MusicVideo musicVideo:
-                        await UpdateMusicVideo(searchRepository, musicVideo);
-                        break;
-                    case Episode episode:
-                        await UpdateEpisode(searchRepository, episode);
-                        break;
-                    case OtherVideo otherVideo:
-                        await UpdateOtherVideo(searchRepository, otherVideo);
-                        break;
-                    case Song song:
-                        await UpdateSong(searchRepository, song);
-                        break;
-                }
             }
         }
 
         return Unit.Default;
     }
 
+    private async Task RebuildItem(ISearchRepository searchRepository, MediaItem mediaItem)
+    {
+        switch (mediaItem)
+        {
+            case Movie movie:
+                await UpdateMovie(searchRepository, movie);
+                break;
+            case Show show:
+                await UpdateShow(searchRepository, show);
+                break;
+            case Season season:
+                await UpdateSeason(searchRepository, season);
+                break;
+            case Artist artist:
+                await UpdateArtist(searchRepository, artist);
+                break;
+            case MusicVideo musicVideo:
+                await UpdateMusicVideo(searchRepository, musicVideo);
+                break;
+            case Episode episode:
+                await UpdateEpisode(searchRepository, episode);
+                break;
+            case OtherVideo otherVideo:
+                await UpdateOtherVideo(searchRepository, otherVideo);
+                break;
+            case Song song:
+                await UpdateSong(searchRepository, song);
+                break;
+        }
+    }
+
     private static Option<SearchPageMap> GetSearchPageMap(
         IndexSearcher searcher,
         Query query,

ErsatzTV/ErsatzTV.csproj

@@ -59,7 +59,7 @@
       <PackageReference Include="FluentValidation.AspNetCore" Version="10.4.0" />
       <PackageReference Include="HtmlSanitizer" Version="7.1.488" />
       <PackageReference Include="LanguageExt.Core" Version="4.0.4" />
-      <PackageReference Include="Markdig" Version="0.30.0" />
+      <PackageReference Include="Markdig" Version="0.30.2" />
       <PackageReference Include="MediatR.Courier.DependencyInjection" Version="5.0.0" />
       <PackageReference Include="MediatR.Extensions.Microsoft.DependencyInjection" Version="10.0.1" />
       <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.4" />
@@ -76,7 +76,7 @@
       <PackageReference Include="NaturalSort.Extension" Version="3.2.0" />
       <PackageReference Include="PPioli.FluentValidation.Blazor" Version="5.0.0" />
       <PackageReference Include="Refit.HttpClientFactory" Version="6.3.2" />
-      <PackageReference Include="Serilog" Version="2.10.0" />
+      <PackageReference Include="Serilog" Version="2.11.0" />
       <PackageReference Include="Serilog.AspNetCore" Version="5.0.0" />
       <PackageReference Include="Serilog.Settings.Configuration" Version="3.3.0" />
       <PackageReference Include="Serilog.Sinks.SQLite" Version="6.0.1" />