optimize memory use during search index rebuild (#88)

ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs

@@ -40,8 +40,8 @@ namespace ErsatzTV.Application.Search.Commands
             {
                 _logger.LogDebug("Migrating search index to version {Version}", _searchIndex.Version);
 
-                List<MediaItem> items = await _searchRepository.GetItemsToIndex();
+                List<int> itemIds = await _searchRepository.GetItemIdsToIndex();
-                await _searchIndex.Rebuild(items);
+                await _searchIndex.Rebuild(itemIds);
 
                 Option<ConfigElement> maybeVersion =
                     await _configElementRepository.Get(ConfigElementKey.SearchIndexVersion);

ErsatzTV.Core/Interfaces/Repositories/ISearchRepository.cs

@@ -1,12 +1,14 @@
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using ErsatzTV.Core.Domain;
+using LanguageExt;
 
 namespace ErsatzTV.Core.Interfaces.Repositories
 {
     public interface ISearchRepository
     {
-        public Task<List<MediaItem>> GetItemsToIndex();
+        public Task<List<int>> GetItemIdsToIndex();
+        public Task<Option<MediaItem>> GetItemToIndex(int id);
         public Task<List<MediaItem>> SearchMediaItemsByTitle(string query);
         public Task<List<MediaItem>> SearchMediaItemsByGenre(string genre);
         public Task<List<MediaItem>> SearchMediaItemsByTag(string tag);

ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs

@@ -10,7 +10,7 @@ namespace ErsatzTV.Core.Interfaces.Search
     {
         public int Version { get; }
         Task<bool> Initialize();
-        Task<Unit> Rebuild(List<MediaItem> items);
+        Task<Unit> Rebuild(List<int> itemIds);
         Task<Unit> AddItems(List<MediaItem> items);
         Task<Unit> UpdateItems(List<MediaItem> items);
         Task<Unit> RemoveItems(List<int> ids);

ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs

@@ -7,6 +7,7 @@ using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Repositories;
 using LanguageExt;
 using Microsoft.EntityFrameworkCore;
+using static LanguageExt.Prelude;
 
 namespace ErsatzTV.Infrastructure.Data.Repositories
 {
@@ -21,7 +22,11 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
             _dbConnection = dbConnection;
         }
 
-        public async Task<List<MediaItem>> GetItemsToIndex()
+        public Task<List<int>> GetItemIdsToIndex() =>
+            _dbConnection.QueryAsync<int>(@"SELECT Id FROM MediaItem")
+                .Map(result => result.ToList());
+
+        public async Task<Option<MediaItem>> GetItemToIndex(int id)
         {
             await using TvContext dbContext = _dbContextFactory.CreateDbContext();
             return await dbContext.MediaItems
@@ -36,7 +41,9 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
                 .ThenInclude(mm => mm.Genres)
                 .Include(mi => (mi as Show).ShowMetadata)
                 .ThenInclude(mm => mm.Tags)
-                .ToListAsync();
+                .OrderBy(mi => mi.Id)
+                .SingleOrDefaultAsync(mi => mi.Id == id)
+                .Map(Optional);
         }
 
         public async Task<List<MediaItem>> SearchMediaItemsByTitle(string query)

ErsatzTV.Infrastructure/Search/SearchIndex.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Metadata;
+using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Core.Interfaces.Search;
 using ErsatzTV.Core.Search;
 using LanguageExt;
@@ -42,8 +43,13 @@ namespace ErsatzTV.Infrastructure.Search
         private readonly ILocalFileSystem _localFileSystem;
 
         private readonly string[] _searchFields = { TitleField, GenreField, TagField };
+        private readonly ISearchRepository _searchRepository;
 
-        public SearchIndex(ILocalFileSystem localFileSystem) => _localFileSystem = localFileSystem;
+        public SearchIndex(ILocalFileSystem localFileSystem, ISearchRepository searchRepository)
+        {
+            _localFileSystem = localFileSystem;
+            _searchRepository = searchRepository;
+        }
 
         public int Version => 1;
 
@@ -53,7 +59,7 @@ namespace ErsatzTV.Infrastructure.Search
             return Task.FromResult(true);
         }
 
-        public async Task<Unit> Rebuild(List<MediaItem> items)
+        public async Task<Unit> Rebuild(List<int> itemIds)
         {
             await Initialize();
 
@@ -62,8 +68,23 @@ namespace ErsatzTV.Infrastructure.Search
             var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.CREATE };
             using var writer = new IndexWriter(dir, indexConfig);
 
-            UpdateMovies(items.OfType<Movie>(), writer);
+            foreach (int id in itemIds)
-            UpdateShows(items.OfType<Show>(), writer);
+            {
+                Option<MediaItem> maybeMediaItem = await _searchRepository.GetItemToIndex(id);
+                if (maybeMediaItem.IsSome)
+                {
+                    MediaItem mediaItem = maybeMediaItem.ValueUnsafe();
+                    switch (mediaItem)
+                    {
+                        case Movie movie:
+                            UpdateMovie(movie, writer);
+                            break;
+                        case Show show:
+                            UpdateShow(show, writer);
+                            break;
+                    }
+                }
+            }
 
             return Unit.Default;
         }
@@ -77,8 +98,18 @@ namespace ErsatzTV.Infrastructure.Search
             var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.APPEND };
             using var writer = new IndexWriter(dir, indexConfig);
 
-            UpdateMovies(items.OfType<Movie>(), writer);
+            foreach (MediaItem item in items)
-            UpdateShows(items.OfType<Show>(), writer);
+            {
+                switch (item)
+                {
+                    case Movie movie:
+                        UpdateMovie(movie, writer);
+                        break;
+                    case Show show:
+                        UpdateShow(show, writer);
+                        break;
+                }
+            }
 
             return Task.FromResult(Unit.Default);
         }
@@ -174,83 +205,77 @@ namespace ErsatzTV.Infrastructure.Search
             return new SearchPageMap(map);
         }
 
-        private static void UpdateMovies(IEnumerable<Movie> movies, IndexWriter writer)
+        private static void UpdateMovie(Movie movie, IndexWriter writer)
         {
-            foreach (Movie movie in movies)
+            Option<MovieMetadata> maybeMetadata = movie.MovieMetadata.HeadOrNone();
+            if (maybeMetadata.IsSome)
             {
-                Option<MovieMetadata> maybeMetadata = movie.MovieMetadata.HeadOrNone();
+                MovieMetadata metadata = maybeMetadata.ValueUnsafe();
-                if (maybeMetadata.IsSome)
-                {
-                    MovieMetadata metadata = maybeMetadata.ValueUnsafe();
-
-                    var doc = new Document
-                    {
-                        new StringField(IdField, movie.Id.ToString(), Field.Store.YES),
-                        new StringField(TypeField, MovieType, Field.Store.NO),
-                        new TextField(TitleField, metadata.Title, Field.Store.NO),
-                        new StringField(SortTitleField, metadata.SortTitle.ToLowerInvariant(), Field.Store.NO),
-                        new TextField(LibraryNameField, movie.LibraryPath.Library.Name, Field.Store.NO),
-                        new StringField(TitleAndYearField, GetTitleAndYear(metadata), Field.Store.NO),
-                        new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
-                    };
-
-                    if (!string.IsNullOrWhiteSpace(metadata.Plot))
-                    {
-                        doc.Add(new TextField(PlotField, metadata.Plot ?? string.Empty, Field.Store.NO));
-                    }
 
-                    foreach (Genre genre in metadata.Genres)
+                var doc = new Document
-                    {
+                {
-                        doc.Add(new TextField(GenreField, genre.Name, Field.Store.NO));
+                    new StringField(IdField, movie.Id.ToString(), Field.Store.YES),
-                    }
+                    new StringField(TypeField, MovieType, Field.Store.NO),
+                    new TextField(TitleField, metadata.Title, Field.Store.NO),
+                    new StringField(SortTitleField, metadata.SortTitle.ToLowerInvariant(), Field.Store.NO),
+                    new TextField(LibraryNameField, movie.LibraryPath.Library.Name, Field.Store.NO),
+                    new StringField(TitleAndYearField, GetTitleAndYear(metadata), Field.Store.NO),
+                    new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
+                };
+
+                if (!string.IsNullOrWhiteSpace(metadata.Plot))
+                {
+                    doc.Add(new TextField(PlotField, metadata.Plot ?? string.Empty, Field.Store.NO));
+                }
 
-                    foreach (Tag tag in metadata.Tags)
+                foreach (Genre genre in metadata.Genres)
-                    {
+                {
-                        doc.Add(new TextField(TagField, tag.Name, Field.Store.NO));
+                    doc.Add(new TextField(GenreField, genre.Name, Field.Store.NO));
-                    }
+                }
 
-                    writer.UpdateDocument(new Term(IdField, movie.Id.ToString()), doc);
+                foreach (Tag tag in metadata.Tags)
+                {
+                    doc.Add(new TextField(TagField, tag.Name, Field.Store.NO));
                 }
+
+                writer.UpdateDocument(new Term(IdField, movie.Id.ToString()), doc);
             }
         }
 
-        private static void UpdateShows(IEnumerable<Show> shows, IndexWriter writer)
+        private static void UpdateShow(Show show, IndexWriter writer)
         {
-            foreach (Show show in shows)
+            Option<ShowMetadata> maybeMetadata = show.ShowMetadata.HeadOrNone();
+            if (maybeMetadata.IsSome)
             {
-                Option<ShowMetadata> maybeMetadata = show.ShowMetadata.HeadOrNone();
+                ShowMetadata metadata = maybeMetadata.ValueUnsafe();
-                if (maybeMetadata.IsSome)
-                {
-                    ShowMetadata metadata = maybeMetadata.ValueUnsafe();
-
-                    var doc = new Document
-                    {
-                        new StringField(IdField, show.Id.ToString(), Field.Store.YES),
-                        new StringField(TypeField, ShowType, Field.Store.NO),
-                        new TextField(TitleField, metadata.Title, Field.Store.NO),
-                        new StringField(SortTitleField, metadata.SortTitle.ToLowerInvariant(), Field.Store.NO),
-                        new TextField(LibraryNameField, show.LibraryPath.Library.Name, Field.Store.NO),
-                        new StringField(TitleAndYearField, GetTitleAndYear(metadata), Field.Store.NO),
-                        new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
-                    };
-
-                    if (!string.IsNullOrWhiteSpace(metadata.Plot))
-                    {
-                        doc.Add(new TextField(PlotField, metadata.Plot ?? string.Empty, Field.Store.NO));
-                    }
 
-                    foreach (Genre genre in metadata.Genres)
+                var doc = new Document
-                    {
+                {
-                        doc.Add(new TextField(GenreField, genre.Name, Field.Store.NO));
+                    new StringField(IdField, show.Id.ToString(), Field.Store.YES),
-                    }
+                    new StringField(TypeField, ShowType, Field.Store.NO),
+                    new TextField(TitleField, metadata.Title, Field.Store.NO),
+                    new StringField(SortTitleField, metadata.SortTitle.ToLowerInvariant(), Field.Store.NO),
+                    new TextField(LibraryNameField, show.LibraryPath.Library.Name, Field.Store.NO),
+                    new StringField(TitleAndYearField, GetTitleAndYear(metadata), Field.Store.NO),
+                    new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
+                };
+
+                if (!string.IsNullOrWhiteSpace(metadata.Plot))
+                {
+                    doc.Add(new TextField(PlotField, metadata.Plot ?? string.Empty, Field.Store.NO));
+                }
 
-                    foreach (Tag tag in metadata.Tags)
+                foreach (Genre genre in metadata.Genres)
-                    {
+                {
-                        doc.Add(new TextField(TagField, tag.Name, Field.Store.NO));
+                    doc.Add(new TextField(GenreField, genre.Name, Field.Store.NO));
-                    }
+                }
 
-                    writer.UpdateDocument(new Term(IdField, show.Id.ToString()), doc);
+                foreach (Tag tag in metadata.Tags)
+                {
+                    doc.Add(new TextField(TagField, tag.Name, Field.Store.NO));
                 }
+
+                writer.UpdateDocument(new Term(IdField, show.Id.ToString()), doc);
             }
         }