pimhwang
/
ErsatzTV
mirror of https://github.com/ErsatzTV/ErsatzTV.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

optimize memory use during search index rebuild (#88)

pull/89/head
Jason Dove 5 years ago committed by GitHub
parent
2efcbca2da
commit
c0a6677861
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 108 additions and 74 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs
  2. 4
      ErsatzTV.Core/Interfaces/Repositories/ISearchRepository.cs
  3. 2
      ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs
  4. 11
      ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs
  5. 161
      ErsatzTV.Infrastructure/Search/SearchIndex.cs

4
ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs
Unescape View File

@ -40,8 +40,8 @@ namespace ErsatzTV.Application.Search.Commands @@ -40,8 +40,8 @@ namespace ErsatzTV.Application.Search.Commands
{
_logger.LogDebug("Migrating search index to version {Version}", _searchIndex.Version);
List<MediaItem> items = await _searchRepository.GetItemsToIndex();
await _searchIndex.Rebuild(items);
List<int> itemIds = await _searchRepository.GetItemIdsToIndex();
await _searchIndex.Rebuild(itemIds);
Option<ConfigElement> maybeVersion =
await _configElementRepository.Get(ConfigElementKey.SearchIndexVersion);

4
ErsatzTV.Core/Interfaces/Repositories/ISearchRepository.cs
Unescape View File

@ -1,12 +1,14 @@ @@ -1,12 +1,14 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using ErsatzTV.Core.Domain;
using LanguageExt;
namespace ErsatzTV.Core.Interfaces.Repositories
{
public interface ISearchRepository
{
public Task<List<MediaItem>> GetItemsToIndex();
public Task<List<int>> GetItemIdsToIndex();
public Task<Option<MediaItem>> GetItemToIndex(int id);
public Task<List<MediaItem>> SearchMediaItemsByTitle(string query);
public Task<List<MediaItem>> SearchMediaItemsByGenre(string genre);
public Task<List<MediaItem>> SearchMediaItemsByTag(string tag);

2
ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs
Unescape View File

@ -10,7 +10,7 @@ namespace ErsatzTV.Core.Interfaces.Search @@ -10,7 +10,7 @@ namespace ErsatzTV.Core.Interfaces.Search
{
public int Version { get; }
Task<bool> Initialize();
Task<Unit> Rebuild(List<MediaItem> items);
Task<Unit> Rebuild(List<int> itemIds);
Task<Unit> AddItems(List<MediaItem> items);
Task<Unit> UpdateItems(List<MediaItem> items);
Task<Unit> RemoveItems(List<int> ids);

11
ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs
Unescape View File

@ -7,6 +7,7 @@ using ErsatzTV.Core.Domain; @@ -7,6 +7,7 @@ using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.Repositories;
using LanguageExt;
using Microsoft.EntityFrameworkCore;
using static LanguageExt.Prelude;
namespace ErsatzTV.Infrastructure.Data.Repositories
{
@ -21,7 +22,11 @@ namespace ErsatzTV.Infrastructure.Data.Repositories @@ -21,7 +22,11 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
_dbConnection = dbConnection;
}
public async Task<List<MediaItem>> GetItemsToIndex()
public Task<List<int>> GetItemIdsToIndex() =>
_dbConnection.QueryAsync<int>(@"SELECT Id FROM MediaItem")
.Map(result => result.ToList());
public async Task<Option<MediaItem>> GetItemToIndex(int id)
{
await using TvContext dbContext = _dbContextFactory.CreateDbContext();
return await dbContext.MediaItems
@ -36,7 +41,9 @@ namespace ErsatzTV.Infrastructure.Data.Repositories @@ -36,7 +41,9 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.ThenInclude(mm => mm.Genres)
.Include(mi => (mi as Show).ShowMetadata)
.ThenInclude(mm => mm.Tags)
.ToListAsync();
.OrderBy(mi => mi.Id)
.SingleOrDefaultAsync(mi => mi.Id == id)
.Map(Optional);
}
public async Task<List<MediaItem>> SearchMediaItemsByTitle(string query)

161
ErsatzTV.Infrastructure/Search/SearchIndex.cs
Unescape View File

@ -5,6 +5,7 @@ using System.Threading.Tasks; @@ -5,6 +5,7 @@ using System.Threading.Tasks;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.Metadata;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Core.Interfaces.Search;
using ErsatzTV.Core.Search;
using LanguageExt;
@ -42,8 +43,13 @@ namespace ErsatzTV.Infrastructure.Search @@ -42,8 +43,13 @@ namespace ErsatzTV.Infrastructure.Search
private readonly ILocalFileSystem _localFileSystem;
private readonly string[] _searchFields = { TitleField, GenreField, TagField };
private readonly ISearchRepository _searchRepository;
public SearchIndex(ILocalFileSystem localFileSystem) => _localFileSystem = localFileSystem;
public SearchIndex(ILocalFileSystem localFileSystem, ISearchRepository searchRepository)
{
_localFileSystem = localFileSystem;
_searchRepository = searchRepository;
}
public int Version => 1;
@ -53,7 +59,7 @@ namespace ErsatzTV.Infrastructure.Search @@ -53,7 +59,7 @@ namespace ErsatzTV.Infrastructure.Search
return Task.FromResult(true);
}
public async Task<Unit> Rebuild(List<MediaItem> items)
public async Task<Unit> Rebuild(List<int> itemIds)
{
await Initialize();
@ -62,8 +68,23 @@ namespace ErsatzTV.Infrastructure.Search @@ -62,8 +68,23 @@ namespace ErsatzTV.Infrastructure.Search
var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.CREATE };
using var writer = new IndexWriter(dir, indexConfig);
UpdateMovies(items.OfType<Movie>(), writer);
UpdateShows(items.OfType<Show>(), writer);
foreach (int id in itemIds)
{
Option<MediaItem> maybeMediaItem = await _searchRepository.GetItemToIndex(id);
if (maybeMediaItem.IsSome)
{
MediaItem mediaItem = maybeMediaItem.ValueUnsafe();
switch (mediaItem)
{
case Movie movie:
UpdateMovie(movie, writer);
break;
case Show show:
UpdateShow(show, writer);
break;
}
}
}
return Unit.Default;
}
@ -77,8 +98,18 @@ namespace ErsatzTV.Infrastructure.Search @@ -77,8 +98,18 @@ namespace ErsatzTV.Infrastructure.Search
var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.APPEND };
using var writer = new IndexWriter(dir, indexConfig);
UpdateMovies(items.OfType<Movie>(), writer);
UpdateShows(items.OfType<Show>(), writer);
foreach (MediaItem item in items)
{
switch (item)
{
case Movie movie:
UpdateMovie(movie, writer);
break;
case Show show:
UpdateShow(show, writer);
break;
}
}
return Task.FromResult(Unit.Default);
}
@ -174,83 +205,77 @@ namespace ErsatzTV.Infrastructure.Search @@ -174,83 +205,77 @@ namespace ErsatzTV.Infrastructure.Search
return new SearchPageMap(map);
}
private static void UpdateMovies(IEnumerable<Movie> movies, IndexWriter writer)
private static void UpdateMovie(Movie movie, IndexWriter writer)
{
foreach (Movie movie in movies)
Option<MovieMetadata> maybeMetadata = movie.MovieMetadata.HeadOrNone();
if (maybeMetadata.IsSome)
{
Option<MovieMetadata> maybeMetadata = movie.MovieMetadata.HeadOrNone();
if (maybeMetadata.IsSome)
{
MovieMetadata metadata = maybeMetadata.ValueUnsafe();
var doc = new Document
{
new StringField(IdField, movie.Id.ToString(), Field.Store.YES),
new StringField(TypeField, MovieType, Field.Store.NO),
new TextField(TitleField, metadata.Title, Field.Store.NO),
new StringField(SortTitleField, metadata.SortTitle.ToLowerInvariant(), Field.Store.NO),
new TextField(LibraryNameField, movie.LibraryPath.Library.Name, Field.Store.NO),
new StringField(TitleAndYearField, GetTitleAndYear(metadata), Field.Store.NO),
new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
};
if (!string.IsNullOrWhiteSpace(metadata.Plot))
{
doc.Add(new TextField(PlotField, metadata.Plot ?? string.Empty, Field.Store.NO));
}
MovieMetadata metadata = maybeMetadata.ValueUnsafe();
foreach (Genre genre in metadata.Genres)
{
doc.Add(new TextField(GenreField, genre.Name, Field.Store.NO));
}
var doc = new Document
{
new StringField(IdField, movie.Id.ToString(), Field.Store.YES),
new StringField(TypeField, MovieType, Field.Store.NO),
new TextField(TitleField, metadata.Title, Field.Store.NO),
new StringField(SortTitleField, metadata.SortTitle.ToLowerInvariant(), Field.Store.NO),
new TextField(LibraryNameField, movie.LibraryPath.Library.Name, Field.Store.NO),
new StringField(TitleAndYearField, GetTitleAndYear(metadata), Field.Store.NO),
new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
};
if (!string.IsNullOrWhiteSpace(metadata.Plot))
{
doc.Add(new TextField(PlotField, metadata.Plot ?? string.Empty, Field.Store.NO));
}
foreach (Tag tag in metadata.Tags)
{
doc.Add(new TextField(TagField, tag.Name, Field.Store.NO));
}
foreach (Genre genre in metadata.Genres)
{
doc.Add(new TextField(GenreField, genre.Name, Field.Store.NO));
}
writer.UpdateDocument(new Term(IdField, movie.Id.ToString()), doc);
foreach (Tag tag in metadata.Tags)
{
doc.Add(new TextField(TagField, tag.Name, Field.Store.NO));
}
writer.UpdateDocument(new Term(IdField, movie.Id.ToString()), doc);
}
}
private static void UpdateShows(IEnumerable<Show> shows, IndexWriter writer)
private static void UpdateShow(Show show, IndexWriter writer)
{
foreach (Show show in shows)
Option<ShowMetadata> maybeMetadata = show.ShowMetadata.HeadOrNone();
if (maybeMetadata.IsSome)
{
Option<ShowMetadata> maybeMetadata = show.ShowMetadata.HeadOrNone();
if (maybeMetadata.IsSome)
{
ShowMetadata metadata = maybeMetadata.ValueUnsafe();
var doc = new Document
{
new StringField(IdField, show.Id.ToString(), Field.Store.YES),
new StringField(TypeField, ShowType, Field.Store.NO),
new TextField(TitleField, metadata.Title, Field.Store.NO),
new StringField(SortTitleField, metadata.SortTitle.ToLowerInvariant(), Field.Store.NO),
new TextField(LibraryNameField, show.LibraryPath.Library.Name, Field.Store.NO),
new StringField(TitleAndYearField, GetTitleAndYear(metadata), Field.Store.NO),
new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
};
if (!string.IsNullOrWhiteSpace(metadata.Plot))
{
doc.Add(new TextField(PlotField, metadata.Plot ?? string.Empty, Field.Store.NO));
}
ShowMetadata metadata = maybeMetadata.ValueUnsafe();
foreach (Genre genre in metadata.Genres)
{
doc.Add(new TextField(GenreField, genre.Name, Field.Store.NO));
}
var doc = new Document
{
new StringField(IdField, show.Id.ToString(), Field.Store.YES),
new StringField(TypeField, ShowType, Field.Store.NO),
new TextField(TitleField, metadata.Title, Field.Store.NO),
new StringField(SortTitleField, metadata.SortTitle.ToLowerInvariant(), Field.Store.NO),
new TextField(LibraryNameField, show.LibraryPath.Library.Name, Field.Store.NO),
new StringField(TitleAndYearField, GetTitleAndYear(metadata), Field.Store.NO),
new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
};
if (!string.IsNullOrWhiteSpace(metadata.Plot))
{
doc.Add(new TextField(PlotField, metadata.Plot ?? string.Empty, Field.Store.NO));
}
foreach (Tag tag in metadata.Tags)
{
doc.Add(new TextField(TagField, tag.Name, Field.Store.NO));
}
foreach (Genre genre in metadata.Genres)
{
doc.Add(new TextField(GenreField, genre.Name, Field.Store.NO));
}
writer.UpdateDocument(new Term(IdField, show.Id.ToString()), doc);
foreach (Tag tag in metadata.Tags)
{
doc.Add(new TextField(TagField, tag.Name, Field.Store.NO));
}
writer.UpdateDocument(new Term(IdField, show.Id.ToString()), doc);
}
}

Write Preview
Loading…
Cancel
Save