From 79bfba642851e0b43c95d87ed168c2bdf9a78fc4 Mon Sep 17 00:00:00 2001 From: Jason Dove Date: Mon, 5 Apr 2021 16:18:17 -0500 Subject: [PATCH] better search index thread fix (#143) * Revert "fix search index threading (#141)" This reverts commit 3fb6da0754e72aa844065bba5c7d814eed7bd3bc. * better search index thread fix --- .../Commands/RebuildSearchIndexHandler.cs | 8 +- .../Interfaces/Search/ISearchIndex.cs | 12 ++- ErsatzTV.Core/Metadata/MovieFolderScanner.cs | 1 + .../Metadata/MusicVideoFolderScanner.cs | 3 +- .../Metadata/TelevisionFolderScanner.cs | 1 + ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs | 1 + .../Plex/PlexTelevisionLibraryScanner.cs | 1 + .../Data/Repositories/SearchRepository.cs | 6 ++ ErsatzTV.Infrastructure/Search/SearchIndex.cs | 95 ++++++++----------- ErsatzTV/Startup.cs | 2 +- 10 files changed, 67 insertions(+), 63 deletions(-) diff --git a/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs b/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs index f8a89d025..19bcf349f 100644 --- a/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs +++ b/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using ErsatzTV.Core; using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Interfaces.Metadata; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Interfaces.Search; using LanguageExt; @@ -14,6 +15,7 @@ namespace ErsatzTV.Application.Search.Commands public class RebuildSearchIndexHandler : MediatR.IRequestHandler { private readonly IConfigElementRepository _configElementRepository; + private readonly ILocalFileSystem _localFileSystem; private readonly ILogger _logger; private readonly ISearchIndex _searchIndex; private readonly ISearchRepository _searchRepository; @@ -22,18 +24,22 @@ namespace ErsatzTV.Application.Search.Commands ISearchIndex searchIndex, ISearchRepository searchRepository, IConfigElementRepository configElementRepository, + ILocalFileSystem localFileSystem, ILogger logger) { _searchIndex = searchIndex; _logger = logger; _searchRepository = searchRepository; _configElementRepository = configElementRepository; + _localFileSystem = localFileSystem; } public async Task Handle(RebuildSearchIndex request, CancellationToken cancellationToken) { bool indexFolderExists = Directory.Exists(FileSystemLayout.SearchIndexFolder); + await _searchIndex.Initialize(_localFileSystem); + if (!indexFolderExists || await _configElementRepository.GetValue(ConfigElementKey.SearchIndexVersion) < _searchIndex.Version) @@ -41,7 +47,7 @@ namespace ErsatzTV.Application.Search.Commands _logger.LogDebug("Migrating search index to version {Version}", _searchIndex.Version); List itemIds = await _searchRepository.GetItemIdsToIndex(); - await _searchIndex.Rebuild(itemIds); + await _searchIndex.Rebuild(_searchRepository, itemIds); Option maybeVersion = await _configElementRepository.Get(ConfigElementKey.SearchIndexVersion); diff --git a/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs b/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs index b5db66091..cdd7ab3ed 100644 --- a/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs +++ b/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs @@ -1,19 +1,23 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Threading.Tasks; using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Interfaces.Metadata; +using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Search; using LanguageExt; namespace ErsatzTV.Core.Interfaces.Search { - public interface ISearchIndex + public interface ISearchIndex : IDisposable { public int Version { get; } - Task Initialize(); - Task Rebuild(List itemIds); + Task Initialize(ILocalFileSystem localFileSystem); + Task Rebuild(ISearchRepository searchRepository, List itemIds); Task AddItems(List items); Task UpdateItems(List items); Task RemoveItems(List ids); Task Search(string query, int skip, int limit, string searchField = ""); + void Commit(); } } diff --git a/ErsatzTV.Core/Metadata/MovieFolderScanner.cs b/ErsatzTV.Core/Metadata/MovieFolderScanner.cs index 508485923..52a04b1d1 100644 --- a/ErsatzTV.Core/Metadata/MovieFolderScanner.cs +++ b/ErsatzTV.Core/Metadata/MovieFolderScanner.cs @@ -140,6 +140,7 @@ namespace ErsatzTV.Core.Metadata } } + _searchIndex.Commit(); return Unit.Default; } diff --git a/ErsatzTV.Core/Metadata/MusicVideoFolderScanner.cs b/ErsatzTV.Core/Metadata/MusicVideoFolderScanner.cs index 4a7449682..d1065b3ee 100644 --- a/ErsatzTV.Core/Metadata/MusicVideoFolderScanner.cs +++ b/ErsatzTV.Core/Metadata/MusicVideoFolderScanner.cs @@ -132,7 +132,8 @@ namespace ErsatzTV.Core.Metadata await _searchIndex.RemoveItems(ids); } } - + + _searchIndex.Commit(); return Unit.Default; } diff --git a/ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs b/ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs index 7c88c4047..9d0046f74 100644 --- a/ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs +++ b/ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs @@ -124,6 +124,7 @@ namespace ErsatzTV.Core.Metadata List ids = await _televisionRepository.DeleteEmptyShows(libraryPath); await _searchIndex.RemoveItems(ids); + _searchIndex.Commit(); return Unit.Default; } diff --git a/ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs b/ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs index fffa6f694..18095de5b 100644 --- a/ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs +++ b/ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs @@ -102,6 +102,7 @@ namespace ErsatzTV.Core.Plex return Task.CompletedTask; }); + _searchIndex.Commit(); return Unit.Default; } diff --git a/ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs b/ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs index 2179a5fc2..2eaa4f838 100644 --- a/ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs +++ b/ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs @@ -95,6 +95,7 @@ namespace ErsatzTV.Core.Plex await _mediator.Publish(new LibraryScanProgress(plexMediaSourceLibrary.Id, 0)); + _searchIndex.Commit(); return Unit.Default; }, error => diff --git a/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs index 44f1ef6dc..59c504e38 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs @@ -45,6 +45,12 @@ namespace ErsatzTV.Infrastructure.Data.Repositories .ThenInclude(mm => mm.Tags) .Include(mi => (mi as Show).ShowMetadata) .ThenInclude(mm => mm.Studios) + .Include(mi => (mi as MusicVideo).MusicVideoMetadata) + .ThenInclude(mm => mm.Genres) + .Include(mi => (mi as MusicVideo).MusicVideoMetadata) + .ThenInclude(mm => mm.Tags) + .Include(mi => (mi as MusicVideo).MusicVideoMetadata) + .ThenInclude(mm => mm.Studios) .OrderBy(mi => mi.Id) .SingleOrDefaultAsync(mi => mi.Id == id) .Map(Optional); diff --git a/ErsatzTV.Infrastructure/Search/SearchIndex.cs b/ErsatzTV.Infrastructure/Search/SearchIndex.cs index 704d80b65..0dd540032 100644 --- a/ErsatzTV.Infrastructure/Search/SearchIndex.cs +++ b/ErsatzTV.Infrastructure/Search/SearchIndex.cs @@ -23,7 +23,7 @@ using Query = Lucene.Net.Search.Query; namespace ErsatzTV.Infrastructure.Search { - public class SearchIndex : ISearchIndex + public sealed class SearchIndex : ISearchIndex { private const LuceneVersion AppLuceneVersion = LuceneVersion.LUCENE_48; @@ -45,65 +45,52 @@ namespace ErsatzTV.Infrastructure.Search private const string ShowType = "show"; private const string MusicVideoType = "music_video"; - private static bool _isRebuilding; - - private readonly ILocalFileSystem _localFileSystem; private readonly ILogger _logger; - private readonly ISearchRepository _searchRepository; + private FSDirectory _directory; + private IndexWriter _writer; - public SearchIndex( - ILocalFileSystem localFileSystem, - ISearchRepository searchRepository, - ILogger logger) - { - _localFileSystem = localFileSystem; - _searchRepository = searchRepository; - _logger = logger; - } + public SearchIndex(ILogger logger) => _logger = logger; public int Version => 2; - public Task Initialize() + public Task Initialize(ILocalFileSystem localFileSystem) { - _localFileSystem.EnsureFolderExists(FileSystemLayout.SearchIndexFolder); + localFileSystem.EnsureFolderExists(FileSystemLayout.SearchIndexFolder); + + _directory = FSDirectory.Open(FileSystemLayout.SearchIndexFolder); + var analyzer = new StandardAnalyzer(AppLuceneVersion); + var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) + { OpenMode = OpenMode.CREATE_OR_APPEND }; + _writer = new IndexWriter(_directory, indexConfig); + return Task.FromResult(true); } - public async Task Rebuild(List itemIds) + public async Task Rebuild(ISearchRepository searchRepository, List itemIds) { - _isRebuilding = true; - - await Initialize(); - - using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder); - var analyzer = new StandardAnalyzer(AppLuceneVersion); - var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.CREATE }; - using var writer = new IndexWriter(dir, indexConfig); - foreach (int id in itemIds) { - Option maybeMediaItem = await _searchRepository.GetItemToIndex(id); + Option maybeMediaItem = await searchRepository.GetItemToIndex(id); if (maybeMediaItem.IsSome) { MediaItem mediaItem = maybeMediaItem.ValueUnsafe(); switch (mediaItem) { case Movie movie: - UpdateMovie(movie, writer); + UpdateMovie(movie); break; case Show show: - UpdateShow(show, writer); + UpdateShow(show); break; case MusicVideo musicVideo: - UpdateMusicVideo(musicVideo, writer); + UpdateMusicVideo(musicVideo); break; } } } - _isRebuilding = false; - + _writer.Commit(); return Unit.Default; } @@ -111,23 +98,18 @@ namespace ErsatzTV.Infrastructure.Search public Task UpdateItems(List items) { - using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder); - var analyzer = new StandardAnalyzer(AppLuceneVersion); - var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.APPEND }; - using var writer = new IndexWriter(dir, indexConfig); - foreach (MediaItem item in items) { switch (item) { case Movie movie: - UpdateMovie(movie, writer); + UpdateMovie(movie); break; case Show show: - UpdateShow(show, writer); + UpdateShow(show); break; case MusicVideo musicVideo: - UpdateMusicVideo(musicVideo, writer); + UpdateMusicVideo(musicVideo); break; } } @@ -137,14 +119,9 @@ namespace ErsatzTV.Infrastructure.Search public Task RemoveItems(List ids) { - using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder); - var analyzer = new StandardAnalyzer(AppLuceneVersion); - var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) { OpenMode = OpenMode.APPEND }; - using var writer = new IndexWriter(dir, indexConfig); - foreach (int id in ids) { - writer.DeleteDocuments(new Term(IdField, id.ToString())); + _writer.DeleteDocuments(new Term(IdField, id.ToString())); } return Task.FromResult(Unit.Default); @@ -152,14 +129,12 @@ namespace ErsatzTV.Infrastructure.Search public Task Search(string searchQuery, int skip, int limit, string searchField = "") { - if (_isRebuilding || - string.IsNullOrWhiteSpace(searchQuery.Replace("*", string.Empty).Replace("?", string.Empty))) + if (string.IsNullOrWhiteSpace(searchQuery.Replace("*", string.Empty).Replace("?", string.Empty))) { return new SearchResult(new List(), 0).AsTask(); } - using var dir = FSDirectory.Open(FileSystemLayout.SearchIndexFolder); - using var reader = DirectoryReader.Open(dir); + using DirectoryReader reader = _writer.GetReader(true); var searcher = new IndexSearcher(reader); int hitsLimit = skip + limit; using var analyzer = new StandardAnalyzer(AppLuceneVersion); @@ -182,6 +157,14 @@ namespace ErsatzTV.Infrastructure.Search return searchResult.AsTask(); } + public void Commit() => _writer.Commit(); + + public void Dispose() + { + _writer?.Dispose(); + _directory?.Dispose(); + } + private static Option GetSearchPageMap( IndexSearcher searcher, Query query, @@ -228,7 +211,7 @@ namespace ErsatzTV.Infrastructure.Search return new SearchPageMap(map); } - private void UpdateMovie(Movie movie, IndexWriter writer) + private void UpdateMovie(Movie movie) { Option maybeMetadata = movie.MovieMetadata.HeadOrNone(); if (maybeMetadata.IsSome) @@ -277,7 +260,7 @@ namespace ErsatzTV.Infrastructure.Search doc.Add(new TextField(StudioField, studio.Name, Field.Store.NO)); } - writer.UpdateDocument(new Term(IdField, movie.Id.ToString()), doc); + _writer.UpdateDocument(new Term(IdField, movie.Id.ToString()), doc); } catch (Exception ex) { @@ -287,7 +270,7 @@ namespace ErsatzTV.Infrastructure.Search } } - private void UpdateShow(Show show, IndexWriter writer) + private void UpdateShow(Show show) { Option maybeMetadata = show.ShowMetadata.HeadOrNone(); if (maybeMetadata.IsSome) @@ -336,7 +319,7 @@ namespace ErsatzTV.Infrastructure.Search doc.Add(new TextField(StudioField, studio.Name, Field.Store.NO)); } - writer.UpdateDocument(new Term(IdField, show.Id.ToString()), doc); + _writer.UpdateDocument(new Term(IdField, show.Id.ToString()), doc); } catch (Exception ex) { @@ -346,7 +329,7 @@ namespace ErsatzTV.Infrastructure.Search } } - private void UpdateMusicVideo(MusicVideo musicVideo, IndexWriter writer) + private void UpdateMusicVideo(MusicVideo musicVideo) { Option maybeMetadata = musicVideo.MusicVideoMetadata.HeadOrNone(); if (maybeMetadata.IsSome) @@ -396,7 +379,7 @@ namespace ErsatzTV.Infrastructure.Search doc.Add(new TextField(StudioField, studio.Name, Field.Store.NO)); } - writer.UpdateDocument(new Term(IdField, musicVideo.Id.ToString()), doc); + _writer.UpdateDocument(new Term(IdField, musicVideo.Id.ToString()), doc); } catch (Exception ex) { diff --git a/ErsatzTV/Startup.cs b/ErsatzTV/Startup.cs index b25b73be7..6a5b4c2a1 100644 --- a/ErsatzTV/Startup.cs +++ b/ErsatzTV/Startup.cs @@ -185,6 +185,7 @@ namespace ErsatzTV services.AddSingleton(); services.AddSingleton(); // TODO: does this need to be singleton? services.AddSingleton(); + services.AddSingleton(); AddChannel(services); AddChannel(services); @@ -217,7 +218,6 @@ namespace ErsatzTV services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped();