diff --git a/CHANGELOG.md b/CHANGELOG.md index 62e03112..23dc0b13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Added +- Automatically rebuild search index after improper shutdown +- Add *experimental* support for Elasticsearch as search index backend + - No query changes should be needed since ES is backed by lucene and supports the same query syntax + - This can be configured using the following env vars (note the double underscore separator `__`) + - `ELASTICSEARCH__URI` (e.g. `http://localhost:9200`) + - `ELASTICSEARCH__INDEXNAME` (default is `ersatztv`) + ### Fixed - Fix subtitle scaling when using QSV hardware acceleration - Fix log viewer crash when log file contains invalid data diff --git a/ErsatzTV.Application/Maintenance/Commands/EmptyTrashHandler.cs b/ErsatzTV.Application/Maintenance/Commands/EmptyTrashHandler.cs index 512d81ff..cf3085b7 100644 --- a/ErsatzTV.Application/Maintenance/Commands/EmptyTrashHandler.cs +++ b/ErsatzTV.Application/Maintenance/Commands/EmptyTrashHandler.cs @@ -29,21 +29,21 @@ public class EmptyTrashHandler : IRequestHandler(); foreach (string type in types) { - SearchResult result = _searchIndex.Search(_client, $"type:{type} AND (state:FileNotFound)", 0, 0); + SearchResult result = await _searchIndex.Search(_client, $"type:{type} AND (state:FileNotFound)", 0, 0); ids.AddRange(result.Items.Map(i => i.Id)); } diff --git a/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs b/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs index 0d3e602b..ff9f2da6 100644 --- a/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs +++ b/ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs @@ -42,13 +42,16 @@ public class RebuildSearchIndexHandler : IRequestHandler { _logger.LogInformation("Initializing search index"); - bool indexFolderExists = Directory.Exists(FileSystemLayout.SearchIndexFolder); + bool indexExists = await _searchIndex.IndexExists(); - await _searchIndex.Initialize(_localFileSystem, _configElementRepository); + if (!await _searchIndex.Initialize(_localFileSystem, _configElementRepository)) + { + indexExists = false; + } _logger.LogInformation("Done initializing search index"); - if (!indexFolderExists || + if (!indexExists || await _configElementRepository.GetValue(ConfigElementKey.SearchIndexVersion) < _searchIndex.Version) { diff --git a/ErsatzTV.Application/Search/Queries/QuerySearchIndexAllItemsHandler.cs b/ErsatzTV.Application/Search/Queries/QuerySearchIndexAllItemsHandler.cs index 66af1c06..5b8c300e 100644 --- a/ErsatzTV.Application/Search/Queries/QuerySearchIndexAllItemsHandler.cs +++ b/ErsatzTV.Application/Search/Queries/QuerySearchIndexAllItemsHandler.cs @@ -15,19 +15,19 @@ public class QuerySearchIndexAllItemsHandler : IRequestHandler Handle( + public async Task Handle( QuerySearchIndexAllItems request, CancellationToken cancellationToken) => - new SearchResultAllItemsViewModel( - GetIds(SearchIndex.MovieType, request.Query), - GetIds(SearchIndex.ShowType, request.Query), - GetIds(SearchIndex.SeasonType, request.Query), - GetIds(SearchIndex.EpisodeType, request.Query), - GetIds(SearchIndex.ArtistType, request.Query), - GetIds(SearchIndex.MusicVideoType, request.Query), - GetIds(SearchIndex.OtherVideoType, request.Query), - GetIds(SearchIndex.SongType, request.Query)).AsTask(); + new( + await GetIds(LuceneSearchIndex.MovieType, request.Query), + await GetIds(LuceneSearchIndex.ShowType, request.Query), + await GetIds(LuceneSearchIndex.SeasonType, request.Query), + await GetIds(LuceneSearchIndex.EpisodeType, request.Query), + await GetIds(LuceneSearchIndex.ArtistType, request.Query), + await GetIds(LuceneSearchIndex.MusicVideoType, request.Query), + await GetIds(LuceneSearchIndex.OtherVideoType, request.Query), + await GetIds(LuceneSearchIndex.SongType, request.Query)); - private List GetIds(string type, string query) => - _searchIndex.Search(_client, $"type:{type} AND ({query})", 0, 0).Items.Map(i => i.Id).ToList(); + private async Task> GetIds(string type, string query) => + (await _searchIndex.Search(_client, $"type:{type} AND ({query})", 0, 0)).Items.Map(i => i.Id).ToList(); } diff --git a/ErsatzTV.Application/Search/Queries/QuerySearchIndexArtistsHandler.cs b/ErsatzTV.Application/Search/Queries/QuerySearchIndexArtistsHandler.cs index d2a818c0..035eaa30 100644 --- a/ErsatzTV.Application/Search/Queries/QuerySearchIndexArtistsHandler.cs +++ b/ErsatzTV.Application/Search/Queries/QuerySearchIndexArtistsHandler.cs @@ -24,7 +24,7 @@ public class QuerySearchIndexArtistsHandler : IRequestHandler(); services.AddScoped(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(_ => Substitute.For()); diff --git a/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs b/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs index cf9be2bb..3f729766 100644 --- a/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs +++ b/ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs @@ -10,6 +10,7 @@ namespace ErsatzTV.Core.Interfaces.Search; public interface ISearchIndex : IDisposable { public int Version { get; } + Task IndexExists(); Task Initialize(ILocalFileSystem localFileSystem, IConfigElementRepository configElementRepository); Task Rebuild(ICachingSearchRepository searchRepository, IFallbackMetadataProvider fallbackMetadataProvider); @@ -24,6 +25,6 @@ public interface ISearchIndex : IDisposable List items); Task RemoveItems(IEnumerable ids); - SearchResult Search(IClient client, string query, int skip, int limit, string searchField = ""); + Task Search(IClient client, string query, int skip, int limit, string searchField = ""); void Commit(); } diff --git a/ErsatzTV.Infrastructure/Data/Repositories/MediaCollectionRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/MediaCollectionRepository.cs index 579c7dd7..3369d649 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/MediaCollectionRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/MediaCollectionRepository.cs @@ -100,51 +100,51 @@ public class MediaCollectionRepository : IMediaCollectionRepository foreach (SmartCollection collection in maybeCollection) { - SearchResult searchResults = _searchIndex.Search(_client, collection.Query, 0, 0); + SearchResult searchResults = await _searchIndex.Search(_client, collection.Query, 0, 0); var movieIds = searchResults.Items - .Filter(i => i.Type == SearchIndex.MovieType) + .Filter(i => i.Type == LuceneSearchIndex.MovieType) .Map(i => i.Id) .ToList(); result.AddRange(await GetMovieItems(dbContext, movieIds)); - foreach (int showId in searchResults.Items.Filter(i => i.Type == SearchIndex.ShowType).Map(i => i.Id)) + foreach (int showId in searchResults.Items.Filter(i => i.Type == LuceneSearchIndex.ShowType).Map(i => i.Id)) { result.AddRange(await GetShowItemsFromShowId(dbContext, showId)); } - foreach (int seasonId in searchResults.Items.Filter(i => i.Type == SearchIndex.SeasonType) + foreach (int seasonId in searchResults.Items.Filter(i => i.Type == LuceneSearchIndex.SeasonType) .Map(i => i.Id)) { result.AddRange(await GetSeasonItemsFromSeasonId(dbContext, seasonId)); } - foreach (int artistId in searchResults.Items.Filter(i => i.Type == SearchIndex.ArtistType) + foreach (int artistId in searchResults.Items.Filter(i => i.Type == LuceneSearchIndex.ArtistType) .Map(i => i.Id)) { result.AddRange(await GetArtistItemsFromArtistId(dbContext, artistId)); } var musicVideoIds = searchResults.Items - .Filter(i => i.Type == SearchIndex.MusicVideoType) + .Filter(i => i.Type == LuceneSearchIndex.MusicVideoType) .Map(i => i.Id) .ToList(); result.AddRange(await GetMusicVideoItems(dbContext, musicVideoIds)); var episodeIds = searchResults.Items - .Filter(i => i.Type == SearchIndex.EpisodeType) + .Filter(i => i.Type == LuceneSearchIndex.EpisodeType) .Map(i => i.Id) .ToList(); result.AddRange(await GetEpisodeItems(dbContext, episodeIds)); var otherVideoIds = searchResults.Items - .Filter(i => i.Type == SearchIndex.OtherVideoType) + .Filter(i => i.Type == LuceneSearchIndex.OtherVideoType) .Map(i => i.Id) .ToList(); result.AddRange(await GetOtherVideoItems(dbContext, otherVideoIds)); var songIds = searchResults.Items - .Filter(i => i.Type == SearchIndex.SongType) + .Filter(i => i.Type == LuceneSearchIndex.SongType) .Map(i => i.Id) .ToList(); result.AddRange(await GetSongItems(dbContext, songIds)); diff --git a/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj b/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj index 86a0d7a7..421dede5 100644 --- a/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj +++ b/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj @@ -11,6 +11,7 @@ + @@ -35,4 +36,5 @@ + diff --git a/ErsatzTV.Infrastructure/Search/CustomMultiFieldQueryParser.cs b/ErsatzTV.Infrastructure/Search/CustomMultiFieldQueryParser.cs index 3130bf9c..470fb8e7 100644 --- a/ErsatzTV.Infrastructure/Search/CustomMultiFieldQueryParser.cs +++ b/ErsatzTV.Infrastructure/Search/CustomMultiFieldQueryParser.cs @@ -29,7 +29,7 @@ public class CustomMultiFieldQueryParser : MultiFieldQueryParser if (field == "released_onthisday") { var todayString = DateTime.Today.ToString("*MMdd"); - return base.GetWildcardQuery(SearchIndex.ReleaseDateField, todayString); + return base.GetWildcardQuery(LuceneSearchIndex.ReleaseDateField, todayString); } if (CustomQueryParser.NumericFields.Contains(field) && int.TryParse(queryText, out int val)) @@ -49,14 +49,14 @@ public class CustomMultiFieldQueryParser : MultiFieldQueryParser var todayString = DateTime.UtcNow.ToString("yyyyMMdd"); var dateString = start.ToString("yyyyMMdd"); - return base.GetRangeQuery(SearchIndex.ReleaseDateField, dateString, todayString, true, true); + return base.GetRangeQuery(LuceneSearchIndex.ReleaseDateField, dateString, todayString, true, true); } if (field == "released_notinthelast" && CustomQueryParser.ParseStart(queryText, out DateTime finish)) { var dateString = finish.ToString("yyyyMMdd"); - return base.GetRangeQuery(SearchIndex.ReleaseDateField, "00000000", dateString, false, false); + return base.GetRangeQuery(LuceneSearchIndex.ReleaseDateField, "00000000", dateString, false, false); } if (field == "added_inthelast" && CustomQueryParser.ParseStart(queryText, out DateTime addedStart)) @@ -64,14 +64,14 @@ public class CustomMultiFieldQueryParser : MultiFieldQueryParser var todayString = DateTime.UtcNow.ToString("yyyyMMdd"); var dateString = addedStart.ToString("yyyyMMdd"); - return base.GetRangeQuery(SearchIndex.AddedDateField, dateString, todayString, true, true); + return base.GetRangeQuery(LuceneSearchIndex.AddedDateField, dateString, todayString, true, true); } if (field == "added_notinthelast" && CustomQueryParser.ParseStart(queryText, out DateTime addedFinish)) { var dateString = addedFinish.ToString("yyyyMMdd"); - return base.GetRangeQuery(SearchIndex.AddedDateField, "00000000", dateString, false, false); + return base.GetRangeQuery(LuceneSearchIndex.AddedDateField, "00000000", dateString, false, false); } return base.GetFieldQuery(field, queryText, slop); diff --git a/ErsatzTV.Infrastructure/Search/CustomQueryParser.cs b/ErsatzTV.Infrastructure/Search/CustomQueryParser.cs index eb60918c..4c143c2c 100644 --- a/ErsatzTV.Infrastructure/Search/CustomQueryParser.cs +++ b/ErsatzTV.Infrastructure/Search/CustomQueryParser.cs @@ -12,12 +12,12 @@ public class CustomQueryParser : QueryParser { internal static readonly List NumericFields = new() { - SearchIndex.MinutesField, - SearchIndex.HeightField, - SearchIndex.WidthField, - SearchIndex.SeasonNumberField, - SearchIndex.EpisodeNumberField, - SearchIndex.VideoBitDepthField + LuceneSearchIndex.MinutesField, + LuceneSearchIndex.HeightField, + LuceneSearchIndex.WidthField, + LuceneSearchIndex.SeasonNumberField, + LuceneSearchIndex.EpisodeNumberField, + LuceneSearchIndex.VideoBitDepthField }; public CustomQueryParser(LuceneVersion matchVersion, string f, Analyzer a) : base(matchVersion, f, a) @@ -37,7 +37,7 @@ public class CustomQueryParser : QueryParser if (field == "released_onthisday") { var todayString = DateTime.Today.ToString("*MMdd"); - return base.GetWildcardQuery(SearchIndex.ReleaseDateField, todayString); + return base.GetWildcardQuery(LuceneSearchIndex.ReleaseDateField, todayString); } if (NumericFields.Contains(field) && int.TryParse(queryText, out int val)) @@ -57,14 +57,14 @@ public class CustomQueryParser : QueryParser var todayString = DateTime.UtcNow.ToString("yyyyMMdd"); var dateString = start.ToString("yyyyMMdd"); - return base.GetRangeQuery(SearchIndex.ReleaseDateField, dateString, todayString, true, true); + return base.GetRangeQuery(LuceneSearchIndex.ReleaseDateField, dateString, todayString, true, true); } if (field == "released_notinthelast" && ParseStart(queryText, out DateTime finish)) { var dateString = finish.ToString("yyyyMMdd"); - return base.GetRangeQuery(SearchIndex.ReleaseDateField, "00000000", dateString, false, false); + return base.GetRangeQuery(LuceneSearchIndex.ReleaseDateField, "00000000", dateString, false, false); } if (field == "added_inthelast" && ParseStart(queryText, out DateTime addedStart)) @@ -72,14 +72,14 @@ public class CustomQueryParser : QueryParser var todayString = DateTime.UtcNow.ToString("yyyyMMdd"); var dateString = addedStart.ToString("yyyyMMdd"); - return base.GetRangeQuery(SearchIndex.AddedDateField, dateString, todayString, true, true); + return base.GetRangeQuery(LuceneSearchIndex.AddedDateField, dateString, todayString, true, true); } if (field == "added_notinthelast" && ParseStart(queryText, out DateTime addedFinish)) { var dateString = addedFinish.ToString("yyyyMMdd"); - return base.GetRangeQuery(SearchIndex.AddedDateField, "00000000", dateString, false, false); + return base.GetRangeQuery(LuceneSearchIndex.AddedDateField, "00000000", dateString, false, false); } return base.GetFieldQuery(field, queryText, slop); diff --git a/ErsatzTV.Infrastructure/Search/ElasticSearchIndex.cs b/ErsatzTV.Infrastructure/Search/ElasticSearchIndex.cs new file mode 100644 index 00000000..c1e92560 --- /dev/null +++ b/ErsatzTV.Infrastructure/Search/ElasticSearchIndex.cs @@ -0,0 +1,828 @@ +using System.Globalization; +using Bugsnag; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Clients.Elasticsearch.IndexManagement; +using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Interfaces.Metadata; +using ErsatzTV.Core.Interfaces.Repositories; +using ErsatzTV.Core.Interfaces.Repositories.Caching; +using ErsatzTV.Core.Interfaces.Search; +using ErsatzTV.Core.Search; +using ErsatzTV.FFmpeg; +using ErsatzTV.FFmpeg.Format; +using ErsatzTV.Infrastructure.Search.Models; +using Microsoft.Extensions.Logging; +using ExistsResponse = Elastic.Clients.Elasticsearch.IndexManagement.ExistsResponse; +using MediaStream = ErsatzTV.Core.Domain.MediaStream; + +namespace ErsatzTV.Infrastructure.Search; + +public class ElasticSearchIndex : ISearchIndex +{ + public static Uri Uri; + public static string IndexName; + private readonly List _cultureInfos; + + private readonly ILogger _logger; + private ElasticsearchClient _client; + + public ElasticSearchIndex(ILogger logger) + { + _logger = logger; + _cultureInfos = CultureInfo.GetCultures(CultureTypes.NeutralCultures).ToList(); + } + + public void Dispose() + { + // do nothing + } + + // not really used by elasticsearch + public Task IndexExists() => Task.FromResult(false); + + public int Version => 36; + + public async Task Initialize( + ILocalFileSystem localFileSystem, + IConfigElementRepository configElementRepository) + { + _client = new ElasticsearchClient(Uri); + ExistsResponse exists = await _client.Indices.ExistsAsync(IndexName); + if (!exists.IsValidResponse) + { + CreateIndexResponse createResponse = await CreateIndex(); + return createResponse.IsValidResponse; + } + + return true; + } + + public async Task Rebuild( + ICachingSearchRepository searchRepository, + IFallbackMetadataProvider fallbackMetadataProvider) + { + DeleteIndexResponse deleteResponse = await _client.Indices.DeleteAsync(IndexName); + if (!deleteResponse.IsValidResponse) + { + return Unit.Default; + } + + CreateIndexResponse createResponse = await CreateIndex(); + if (!createResponse.IsValidResponse) + { + return Unit.Default; + } + + await foreach (MediaItem mediaItem in searchRepository.GetAllMediaItems()) + { + await RebuildItem(searchRepository, fallbackMetadataProvider, mediaItem); + } + + return Unit.Default; + } + + public async Task RebuildItems( + ICachingSearchRepository searchRepository, + IFallbackMetadataProvider fallbackMetadataProvider, + IEnumerable itemIds) + { + foreach (int id in itemIds) + { + foreach (MediaItem mediaItem in await searchRepository.GetItemToIndex(id)) + { + await RebuildItem(searchRepository, fallbackMetadataProvider, mediaItem); + } + } + + return Unit.Default; + } + + public async Task UpdateItems( + ICachingSearchRepository searchRepository, + IFallbackMetadataProvider fallbackMetadataProvider, + List items) + { + foreach (MediaItem item in items) + { + switch (item) + { + case Movie movie: + await UpdateMovie(searchRepository, movie); + break; + case Show show: + await UpdateShow(searchRepository, show); + break; + case Season season: + await UpdateSeason(searchRepository, season); + break; + case Artist artist: + await UpdateArtist(searchRepository, artist); + break; + case MusicVideo musicVideo: + await UpdateMusicVideo(searchRepository, musicVideo); + break; + case Episode episode: + await UpdateEpisode(searchRepository, fallbackMetadataProvider, episode); + break; + case OtherVideo otherVideo: + await UpdateOtherVideo(searchRepository, otherVideo); + break; + case Song song: + await UpdateSong(searchRepository, song); + break; + } + } + + return Unit.Default; + } + + public async Task RemoveItems(IEnumerable ids) + { + await _client.BulkAsync( + descriptor => descriptor + .Index(IndexName) + .DeleteMany(ids.Map(id => new Id(id))) + ); + + return Unit.Default; + } + + public async Task Search(IClient client, string query, int skip, int limit, string searchField = "") + { + var items = new List(); + var totalCount = 0; + + SearchResponse response = await _client.SearchAsync( + s => s.Index(IndexName) + .Sort(ss => ss.Field(f => f.SortTitle, fs => fs.Order(SortOrder.Asc))) + .From(skip) + .Size(limit) + .QueryLuceneSyntax(query)); + if (response.IsValidResponse) + { + items.AddRange(response.Documents); + totalCount = (int)response.Total; + } + + var searchResult = new SearchResult(items.Map(i => new SearchItem(i.Type, i.Id)).ToList(), totalCount); + + if (limit > 0) + { + searchResult.PageMap = await GetSearchPageMap(query, limit); + } + + return searchResult; + } + + public void Commit() + { + // do nothing + } + + private async Task CreateIndex() => + await _client.Indices.CreateAsync( + IndexName, + i => i.Mappings( + m => m.Properties( + p => p + .Keyword(t => t.Type, t => t.Store()) + .Text(t => t.Title, t => t.Store(false)) + .Keyword(t => t.SortTitle, t => t.Store(false)) + .Text(t => t.LibraryName, t => t.Store(false)) + .Keyword(t => t.LibraryId, t => t.Store(false)) + .Keyword(t => t.TitleAndYear, t => t.Store(false)) + .Keyword(t => t.JumpLetter, t => t.Store()) + .Keyword(t => t.State, t => t.Store(false)) + .Text(t => t.MetadataKind, t => t.Store(false)) + .Text(t => t.Language, t => t.Store(false)) + .IntegerNumber(t => t.Height, t => t.Store(false)) + .IntegerNumber(t => t.Width, t => t.Store(false)) + .Keyword(t => t.VideoCodec, t => t.Store(false)) + .IntegerNumber(t => t.VideoBitDepth, t => t.Store(false)) + .Keyword(t => t.VideoDynamicRange, t => t.Store(false)) + .Keyword(t => t.ContentRating, t => t.Store(false)) + .Keyword(t => t.ReleaseDate, t => t.Store(false)) + .Keyword(t => t.AddedDate, t => t.Store(false)) + .Text(t => t.Plot, t => t.Store(false)) + .Text(t => t.Genre, t => t.Store(false)) + .Text(t => t.Tag, t => t.Store(false)) + .Text(t => t.Studio, t => t.Store(false)) + .Text(t => t.Actor, t => t.Store(false)) + .Text(t => t.Director, t => t.Store(false)) + .Text(t => t.Writer, t => t.Store(false)) + .Keyword(t => t.TraktList, t => t.Store(false)) + .IntegerNumber(t => t.SeasonNumber, t => t.Store(false)) + .Text(t => t.ShowTitle, t => t.Store(false)) + .Text(t => t.ShowGenre, t => t.Store(false)) + .Text(t => t.ShowTag, t => t.Store(false)) + .Text(t => t.Style, t => t.Store(false)) + .Text(t => t.Mood, t => t.Store(false)) + .Text(t => t.Album, t => t.Store(false)) + .Text(t => t.Artist, t => t.Store(false)) + .IntegerNumber(t => t.EpisodeNumber, t => t.Store(false)) + .Text(t => t.AlbumArtist, t => t.Store(false)) + ))); + + private async Task RebuildItem( + ISearchRepository searchRepository, + IFallbackMetadataProvider fallbackMetadataProvider, + MediaItem mediaItem) + { + switch (mediaItem) + { + case Movie movie: + await UpdateMovie(searchRepository, movie); + break; + case Show show: + await UpdateShow(searchRepository, show); + break; + case Season season: + await UpdateSeason(searchRepository, season); + break; + case Artist artist: + await UpdateArtist(searchRepository, artist); + break; + case MusicVideo musicVideo: + await UpdateMusicVideo(searchRepository, musicVideo); + break; + case Episode episode: + await UpdateEpisode(searchRepository, fallbackMetadataProvider, episode); + break; + case OtherVideo otherVideo: + await UpdateOtherVideo(searchRepository, otherVideo); + break; + case Song song: + await UpdateSong(searchRepository, song); + break; + } + } + + private async Task UpdateMovie(ISearchRepository searchRepository, Movie movie) + { + foreach (MovieMetadata metadata in movie.MovieMetadata.HeadOrNone()) + { + try + { + var doc = new ElasticSearchItem + { + Id = movie.Id, + Type = LuceneSearchIndex.MovieType, + Title = metadata.Title, + SortTitle = metadata.SortTitle.ToLowerInvariant(), + LibraryName = movie.LibraryPath.Library.Name, + LibraryId = movie.LibraryPath.Library.Id, + TitleAndYear = LuceneSearchIndex.GetTitleAndYear(metadata), + JumpLetter = LuceneSearchIndex.GetJumpLetter(metadata), + State = movie.State.ToString(), + MetadataKind = metadata.MetadataKind.ToString(), + Language = await GetLanguages(searchRepository, movie.MediaVersions), + ContentRating = GetContentRatings(metadata.ContentRating), + ReleaseDate = GetReleaseDate(metadata.ReleaseDate), + AddedDate = GetAddedDate(metadata.DateAdded), + Plot = metadata.Plot ?? string.Empty, + Genre = metadata.Genres.Map(g => g.Name).ToList(), + Tag = metadata.Tags.Map(t => t.Name).ToList(), + Studio = metadata.Studios.Map(s => s.Name).ToList(), + Actor = metadata.Actors.Map(a => a.Name).ToList(), + Director = metadata.Directors.Map(d => d.Name).ToList(), + Writer = metadata.Writers.Map(w => w.Name).ToList(), + TraktList = movie.TraktListItems.Map(t => t.TraktList.TraktId.ToString()).ToList() + }; + + AddStatistics(doc, movie.MediaVersions); + + foreach ((string key, List value) in GetMetadataGuids(metadata)) + { + doc.AdditionalProperties.Add(key, value); + } + + await _client.IndexAsync(doc, IndexName); + } + catch (Exception ex) + { + metadata.Movie = null; + _logger.LogWarning(ex, "Error indexing movie with metadata {@Metadata}", metadata); + } + } + } + + private async Task UpdateShow(ISearchRepository searchRepository, Show show) + { + foreach (ShowMetadata metadata in show.ShowMetadata.HeadOrNone()) + { + try + { + var doc = new ElasticSearchItem + { + Id = show.Id, + Type = LuceneSearchIndex.ShowType, + Title = metadata.Title, + SortTitle = metadata.SortTitle.ToLowerInvariant(), + LibraryName = show.LibraryPath.Library.Name, + LibraryId = show.LibraryPath.Library.Id, + TitleAndYear = LuceneSearchIndex.GetTitleAndYear(metadata), + JumpLetter = LuceneSearchIndex.GetJumpLetter(metadata), + State = show.State.ToString(), + MetadataKind = metadata.MetadataKind.ToString(), + Language = await GetLanguages(searchRepository, await searchRepository.GetLanguagesForShow(show)), + ContentRating = GetContentRatings(metadata.ContentRating), + ReleaseDate = GetReleaseDate(metadata.ReleaseDate), + AddedDate = GetAddedDate(metadata.DateAdded), + Plot = metadata.Plot ?? string.Empty, + Genre = metadata.Genres.Map(g => g.Name).ToList(), + Tag = metadata.Tags.Map(t => t.Name).ToList(), + Studio = metadata.Studios.Map(s => s.Name).ToList(), + Actor = metadata.Actors.Map(a => a.Name).ToList(), + TraktList = show.TraktListItems.Map(t => t.TraktList.TraktId.ToString()).ToList() + }; + + foreach ((string key, List value) in GetMetadataGuids(metadata)) + { + doc.AdditionalProperties.Add(key, value); + } + + await _client.IndexAsync(doc, IndexName); + } + catch (Exception ex) + { + metadata.Show = null; + _logger.LogWarning(ex, "Error indexing show with metadata {@Metadata}", metadata); + } + } + } + + private async Task UpdateSeason(ISearchRepository searchRepository, Season season) + { + foreach (SeasonMetadata metadata in season.SeasonMetadata.HeadOrNone()) + foreach (ShowMetadata showMetadata in season.Show.ShowMetadata.HeadOrNone()) + { + try + { + var seasonTitle = $"{showMetadata.Title} - S{season.SeasonNumber}"; + string sortTitle = $"{showMetadata.SortTitle}_{season.SeasonNumber:0000}" + .ToLowerInvariant(); + string titleAndYear = $"{showMetadata.Title}_{showMetadata.Year}_{season.SeasonNumber}" + .ToLowerInvariant(); + + var doc = new ElasticSearchItem + { + Id = season.Id, + Type = LuceneSearchIndex.SeasonType, + Title = seasonTitle, + SortTitle = sortTitle, + LibraryName = season.LibraryPath.Library.Name, + LibraryId = season.LibraryPath.Library.Id, + TitleAndYear = titleAndYear, + JumpLetter = LuceneSearchIndex.GetJumpLetter(showMetadata), + State = season.State.ToString(), + SeasonNumber = season.SeasonNumber, + ShowTitle = showMetadata.Title, + ShowGenre = showMetadata.Genres.Map(g => g.Name).ToList(), + ShowTag = showMetadata.Tags.Map(t => t.Name).ToList(), + Language = await GetLanguages( + searchRepository, + await searchRepository.GetLanguagesForSeason(season)), + ContentRating = GetContentRatings(showMetadata.ContentRating), + ReleaseDate = GetReleaseDate(metadata.ReleaseDate), + AddedDate = GetAddedDate(metadata.DateAdded), + TraktList = season.TraktListItems.Map(t => t.TraktList.TraktId.ToString()).ToList(), + Tag = metadata.Tags.Map(a => a.Name).ToList() + }; + + foreach ((string key, List value) in GetMetadataGuids(metadata)) + { + doc.AdditionalProperties.Add(key, value); + } + + await _client.IndexAsync(doc, IndexName); + } + catch (Exception ex) + { + metadata.Season = null; + _logger.LogWarning(ex, "Error indexing season with metadata {@Metadata}", metadata); + } + } + } + + private async Task UpdateArtist(ISearchRepository searchRepository, Artist artist) + { + foreach (ArtistMetadata metadata in artist.ArtistMetadata.HeadOrNone()) + { + try + { + var doc = new ElasticSearchItem + { + Id = artist.Id, + Type = LuceneSearchIndex.ArtistType, + Title = metadata.Title, + SortTitle = metadata.SortTitle.ToLowerInvariant(), + LibraryName = artist.LibraryPath.Library.Name, + LibraryId = artist.LibraryPath.Library.Id, + TitleAndYear = LuceneSearchIndex.GetTitleAndYear(metadata), + JumpLetter = LuceneSearchIndex.GetJumpLetter(metadata), + State = artist.State.ToString(), + MetadataKind = metadata.MetadataKind.ToString(), + Language = await GetLanguages( + searchRepository, + await searchRepository.GetLanguagesForArtist(artist)), + AddedDate = GetAddedDate(metadata.DateAdded), + Genre = metadata.Genres.Map(g => g.Name).ToList(), + Style = metadata.Styles.Map(t => t.Name).ToList(), + Mood = metadata.Moods.Map(s => s.Name).ToList() + }; + + foreach ((string key, List value) in GetMetadataGuids(metadata)) + { + doc.AdditionalProperties.Add(key, value); + } + + await _client.IndexAsync(doc, IndexName); + } + catch (Exception ex) + { + metadata.Artist = null; + _logger.LogWarning(ex, "Error indexing artist with metadata {@Metadata}", metadata); + } + } + } + + private async Task UpdateMusicVideo(ISearchRepository searchRepository, MusicVideo musicVideo) + { + foreach (MusicVideoMetadata metadata in musicVideo.MusicVideoMetadata.HeadOrNone()) + { + try + { + var doc = new ElasticSearchItem + { + Id = musicVideo.Id, + Type = LuceneSearchIndex.MusicVideoType, + Title = metadata.Title, + SortTitle = metadata.SortTitle.ToLowerInvariant(), + LibraryName = musicVideo.LibraryPath.Library.Name, + LibraryId = musicVideo.LibraryPath.Library.Id, + TitleAndYear = LuceneSearchIndex.GetTitleAndYear(metadata), + JumpLetter = LuceneSearchIndex.GetJumpLetter(metadata), + State = musicVideo.State.ToString(), + MetadataKind = metadata.MetadataKind.ToString(), + Language = await GetLanguages(searchRepository, musicVideo.MediaVersions), + ReleaseDate = GetReleaseDate(metadata.ReleaseDate), + AddedDate = GetAddedDate(metadata.DateAdded), + Album = metadata.Album ?? string.Empty, + Plot = metadata.Plot ?? string.Empty, + Genre = metadata.Genres.Map(g => g.Name).ToList(), + Tag = metadata.Tags.Map(t => t.Name).ToList(), + Studio = metadata.Studios.Map(s => s.Name).ToList() + }; + + var artists = new System.Collections.Generic.HashSet(); + + if (musicVideo.Artist != null) + { + foreach (ArtistMetadata artistMetadata in musicVideo.Artist.ArtistMetadata) + { + artists.Add(artistMetadata.Title); + } + } + + foreach (MusicVideoArtist artist in metadata.Artists) + { + artists.Add(artist.Name); + } + + doc.Artist = artists.ToList(); + + AddStatistics(doc, musicVideo.MediaVersions); + + foreach ((string key, List value) in GetMetadataGuids(metadata)) + { + doc.AdditionalProperties.Add(key, value); + } + + await _client.IndexAsync(doc, IndexName); + } + catch (Exception ex) + { + metadata.MusicVideo = null; + _logger.LogWarning(ex, "Error indexing music video with metadata {@Metadata}", metadata); + } + } + } + + private async Task UpdateEpisode( + ISearchRepository searchRepository, + IFallbackMetadataProvider fallbackMetadataProvider, + Episode episode) + { + // try to load metadata here, since episodes without metadata won't index + if (episode.EpisodeMetadata.Count == 0) + { + episode.EpisodeMetadata ??= new List(); + episode.EpisodeMetadata = fallbackMetadataProvider.GetFallbackMetadata(episode); + foreach (EpisodeMetadata metadata in episode.EpisodeMetadata) + { + metadata.Episode = episode; + } + } + + foreach (EpisodeMetadata metadata in episode.EpisodeMetadata.HeadOrNone()) + { + try + { + var doc = new ElasticSearchItem + { + Id = episode.Id, + Type = LuceneSearchIndex.EpisodeType, + Title = metadata.Title, + SortTitle = metadata.SortTitle.ToLowerInvariant(), + LibraryName = episode.LibraryPath.Library.Name, + LibraryId = episode.LibraryPath.Library.Id, + TitleAndYear = LuceneSearchIndex.GetTitleAndYear(metadata), + JumpLetter = LuceneSearchIndex.GetJumpLetter(metadata), + State = episode.State.ToString(), + MetadataKind = metadata.MetadataKind.ToString(), + SeasonNumber = episode.Season?.SeasonNumber ?? 0, + EpisodeNumber = metadata.EpisodeNumber, + Language = await GetLanguages(searchRepository, episode.MediaVersions), + ReleaseDate = GetReleaseDate(metadata.ReleaseDate), + AddedDate = GetAddedDate(metadata.DateAdded), + Plot = metadata.Plot ?? string.Empty, + Genre = metadata.Genres.Map(g => g.Name).ToList(), + Tag = metadata.Tags.Map(t => t.Name).ToList(), + Studio = metadata.Studios.Map(s => s.Name).ToList(), + Actor = metadata.Actors.Map(a => a.Name).ToList(), + Director = metadata.Directors.Map(d => d.Name).ToList(), + Writer = metadata.Writers.Map(w => w.Name).ToList(), + TraktList = episode.TraktListItems.Map(t => t.TraktList.TraktId.ToString()).ToList() + }; + + // add some show fields to help filter episodes within a particular show + foreach (ShowMetadata showMetadata in Optional(episode.Season?.Show?.ShowMetadata).Flatten()) + { + doc.ShowTitle = showMetadata.Title; + doc.ShowGenre = showMetadata.Genres.Map(g => g.Name).ToList(); + doc.ShowTag = showMetadata.Tags.Map(t => t.Name).ToList(); + } + + AddStatistics(doc, episode.MediaVersions); + + foreach ((string key, List value) in GetMetadataGuids(metadata)) + { + doc.AdditionalProperties.Add(key, value); + } + + await _client.IndexAsync(doc, IndexName); + } + catch (Exception ex) + { + metadata.Episode = null; + _logger.LogWarning(ex, "Error indexing episode with metadata {@Metadata}", metadata); + } + } + } + + private async Task UpdateOtherVideo(ISearchRepository searchRepository, OtherVideo otherVideo) + { + foreach (OtherVideoMetadata metadata in otherVideo.OtherVideoMetadata.HeadOrNone()) + { + try + { + var doc = new ElasticSearchItem + { + Id = otherVideo.Id, + Type = LuceneSearchIndex.OtherVideoType, + Title = metadata.Title, + SortTitle = metadata.SortTitle.ToLowerInvariant(), + LibraryName = otherVideo.LibraryPath.Library.Name, + LibraryId = otherVideo.LibraryPath.Library.Id, + TitleAndYear = LuceneSearchIndex.GetTitleAndYear(metadata), + JumpLetter = LuceneSearchIndex.GetJumpLetter(metadata), + State = otherVideo.State.ToString(), + MetadataKind = metadata.MetadataKind.ToString(), + Language = await GetLanguages(searchRepository, otherVideo.MediaVersions), + ContentRating = GetContentRatings(metadata.ContentRating), + ReleaseDate = GetReleaseDate(metadata.ReleaseDate), + AddedDate = GetAddedDate(metadata.DateAdded), + Plot = metadata.Plot ?? string.Empty, + Genre = metadata.Genres.Map(g => g.Name).ToList(), + Tag = metadata.Tags.Map(t => t.Name).ToList(), + Studio = metadata.Studios.Map(s => s.Name).ToList(), + Actor = metadata.Actors.Map(a => a.Name).ToList(), + Director = metadata.Directors.Map(d => d.Name).ToList(), + Writer = metadata.Writers.Map(w => w.Name).ToList() + }; + + AddStatistics(doc, otherVideo.MediaVersions); + + foreach ((string key, List value) in GetMetadataGuids(metadata)) + { + doc.AdditionalProperties.Add(key, value); + } + + await _client.IndexAsync(doc, IndexName); + } + catch (Exception ex) + { + metadata.OtherVideo = null; + _logger.LogWarning(ex, "Error indexing other video with metadata {@Metadata}", metadata); + } + } + } + + private async Task UpdateSong(ISearchRepository searchRepository, Song song) + { + foreach (SongMetadata metadata in song.SongMetadata.HeadOrNone()) + { + try + { + var doc = new ElasticSearchItem + { + Id = song.Id, + Type = LuceneSearchIndex.SongType, + Title = metadata.Title, + SortTitle = metadata.SortTitle.ToLowerInvariant(), + LibraryName = song.LibraryPath.Library.Name, + LibraryId = song.LibraryPath.Library.Id, + TitleAndYear = LuceneSearchIndex.GetTitleAndYear(metadata), + JumpLetter = LuceneSearchIndex.GetJumpLetter(metadata), + State = song.State.ToString(), + MetadataKind = metadata.MetadataKind.ToString(), + Language = await GetLanguages(searchRepository, song.MediaVersions), + AddedDate = GetAddedDate(metadata.DateAdded), + Album = metadata.Album ?? string.Empty, + Artist = !string.IsNullOrWhiteSpace(metadata.Artist) ? new List { metadata.Artist } : null, + AlbumArtist = metadata.AlbumArtist, + Genre = metadata.Genres.Map(g => g.Name).ToList(), + Tag = metadata.Tags.Map(t => t.Name).ToList() + }; + + AddStatistics(doc, song.MediaVersions); + + foreach ((string key, List value) in GetMetadataGuids(metadata)) + { + doc.AdditionalProperties.Add(key, value); + } + + await _client.IndexAsync(doc, IndexName); + } + catch (Exception ex) + { + metadata.Song = null; + _logger.LogWarning(ex, "Error indexing song with metadata {@Metadata}", metadata); + } + } + } + + private static string GetReleaseDate(DateTime? metadataReleaseDate) => metadataReleaseDate?.ToString("yyyyMMdd"); + + private static string GetAddedDate(DateTime metadataAddedDate) => metadataAddedDate.ToString("yyyyMMdd"); + + private static List GetContentRatings(string metadataContentRating) + { + var contentRatings = new List(); + + if (!string.IsNullOrWhiteSpace(metadataContentRating)) + { + foreach (string contentRating in metadataContentRating.Split("/") + .Map(x => x.Trim()).Where(x => !string.IsNullOrWhiteSpace(x))) + { + contentRatings.Add(contentRating); + } + } + + return contentRatings; + } + + private async Task> GetLanguages( + ISearchRepository searchRepository, + IEnumerable mediaVersions) + { + var result = new List(); + + foreach (MediaVersion version in mediaVersions.HeadOrNone()) + { + var mediaCodes = version.Streams + .Filter(ms => ms.MediaStreamKind == MediaStreamKind.Audio) + .Map(ms => ms.Language) + .Distinct() + .ToList(); + + result.AddRange(await GetLanguages(searchRepository, mediaCodes)); + } + + return result; + } + + private async Task> GetLanguages(ISearchRepository searchRepository, List mediaCodes) + { + var englishNames = new System.Collections.Generic.HashSet(); + foreach (string code in await searchRepository.GetAllLanguageCodes(mediaCodes)) + { + Option maybeCultureInfo = _cultureInfos.Find( + ci => string.Equals(ci.ThreeLetterISOLanguageName, code, StringComparison.OrdinalIgnoreCase)); + foreach (CultureInfo cultureInfo in maybeCultureInfo) + { + englishNames.Add(cultureInfo.EnglishName); + } + } + + return englishNames.ToList(); + } + + private static void AddStatistics(ElasticSearchItem doc, IEnumerable mediaVersions) + { + foreach (MediaVersion version in mediaVersions.HeadOrNone()) + { + doc.Minutes = (int)Math.Ceiling(version.Duration.TotalMinutes); + + foreach (MediaStream videoStream in version.Streams + .Filter(s => s.MediaStreamKind == MediaStreamKind.Video) + .HeadOrNone()) + { + doc.Height = version.Height; + doc.Width = version.Width; + + if (!string.IsNullOrWhiteSpace(videoStream.Codec)) + { + doc.VideoCodec = videoStream.Codec; + } + + Option maybePixelFormat = + AvailablePixelFormats.ForPixelFormat(videoStream.PixelFormat, null); + foreach (IPixelFormat pixelFormat in maybePixelFormat) + { + doc.VideoBitDepth = pixelFormat.BitDepth; + } + + var colorParams = new ColorParams( + videoStream.ColorRange, + videoStream.ColorSpace, + videoStream.ColorTransfer, + videoStream.ColorPrimaries); + + doc.VideoDynamicRange = colorParams.IsHdr ? "hdr" : "sdr"; + } + } + } + + private static Dictionary> GetMetadataGuids(Metadata metadata) + { + var result = new Dictionary>(); + + foreach (MetadataGuid guid in metadata.Guids) + { + string[] split = (guid.Guid ?? string.Empty).Split("://"); + if (split.Length == 2 && !string.IsNullOrWhiteSpace(split[1])) + { + string key = split[0]; + string value = split[1].ToLowerInvariant(); + if (!result.ContainsKey(key)) + { + result.Add(key, new List()); + } + + result[key].Add(value); + } + } + + return result; + } + + private async Task> GetSearchPageMap(string query, int limit) + { + SearchResponse response = await _client.SearchAsync( + s => s.Index(IndexName) + .Size(0) + .Sort(ss => ss.Field(f => f.SortTitle, fs => fs.Order(SortOrder.Asc))) + .Aggregations(a => a.Terms("count", v => v.Field(i => i.JumpLetter).Size(30))) + .QueryLuceneSyntax(query)); + + if (!response.IsValidResponse) + { + return Option.None; + } + + var letters = new List + { + '#', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z' + }; + var map = letters.ToDictionary(letter => letter, _ => 0); + + if (response.Aggregations?.Values.Head() is StringTermsAggregate aggregate) + { + // start on page 1 + int total = limit; + + foreach (char letter in letters) + { + map[letter] = total / limit; + + Option maybeBucket = aggregate.Buckets.Find(b => b.Key == letter.ToString()); + total += maybeBucket.Sum(bucket => (int)bucket.DocCount); + } + } + + return new SearchPageMap(map); + } +} diff --git a/ErsatzTV.Infrastructure/Search/SearchIndex.cs b/ErsatzTV.Infrastructure/Search/LuceneSearchIndex.cs similarity index 93% rename from ErsatzTV.Infrastructure/Search/SearchIndex.cs rename to ErsatzTV.Infrastructure/Search/LuceneSearchIndex.cs index c8bc9391..ad0463ca 100644 --- a/ErsatzTV.Infrastructure/Search/SearchIndex.cs +++ b/ErsatzTV.Infrastructure/Search/LuceneSearchIndex.cs @@ -28,40 +28,40 @@ using Query = Lucene.Net.Search.Query; namespace ErsatzTV.Infrastructure.Search; -public sealed class SearchIndex : ISearchIndex +public sealed class LuceneSearchIndex : ISearchIndex { private const LuceneVersion AppLuceneVersion = LuceneVersion.LUCENE_48; - private const string IdField = "id"; - private const string TypeField = "type"; - private const string TitleField = "title"; - private const string SortTitleField = "sort_title"; - private const string GenreField = "genre"; - private const string TagField = "tag"; - private const string PlotField = "plot"; - private const string LibraryNameField = "library_name"; - private const string LibraryIdField = "library_id"; - private const string TitleAndYearField = "title_and_year"; - private const string JumpLetterField = "jump_letter"; - private const string StudioField = "studio"; - private const string LanguageField = "language"; - private const string StyleField = "style"; - private const string MoodField = "mood"; - private const string ActorField = "actor"; - private const string ContentRatingField = "content_rating"; - private const string DirectorField = "director"; - private const string WriterField = "writer"; - private const string TraktListField = "trakt_list"; - private const string AlbumField = "album"; - private const string ArtistField = "artist"; - private const string StateField = "state"; - private const string AlbumArtistField = "album_artist"; - private const string ShowTitleField = "show_title"; - private const string ShowGenreField = "show_genre"; - private const string ShowTagField = "show_tag"; - private const string MetadataKindField = "metadata_kind"; - private const string VideoCodecField = "video_codec"; - private const string VideoDynamicRange = "video_dynamic_range"; + internal const string IdField = "id"; + internal const string TypeField = "type"; + internal const string TitleField = "title"; + internal const string SortTitleField = "sort_title"; + internal const string GenreField = "genre"; + internal const string TagField = "tag"; + internal const string PlotField = "plot"; + internal const string LibraryNameField = "library_name"; + internal const string LibraryIdField = "library_id"; + internal const string TitleAndYearField = "title_and_year"; + internal const string JumpLetterField = "jump_letter"; + internal const string StudioField = "studio"; + internal const string LanguageField = "language"; + internal const string StyleField = "style"; + internal const string MoodField = "mood"; + internal const string ActorField = "actor"; + internal const string ContentRatingField = "content_rating"; + internal const string DirectorField = "director"; + internal const string WriterField = "writer"; + internal const string TraktListField = "trakt_list"; + internal const string AlbumField = "album"; + internal const string ArtistField = "artist"; + internal const string StateField = "state"; + internal const string AlbumArtistField = "album_artist"; + internal const string ShowTitleField = "show_title"; + internal const string ShowGenreField = "show_genre"; + internal const string ShowTagField = "show_tag"; + internal const string MetadataKindField = "metadata_kind"; + internal const string VideoCodecField = "video_codec"; + internal const string VideoDynamicRange = "video_dynamic_range"; internal const string MinutesField = "minutes"; internal const string HeightField = "height"; @@ -82,21 +82,31 @@ public sealed class SearchIndex : ISearchIndex public const string SongType = "song"; private readonly List _cultureInfos; + private readonly string _cleanShutdownPath; - private readonly ILogger _logger; + private readonly ILogger _logger; private FSDirectory _directory; private bool _initialized; private IndexWriter _writer; - public SearchIndex(ILogger logger) + public LuceneSearchIndex(ILogger logger) { _logger = logger; _cultureInfos = CultureInfo.GetCultures(CultureTypes.NeutralCultures).ToList(); + _cleanShutdownPath = Path.Combine(FileSystemLayout.SearchIndexFolder, ".clean-shutdown"); _initialized = false; } - public int Version => 35; + public Task IndexExists() + { + bool directoryExists = Directory.Exists(FileSystemLayout.SearchIndexFolder); + bool fileExists = File.Exists(_cleanShutdownPath); + + return Task.FromResult(directoryExists && fileExists); + } + + public int Version => 36; public async Task Initialize( ILocalFileSystem localFileSystem, @@ -114,6 +124,11 @@ public sealed class SearchIndex : ISearchIndex localFileSystem.EnsureFolderExists(FileSystemLayout.SearchIndexFolder); } + if (File.Exists(_cleanShutdownPath)) + { + File.Delete(_cleanShutdownPath); + } + _directory = FSDirectory.Open(FileSystemLayout.SearchIndexFolder); var analyzer = new StandardAnalyzer(AppLuceneVersion); var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer) @@ -174,7 +189,7 @@ public sealed class SearchIndex : ISearchIndex return Task.FromResult(Unit.Default); } - public SearchResult Search(IClient client, string searchQuery, int skip, int limit, string searchField = "") + public Task Search(IClient client, string searchQuery, int skip, int limit, string searchField = "") { var metadata = new Dictionary { @@ -189,7 +204,7 @@ public sealed class SearchIndex : ISearchIndex if (string.IsNullOrWhiteSpace(searchQuery.Replace("*", string.Empty).Replace("?", string.Empty)) || _writer.MaxDoc == 0) { - return new SearchResult(new List(), 0); + return Task.FromResult(new SearchResult(new List(), 0)); } using DirectoryReader reader = _writer.GetReader(true); @@ -227,7 +242,7 @@ public sealed class SearchIndex : ISearchIndex searchResult.PageMap = GetSearchPageMap(searcher, query, null, sort, limit); } - return searchResult; + return Task.FromResult(searchResult); } public void Commit() => _writer.Commit(); @@ -236,6 +251,11 @@ public sealed class SearchIndex : ISearchIndex { _writer?.Dispose(); _directory?.Dispose(); + + using (File.Create(_cleanShutdownPath)) + { + // do nothing + } } public async Task Rebuild( @@ -270,7 +290,7 @@ public sealed class SearchIndex : ISearchIndex return Unit.Default; } - private static bool ValidateDirectory(string folder) + private bool ValidateDirectory(string folder) { try { @@ -284,7 +304,7 @@ public sealed class SearchIndex : ISearchIndex { using (DirectoryReader _ = w.GetReader(true)) { - return true; + return File.Exists(_cleanShutdownPath); } } } @@ -629,7 +649,7 @@ public sealed class SearchIndex : ISearchIndex new TextField(ShowTitleField, showMetadata.Title, Field.Store.NO) }; - // add some show fields to help filter shows within a particular show + // add some show fields to help filter seasons within a particular show foreach (Genre genre in showMetadata.Genres) { doc.Add(new TextField(ShowGenreField, genre.Name, Field.Store.NO)); @@ -680,7 +700,7 @@ public sealed class SearchIndex : ISearchIndex catch (Exception ex) { metadata.Season = null; - _logger.LogWarning(ex, "Error indexing show with metadata {@Metadata}", metadata); + _logger.LogWarning(ex, "Error indexing season with metadata {@Metadata}", metadata); } } } @@ -1174,7 +1194,7 @@ public sealed class SearchIndex : ISearchIndex string dynamicRange = colorParams.IsHdr ? "hdr" : "sdr"; - doc.Add(new TextField(VideoDynamicRange, dynamicRange, Field.Store.NO)); + doc.Add(new StringField(VideoDynamicRange, dynamicRange, Field.Store.NO)); } } } @@ -1192,7 +1212,7 @@ public sealed class SearchIndex : ISearchIndex } // this is used for filtering duplicate search results - private static string GetTitleAndYear(Metadata metadata) => + internal static string GetTitleAndYear(Metadata metadata) => metadata switch { EpisodeMetadata em => @@ -1212,7 +1232,7 @@ public sealed class SearchIndex : ISearchIndex private static string Title(Metadata metadata) => (metadata.Title ?? string.Empty).Replace(' ', '_'); - private static string GetJumpLetter(Metadata metadata) + internal static string GetJumpLetter(Metadata metadata) { char c = (metadata.SortTitle ?? " ").ToLowerInvariant().Head(); return c switch diff --git a/ErsatzTV.Infrastructure/Search/Models/ElasticSearchItem.cs b/ErsatzTV.Infrastructure/Search/Models/ElasticSearchItem.cs new file mode 100644 index 00000000..e008813a --- /dev/null +++ b/ErsatzTV.Infrastructure/Search/Models/ElasticSearchItem.cs @@ -0,0 +1,111 @@ +using System.Text.Json.Serialization; + +namespace ErsatzTV.Infrastructure.Search.Models; + +public class ElasticSearchItem : MinimalElasticSearchItem +{ + [JsonExtensionData] + public Dictionary AdditionalProperties { get; set; } = new(); + + [JsonPropertyName(LuceneSearchIndex.TitleField)] + public string Title { get; set; } + + [JsonPropertyName(LuceneSearchIndex.LibraryNameField)] + public string LibraryName { get; set; } + + [JsonPropertyName(LuceneSearchIndex.LibraryIdField)] + public int LibraryId { get; set; } + + [JsonPropertyName(LuceneSearchIndex.TitleAndYearField)] + public string TitleAndYear { get; set; } + + [JsonPropertyName(LuceneSearchIndex.StateField)] + public string State { get; set; } + + [JsonPropertyName(LuceneSearchIndex.MetadataKindField)] + public string MetadataKind { get; set; } + + [JsonPropertyName(LuceneSearchIndex.LanguageField)] + public List Language { get; set; } + + [JsonPropertyName(LuceneSearchIndex.MinutesField)] + public int Minutes { get; set; } + + [JsonPropertyName(LuceneSearchIndex.HeightField)] + public int Height { get; set; } + + [JsonPropertyName(LuceneSearchIndex.WidthField)] + public int Width { get; set; } + + [JsonPropertyName(LuceneSearchIndex.VideoCodecField)] + public string VideoCodec { get; set; } + + [JsonPropertyName(LuceneSearchIndex.VideoBitDepthField)] + public int VideoBitDepth { get; set; } + + [JsonPropertyName(LuceneSearchIndex.VideoDynamicRange)] + public string VideoDynamicRange { get; set; } + + [JsonPropertyName(LuceneSearchIndex.ContentRatingField)] + public List ContentRating { get; set; } + + [JsonPropertyName(LuceneSearchIndex.ReleaseDateField)] + public string ReleaseDate { get; set; } + + [JsonPropertyName(LuceneSearchIndex.AddedDateField)] + public string AddedDate { get; set; } + + [JsonPropertyName(LuceneSearchIndex.AlbumField)] + public string Album { get; set; } + + [JsonPropertyName(LuceneSearchIndex.AlbumArtistField)] + public string AlbumArtist { get; set; } + + [JsonPropertyName(LuceneSearchIndex.PlotField)] + public string Plot { get; set; } + + [JsonPropertyName(LuceneSearchIndex.GenreField)] + public List Genre { get; set; } + + [JsonPropertyName(LuceneSearchIndex.TagField)] + public List Tag { get; set; } + + [JsonPropertyName(LuceneSearchIndex.StudioField)] + public List Studio { get; set; } + + [JsonPropertyName(LuceneSearchIndex.ArtistField)] + public List Artist { get; set; } + + [JsonPropertyName(LuceneSearchIndex.ActorField)] + public List Actor { get; set; } + + [JsonPropertyName(LuceneSearchIndex.DirectorField)] + public List Director { get; set; } + + [JsonPropertyName(LuceneSearchIndex.WriterField)] + public List Writer { get; set; } + + [JsonPropertyName(LuceneSearchIndex.TraktListField)] + public List TraktList { get; set; } + + [JsonPropertyName(LuceneSearchIndex.SeasonNumberField)] + public int SeasonNumber { get; set; } + + [JsonPropertyName(LuceneSearchIndex.EpisodeNumberField)] + public int EpisodeNumber { get; set; } + + [JsonPropertyName(LuceneSearchIndex.ShowTitleField)] + public string ShowTitle { get; set; } + + [JsonPropertyName(LuceneSearchIndex.ShowGenreField)] + public List ShowGenre { get; set; } + + [JsonPropertyName(LuceneSearchIndex.ShowTagField)] + public List ShowTag { get; set; } + + [JsonPropertyName(LuceneSearchIndex.StyleField)] + public List Style { get; set; } + + [JsonPropertyName(LuceneSearchIndex.MoodField)] + public List Mood { get; set; } +} diff --git a/ErsatzTV.Infrastructure/Search/Models/MinimalElasticSearchItem.cs b/ErsatzTV.Infrastructure/Search/Models/MinimalElasticSearchItem.cs new file mode 100644 index 00000000..46dadfd4 --- /dev/null +++ b/ErsatzTV.Infrastructure/Search/Models/MinimalElasticSearchItem.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace ErsatzTV.Infrastructure.Search.Models; + +public class MinimalElasticSearchItem +{ + [JsonPropertyName(LuceneSearchIndex.IdField)] + public int Id { get; set; } + + [JsonPropertyName(LuceneSearchIndex.TypeField)] + public string Type { get; set; } + + [JsonPropertyName(LuceneSearchIndex.SortTitleField)] + public string SortTitle { get; set; } + + [JsonPropertyName(LuceneSearchIndex.JumpLetterField)] + public string JumpLetter { get; set; } +} diff --git a/ErsatzTV.Scanner/Program.cs b/ErsatzTV.Scanner/Program.cs index 60aa7fee..adb16cc0 100644 --- a/ErsatzTV.Scanner/Program.cs +++ b/ErsatzTV.Scanner/Program.cs @@ -162,7 +162,7 @@ public class Program services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); // TODO: real bugsnag? services.AddSingleton(_ => new BugsnagNoopClient()); diff --git a/ErsatzTV/Extensions/NavigationManagerExtensions.cs b/ErsatzTV/Extensions/NavigationManagerExtensions.cs index 64fc03b6..43abebe9 100644 --- a/ErsatzTV/Extensions/NavigationManagerExtensions.cs +++ b/ErsatzTV/Extensions/NavigationManagerExtensions.cs @@ -5,15 +5,14 @@ namespace ErsatzTV.Extensions; public static class NavigationManagerExtensions { - public static ValueTask NavigateToFragmentAsync(this NavigationManager navigationManager, IJSRuntime jSRuntime) + public static async Task NavigateToFragmentAsync(this NavigationManager navigationManager, IJSRuntime jSRuntime) { Uri uri = navigationManager.ToAbsoluteUri(navigationManager.Uri); - if (uri.Fragment.Length == 0) + if (uri.Fragment.Length > 0) { - return default; + await Task.Delay(250); + await jSRuntime.InvokeVoidAsync("blazorHelpers.scrollToFragment", uri.Fragment.Substring(1)); } - - return jSRuntime.InvokeVoidAsync("blazorHelpers.scrollToFragment", uri.Fragment.Substring(1)); } } diff --git a/ErsatzTV/Pages/ArtistList.razor b/ErsatzTV/Pages/ArtistList.razor index 50289c6d..8818b2f5 100644 --- a/ErsatzTV/Pages/ArtistList.razor +++ b/ErsatzTV/Pages/ArtistList.razor @@ -77,10 +77,10 @@ [Parameter] public int PageNumber { get; set; } - private ArtistCardResultsViewModel _data; + private ArtistCardResultsViewModel _data = new(0, new List(), None); private string _query; - protected override Task OnParametersSetAsync() + protected override async Task OnParametersSetAsync() { if (PageNumber == 0) { @@ -88,7 +88,8 @@ } _query = _navigationManager.Uri.GetSearchQuery(); - return RefreshData(); + + await RefreshData(); } protected override async Task RefreshData() diff --git a/ErsatzTV/Pages/EpisodeList.razor b/ErsatzTV/Pages/EpisodeList.razor index b8f4aa7a..10224c41 100644 --- a/ErsatzTV/Pages/EpisodeList.razor +++ b/ErsatzTV/Pages/EpisodeList.razor @@ -77,10 +77,10 @@ [Parameter] public int PageNumber { get; set; } - private TelevisionEpisodeCardResultsViewModel _data; + private TelevisionEpisodeCardResultsViewModel _data = new(0, new List(), None); private string _query; - protected override Task OnParametersSetAsync() + protected override async Task OnParametersSetAsync() { if (PageNumber == 0) { @@ -88,7 +88,8 @@ } _query = _navigationManager.Uri.GetSearchQuery(); - return RefreshData(); + + await RefreshData(); } protected override async Task RefreshData() diff --git a/ErsatzTV/Pages/MovieList.razor b/ErsatzTV/Pages/MovieList.razor index 981a7460..f112951a 100644 --- a/ErsatzTV/Pages/MovieList.razor +++ b/ErsatzTV/Pages/MovieList.razor @@ -6,8 +6,7 @@ @using ErsatzTV.Application.MediaCollections @using ErsatzTV.Application.Search @inherits MultiSelectBase -@inject NavigationManager _navigationManager -@inject ChannelWriter _channel +@inject NavigationManager NavigationManager
@@ -76,18 +75,19 @@ [Parameter] public int PageNumber { get; set; } - private MovieCardResultsViewModel _data; + private MovieCardResultsViewModel _data = new(0, new List(), None); private string _query; - protected override Task OnParametersSetAsync() + protected override async Task OnParametersSetAsync() { if (PageNumber == 0) { PageNumber = 1; } - _query = _navigationManager.Uri.GetSearchQuery(); - return RefreshData(); + _query = NavigationManager.Uri.GetSearchQuery(); + + await RefreshData(); } protected override async Task RefreshData() @@ -104,7 +104,7 @@ (string key, string value) = _query.EncodeQuery(); uri = $"{uri}?{key}={value}"; } - _navigationManager.NavigateTo(uri); + NavigationManager.NavigateTo(uri); } private void NextPage() @@ -115,7 +115,7 @@ (string key, string value) = _query.EncodeQuery(); uri = $"{uri}?{key}={value}"; } - _navigationManager.NavigateTo(uri); + NavigationManager.NavigateTo(uri); } private void SelectClicked(MediaCardViewModel card, MouseEventArgs e) diff --git a/ErsatzTV/Pages/MusicVideoList.razor b/ErsatzTV/Pages/MusicVideoList.razor index d50a7cf3..53e65180 100644 --- a/ErsatzTV/Pages/MusicVideoList.razor +++ b/ErsatzTV/Pages/MusicVideoList.razor @@ -77,10 +77,10 @@ [Parameter] public int PageNumber { get; set; } - private MusicVideoCardResultsViewModel _data; + private MusicVideoCardResultsViewModel _data = new(0, new List(), None); private string _query; - protected override Task OnParametersSetAsync() + protected override async Task OnParametersSetAsync() { if (PageNumber == 0) { @@ -88,7 +88,8 @@ } _query = _navigationManager.Uri.GetSearchQuery(); - return RefreshData(); + + await RefreshData(); } protected override async Task RefreshData() diff --git a/ErsatzTV/Pages/OtherVideoList.razor b/ErsatzTV/Pages/OtherVideoList.razor index 621e16f3..e19b7cc6 100644 --- a/ErsatzTV/Pages/OtherVideoList.razor +++ b/ErsatzTV/Pages/OtherVideoList.razor @@ -77,10 +77,10 @@ [Parameter] public int PageNumber { get; set; } - private OtherVideoCardResultsViewModel _data; + private OtherVideoCardResultsViewModel _data = new(0, new List(), None); private string _query; - protected override Task OnParametersSetAsync() + protected override async Task OnParametersSetAsync() { if (PageNumber == 0) { @@ -88,7 +88,8 @@ } _query = _navigationManager.Uri.GetSearchQuery(); - return RefreshData(); + + await RefreshData(); } protected override async Task RefreshData() diff --git a/ErsatzTV/Pages/SongList.razor b/ErsatzTV/Pages/SongList.razor index 68116855..116d76cf 100644 --- a/ErsatzTV/Pages/SongList.razor +++ b/ErsatzTV/Pages/SongList.razor @@ -77,10 +77,10 @@ [Parameter] public int PageNumber { get; set; } - private SongCardResultsViewModel _data; + private SongCardResultsViewModel _data = new(0, new List(), None); private string _query; - protected override Task OnParametersSetAsync() + protected override async Task OnParametersSetAsync() { if (PageNumber == 0) { @@ -88,7 +88,8 @@ } _query = _navigationManager.Uri.GetSearchQuery(); - return RefreshData(); + + await RefreshData(); } protected override async Task RefreshData() diff --git a/ErsatzTV/Pages/TelevisionSeasonSearchResults.razor b/ErsatzTV/Pages/TelevisionSeasonSearchResults.razor index ea816dbc..452665c0 100644 --- a/ErsatzTV/Pages/TelevisionSeasonSearchResults.razor +++ b/ErsatzTV/Pages/TelevisionSeasonSearchResults.razor @@ -76,10 +76,10 @@ [Parameter] public int PageNumber { get; set; } - private TelevisionSeasonCardResultsViewModel _data; + private TelevisionSeasonCardResultsViewModel _data = new(0, new List(), None); private string _query; - protected override Task OnParametersSetAsync() + protected override async Task OnParametersSetAsync() { if (PageNumber == 0) { @@ -87,7 +87,8 @@ } _query = _navigationManager.Uri.GetSearchQuery(); - return RefreshData(); + + await RefreshData(); } protected override async Task RefreshData() diff --git a/ErsatzTV/Pages/TelevisionShowList.razor b/ErsatzTV/Pages/TelevisionShowList.razor index 0bd9fecf..4672b3f6 100644 --- a/ErsatzTV/Pages/TelevisionShowList.razor +++ b/ErsatzTV/Pages/TelevisionShowList.razor @@ -76,10 +76,10 @@ [Parameter] public int PageNumber { get; set; } - private TelevisionShowCardResultsViewModel _data; + private TelevisionShowCardResultsViewModel _data = new(0, new List(), None); private string _query; - protected override Task OnParametersSetAsync() + protected override async Task OnParametersSetAsync() { if (PageNumber == 0) { @@ -87,7 +87,8 @@ } _query = _navigationManager.Uri.GetSearchQuery(); - return RefreshData(); + + await RefreshData(); } protected override async Task RefreshData() diff --git a/ErsatzTV/SearchHelper.cs b/ErsatzTV/SearchHelper.cs new file mode 100644 index 00000000..b6b1efdf --- /dev/null +++ b/ErsatzTV/SearchHelper.cs @@ -0,0 +1,20 @@ +namespace ErsatzTV; + +public static class SearchHelper +{ + public static string ElasticSearchUri { get; private set; } + public static string ElasticSearchIndexName { get; private set; } + public static bool IsElasticSearchEnabled { get; private set; } + + public static void Init(IConfiguration configuration) + { + ElasticSearchUri = configuration["ElasticSearch:Uri"]; + ElasticSearchIndexName = configuration["ElasticSearch:IndexName"]; + if (string.IsNullOrWhiteSpace(ElasticSearchIndexName)) + { + ElasticSearchIndexName = "ersatztv"; + } + + IsElasticSearchEnabled = !string.IsNullOrWhiteSpace(ElasticSearchUri); + } +} diff --git a/ErsatzTV/Startup.cs b/ErsatzTV/Startup.cs index 5ea87fde..2cf51b02 100644 --- a/ErsatzTV/Startup.cs +++ b/ErsatzTV/Startup.cs @@ -127,6 +127,7 @@ public class Startup OidcHelper.Init(Configuration); JwtHelper.Init(Configuration); + SearchHelper.Init(Configuration); if (OidcHelper.IsEnabled) { @@ -516,7 +517,18 @@ public class Startup services.AddSingleton(); // TODO: does this need to be singleton? services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + + if (SearchHelper.IsElasticSearchEnabled) + { + ElasticSearchIndex.Uri = new Uri(SearchHelper.ElasticSearchUri); + ElasticSearchIndex.IndexName = SearchHelper.ElasticSearchIndexName; + services.AddSingleton(); + } + else + { + services.AddSingleton(); + } + services.AddSingleton(); services.AddSingleton(); services.AddSingleton();