pimhwang
/
ErsatzTV
mirror of https://github.com/ErsatzTV/ErsatzTV.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

optimize search-index rebuilding (#765) 

* update dependencies

* optimize search-index rebuilding

* cleanup logging
pull/766/head
Jason Dove 4 years ago committed by GitHub
parent
2e1073eb53
commit
a9a89d04ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 146 additions and 54 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      CHANGELOG.md
  2. 3
      ErsatzTV.Application/ErsatzTV.Application.csproj
  3. 11
      ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs
  4. 4
      ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj
  5. 4
      ErsatzTV.Core/ErsatzTV.Core.csproj
  6. 2
      ErsatzTV.Core/Interfaces/Repositories/ISearchRepository.cs
  7. 2
      ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs
  8. 2
      ErsatzTV.FFmpeg/ErsatzTV.FFmpeg.csproj
  9. 98
      ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs
  10. 2
      ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj
  11. 67
      ErsatzTV.Infrastructure/Search/SearchIndex.cs
  12. 4
      ErsatzTV/ErsatzTV.csproj

1
CHANGELOG.md
Unescape View File

@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Changed
- Update Plex, Jellyfin and Emby movie library scanners to share a significant amount of code
- This should help maintain feature parity going forward
- Optimize search-index rebuilding to complete 100x faster
### Added
- Add `unavailable` state for Emby movie libraries

3
ErsatzTV.Application/ErsatzTV.Application.csproj
Unescape View File

@ -8,7 +8,8 @@ @@ -8,7 +8,8 @@
<ItemGroup>
<PackageReference Include="Bugsnag" Version="3.0.1" />
<PackageReference Include="CliWrap" Version="3.4.3" />
<PackageReference Include="CliWrap" Version="3.4.4" />
<PackageReference Include="Humanizer.Core" Version="2.14.1" />
<PackageReference Include="MediatR" Version="10.0.1" />
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="6.0.0" />
<PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.1.46">

11
ErsatzTV.Application/Search/Commands/RebuildSearchIndexHandler.cs
Unescape View File

@ -1,8 +1,10 @@ @@ -1,8 +1,10 @@
using ErsatzTV.Core;
using System.Diagnostics;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.Metadata;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Core.Interfaces.Search;
using Humanizer;
using Microsoft.Extensions.Logging;
namespace ErsatzTV.Application.Search;
@ -41,12 +43,13 @@ public class RebuildSearchIndexHandler : IRequestHandler<RebuildSearchIndex, Uni @@ -41,12 +43,13 @@ public class RebuildSearchIndexHandler : IRequestHandler<RebuildSearchIndex, Uni
{
_logger.LogInformation("Migrating search index to version {Version}", _searchIndex.Version);
List<int> itemIds = await _searchRepository.GetItemIdsToIndex();
await _searchIndex.Rebuild(_searchRepository, itemIds);
var sw = Stopwatch.StartNew();
await _searchIndex.Rebuild(_searchRepository);
await _configElementRepository.Upsert(ConfigElementKey.SearchIndexVersion, _searchIndex.Version);
sw.Stop();
_logger.LogInformation("Done migrating search index");
_logger.LogInformation("Done migrating search index in {Duration}", sw.Elapsed.Humanize());
}
else
{

4
ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj
Unescape View File

@ -8,7 +8,7 @@ @@ -8,7 +8,7 @@
<ItemGroup>
<PackageReference Include="Bugsnag" Version="3.0.1" />
<PackageReference Include="CliWrap" Version="3.4.3" />
<PackageReference Include="CliWrap" Version="3.4.4" />
<PackageReference Include="FluentAssertions" Version="6.6.0" />
<PackageReference Include="LanguageExt.Core" Version="4.0.4" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="6.0.1" />
@ -24,7 +24,7 @@ @@ -24,7 +24,7 @@
<PackageReference Include="Moq" Version="4.17.2" />
<PackageReference Include="NUnit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
<PackageReference Include="Serilog" Version="2.10.0" />
<PackageReference Include="Serilog" Version="2.11.0" />
<PackageReference Include="Serilog.Extensions.Logging" Version="3.1.0" />
<PackageReference Include="Serilog.Sinks.Debug" Version="2.0.0" />
</ItemGroup>

4
ErsatzTV.Core/ErsatzTV.Core.csproj
Unescape View File

@ -9,7 +9,7 @@ @@ -9,7 +9,7 @@
<ItemGroup>
<PackageReference Include="Bugsnag" Version="3.0.1" />
<PackageReference Include="Destructurama.Attributed" Version="3.0.0" />
<PackageReference Include="Flurl" Version="3.0.4" />
<PackageReference Include="Flurl" Version="3.0.5" />
<PackageReference Include="LanguageExt.Core" Version="4.0.4" />
<PackageReference Include="MediatR" Version="10.0.1" />
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="6.0.0" />
@ -21,7 +21,7 @@ @@ -21,7 +21,7 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="Serilog" Version="2.10.0" />
<PackageReference Include="Serilog" Version="2.11.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="4.0.1" />
</ItemGroup>

2
ErsatzTV.Core/Interfaces/Repositories/ISearchRepository.cs
Unescape View File

@ -4,10 +4,10 @@ namespace ErsatzTV.Core.Interfaces.Repositories; @@ -4,10 +4,10 @@ namespace ErsatzTV.Core.Interfaces.Repositories;
public interface ISearchRepository
{
Task<List<int>> GetItemIdsToIndex();
Task<Option<MediaItem>> GetItemToIndex(int id);
Task<List<string>> GetLanguagesForShow(Show show);
Task<List<string>> GetLanguagesForSeason(Season season);
Task<List<string>> GetLanguagesForArtist(Artist artist);
Task<List<string>> GetAllLanguageCodes(List<string> mediaCodes);
IAsyncEnumerable<MediaItem> GetAllMediaItems();
}

2
ErsatzTV.Core/Interfaces/Search/ISearchIndex.cs
Unescape View File

@ -9,7 +9,7 @@ public interface ISearchIndex : IDisposable @@ -9,7 +9,7 @@ public interface ISearchIndex : IDisposable
{
public int Version { get; }
Task<bool> Initialize(ILocalFileSystem localFileSystem);
Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds);
Task<Unit> Rebuild(ISearchRepository searchRepository);
Task<Unit> RebuildItems(ISearchRepository searchRepository, List<int> itemIds);
Task<Unit> AddItems(ISearchRepository searchRepository, List<MediaItem> items);
Task<Unit> UpdateItems(ISearchRepository searchRepository, List<MediaItem> items);

2
ErsatzTV.FFmpeg/ErsatzTV.FFmpeg.csproj
Unescape View File

@ -7,7 +7,7 @@ @@ -7,7 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CliWrap" Version="3.4.3" />
<PackageReference Include="CliWrap" Version="3.4.4" />
<PackageReference Include="LanguageExt.Core" Version="4.0.4" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
</ItemGroup>

98
ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs
Unescape View File

@ -12,13 +12,6 @@ public class SearchRepository : ISearchRepository @@ -12,13 +12,6 @@ public class SearchRepository : ISearchRepository
public SearchRepository(IDbContextFactory<TvContext> dbContextFactory) => _dbContextFactory = dbContextFactory;
public async Task<List<int>> GetItemIdsToIndex()
{
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync();
return await dbContext.Connection.QueryAsync<int>(@"SELECT Id FROM MediaItem")
.Map(result => result.ToList());
}
public async Task<Option<MediaItem>> GetItemToIndex(int id)
{
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync();
@ -103,9 +96,7 @@ public class SearchRepository : ISearchRepository @@ -103,9 +96,7 @@ public class SearchRepository : ISearchRepository
.ThenInclude(mm => mm.Streams)
.Include(mi => mi.TraktListItems)
.ThenInclude(tli => tli.TraktList)
.OrderBy(mi => mi.Id)
.SingleOrDefaultAsync(mi => mi.Id == id)
.Map(Optional);
.SelectOneAsync(mi => mi.Id, mi => mi.Id == id);
}
public async Task<List<string>> GetLanguagesForShow(Show show)
@ -151,4 +142,91 @@ public class SearchRepository : ISearchRepository @@ -151,4 +142,91 @@ public class SearchRepository : ISearchRepository
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync();
return await dbContext.LanguageCodes.GetAllLanguageCodes(mediaCodes);
}
public IAsyncEnumerable<MediaItem> GetAllMediaItems()
{
TvContext dbContext = _dbContextFactory.CreateDbContext();
return dbContext.MediaItems
.AsNoTracking()
.Include(mi => mi.LibraryPath)
.ThenInclude(lp => lp.Library)
.Include(mi => (mi as Movie).MovieMetadata)
.ThenInclude(mm => mm.Genres)
.Include(mi => (mi as Movie).MovieMetadata)
.ThenInclude(mm => mm.Tags)
.Include(mi => (mi as Movie).MovieMetadata)
.ThenInclude(mm => mm.Studios)
.Include(mi => (mi as Movie).MovieMetadata)
.ThenInclude(mm => mm.Actors)
.Include(mi => (mi as Movie).MovieMetadata)
.ThenInclude(mm => mm.Directors)
.Include(mi => (mi as Movie).MovieMetadata)
.ThenInclude(mm => mm.Writers)
.Include(mi => (mi as Movie).MediaVersions)
.ThenInclude(mm => mm.Streams)
.Include(mi => (mi as Episode).EpisodeMetadata)
.ThenInclude(em => em.Genres)
.Include(mi => (mi as Episode).EpisodeMetadata)
.ThenInclude(em => em.Tags)
.Include(mi => (mi as Episode).EpisodeMetadata)
.ThenInclude(em => em.Studios)
.Include(mi => (mi as Episode).EpisodeMetadata)
.ThenInclude(em => em.Actors)
.Include(mi => (mi as Episode).EpisodeMetadata)
.ThenInclude(em => em.Directors)
.Include(mi => (mi as Episode).EpisodeMetadata)
.ThenInclude(em => em.Writers)
.Include(mi => (mi as Episode).EpisodeMetadata)
.ThenInclude(em => em.Guids)
.Include(mi => (mi as Episode).MediaVersions)
.ThenInclude(em => em.Streams)
.Include(mi => (mi as Episode).Season)
.Include(mi => (mi as Season).SeasonMetadata)
.ThenInclude(sm => sm.Genres)
.Include(mi => (mi as Season).SeasonMetadata)
.ThenInclude(sm => sm.Tags)
.Include(mi => (mi as Season).SeasonMetadata)
.ThenInclude(sm => sm.Studios)
.Include(mi => (mi as Season).SeasonMetadata)
.ThenInclude(sm => sm.Actors)
.Include(mi => (mi as Season).Show)
.ThenInclude(sm => sm.ShowMetadata)
.Include(mi => (mi as Show).ShowMetadata)
.ThenInclude(mm => mm.Genres)
.Include(mi => (mi as Show).ShowMetadata)
.ThenInclude(mm => mm.Tags)
.Include(mi => (mi as Show).ShowMetadata)
.ThenInclude(mm => mm.Studios)
.Include(mi => (mi as Show).ShowMetadata)
.ThenInclude(mm => mm.Actors)
.Include(mi => (mi as MusicVideo).Artist)
.ThenInclude(mm => mm.ArtistMetadata)
.Include(mi => (mi as MusicVideo).MusicVideoMetadata)
.ThenInclude(mm => mm.Genres)
.Include(mi => (mi as MusicVideo).MusicVideoMetadata)
.ThenInclude(mm => mm.Tags)
.Include(mi => (mi as MusicVideo).MusicVideoMetadata)
.ThenInclude(mm => mm.Studios)
.Include(mi => (mi as MusicVideo).MediaVersions)
.ThenInclude(mm => mm.Streams)
.Include(mi => (mi as Artist).ArtistMetadata)
.ThenInclude(mm => mm.Genres)
.Include(mi => (mi as Artist).ArtistMetadata)
.ThenInclude(mm => mm.Styles)
.Include(mi => (mi as Artist).ArtistMetadata)
.ThenInclude(mm => mm.Moods)
.Include(mi => (mi as OtherVideo).OtherVideoMetadata)
.ThenInclude(mm => mm.Tags)
.Include(mi => (mi as OtherVideo).MediaVersions)
.ThenInclude(mm => mm.Streams)
.Include(mi => (mi as Song).SongMetadata)
.ThenInclude(mm => mm.Tags)
.Include(mi => (mi as Song).SongMetadata)
.ThenInclude(mm => mm.Genres)
.Include(mi => (mi as Song).MediaVersions)
.ThenInclude(mm => mm.Streams)
.Include(mi => mi.TraktListItems)
.ThenInclude(tli => tli.TraktList)
.AsAsyncEnumerable();
}
}

2
ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj
Unescape View File

@ -9,7 +9,7 @@ @@ -9,7 +9,7 @@
<ItemGroup>
<PackageReference Include="Blurhash.System.Drawing.Common" Version="2.2.0" />
<PackageReference Include="CliWrap" Version="3.4.3" />
<PackageReference Include="CliWrap" Version="3.4.4" />
<PackageReference Include="Dapper" Version="2.0.123" />
<PackageReference Include="Lucene.Net" Version="4.8.0-beta00016" />
<PackageReference Include="Lucene.Net.Analysis.Common" Version="4.8.0-beta00016" />

67
ErsatzTV.Infrastructure/Search/SearchIndex.cs
Unescape View File

@ -197,11 +197,15 @@ public sealed class SearchIndex : ISearchIndex @@ -197,11 +197,15 @@ public sealed class SearchIndex : ISearchIndex
_directory?.Dispose();
}
public async Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds)
public async Task<Unit> Rebuild(ISearchRepository searchRepository)
{
_writer.DeleteAll();
_writer.Commit();
await RebuildItems(searchRepository, itemIds);
await foreach (MediaItem mediaItem in searchRepository.GetAllMediaItems())
{
await RebuildItem(searchRepository, mediaItem);
}
_writer.Commit();
return Unit.Default;
@ -213,39 +217,44 @@ public sealed class SearchIndex : ISearchIndex @@ -213,39 +217,44 @@ public sealed class SearchIndex : ISearchIndex
{
foreach (MediaItem mediaItem in await searchRepository.GetItemToIndex(id))
{
switch (mediaItem)
{
case Movie movie:
await UpdateMovie(searchRepository, movie);
break;
case Show show:
await UpdateShow(searchRepository, show);
break;
case Season season:
await UpdateSeason(searchRepository, season);
break;
case Artist artist:
await UpdateArtist(searchRepository, artist);
break;
case MusicVideo musicVideo:
await UpdateMusicVideo(searchRepository, musicVideo);
break;
case Episode episode:
await UpdateEpisode(searchRepository, episode);
break;
case OtherVideo otherVideo:
await UpdateOtherVideo(searchRepository, otherVideo);
break;
case Song song:
await UpdateSong(searchRepository, song);
break;
}
await RebuildItem(searchRepository, mediaItem);
}
}
return Unit.Default;
}
private async Task RebuildItem(ISearchRepository searchRepository, MediaItem mediaItem)
{
switch (mediaItem)
{
case Movie movie:
await UpdateMovie(searchRepository, movie);
break;
case Show show:
await UpdateShow(searchRepository, show);
break;
case Season season:
await UpdateSeason(searchRepository, season);
break;
case Artist artist:
await UpdateArtist(searchRepository, artist);
break;
case MusicVideo musicVideo:
await UpdateMusicVideo(searchRepository, musicVideo);
break;
case Episode episode:
await UpdateEpisode(searchRepository, episode);
break;
case OtherVideo otherVideo:
await UpdateOtherVideo(searchRepository, otherVideo);
break;
case Song song:
await UpdateSong(searchRepository, song);
break;
}
}
private static Option<SearchPageMap> GetSearchPageMap(
IndexSearcher searcher,
Query query,

4
ErsatzTV/ErsatzTV.csproj
Unescape View File

@ -59,7 +59,7 @@ @@ -59,7 +59,7 @@
<PackageReference Include="FluentValidation.AspNetCore" Version="10.4.0" />
<PackageReference Include="HtmlSanitizer" Version="7.1.488" />
<PackageReference Include="LanguageExt.Core" Version="4.0.4" />
<PackageReference Include="Markdig" Version="0.30.0" />
<PackageReference Include="Markdig" Version="0.30.2" />
<PackageReference Include="MediatR.Courier.DependencyInjection" Version="5.0.0" />
<PackageReference Include="MediatR.Extensions.Microsoft.DependencyInjection" Version="10.0.1" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.4" />
@ -76,7 +76,7 @@ @@ -76,7 +76,7 @@
<PackageReference Include="NaturalSort.Extension" Version="3.2.0" />
<PackageReference Include="PPioli.FluentValidation.Blazor" Version="5.0.0" />
<PackageReference Include="Refit.HttpClientFactory" Version="6.3.2" />
<PackageReference Include="Serilog" Version="2.10.0" />
<PackageReference Include="Serilog" Version="2.11.0" />
<PackageReference Include="Serilog.AspNetCore" Version="5.0.0" />
<PackageReference Include="Serilog.Settings.Configuration" Version="3.3.0" />
<PackageReference Include="Serilog.Sinks.SQLite" Version="6.0.1" />

Write Preview
Loading…
Cancel
Save