From 474e647d6d02d15bff306c8bfe32d3b3a2fff351 Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Wed, 31 Dec 2025 22:28:15 -0600 Subject: [PATCH] more jellyfin performance improvements (#2747) * fix slow db and api logging so it also works in scanner project * don't request people from jellyfin by default --- CHANGELOG.md | 2 + .../Interfaces/Jellyfin/IJellyfinApiClient.cs | 16 +- .../JellyfinTelevisionRepository.cs | 5 + .../Data/Repositories/SearchRepository.cs | 21 ++ ErsatzTV.Infrastructure/Data/TvContext.cs | 16 +- .../Jellyfin/IJellyfinApi.cs | 39 ++++ .../Jellyfin/JellyfinApiClient.cs | 53 ++++- .../SlowApiHandler.cs | 3 +- .../SlowQueryInterceptor.cs | 10 +- .../Core/Emby/EmbyTelevisionLibraryScanner.cs | 3 +- .../JellyfinTelevisionLibraryScanner.cs | 218 ++++++++++++++++-- .../MediaServerTelevisionLibraryScanner.cs | 5 +- .../Core/Plex/PlexTelevisionLibraryScanner.cs | 3 +- ErsatzTV.Scanner/Program.cs | 6 + ErsatzTV/Startup.cs | 18 +- 15 files changed, 368 insertions(+), 50 deletions(-) rename {ErsatzTV => ErsatzTV.Infrastructure}/SlowApiHandler.cs (93%) rename {ErsatzTV => ErsatzTV.Infrastructure}/SlowQueryInterceptor.cs (63%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e4489472..ae0177a18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Hardware acceleration will now be used - Items can "work ahead" (transcode faster than realtime) when less than 3 minutes in duration - Optimize Jellyfin database fields and indexes +- Optimize Jellyfin show library scans by only requesting `People` (actors, directors, writers) when etags don't match + - This should significantly speed up periodic library scans, particularly against Jellyfin 10.11.x ## [25.9.0] - 2025-11-29 ### Added diff --git a/ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs b/ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs index db964bc64..b3b64ef5a 100644 --- a/ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs +++ b/ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs @@ -13,7 +13,7 @@ public interface IJellyfinApiClient string apiKey, JellyfinLibrary library); - IAsyncEnumerable> GetShowLibraryItems( + IAsyncEnumerable> GetShowLibraryItemsWithoutPeople( string address, string apiKey, JellyfinLibrary library); @@ -30,6 +30,12 @@ public interface IJellyfinApiClient JellyfinLibrary library, string seasonId); + IAsyncEnumerable> GetEpisodeLibraryItemsWithoutPeople( + string address, + string apiKey, + JellyfinLibrary library, + string seasonId); + IAsyncEnumerable> GetCollectionLibraryItems( string address, string apiKey, @@ -58,4 +64,12 @@ public interface IJellyfinApiClient string apiKey, JellyfinLibrary library, string showTitle); + + Task>> GetSingleEpisode( + string address, + string apiKey, + JellyfinLibrary library, + string seasonId, + string episodeId); + } diff --git a/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs index b53ff14ee..38c0be919 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs @@ -88,6 +88,7 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository { await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); Option maybeExisting = await dbContext.JellyfinShows + .TagWithCallSite() .Include(m => m.LibraryPath) .ThenInclude(lp => lp.Library) .Include(m => m.ShowMetadata) @@ -128,6 +129,7 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository { await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); Option maybeExisting = await dbContext.JellyfinSeasons + .TagWithCallSite() .Include(m => m.LibraryPath) .Include(m => m.SeasonMetadata) .ThenInclude(mm => mm.Artwork) @@ -158,6 +160,7 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository { await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); Option maybeExisting = await dbContext.JellyfinEpisodes + .TagWithCallSite() .Include(m => m.LibraryPath) .ThenInclude(lp => lp.Library) .Include(m => m.MediaVersions) @@ -511,6 +514,8 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); Option maybeShow = await dbContext.JellyfinShows + .AsNoTracking() + .TagWithCallSite() .Where(s => s.Id == showId) .Where(s => s.LibraryPath.LibraryId == libraryId) .Include(s => s.ShowMetadata) diff --git a/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs index 21129a4fa..26cb1d44a 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs @@ -15,6 +15,7 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I var baseItem = await dbContext.MediaItems .AsNoTracking() + .TagWithCallSite() .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); if (baseItem is null) @@ -27,60 +28,70 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I case Movie: return await dbContext.Movies .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); case Episode: return await dbContext.Episodes .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); case Season: return await dbContext.Seasons .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); case Show: return await dbContext.Shows .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); case MusicVideo: return await dbContext.MusicVideos .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); case Artist: return await dbContext.Artists .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); case OtherVideo: return await dbContext.OtherVideos .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); case Song: return await dbContext.Songs .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); case Image: return await dbContext.Images .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); case RemoteStream: return await dbContext.RemoteStreams .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); @@ -225,6 +236,7 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I ConfiguredCancelableAsyncEnumerable movies = dbContext.Movies .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .AsAsyncEnumerable() @@ -242,6 +254,7 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I ConfiguredCancelableAsyncEnumerable shows = dbContext.Shows .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .AsAsyncEnumerable() @@ -259,6 +272,7 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I ConfiguredCancelableAsyncEnumerable seasons = dbContext.Seasons .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .AsAsyncEnumerable() @@ -276,6 +290,7 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I ConfiguredCancelableAsyncEnumerable episodes = dbContext.Episodes .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .AsAsyncEnumerable() @@ -294,6 +309,7 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I ConfiguredCancelableAsyncEnumerable musicVideos = dbContext.MusicVideos .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .AsAsyncEnumerable() @@ -311,6 +327,7 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I ConfiguredCancelableAsyncEnumerable artists = dbContext.Artists .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .AsAsyncEnumerable() @@ -329,6 +346,7 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I ConfiguredCancelableAsyncEnumerable otherVideos = dbContext.OtherVideos .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .AsAsyncEnumerable() @@ -346,6 +364,7 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I ConfiguredCancelableAsyncEnumerable songs = dbContext.Songs .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .AsAsyncEnumerable() @@ -363,6 +382,7 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I ConfiguredCancelableAsyncEnumerable images = dbContext.Images .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .AsAsyncEnumerable() @@ -381,6 +401,7 @@ public class SearchRepository(IDbContextFactory dbContextFactory) : I ConfiguredCancelableAsyncEnumerable remoteStreams = dbContext.RemoteStreams .AsNoTracking() + .TagWithCallSite() .IncludeForSearch() .AsSplitQuery() .AsAsyncEnumerable() diff --git a/ErsatzTV.Infrastructure/Data/TvContext.cs b/ErsatzTV.Infrastructure/Data/TvContext.cs index 912ff1810..f1da5ed20 100644 --- a/ErsatzTV.Infrastructure/Data/TvContext.cs +++ b/ErsatzTV.Infrastructure/Data/TvContext.cs @@ -10,10 +10,17 @@ namespace ErsatzTV.Infrastructure.Data; public class TvContext : DbContext { private readonly ILoggerFactory _loggerFactory; + private readonly SlowQueryInterceptor _slowQueryInterceptor; - public TvContext(DbContextOptions options, ILoggerFactory loggerFactory) - : base(options) => + public TvContext( + DbContextOptions options, + ILoggerFactory loggerFactory, + SlowQueryInterceptor slowQueryInterceptor) + : base(options) + { _loggerFactory = loggerFactory; + _slowQueryInterceptor = slowQueryInterceptor; + } public static string LastInsertedRowId { get; set; } = "last_insert_rowid()"; public static string CaseInsensitiveCollation { get; set; } = "NOCASE"; @@ -119,8 +126,11 @@ public class TvContext : DbContext public DbSet Subtitles { get; set; } public DbSet GraphicsElements { get; set; } - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { optionsBuilder.UseLoggerFactory(_loggerFactory); + optionsBuilder.AddInterceptors(_slowQueryInterceptor); + } protected override void OnModelCreating(ModelBuilder modelBuilder) { diff --git a/ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs b/ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs index f82c3bb2f..af43b2a4d 100644 --- a/ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs +++ b/ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs @@ -42,6 +42,26 @@ public interface IJellyfinApi [Query] int limit = 0); + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] + Task GetShowLibraryItemsWithoutPeople( + [Header("X-Emby-Token")] + string apiKey, + [Query] + string parentId, + [Query] + string fields = + "Path,Genres,Tags,DateCreated,Etag,Overview,Taglines,Studios,OfficialRating,ProviderIds", + [Query] + string includeItemTypes = "Series", + [Query] + bool recursive = true, + [Query] + int startIndex = 0, + [Query] + int limit = 0, + [Query] + string ids = null); + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] Task GetShowLibraryItems( [Header("X-Emby-Token")] @@ -94,6 +114,25 @@ public interface IJellyfinApi [Query] int startIndex = 0, [Query] + int limit = 0, + [Query] + string ids = null); + + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] + Task GetEpisodeLibraryItemsWithoutPeople( + [Header("X-Emby-Token")] + string apiKey, + [Query] + string parentId, + [Query] + string fields = "Path,Genres,Tags,DateCreated,Etag,Overview,ProviderIds,Chapters", + [Query] + string includeItemTypes = "Episode", + [Query] + bool recursive = true, + [Query] + int startIndex = 0, + [Query] int limit = 0); [Get("/Items?sortOrder=Ascending&sortBy=SortName")] diff --git a/ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs b/ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs index a71be3484..ebbbcad53 100644 --- a/ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs +++ b/ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs @@ -92,7 +92,7 @@ public class JellyfinApiClient : IJellyfinApiClient limit: pageSize), (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToMovie(lib, item)).Flatten()); - public IAsyncEnumerable> GetShowLibraryItems( + public IAsyncEnumerable> GetShowLibraryItemsWithoutPeople( string address, string apiKey, JellyfinLibrary library) => @@ -101,7 +101,7 @@ public class JellyfinApiClient : IJellyfinApiClient library, library.MediaSourceId, library.ItemId, - (service, itemId, skip, pageSize) => service.GetShowLibraryItems( + (service, itemId, skip, pageSize) => service.GetShowLibraryItemsWithoutPeople( apiKey, itemId, startIndex: skip, @@ -142,6 +142,23 @@ public class JellyfinApiClient : IJellyfinApiClient limit: pageSize), (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToEpisode(lib, item)).Flatten()); + public IAsyncEnumerable> GetEpisodeLibraryItemsWithoutPeople( + string address, + string apiKey, + JellyfinLibrary library, + string seasonId) => + GetPagedLibraryItems( + address, + library, + library.MediaSourceId, + seasonId, + (service, _, skip, pageSize) => service.GetEpisodeLibraryItemsWithoutPeople( + apiKey, + seasonId, + startIndex: skip, + limit: pageSize), + (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToEpisode(lib, item)).Flatten()); + public IAsyncEnumerable> GetCollectionLibraryItems( string address, string apiKey, @@ -284,6 +301,38 @@ public class JellyfinApiClient : IJellyfinApiClient } } + public async Task>> GetSingleEpisode( + string address, + string apiKey, + JellyfinLibrary library, + string seasonId, + string episodeId) + { + try + { + IJellyfinApi service = ServiceForAddress(address); + JellyfinLibraryItemsResponse itemsResponse = await service.GetEpisodeLibraryItems( + apiKey, + parentId: seasonId, + recursive: false, + startIndex: 0, + limit: 1, + ids: episodeId); + + foreach (JellyfinLibraryItemResponse item in itemsResponse.Items) + { + return ProjectToEpisode(library, item); + } + + return BaseError.New($"Unable to locate episode with id {episodeId}"); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error searching Jellyfin episodes by id"); + return BaseError.New(ex.Message); + } + } + private async IAsyncEnumerable> GetPagedLibraryItems( string address, Option maybeLibrary, diff --git a/ErsatzTV/SlowApiHandler.cs b/ErsatzTV.Infrastructure/SlowApiHandler.cs similarity index 93% rename from ErsatzTV/SlowApiHandler.cs rename to ErsatzTV.Infrastructure/SlowApiHandler.cs index 997250b58..7b5a3bc87 100644 --- a/ErsatzTV/SlowApiHandler.cs +++ b/ErsatzTV.Infrastructure/SlowApiHandler.cs @@ -1,6 +1,7 @@ using ErsatzTV.Core; +using Microsoft.Extensions.Logging; -namespace ErsatzTV; +namespace ErsatzTV.Infrastructure; using System.Diagnostics; diff --git a/ErsatzTV/SlowQueryInterceptor.cs b/ErsatzTV.Infrastructure/SlowQueryInterceptor.cs similarity index 63% rename from ErsatzTV/SlowQueryInterceptor.cs rename to ErsatzTV.Infrastructure/SlowQueryInterceptor.cs index 52ce0fb19..54c8b72e3 100644 --- a/ErsatzTV/SlowQueryInterceptor.cs +++ b/ErsatzTV.Infrastructure/SlowQueryInterceptor.cs @@ -1,9 +1,11 @@ using System.Data.Common; +using ErsatzTV.Core; using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.Logging; -namespace ErsatzTV; +namespace ErsatzTV.Infrastructure; -public class SlowQueryInterceptor(int threshold) : DbCommandInterceptor +public class SlowQueryInterceptor(ILogger logger) : DbCommandInterceptor { public override ValueTask ReaderExecutedAsync( DbCommand command, @@ -11,9 +13,9 @@ public class SlowQueryInterceptor(int threshold) : DbCommandInterceptor DbDataReader result, CancellationToken cancellationToken = default) { - if (eventData.Duration.TotalMilliseconds > threshold) + if (SystemEnvironment.SlowDbMs > 0 && eventData.Duration.TotalMilliseconds > SystemEnvironment.SlowDbMs) { - Serilog.Log.Logger.Debug( + logger.LogDebug( "[SLOW QUERY] ({Milliseconds}ms): {Command}", eventData.Duration.TotalMilliseconds, command.CommandText); diff --git a/ErsatzTV.Scanner/Core/Emby/EmbyTelevisionLibraryScanner.cs b/ErsatzTV.Scanner/Core/Emby/EmbyTelevisionLibraryScanner.cs index 2f3a2f0eb..aa625fbe2 100644 --- a/ErsatzTV.Scanner/Core/Emby/EmbyTelevisionLibraryScanner.cs +++ b/ErsatzTV.Scanner/Core/Emby/EmbyTelevisionLibraryScanner.cs @@ -158,7 +158,8 @@ public class EmbyTelevisionLibraryScanner : MediaServerTelevisionLibraryScanner< EmbyLibrary library, EmbyConnectionParameters connectionParameters, EmbyShow show, - EmbySeason season) => + EmbySeason season, + bool isNewSeason) => _embyApiClient.GetEpisodeLibraryItems( connectionParameters.Address, connectionParameters.ApiKey, diff --git a/ErsatzTV.Scanner/Core/Jellyfin/JellyfinTelevisionLibraryScanner.cs b/ErsatzTV.Scanner/Core/Jellyfin/JellyfinTelevisionLibraryScanner.cs index 82957f505..4c6f603af 100644 --- a/ErsatzTV.Scanner/Core/Jellyfin/JellyfinTelevisionLibraryScanner.cs +++ b/ErsatzTV.Scanner/Core/Jellyfin/JellyfinTelevisionLibraryScanner.cs @@ -23,13 +23,16 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan private readonly ILogger _logger; private readonly IMediaSourceRepository _mediaSourceRepository; private readonly IJellyfinPathReplacementService _pathReplacementService; - private readonly IJellyfinTelevisionRepository _televisionRepository; + private readonly IMetadataRepository _metadataRepository; + private readonly IJellyfinTelevisionRepository _jellyfinTelevisionRepository; + private readonly ITelevisionRepository _televisionRepository; public JellyfinTelevisionLibraryScanner( IScannerProxy scannerProxy, IJellyfinApiClient jellyfinApiClient, IMediaSourceRepository mediaSourceRepository, - IJellyfinTelevisionRepository televisionRepository, + IJellyfinTelevisionRepository jellyfinTelevisionRepository, + ITelevisionRepository televisionRepository, IJellyfinPathReplacementService pathReplacementService, IFileSystem fileSystem, ILocalChaptersProvider localChaptersProvider, @@ -44,8 +47,10 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan { _jellyfinApiClient = jellyfinApiClient; _mediaSourceRepository = mediaSourceRepository; + _jellyfinTelevisionRepository = jellyfinTelevisionRepository; _televisionRepository = televisionRepository; _pathReplacementService = pathReplacementService; + _metadataRepository = metadataRepository; _logger = logger; } @@ -70,7 +75,7 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan } return await ScanLibrary( - _televisionRepository, + _jellyfinTelevisionRepository, new JellyfinConnectionParameters(address, apiKey, library.MediaSourceId), library, GetLocalPath, @@ -116,7 +121,7 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan show.ItemId); return await ScanSingleShowInternal( - _televisionRepository, + _jellyfinTelevisionRepository, new JellyfinConnectionParameters(address, apiKey, library.MediaSourceId), library, show, @@ -135,7 +140,10 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan protected override IAsyncEnumerable> GetShowLibraryItems( JellyfinConnectionParameters connectionParameters, JellyfinLibrary library) => - _jellyfinApiClient.GetShowLibraryItems(connectionParameters.Address, connectionParameters.ApiKey, library); + _jellyfinApiClient.GetShowLibraryItemsWithoutPeople( + connectionParameters.Address, + connectionParameters.ApiKey, + library); protected override string MediaServerItemId(JellyfinShow show) => show.ItemId; protected override string MediaServerItemId(JellyfinSeason season) => season.ItemId; @@ -159,20 +167,56 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan JellyfinLibrary library, JellyfinConnectionParameters connectionParameters, JellyfinShow show, - JellyfinSeason season) => - _jellyfinApiClient.GetEpisodeLibraryItems( + JellyfinSeason season, + bool isNewSeason) + { + if (isNewSeason) + { + return _jellyfinApiClient.GetEpisodeLibraryItems( + connectionParameters.Address, + connectionParameters.ApiKey, + library, + season.ItemId); + } + + return _jellyfinApiClient.GetEpisodeLibraryItemsWithoutPeople( connectionParameters.Address, connectionParameters.ApiKey, library, season.ItemId); + } - protected override Task> GetFullMetadata( + protected override async Task> GetFullMetadata( JellyfinConnectionParameters connectionParameters, JellyfinLibrary library, MediaItemScanResult result, JellyfinShow incoming, - bool deepScan) => - Task.FromResult(Option.None); + bool deepScan) + { + if (result.IsAdded || result.Item.Etag != incoming.Etag || deepScan) + { + Either> maybeShowResult = await _jellyfinApiClient.GetSingleShow( + connectionParameters.Address, + connectionParameters.ApiKey, + library, + incoming.ItemId); + + foreach (BaseError error in maybeShowResult.LeftToSeq()) + { + _logger.LogWarning("Failed to get show metadata from Jellyfin: {Error}", error.ToString()); + } + + foreach (Option maybeShow in maybeShowResult.RightToSeq()) + { + foreach (JellyfinShow show in maybeShow) + { + return show.ShowMetadata.HeadOrNone(); + } + } + } + + return None; + } protected override Task> GetFullMetadata( JellyfinConnectionParameters connectionParameters, @@ -182,13 +226,39 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan bool deepScan) => Task.FromResult(Option.None); - protected override Task> GetFullMetadata( + protected override async Task> GetFullMetadata( JellyfinConnectionParameters connectionParameters, JellyfinLibrary library, MediaItemScanResult result, JellyfinEpisode incoming, - bool deepScan) => - Task.FromResult(Option.None); + bool deepScan) + { + if (result.Item.Season is JellyfinSeason jellyfinSeason && + (result.IsAdded || result.Item.Etag != incoming.Etag || deepScan)) + { + Either> maybeEpisodeResult = await _jellyfinApiClient.GetSingleEpisode( + connectionParameters.Address, + connectionParameters.ApiKey, + library, + jellyfinSeason.ItemId, + incoming.ItemId); + + foreach (BaseError error in maybeEpisodeResult.LeftToSeq()) + { + _logger.LogWarning("Failed to get episode metadata from Jellyfin: {Error}", error.ToString()); + } + + foreach (Option maybeEpisode in maybeEpisodeResult.RightToSeq()) + { + foreach (JellyfinEpisode episode in maybeEpisode) + { + return episode.EpisodeMetadata.HeadOrNone(); + } + } + } + + return None; + } protected override Task>> GetFullMetadataAndStatistics( JellyfinConnectionParameters connectionParameters, @@ -225,21 +295,131 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan return maybeVersion.ToOption(); } - protected override Task>> UpdateMetadata( + protected override async Task>> UpdateMetadata( MediaItemScanResult result, - ShowMetadata fullMetadata) => - Task.FromResult>>(result); + ShowMetadata fullMetadata) + { + JellyfinShow existing = result.Item; + ShowMetadata existingMetadata = existing.ShowMetadata.Head(); + + foreach (Actor actor in existingMetadata.Actors + .Filter(a => + fullMetadata.Actors.All(a2 => a2.Name != a.Name || a.Artwork == null && a2.Artwork != null)) + .ToList()) + { + existingMetadata.Actors.Remove(actor); + if (await _metadataRepository.RemoveActor(actor)) + { + result.IsUpdated = true; + } + } + + foreach (Actor actor in fullMetadata.Actors + .Filter(a => existingMetadata.Actors.All(a2 => a2.Name != a.Name)) + .ToList()) + { + existingMetadata.Actors.Add(actor); + if (await _televisionRepository.AddActor(existingMetadata, actor)) + { + result.IsUpdated = true; + } + } + + if (result.IsUpdated) + { + await _metadataRepository.MarkAsUpdated(existingMetadata, fullMetadata.DateUpdated); + } + + return result; + } protected override Task>> UpdateMetadata( MediaItemScanResult result, SeasonMetadata fullMetadata) => Task.FromResult>>(result); - protected override Task>> UpdateMetadata( + protected override async Task>> UpdateMetadata( MediaItemScanResult result, EpisodeMetadata fullMetadata, - CancellationToken cancellationToken) => - Task.FromResult>>(result); + CancellationToken cancellationToken) + { + JellyfinEpisode existing = result.Item; + EpisodeMetadata existingMetadata = existing.EpisodeMetadata.Head(); + + foreach (Actor actor in existingMetadata.Actors + .Filter(a => + fullMetadata.Actors.All(a2 => a2.Name != a.Name || a.Artwork == null && a2.Artwork != null)) + .ToList()) + { + existingMetadata.Actors.Remove(actor); + if (await _metadataRepository.RemoveActor(actor)) + { + result.IsUpdated = true; + } + } + + foreach (Actor actor in fullMetadata.Actors + .Filter(a => existingMetadata.Actors.All(a2 => a2.Name != a.Name)) + .ToList()) + { + existingMetadata.Actors.Add(actor); + if (await _televisionRepository.AddActor(existingMetadata, actor)) + { + result.IsUpdated = true; + } + } + + foreach (Director director in existingMetadata.Directors + .Filter(g => fullMetadata.Directors.All(g2 => g2.Name != g.Name)) + .ToList()) + { + existingMetadata.Directors.Remove(director); + if (await _metadataRepository.RemoveDirector(director)) + { + result.IsUpdated = true; + } + } + + foreach (Director director in fullMetadata.Directors + .Filter(g => existingMetadata.Directors.All(g2 => g2.Name != g.Name)) + .ToList()) + { + existingMetadata.Directors.Add(director); + if (await _televisionRepository.AddDirector(existingMetadata, director)) + { + result.IsUpdated = true; + } + } + + foreach (Writer writer in existingMetadata.Writers + .Filter(g => fullMetadata.Writers.All(g2 => g2.Name != g.Name)) + .ToList()) + { + existingMetadata.Writers.Remove(writer); + if (await _metadataRepository.RemoveWriter(writer)) + { + result.IsUpdated = true; + } + } + + foreach (Writer writer in fullMetadata.Writers + .Filter(g => existingMetadata.Writers.All(g2 => g2.Name != g.Name)) + .ToList()) + { + existingMetadata.Writers.Add(writer); + if (await _televisionRepository.AddWriter(existingMetadata, writer)) + { + result.IsUpdated = true; + } + } + + if (result.IsUpdated) + { + await _metadataRepository.MarkAsUpdated(existingMetadata, fullMetadata.DateUpdated); + } + + return result; + } private async Task> ScanSingleShowInternal( IJellyfinTelevisionRepository televisionRepository, diff --git a/ErsatzTV.Scanner/Core/Metadata/MediaServerTelevisionLibraryScanner.cs b/ErsatzTV.Scanner/Core/Metadata/MediaServerTelevisionLibraryScanner.cs index 90431117f..1a1370082 100644 --- a/ErsatzTV.Scanner/Core/Metadata/MediaServerTelevisionLibraryScanner.cs +++ b/ErsatzTV.Scanner/Core/Metadata/MediaServerTelevisionLibraryScanner.cs @@ -220,7 +220,8 @@ public abstract class MediaServerTelevisionLibraryScanner> GetFullMetadata( TConnectionParameters connectionParameters, @@ -321,7 +322,7 @@ public abstract class MediaServerTelevisionLibraryScanner + PlexSeason season, + bool isNewSeason) => _plexServerApiClient.GetSeasonEpisodes( library, season, diff --git a/ErsatzTV.Scanner/Program.cs b/ErsatzTV.Scanner/Program.cs index 478b83173..d5e09b62e 100644 --- a/ErsatzTV.Scanner/Program.cs +++ b/ErsatzTV.Scanner/Program.cs @@ -19,6 +19,7 @@ using ErsatzTV.Core.Plex; using ErsatzTV.Core.Search; using ErsatzTV.FFmpeg.Capabilities; using ErsatzTV.FFmpeg.Runtime; +using ErsatzTV.Infrastructure; using ErsatzTV.Infrastructure.Data; using ErsatzTV.Infrastructure.Data.Repositories; using ErsatzTV.Infrastructure.Emby; @@ -171,6 +172,8 @@ public class Program services.AddHttpClient(); + services.AddHttpClient("RefitCustomClient").AddHttpMessageHandler(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); @@ -256,6 +259,9 @@ public class Program services.AddSingleton(); + services.AddTransient(); + services.AddTransient(); + services.AddMediatR(config => config.RegisterServicesFromAssemblyContaining()); services.AddMemoryCache(); diff --git a/ErsatzTV/Startup.cs b/ErsatzTV/Startup.cs index b14b7f506..1127336a3 100644 --- a/ErsatzTV/Startup.cs +++ b/ErsatzTV/Startup.cs @@ -51,6 +51,7 @@ using ErsatzTV.FFmpeg.Pipeline; using ErsatzTV.FFmpeg.Runtime; using ErsatzTV.Filters; using ErsatzTV.Formatters; +using ErsatzTV.Infrastructure; using ErsatzTV.Infrastructure.Data; using ErsatzTV.Infrastructure.Data.Repositories; using ErsatzTV.Infrastructure.Database; @@ -408,12 +409,6 @@ public class Startup var sqliteConnectionString = $"Data Source={FileSystemLayout.DatabasePath};foreign keys=true;"; string mySqlConnectionString = Configuration.GetValue("MySql:ConnectionString"); - SlowQueryInterceptor interceptor = null; - if (SystemEnvironment.SlowDbMs.HasValue) - { - interceptor = new SlowQueryInterceptor(SystemEnvironment.SlowDbMs.Value); - } - services.AddDbContext( options => { @@ -444,11 +439,6 @@ public class Startup } ); } - - if (interceptor != null) - { - options.AddInterceptors(interceptor); - } }, ServiceLifetime.Scoped, ServiceLifetime.Singleton); @@ -478,11 +468,6 @@ public class Startup } ); } - - if (interceptor != null) - { - options.AddInterceptors(interceptor); - } }); if (databaseProvider == Provider.Sqlite.Name) @@ -870,6 +855,7 @@ public class Startup // services.AddTransient(typeof(IRequestHandler<,>), typeof(GetRecentLogEntriesHandler<>)); services.AddTransient(); + services.AddTransient(); // run-once/blocking startup services services.AddHostedService();