Browse Source

more jellyfin performance improvements (#2747)

* fix slow db and api logging so it also works in scanner project

* don't request people from jellyfin by default
main
Jason Dove 16 hours ago committed by GitHub
parent
commit
474e647d6d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 2
      CHANGELOG.md
  2. 16
      ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs
  3. 5
      ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs
  4. 21
      ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs
  5. 16
      ErsatzTV.Infrastructure/Data/TvContext.cs
  6. 39
      ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs
  7. 53
      ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs
  8. 3
      ErsatzTV.Infrastructure/SlowApiHandler.cs
  9. 10
      ErsatzTV.Infrastructure/SlowQueryInterceptor.cs
  10. 3
      ErsatzTV.Scanner/Core/Emby/EmbyTelevisionLibraryScanner.cs
  11. 218
      ErsatzTV.Scanner/Core/Jellyfin/JellyfinTelevisionLibraryScanner.cs
  12. 5
      ErsatzTV.Scanner/Core/Metadata/MediaServerTelevisionLibraryScanner.cs
  13. 3
      ErsatzTV.Scanner/Core/Plex/PlexTelevisionLibraryScanner.cs
  14. 6
      ErsatzTV.Scanner/Program.cs
  15. 18
      ErsatzTV/Startup.cs

2
CHANGELOG.md

@ -82,6 +82,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Hardware acceleration will now be used - Hardware acceleration will now be used
- Items can "work ahead" (transcode faster than realtime) when less than 3 minutes in duration - Items can "work ahead" (transcode faster than realtime) when less than 3 minutes in duration
- Optimize Jellyfin database fields and indexes - Optimize Jellyfin database fields and indexes
- Optimize Jellyfin show library scans by only requesting `People` (actors, directors, writers) when etags don't match
- This should significantly speed up periodic library scans, particularly against Jellyfin 10.11.x
## [25.9.0] - 2025-11-29 ## [25.9.0] - 2025-11-29
### Added ### Added

16
ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs

@ -13,7 +13,7 @@ public interface IJellyfinApiClient
string apiKey, string apiKey,
JellyfinLibrary library); JellyfinLibrary library);
IAsyncEnumerable<Tuple<JellyfinShow, int>> GetShowLibraryItems( IAsyncEnumerable<Tuple<JellyfinShow, int>> GetShowLibraryItemsWithoutPeople(
string address, string address,
string apiKey, string apiKey,
JellyfinLibrary library); JellyfinLibrary library);
@ -30,6 +30,12 @@ public interface IJellyfinApiClient
JellyfinLibrary library, JellyfinLibrary library,
string seasonId); string seasonId);
IAsyncEnumerable<Tuple<JellyfinEpisode, int>> GetEpisodeLibraryItemsWithoutPeople(
string address,
string apiKey,
JellyfinLibrary library,
string seasonId);
IAsyncEnumerable<Tuple<JellyfinCollection, int>> GetCollectionLibraryItems( IAsyncEnumerable<Tuple<JellyfinCollection, int>> GetCollectionLibraryItems(
string address, string address,
string apiKey, string apiKey,
@ -58,4 +64,12 @@ public interface IJellyfinApiClient
string apiKey, string apiKey,
JellyfinLibrary library, JellyfinLibrary library,
string showTitle); string showTitle);
Task<Either<BaseError, Option<JellyfinEpisode>>> GetSingleEpisode(
string address,
string apiKey,
JellyfinLibrary library,
string seasonId,
string episodeId);
} }

5
ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs

@ -88,6 +88,7 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository
{ {
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
Option<JellyfinShow> maybeExisting = await dbContext.JellyfinShows Option<JellyfinShow> maybeExisting = await dbContext.JellyfinShows
.TagWithCallSite()
.Include(m => m.LibraryPath) .Include(m => m.LibraryPath)
.ThenInclude(lp => lp.Library) .ThenInclude(lp => lp.Library)
.Include(m => m.ShowMetadata) .Include(m => m.ShowMetadata)
@ -128,6 +129,7 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository
{ {
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
Option<JellyfinSeason> maybeExisting = await dbContext.JellyfinSeasons Option<JellyfinSeason> maybeExisting = await dbContext.JellyfinSeasons
.TagWithCallSite()
.Include(m => m.LibraryPath) .Include(m => m.LibraryPath)
.Include(m => m.SeasonMetadata) .Include(m => m.SeasonMetadata)
.ThenInclude(mm => mm.Artwork) .ThenInclude(mm => mm.Artwork)
@ -158,6 +160,7 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository
{ {
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
Option<JellyfinEpisode> maybeExisting = await dbContext.JellyfinEpisodes Option<JellyfinEpisode> maybeExisting = await dbContext.JellyfinEpisodes
.TagWithCallSite()
.Include(m => m.LibraryPath) .Include(m => m.LibraryPath)
.ThenInclude(lp => lp.Library) .ThenInclude(lp => lp.Library)
.Include(m => m.MediaVersions) .Include(m => m.MediaVersions)
@ -511,6 +514,8 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
Option<JellyfinShow> maybeShow = await dbContext.JellyfinShows Option<JellyfinShow> maybeShow = await dbContext.JellyfinShows
.AsNoTracking()
.TagWithCallSite()
.Where(s => s.Id == showId) .Where(s => s.Id == showId)
.Where(s => s.LibraryPath.LibraryId == libraryId) .Where(s => s.LibraryPath.LibraryId == libraryId)
.Include(s => s.ShowMetadata) .Include(s => s.ShowMetadata)

21
ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs

@ -15,6 +15,7 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
var baseItem = await dbContext.MediaItems var baseItem = await dbContext.MediaItems
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken);
if (baseItem is null) if (baseItem is null)
@ -27,60 +28,70 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
case Movie: case Movie:
return await dbContext.Movies return await dbContext.Movies
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken);
case Episode: case Episode:
return await dbContext.Episodes return await dbContext.Episodes
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken);
case Season: case Season:
return await dbContext.Seasons return await dbContext.Seasons
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken);
case Show: case Show:
return await dbContext.Shows return await dbContext.Shows
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken);
case MusicVideo: case MusicVideo:
return await dbContext.MusicVideos return await dbContext.MusicVideos
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken);
case Artist: case Artist:
return await dbContext.Artists return await dbContext.Artists
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken);
case OtherVideo: case OtherVideo:
return await dbContext.OtherVideos return await dbContext.OtherVideos
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken);
case Song: case Song:
return await dbContext.Songs return await dbContext.Songs
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken);
case Image: case Image:
return await dbContext.Images return await dbContext.Images
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken);
case RemoteStream: case RemoteStream:
return await dbContext.RemoteStreams return await dbContext.RemoteStreams
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken); .SingleOrDefaultAsync(mi => mi.Id == id, cancellationToken);
@ -225,6 +236,7 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
ConfiguredCancelableAsyncEnumerable<Movie> movies = dbContext.Movies ConfiguredCancelableAsyncEnumerable<Movie> movies = dbContext.Movies
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.AsAsyncEnumerable() .AsAsyncEnumerable()
@ -242,6 +254,7 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
ConfiguredCancelableAsyncEnumerable<Show> shows = dbContext.Shows ConfiguredCancelableAsyncEnumerable<Show> shows = dbContext.Shows
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.AsAsyncEnumerable() .AsAsyncEnumerable()
@ -259,6 +272,7 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
ConfiguredCancelableAsyncEnumerable<Season> seasons = dbContext.Seasons ConfiguredCancelableAsyncEnumerable<Season> seasons = dbContext.Seasons
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.AsAsyncEnumerable() .AsAsyncEnumerable()
@ -276,6 +290,7 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
ConfiguredCancelableAsyncEnumerable<Episode> episodes = dbContext.Episodes ConfiguredCancelableAsyncEnumerable<Episode> episodes = dbContext.Episodes
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.AsAsyncEnumerable() .AsAsyncEnumerable()
@ -294,6 +309,7 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
ConfiguredCancelableAsyncEnumerable<MusicVideo> musicVideos = dbContext.MusicVideos ConfiguredCancelableAsyncEnumerable<MusicVideo> musicVideos = dbContext.MusicVideos
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.AsAsyncEnumerable() .AsAsyncEnumerable()
@ -311,6 +327,7 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
ConfiguredCancelableAsyncEnumerable<Artist> artists = dbContext.Artists ConfiguredCancelableAsyncEnumerable<Artist> artists = dbContext.Artists
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.AsAsyncEnumerable() .AsAsyncEnumerable()
@ -329,6 +346,7 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
ConfiguredCancelableAsyncEnumerable<OtherVideo> otherVideos = dbContext.OtherVideos ConfiguredCancelableAsyncEnumerable<OtherVideo> otherVideos = dbContext.OtherVideos
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.AsAsyncEnumerable() .AsAsyncEnumerable()
@ -346,6 +364,7 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
ConfiguredCancelableAsyncEnumerable<Song> songs = dbContext.Songs ConfiguredCancelableAsyncEnumerable<Song> songs = dbContext.Songs
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.AsAsyncEnumerable() .AsAsyncEnumerable()
@ -363,6 +382,7 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
ConfiguredCancelableAsyncEnumerable<Image> images = dbContext.Images ConfiguredCancelableAsyncEnumerable<Image> images = dbContext.Images
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.AsAsyncEnumerable() .AsAsyncEnumerable()
@ -381,6 +401,7 @@ public class SearchRepository(IDbContextFactory<TvContext> dbContextFactory) : I
ConfiguredCancelableAsyncEnumerable<RemoteStream> remoteStreams = dbContext.RemoteStreams ConfiguredCancelableAsyncEnumerable<RemoteStream> remoteStreams = dbContext.RemoteStreams
.AsNoTracking() .AsNoTracking()
.TagWithCallSite()
.IncludeForSearch() .IncludeForSearch()
.AsSplitQuery() .AsSplitQuery()
.AsAsyncEnumerable() .AsAsyncEnumerable()

16
ErsatzTV.Infrastructure/Data/TvContext.cs

@ -10,10 +10,17 @@ namespace ErsatzTV.Infrastructure.Data;
public class TvContext : DbContext public class TvContext : DbContext
{ {
private readonly ILoggerFactory _loggerFactory; private readonly ILoggerFactory _loggerFactory;
private readonly SlowQueryInterceptor _slowQueryInterceptor;
public TvContext(DbContextOptions<TvContext> options, ILoggerFactory loggerFactory) public TvContext(
: base(options) => DbContextOptions<TvContext> options,
ILoggerFactory loggerFactory,
SlowQueryInterceptor slowQueryInterceptor)
: base(options)
{
_loggerFactory = loggerFactory; _loggerFactory = loggerFactory;
_slowQueryInterceptor = slowQueryInterceptor;
}
public static string LastInsertedRowId { get; set; } = "last_insert_rowid()"; public static string LastInsertedRowId { get; set; } = "last_insert_rowid()";
public static string CaseInsensitiveCollation { get; set; } = "NOCASE"; public static string CaseInsensitiveCollation { get; set; } = "NOCASE";
@ -119,8 +126,11 @@ public class TvContext : DbContext
public DbSet<Subtitle> Subtitles { get; set; } public DbSet<Subtitle> Subtitles { get; set; }
public DbSet<GraphicsElement> GraphicsElements { get; set; } public DbSet<GraphicsElement> GraphicsElements { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
optionsBuilder.UseLoggerFactory(_loggerFactory); optionsBuilder.UseLoggerFactory(_loggerFactory);
optionsBuilder.AddInterceptors(_slowQueryInterceptor);
}
protected override void OnModelCreating(ModelBuilder modelBuilder) protected override void OnModelCreating(ModelBuilder modelBuilder)
{ {

39
ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs

@ -42,6 +42,26 @@ public interface IJellyfinApi
[Query] [Query]
int limit = 0); int limit = 0);
[Get("/Items?sortOrder=Ascending&sortBy=SortName")]
Task<JellyfinLibraryItemsResponse> GetShowLibraryItemsWithoutPeople(
[Header("X-Emby-Token")]
string apiKey,
[Query]
string parentId,
[Query]
string fields =
"Path,Genres,Tags,DateCreated,Etag,Overview,Taglines,Studios,OfficialRating,ProviderIds",
[Query]
string includeItemTypes = "Series",
[Query]
bool recursive = true,
[Query]
int startIndex = 0,
[Query]
int limit = 0,
[Query]
string ids = null);
[Get("/Items?sortOrder=Ascending&sortBy=SortName")] [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
Task<JellyfinLibraryItemsResponse> GetShowLibraryItems( Task<JellyfinLibraryItemsResponse> GetShowLibraryItems(
[Header("X-Emby-Token")] [Header("X-Emby-Token")]
@ -94,6 +114,25 @@ public interface IJellyfinApi
[Query] [Query]
int startIndex = 0, int startIndex = 0,
[Query] [Query]
int limit = 0,
[Query]
string ids = null);
[Get("/Items?sortOrder=Ascending&sortBy=SortName")]
Task<JellyfinLibraryItemsResponse> GetEpisodeLibraryItemsWithoutPeople(
[Header("X-Emby-Token")]
string apiKey,
[Query]
string parentId,
[Query]
string fields = "Path,Genres,Tags,DateCreated,Etag,Overview,ProviderIds,Chapters",
[Query]
string includeItemTypes = "Episode",
[Query]
bool recursive = true,
[Query]
int startIndex = 0,
[Query]
int limit = 0); int limit = 0);
[Get("/Items?sortOrder=Ascending&sortBy=SortName")] [Get("/Items?sortOrder=Ascending&sortBy=SortName")]

53
ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs

@ -92,7 +92,7 @@ public class JellyfinApiClient : IJellyfinApiClient
limit: pageSize), limit: pageSize),
(maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToMovie(lib, item)).Flatten()); (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToMovie(lib, item)).Flatten());
public IAsyncEnumerable<Tuple<JellyfinShow, int>> GetShowLibraryItems( public IAsyncEnumerable<Tuple<JellyfinShow, int>> GetShowLibraryItemsWithoutPeople(
string address, string address,
string apiKey, string apiKey,
JellyfinLibrary library) => JellyfinLibrary library) =>
@ -101,7 +101,7 @@ public class JellyfinApiClient : IJellyfinApiClient
library, library,
library.MediaSourceId, library.MediaSourceId,
library.ItemId, library.ItemId,
(service, itemId, skip, pageSize) => service.GetShowLibraryItems( (service, itemId, skip, pageSize) => service.GetShowLibraryItemsWithoutPeople(
apiKey, apiKey,
itemId, itemId,
startIndex: skip, startIndex: skip,
@ -142,6 +142,23 @@ public class JellyfinApiClient : IJellyfinApiClient
limit: pageSize), limit: pageSize),
(maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToEpisode(lib, item)).Flatten()); (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToEpisode(lib, item)).Flatten());
public IAsyncEnumerable<Tuple<JellyfinEpisode, int>> GetEpisodeLibraryItemsWithoutPeople(
string address,
string apiKey,
JellyfinLibrary library,
string seasonId) =>
GetPagedLibraryItems(
address,
library,
library.MediaSourceId,
seasonId,
(service, _, skip, pageSize) => service.GetEpisodeLibraryItemsWithoutPeople(
apiKey,
seasonId,
startIndex: skip,
limit: pageSize),
(maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToEpisode(lib, item)).Flatten());
public IAsyncEnumerable<Tuple<JellyfinCollection, int>> GetCollectionLibraryItems( public IAsyncEnumerable<Tuple<JellyfinCollection, int>> GetCollectionLibraryItems(
string address, string address,
string apiKey, string apiKey,
@ -284,6 +301,38 @@ public class JellyfinApiClient : IJellyfinApiClient
} }
} }
public async Task<Either<BaseError, Option<JellyfinEpisode>>> GetSingleEpisode(
string address,
string apiKey,
JellyfinLibrary library,
string seasonId,
string episodeId)
{
try
{
IJellyfinApi service = ServiceForAddress(address);
JellyfinLibraryItemsResponse itemsResponse = await service.GetEpisodeLibraryItems(
apiKey,
parentId: seasonId,
recursive: false,
startIndex: 0,
limit: 1,
ids: episodeId);
foreach (JellyfinLibraryItemResponse item in itemsResponse.Items)
{
return ProjectToEpisode(library, item);
}
return BaseError.New($"Unable to locate episode with id {episodeId}");
}
catch (Exception ex)
{
_logger.LogError(ex, "Error searching Jellyfin episodes by id");
return BaseError.New(ex.Message);
}
}
private async IAsyncEnumerable<Tuple<TItem, int>> GetPagedLibraryItems<TItem>( private async IAsyncEnumerable<Tuple<TItem, int>> GetPagedLibraryItems<TItem>(
string address, string address,
Option<JellyfinLibrary> maybeLibrary, Option<JellyfinLibrary> maybeLibrary,

3
ErsatzTV/SlowApiHandler.cs → ErsatzTV.Infrastructure/SlowApiHandler.cs

@ -1,6 +1,7 @@
using ErsatzTV.Core; using ErsatzTV.Core;
using Microsoft.Extensions.Logging;
namespace ErsatzTV; namespace ErsatzTV.Infrastructure;
using System.Diagnostics; using System.Diagnostics;

10
ErsatzTV/SlowQueryInterceptor.cs → ErsatzTV.Infrastructure/SlowQueryInterceptor.cs

@ -1,9 +1,11 @@
using System.Data.Common; using System.Data.Common;
using ErsatzTV.Core;
using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.EntityFrameworkCore.Diagnostics;
using Microsoft.Extensions.Logging;
namespace ErsatzTV; namespace ErsatzTV.Infrastructure;
public class SlowQueryInterceptor(int threshold) : DbCommandInterceptor public class SlowQueryInterceptor(ILogger<SlowQueryInterceptor> logger) : DbCommandInterceptor
{ {
public override ValueTask<DbDataReader> ReaderExecutedAsync( public override ValueTask<DbDataReader> ReaderExecutedAsync(
DbCommand command, DbCommand command,
@ -11,9 +13,9 @@ public class SlowQueryInterceptor(int threshold) : DbCommandInterceptor
DbDataReader result, DbDataReader result,
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
{ {
if (eventData.Duration.TotalMilliseconds > threshold) if (SystemEnvironment.SlowDbMs > 0 && eventData.Duration.TotalMilliseconds > SystemEnvironment.SlowDbMs)
{ {
Serilog.Log.Logger.Debug( logger.LogDebug(
"[SLOW QUERY] ({Milliseconds}ms): {Command}", "[SLOW QUERY] ({Milliseconds}ms): {Command}",
eventData.Duration.TotalMilliseconds, eventData.Duration.TotalMilliseconds,
command.CommandText); command.CommandText);

3
ErsatzTV.Scanner/Core/Emby/EmbyTelevisionLibraryScanner.cs

@ -158,7 +158,8 @@ public class EmbyTelevisionLibraryScanner : MediaServerTelevisionLibraryScanner<
EmbyLibrary library, EmbyLibrary library,
EmbyConnectionParameters connectionParameters, EmbyConnectionParameters connectionParameters,
EmbyShow show, EmbyShow show,
EmbySeason season) => EmbySeason season,
bool isNewSeason) =>
_embyApiClient.GetEpisodeLibraryItems( _embyApiClient.GetEpisodeLibraryItems(
connectionParameters.Address, connectionParameters.Address,
connectionParameters.ApiKey, connectionParameters.ApiKey,

218
ErsatzTV.Scanner/Core/Jellyfin/JellyfinTelevisionLibraryScanner.cs

@ -23,13 +23,16 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan
private readonly ILogger<JellyfinTelevisionLibraryScanner> _logger; private readonly ILogger<JellyfinTelevisionLibraryScanner> _logger;
private readonly IMediaSourceRepository _mediaSourceRepository; private readonly IMediaSourceRepository _mediaSourceRepository;
private readonly IJellyfinPathReplacementService _pathReplacementService; private readonly IJellyfinPathReplacementService _pathReplacementService;
private readonly IJellyfinTelevisionRepository _televisionRepository; private readonly IMetadataRepository _metadataRepository;
private readonly IJellyfinTelevisionRepository _jellyfinTelevisionRepository;
private readonly ITelevisionRepository _televisionRepository;
public JellyfinTelevisionLibraryScanner( public JellyfinTelevisionLibraryScanner(
IScannerProxy scannerProxy, IScannerProxy scannerProxy,
IJellyfinApiClient jellyfinApiClient, IJellyfinApiClient jellyfinApiClient,
IMediaSourceRepository mediaSourceRepository, IMediaSourceRepository mediaSourceRepository,
IJellyfinTelevisionRepository televisionRepository, IJellyfinTelevisionRepository jellyfinTelevisionRepository,
ITelevisionRepository televisionRepository,
IJellyfinPathReplacementService pathReplacementService, IJellyfinPathReplacementService pathReplacementService,
IFileSystem fileSystem, IFileSystem fileSystem,
ILocalChaptersProvider localChaptersProvider, ILocalChaptersProvider localChaptersProvider,
@ -44,8 +47,10 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan
{ {
_jellyfinApiClient = jellyfinApiClient; _jellyfinApiClient = jellyfinApiClient;
_mediaSourceRepository = mediaSourceRepository; _mediaSourceRepository = mediaSourceRepository;
_jellyfinTelevisionRepository = jellyfinTelevisionRepository;
_televisionRepository = televisionRepository; _televisionRepository = televisionRepository;
_pathReplacementService = pathReplacementService; _pathReplacementService = pathReplacementService;
_metadataRepository = metadataRepository;
_logger = logger; _logger = logger;
} }
@ -70,7 +75,7 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan
} }
return await ScanLibrary( return await ScanLibrary(
_televisionRepository, _jellyfinTelevisionRepository,
new JellyfinConnectionParameters(address, apiKey, library.MediaSourceId), new JellyfinConnectionParameters(address, apiKey, library.MediaSourceId),
library, library,
GetLocalPath, GetLocalPath,
@ -116,7 +121,7 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan
show.ItemId); show.ItemId);
return await ScanSingleShowInternal( return await ScanSingleShowInternal(
_televisionRepository, _jellyfinTelevisionRepository,
new JellyfinConnectionParameters(address, apiKey, library.MediaSourceId), new JellyfinConnectionParameters(address, apiKey, library.MediaSourceId),
library, library,
show, show,
@ -135,7 +140,10 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan
protected override IAsyncEnumerable<Tuple<JellyfinShow, int>> GetShowLibraryItems( protected override IAsyncEnumerable<Tuple<JellyfinShow, int>> GetShowLibraryItems(
JellyfinConnectionParameters connectionParameters, JellyfinConnectionParameters connectionParameters,
JellyfinLibrary library) => JellyfinLibrary library) =>
_jellyfinApiClient.GetShowLibraryItems(connectionParameters.Address, connectionParameters.ApiKey, library); _jellyfinApiClient.GetShowLibraryItemsWithoutPeople(
connectionParameters.Address,
connectionParameters.ApiKey,
library);
protected override string MediaServerItemId(JellyfinShow show) => show.ItemId; protected override string MediaServerItemId(JellyfinShow show) => show.ItemId;
protected override string MediaServerItemId(JellyfinSeason season) => season.ItemId; protected override string MediaServerItemId(JellyfinSeason season) => season.ItemId;
@ -159,20 +167,56 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan
JellyfinLibrary library, JellyfinLibrary library,
JellyfinConnectionParameters connectionParameters, JellyfinConnectionParameters connectionParameters,
JellyfinShow show, JellyfinShow show,
JellyfinSeason season) => JellyfinSeason season,
_jellyfinApiClient.GetEpisodeLibraryItems( bool isNewSeason)
{
if (isNewSeason)
{
return _jellyfinApiClient.GetEpisodeLibraryItems(
connectionParameters.Address,
connectionParameters.ApiKey,
library,
season.ItemId);
}
return _jellyfinApiClient.GetEpisodeLibraryItemsWithoutPeople(
connectionParameters.Address, connectionParameters.Address,
connectionParameters.ApiKey, connectionParameters.ApiKey,
library, library,
season.ItemId); season.ItemId);
}
protected override Task<Option<ShowMetadata>> GetFullMetadata( protected override async Task<Option<ShowMetadata>> GetFullMetadata(
JellyfinConnectionParameters connectionParameters, JellyfinConnectionParameters connectionParameters,
JellyfinLibrary library, JellyfinLibrary library,
MediaItemScanResult<JellyfinShow> result, MediaItemScanResult<JellyfinShow> result,
JellyfinShow incoming, JellyfinShow incoming,
bool deepScan) => bool deepScan)
Task.FromResult(Option<ShowMetadata>.None); {
if (result.IsAdded || result.Item.Etag != incoming.Etag || deepScan)
{
Either<BaseError, Option<JellyfinShow>> maybeShowResult = await _jellyfinApiClient.GetSingleShow(
connectionParameters.Address,
connectionParameters.ApiKey,
library,
incoming.ItemId);
foreach (BaseError error in maybeShowResult.LeftToSeq())
{
_logger.LogWarning("Failed to get show metadata from Jellyfin: {Error}", error.ToString());
}
foreach (Option<JellyfinShow> maybeShow in maybeShowResult.RightToSeq())
{
foreach (JellyfinShow show in maybeShow)
{
return show.ShowMetadata.HeadOrNone();
}
}
}
return None;
}
protected override Task<Option<SeasonMetadata>> GetFullMetadata( protected override Task<Option<SeasonMetadata>> GetFullMetadata(
JellyfinConnectionParameters connectionParameters, JellyfinConnectionParameters connectionParameters,
@ -182,13 +226,39 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan
bool deepScan) => bool deepScan) =>
Task.FromResult(Option<SeasonMetadata>.None); Task.FromResult(Option<SeasonMetadata>.None);
protected override Task<Option<EpisodeMetadata>> GetFullMetadata( protected override async Task<Option<EpisodeMetadata>> GetFullMetadata(
JellyfinConnectionParameters connectionParameters, JellyfinConnectionParameters connectionParameters,
JellyfinLibrary library, JellyfinLibrary library,
MediaItemScanResult<JellyfinEpisode> result, MediaItemScanResult<JellyfinEpisode> result,
JellyfinEpisode incoming, JellyfinEpisode incoming,
bool deepScan) => bool deepScan)
Task.FromResult(Option<EpisodeMetadata>.None); {
if (result.Item.Season is JellyfinSeason jellyfinSeason &&
(result.IsAdded || result.Item.Etag != incoming.Etag || deepScan))
{
Either<BaseError, Option<JellyfinEpisode>> maybeEpisodeResult = await _jellyfinApiClient.GetSingleEpisode(
connectionParameters.Address,
connectionParameters.ApiKey,
library,
jellyfinSeason.ItemId,
incoming.ItemId);
foreach (BaseError error in maybeEpisodeResult.LeftToSeq())
{
_logger.LogWarning("Failed to get episode metadata from Jellyfin: {Error}", error.ToString());
}
foreach (Option<JellyfinEpisode> maybeEpisode in maybeEpisodeResult.RightToSeq())
{
foreach (JellyfinEpisode episode in maybeEpisode)
{
return episode.EpisodeMetadata.HeadOrNone();
}
}
}
return None;
}
protected override Task<Option<Tuple<EpisodeMetadata, MediaVersion>>> GetFullMetadataAndStatistics( protected override Task<Option<Tuple<EpisodeMetadata, MediaVersion>>> GetFullMetadataAndStatistics(
JellyfinConnectionParameters connectionParameters, JellyfinConnectionParameters connectionParameters,
@ -225,21 +295,131 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan
return maybeVersion.ToOption(); return maybeVersion.ToOption();
} }
protected override Task<Either<BaseError, MediaItemScanResult<JellyfinShow>>> UpdateMetadata( protected override async Task<Either<BaseError, MediaItemScanResult<JellyfinShow>>> UpdateMetadata(
MediaItemScanResult<JellyfinShow> result, MediaItemScanResult<JellyfinShow> result,
ShowMetadata fullMetadata) => ShowMetadata fullMetadata)
Task.FromResult<Either<BaseError, MediaItemScanResult<JellyfinShow>>>(result); {
JellyfinShow existing = result.Item;
ShowMetadata existingMetadata = existing.ShowMetadata.Head();
foreach (Actor actor in existingMetadata.Actors
.Filter(a =>
fullMetadata.Actors.All(a2 => a2.Name != a.Name || a.Artwork == null && a2.Artwork != null))
.ToList())
{
existingMetadata.Actors.Remove(actor);
if (await _metadataRepository.RemoveActor(actor))
{
result.IsUpdated = true;
}
}
foreach (Actor actor in fullMetadata.Actors
.Filter(a => existingMetadata.Actors.All(a2 => a2.Name != a.Name))
.ToList())
{
existingMetadata.Actors.Add(actor);
if (await _televisionRepository.AddActor(existingMetadata, actor))
{
result.IsUpdated = true;
}
}
if (result.IsUpdated)
{
await _metadataRepository.MarkAsUpdated(existingMetadata, fullMetadata.DateUpdated);
}
return result;
}
protected override Task<Either<BaseError, MediaItemScanResult<JellyfinSeason>>> UpdateMetadata( protected override Task<Either<BaseError, MediaItemScanResult<JellyfinSeason>>> UpdateMetadata(
MediaItemScanResult<JellyfinSeason> result, MediaItemScanResult<JellyfinSeason> result,
SeasonMetadata fullMetadata) => SeasonMetadata fullMetadata) =>
Task.FromResult<Either<BaseError, MediaItemScanResult<JellyfinSeason>>>(result); Task.FromResult<Either<BaseError, MediaItemScanResult<JellyfinSeason>>>(result);
protected override Task<Either<BaseError, MediaItemScanResult<JellyfinEpisode>>> UpdateMetadata( protected override async Task<Either<BaseError, MediaItemScanResult<JellyfinEpisode>>> UpdateMetadata(
MediaItemScanResult<JellyfinEpisode> result, MediaItemScanResult<JellyfinEpisode> result,
EpisodeMetadata fullMetadata, EpisodeMetadata fullMetadata,
CancellationToken cancellationToken) => CancellationToken cancellationToken)
Task.FromResult<Either<BaseError, MediaItemScanResult<JellyfinEpisode>>>(result); {
JellyfinEpisode existing = result.Item;
EpisodeMetadata existingMetadata = existing.EpisodeMetadata.Head();
foreach (Actor actor in existingMetadata.Actors
.Filter(a =>
fullMetadata.Actors.All(a2 => a2.Name != a.Name || a.Artwork == null && a2.Artwork != null))
.ToList())
{
existingMetadata.Actors.Remove(actor);
if (await _metadataRepository.RemoveActor(actor))
{
result.IsUpdated = true;
}
}
foreach (Actor actor in fullMetadata.Actors
.Filter(a => existingMetadata.Actors.All(a2 => a2.Name != a.Name))
.ToList())
{
existingMetadata.Actors.Add(actor);
if (await _televisionRepository.AddActor(existingMetadata, actor))
{
result.IsUpdated = true;
}
}
foreach (Director director in existingMetadata.Directors
.Filter(g => fullMetadata.Directors.All(g2 => g2.Name != g.Name))
.ToList())
{
existingMetadata.Directors.Remove(director);
if (await _metadataRepository.RemoveDirector(director))
{
result.IsUpdated = true;
}
}
foreach (Director director in fullMetadata.Directors
.Filter(g => existingMetadata.Directors.All(g2 => g2.Name != g.Name))
.ToList())
{
existingMetadata.Directors.Add(director);
if (await _televisionRepository.AddDirector(existingMetadata, director))
{
result.IsUpdated = true;
}
}
foreach (Writer writer in existingMetadata.Writers
.Filter(g => fullMetadata.Writers.All(g2 => g2.Name != g.Name))
.ToList())
{
existingMetadata.Writers.Remove(writer);
if (await _metadataRepository.RemoveWriter(writer))
{
result.IsUpdated = true;
}
}
foreach (Writer writer in fullMetadata.Writers
.Filter(g => existingMetadata.Writers.All(g2 => g2.Name != g.Name))
.ToList())
{
existingMetadata.Writers.Add(writer);
if (await _televisionRepository.AddWriter(existingMetadata, writer))
{
result.IsUpdated = true;
}
}
if (result.IsUpdated)
{
await _metadataRepository.MarkAsUpdated(existingMetadata, fullMetadata.DateUpdated);
}
return result;
}
private async Task<Either<BaseError, Unit>> ScanSingleShowInternal( private async Task<Either<BaseError, Unit>> ScanSingleShowInternal(
IJellyfinTelevisionRepository televisionRepository, IJellyfinTelevisionRepository televisionRepository,

5
ErsatzTV.Scanner/Core/Metadata/MediaServerTelevisionLibraryScanner.cs

@ -220,7 +220,8 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
TLibrary library, TLibrary library,
TConnectionParameters connectionParameters, TConnectionParameters connectionParameters,
TShow show, TShow show,
TSeason season); TSeason season,
bool isNewSeason);
protected abstract Task<Option<ShowMetadata>> GetFullMetadata( protected abstract Task<Option<ShowMetadata>> GetFullMetadata(
TConnectionParameters connectionParameters, TConnectionParameters connectionParameters,
@ -321,7 +322,7 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
showIsUpdated, showIsUpdated,
result.Item, result.Item,
connectionParameters, connectionParameters,
GetEpisodeLibraryItems(library, connectionParameters, show, result.Item), GetEpisodeLibraryItems(library, connectionParameters, show, result.Item, result.IsAdded),
deepScan, deepScan,
cancellationToken); cancellationToken);

3
ErsatzTV.Scanner/Core/Plex/PlexTelevisionLibraryScanner.cs

@ -252,7 +252,8 @@ public partial class PlexTelevisionLibraryScanner :
PlexLibrary library, PlexLibrary library,
PlexConnectionParameters connectionParameters, PlexConnectionParameters connectionParameters,
PlexShow show, PlexShow show,
PlexSeason season) => PlexSeason season,
bool isNewSeason) =>
_plexServerApiClient.GetSeasonEpisodes( _plexServerApiClient.GetSeasonEpisodes(
library, library,
season, season,

6
ErsatzTV.Scanner/Program.cs

@ -19,6 +19,7 @@ using ErsatzTV.Core.Plex;
using ErsatzTV.Core.Search; using ErsatzTV.Core.Search;
using ErsatzTV.FFmpeg.Capabilities; using ErsatzTV.FFmpeg.Capabilities;
using ErsatzTV.FFmpeg.Runtime; using ErsatzTV.FFmpeg.Runtime;
using ErsatzTV.Infrastructure;
using ErsatzTV.Infrastructure.Data; using ErsatzTV.Infrastructure.Data;
using ErsatzTV.Infrastructure.Data.Repositories; using ErsatzTV.Infrastructure.Data.Repositories;
using ErsatzTV.Infrastructure.Emby; using ErsatzTV.Infrastructure.Emby;
@ -171,6 +172,8 @@ public class Program
services.AddHttpClient(); services.AddHttpClient();
services.AddHttpClient("RefitCustomClient").AddHttpMessageHandler<SlowApiHandler>();
services.AddScoped<IConfigElementRepository, ConfigElementRepository>(); services.AddScoped<IConfigElementRepository, ConfigElementRepository>();
services.AddScoped<IMetadataRepository, MetadataRepository>(); services.AddScoped<IMetadataRepository, MetadataRepository>();
services.AddScoped<IMediaSourceRepository, MediaSourceRepository>(); services.AddScoped<IMediaSourceRepository, MediaSourceRepository>();
@ -256,6 +259,9 @@ public class Program
services.AddSingleton<IFileSystem, RealFileSystem>(); services.AddSingleton<IFileSystem, RealFileSystem>();
services.AddTransient<SlowApiHandler>();
services.AddTransient<SlowQueryInterceptor>();
services.AddMediatR(config => config.RegisterServicesFromAssemblyContaining<Worker>()); services.AddMediatR(config => config.RegisterServicesFromAssemblyContaining<Worker>());
services.AddMemoryCache(); services.AddMemoryCache();

18
ErsatzTV/Startup.cs

@ -51,6 +51,7 @@ using ErsatzTV.FFmpeg.Pipeline;
using ErsatzTV.FFmpeg.Runtime; using ErsatzTV.FFmpeg.Runtime;
using ErsatzTV.Filters; using ErsatzTV.Filters;
using ErsatzTV.Formatters; using ErsatzTV.Formatters;
using ErsatzTV.Infrastructure;
using ErsatzTV.Infrastructure.Data; using ErsatzTV.Infrastructure.Data;
using ErsatzTV.Infrastructure.Data.Repositories; using ErsatzTV.Infrastructure.Data.Repositories;
using ErsatzTV.Infrastructure.Database; using ErsatzTV.Infrastructure.Database;
@ -408,12 +409,6 @@ public class Startup
var sqliteConnectionString = $"Data Source={FileSystemLayout.DatabasePath};foreign keys=true;"; var sqliteConnectionString = $"Data Source={FileSystemLayout.DatabasePath};foreign keys=true;";
string mySqlConnectionString = Configuration.GetValue<string>("MySql:ConnectionString"); string mySqlConnectionString = Configuration.GetValue<string>("MySql:ConnectionString");
SlowQueryInterceptor interceptor = null;
if (SystemEnvironment.SlowDbMs.HasValue)
{
interceptor = new SlowQueryInterceptor(SystemEnvironment.SlowDbMs.Value);
}
services.AddDbContext<TvContext>( services.AddDbContext<TvContext>(
options => options =>
{ {
@ -444,11 +439,6 @@ public class Startup
} }
); );
} }
if (interceptor != null)
{
options.AddInterceptors(interceptor);
}
}, },
ServiceLifetime.Scoped, ServiceLifetime.Scoped,
ServiceLifetime.Singleton); ServiceLifetime.Singleton);
@ -478,11 +468,6 @@ public class Startup
} }
); );
} }
if (interceptor != null)
{
options.AddInterceptors(interceptor);
}
}); });
if (databaseProvider == Provider.Sqlite.Name) if (databaseProvider == Provider.Sqlite.Name)
@ -870,6 +855,7 @@ public class Startup
// services.AddTransient(typeof(IRequestHandler<,>), typeof(GetRecentLogEntriesHandler<>)); // services.AddTransient(typeof(IRequestHandler<,>), typeof(GetRecentLogEntriesHandler<>));
services.AddTransient<SlowApiHandler>(); services.AddTransient<SlowApiHandler>();
services.AddTransient<SlowQueryInterceptor>();
// run-once/blocking startup services // run-once/blocking startup services
services.AddHostedService<EndpointValidatorService>(); services.AddHostedService<EndpointValidatorService>();

Loading…
Cancel
Save