pimhwang
/
ErsatzTV
mirror of https://github.com/ErsatzTV/ErsatzTV.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

collect and print timings during jellyfin show library scans (#2748) 

* collect and print timings during jellyfin show library scans

* update p99
pull/2750/head
Jason Dove 7 months ago committed by GitHub
parent
474e647d6d
commit
80f6e468eb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 550 additions and 410 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 187
      ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs
  2. 92
      ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs
  3. 61
      ErsatzTV.Infrastructure/ScanProfiler.cs
  4. 620
      ErsatzTV.Scanner/Core/Metadata/MediaServerTelevisionLibraryScanner.cs

187
ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs
Unescape View File

@ -86,40 +86,43 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository @@ -86,40 +86,43 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository
JellyfinShow item,
CancellationToken cancellationToken)
{
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
Option<JellyfinShow> maybeExisting = await dbContext.JellyfinShows
.TagWithCallSite()
.Include(m => m.LibraryPath)
.ThenInclude(lp => lp.Library)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Genres)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Tags)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Studios)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Actors)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Artwork)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Guids)
.Include(m => m.TraktListItems)
.ThenInclude(tli => tli.TraktList)
.SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken);
foreach (JellyfinShow jellyfinShow in maybeExisting)
{
var result = new MediaItemScanResult<JellyfinShow>(jellyfinShow) { IsAdded = false };
if (jellyfinShow.Etag != item.Etag)
using (ScanProfiler.Measure("DB Ins/Upd Show"))
{
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
Option<JellyfinShow> maybeExisting = await dbContext.JellyfinShows
.TagWithCallSite()
.Include(m => m.LibraryPath)
.ThenInclude(lp => lp.Library)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Genres)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Tags)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Studios)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Actors)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Artwork)
.Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Guids)
.Include(m => m.TraktListItems)
.ThenInclude(tli => tli.TraktList)
.SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken);
foreach (JellyfinShow jellyfinShow in maybeExisting)
{
await UpdateShow(dbContext, jellyfinShow, item, cancellationToken);
result.IsUpdated = true;
var result = new MediaItemScanResult<JellyfinShow>(jellyfinShow) { IsAdded = false };
if (jellyfinShow.Etag != item.Etag)
{
await UpdateShow(dbContext, jellyfinShow, item, cancellationToken);
result.IsUpdated = true;
}
return result;
}
return result;
return await AddShow(dbContext, library, item, cancellationToken);
}
return await AddShow(dbContext, library, item, cancellationToken);
}
public async Task<Either<BaseError, MediaItemScanResult<JellyfinSeason>>> GetOrAdd(
@ -127,29 +130,32 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository @@ -127,29 +130,32 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository
JellyfinSeason item,
CancellationToken cancellationToken)
{
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
Option<JellyfinSeason> maybeExisting = await dbContext.JellyfinSeasons
.TagWithCallSite()
.Include(m => m.LibraryPath)
.Include(m => m.SeasonMetadata)
.ThenInclude(mm => mm.Artwork)
.Include(m => m.SeasonMetadata)
.ThenInclude(mm => mm.Guids)
.SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken);
foreach (JellyfinSeason jellyfinSeason in maybeExisting)
using (ScanProfiler.Measure("DB Ins/Upd Season"))
{
var result = new MediaItemScanResult<JellyfinSeason>(jellyfinSeason) { IsAdded = false };
if (jellyfinSeason.Etag != item.Etag)
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
Option<JellyfinSeason> maybeExisting = await dbContext.JellyfinSeasons
.TagWithCallSite()
.Include(m => m.LibraryPath)
.Include(m => m.SeasonMetadata)
.ThenInclude(mm => mm.Artwork)
.Include(m => m.SeasonMetadata)
.ThenInclude(mm => mm.Guids)
.SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken);
foreach (JellyfinSeason jellyfinSeason in maybeExisting)
{
await UpdateSeason(dbContext, jellyfinSeason, item, cancellationToken);
result.IsUpdated = true;
var result = new MediaItemScanResult<JellyfinSeason>(jellyfinSeason) { IsAdded = false };
if (jellyfinSeason.Etag != item.Etag)
{
await UpdateSeason(dbContext, jellyfinSeason, item, cancellationToken);
result.IsUpdated = true;
}
return result;
}
return result;
return await AddSeason(dbContext, library, item, cancellationToken);
}
return await AddSeason(dbContext, library, item, cancellationToken);
}
public async Task<Either<BaseError, MediaItemScanResult<JellyfinEpisode>>> GetOrAdd(
@ -158,51 +164,54 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository @@ -158,51 +164,54 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository
bool deepScan,
CancellationToken cancellationToken)
{
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
Option<JellyfinEpisode> maybeExisting = await dbContext.JellyfinEpisodes
.TagWithCallSite()
.Include(m => m.LibraryPath)
.ThenInclude(lp => lp.Library)
.Include(m => m.MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(m => m.MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(m => m.MediaVersions)
.ThenInclude(mv => mv.Chapters)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Artwork)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Guids)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Genres)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Tags)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Studios)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Actors)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Directors)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Writers)
.Include(m => m.Season)
.Include(m => m.TraktListItems)
.ThenInclude(tli => tli.TraktList)
.SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken);
foreach (JellyfinEpisode jellyfinEpisode in maybeExisting)
{
var result = new MediaItemScanResult<JellyfinEpisode>(jellyfinEpisode) { IsAdded = false };
if (jellyfinEpisode.Etag != item.Etag || deepScan)
using (ScanProfiler.Measure("DB Ins/Upd Episode"))
{
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
Option<JellyfinEpisode> maybeExisting = await dbContext.JellyfinEpisodes
.TagWithCallSite()
.Include(m => m.LibraryPath)
.ThenInclude(lp => lp.Library)
.Include(m => m.MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(m => m.MediaVersions)
.ThenInclude(mv => mv.Streams)
.Include(m => m.MediaVersions)
.ThenInclude(mv => mv.Chapters)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Artwork)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Guids)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Genres)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Tags)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Studios)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Actors)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Directors)
.Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Writers)
.Include(m => m.Season)
.Include(m => m.TraktListItems)
.ThenInclude(tli => tli.TraktList)
.SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken);
foreach (JellyfinEpisode jellyfinEpisode in maybeExisting)
{
await UpdateEpisode(dbContext, jellyfinEpisode, item, cancellationToken);
result.IsUpdated = true;
var result = new MediaItemScanResult<JellyfinEpisode>(jellyfinEpisode) { IsAdded = false };
if (jellyfinEpisode.Etag != item.Etag || deepScan)
{
await UpdateEpisode(dbContext, jellyfinEpisode, item, cancellationToken);
result.IsUpdated = true;
}
return result;
}
return result;
return await AddEpisode(dbContext, library, item, cancellationToken);
}
return await AddEpisode(dbContext, library, item, cancellationToken);
}
public async Task<Unit> SetEtag(JellyfinShow show, string etag, CancellationToken cancellationToken)

92
ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs
Unescape View File

@ -81,6 +81,7 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -81,6 +81,7 @@ public class JellyfinApiClient : IJellyfinApiClient
string apiKey,
JellyfinLibrary library) =>
GetPagedLibraryItems(
"JF Movies",
address,
library,
library.MediaSourceId,
@ -97,6 +98,7 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -97,6 +98,7 @@ public class JellyfinApiClient : IJellyfinApiClient
string apiKey,
JellyfinLibrary library) =>
GetPagedLibraryItems(
"JF Shows",
address,
library,
library.MediaSourceId,
@ -114,6 +116,7 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -114,6 +116,7 @@ public class JellyfinApiClient : IJellyfinApiClient
JellyfinLibrary library,
string showId) =>
GetPagedLibraryItems(
"JF Seasons",
address,
library,
library.MediaSourceId,
@ -131,6 +134,7 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -131,6 +134,7 @@ public class JellyfinApiClient : IJellyfinApiClient
JellyfinLibrary library,
string seasonId) =>
GetPagedLibraryItems(
"JF Episodes*",
address,
library,
library.MediaSourceId,
@ -148,6 +152,7 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -148,6 +152,7 @@ public class JellyfinApiClient : IJellyfinApiClient
JellyfinLibrary library,
string seasonId) =>
GetPagedLibraryItems(
"JF Episodes",
address,
library,
library.MediaSourceId,
@ -169,6 +174,7 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -169,6 +174,7 @@ public class JellyfinApiClient : IJellyfinApiClient
if (_memoryCache.TryGetValue("jellyfin_collections_library_item_id", out string itemId))
{
return GetPagedLibraryItems(
"JF Collections",
address,
None,
mediaSourceId,
@ -190,6 +196,7 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -190,6 +196,7 @@ public class JellyfinApiClient : IJellyfinApiClient
int mediaSourceId,
string collectionId) =>
GetPagedLibraryItems(
"JF Collection Items",
address,
None,
mediaSourceId,
@ -209,10 +216,13 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -209,10 +216,13 @@ public class JellyfinApiClient : IJellyfinApiClient
{
try
{
IJellyfinApi service = ServiceForAddress(address);
JellyfinPlaybackInfoResponse playbackInfo = await service.GetPlaybackInfo(apiKey, itemId);
Option<MediaVersion> maybeVersion = ProjectToMediaVersion(playbackInfo);
return maybeVersion.ToEither(() => BaseError.New("Unable to locate Jellyfin statistics"));
using (ScanProfiler.Measure("JF Playback Info"))
{
IJellyfinApi service = ServiceForAddress(address);
JellyfinPlaybackInfoResponse playbackInfo = await service.GetPlaybackInfo(apiKey, itemId);
Option<MediaVersion> maybeVersion = ProjectToMediaVersion(playbackInfo);
return maybeVersion.ToEither(() => BaseError.New("Unable to locate Jellyfin statistics"));
}
}
catch (Exception ex)
{
@ -229,21 +239,24 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -229,21 +239,24 @@ public class JellyfinApiClient : IJellyfinApiClient
{
try
{
IJellyfinApi service = ServiceForAddress(address);
JellyfinLibraryItemsResponse itemsResponse = await service.GetShowLibraryItems(
apiKey,
parentId: library.ItemId,
recursive: false,
startIndex: 0,
limit: 1,
ids: showId);
foreach (JellyfinLibraryItemResponse item in itemsResponse.Items)
using (ScanProfiler.Measure("JF Single Show"))
{
return ProjectToShow(item);
}
IJellyfinApi service = ServiceForAddress(address);
JellyfinLibraryItemsResponse itemsResponse = await service.GetShowLibraryItems(
apiKey,
parentId: library.ItemId,
recursive: false,
startIndex: 0,
limit: 1,
ids: showId);
foreach (JellyfinLibraryItemResponse item in itemsResponse.Items)
{
return ProjectToShow(item);
}
return BaseError.New($"Unable to locate show with id {showId}");
return BaseError.New($"Unable to locate show with id {showId}");
}
}
catch (Exception ex)
{
@ -310,21 +323,24 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -310,21 +323,24 @@ public class JellyfinApiClient : IJellyfinApiClient
{
try
{
IJellyfinApi service = ServiceForAddress(address);
JellyfinLibraryItemsResponse itemsResponse = await service.GetEpisodeLibraryItems(
apiKey,
parentId: seasonId,
recursive: false,
startIndex: 0,
limit: 1,
ids: episodeId);
foreach (JellyfinLibraryItemResponse item in itemsResponse.Items)
using (ScanProfiler.Measure("JF Single Episode"))
{
return ProjectToEpisode(library, item);
}
IJellyfinApi service = ServiceForAddress(address);
JellyfinLibraryItemsResponse itemsResponse = await service.GetEpisodeLibraryItems(
apiKey,
parentId: seasonId,
recursive: false,
startIndex: 0,
limit: 1,
ids: episodeId);
foreach (JellyfinLibraryItemResponse item in itemsResponse.Items)
{
return ProjectToEpisode(library, item);
}
return BaseError.New($"Unable to locate episode with id {episodeId}");
return BaseError.New($"Unable to locate episode with id {episodeId}");
}
}
catch (Exception ex)
{
@ -334,6 +350,7 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -334,6 +350,7 @@ public class JellyfinApiClient : IJellyfinApiClient
}
private async IAsyncEnumerable<Tuple<TItem, int>> GetPagedLibraryItems<TItem>(
string pageDescription,
string address,
Option<JellyfinLibrary> maybeLibrary,
int mediaSourceId,
@ -348,11 +365,16 @@ public class JellyfinApiClient : IJellyfinApiClient @@ -348,11 +365,16 @@ public class JellyfinApiClient : IJellyfinApiClient
{
int skip = i * SystemEnvironment.JellyfinPageSize;
JellyfinLibraryItemsResponse result = await getItems(
service,
parentId,
skip,
SystemEnvironment.JellyfinPageSize);
JellyfinLibraryItemsResponse result;
using (ScanProfiler.Measure(pageDescription))
{
result = await getItems(
service,
parentId,
skip,
SystemEnvironment.JellyfinPageSize);
}
// update page count
pages = Math.Min(pages, (result.TotalRecordCount - 1) / SystemEnvironment.JellyfinPageSize + 1);

61
ErsatzTV.Infrastructure/ScanProfiler.cs
Unescape View File

@ -0,0 +1,61 @@ @@ -0,0 +1,61 @@
using System.Collections.Concurrent;
using System.Diagnostics;
namespace ErsatzTV.Infrastructure;
public static class ScanProfiler
{
private static readonly ConcurrentDictionary<string, ConcurrentBag<long>> Measurements = new();
public static IDisposable Measure(string operationName)
{
return new TimerToken(operationName);
}
public static void Reset() => Measurements.Clear();
public static void LogStatistics(Action<string> logAction)
{
if (Measurements.IsEmpty)
{
return;
}
var sb = new System.Text.StringBuilder();
sb.AppendLine("Scan Performance Summary:");
sb.AppendLine(FormattableString.Invariant($"{"Operation",-25} | {"Count",-6} | {"Avg (ms)",-8} | {"Min",-6} | {"Max",-6} | {"P99",-6} | {"Total (s)",-8}"));
sb.AppendLine(new string('-', 85));
foreach (string key in Measurements.Keys.OrderBy(k => k))
{
var times = Measurements[key].ToList();
if (times.Count == 0) continue;
times.Sort();
double avg = times.Average();
long min = times.Min();
long max = times.Max();
int n = times.Count;
int idx = (int)Math.Ceiling(0.99 * n) - 1;
idx = Math.Clamp(idx, 0, n - 1);
long p99 = times[idx];
double totalSec = times.Sum() / 1000.0;
sb.AppendLine(FormattableString.Invariant($"{key,-25} | {times.Count,-6} | {avg,-8:F1} | {min,-6} | {max,-6} | {p99,-6} | {totalSec,-8:F2}"));
}
logAction(sb.ToString());
}
private readonly struct TimerToken(string name) : IDisposable
{
private readonly long _startTime = Stopwatch.GetTimestamp();
public void Dispose()
{
long elapsedMs = (long)Stopwatch.GetElapsedTime(_startTime).TotalMilliseconds;
ConcurrentBag<long> bag = Measurements.GetOrAdd(name, _ => []);
bag.Add(elapsedMs);
}
}
}

620
ErsatzTV.Scanner/Core/Metadata/MediaServerTelevisionLibraryScanner.cs
Unescape View File

@ -6,6 +6,7 @@ using ErsatzTV.Core.Errors; @@ -6,6 +6,7 @@ using ErsatzTV.Core.Errors;
using ErsatzTV.Core.Extensions;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Core.Metadata;
using ErsatzTV.Infrastructure;
using ErsatzTV.Scanner.Core.Interfaces;
using ErsatzTV.Scanner.Core.Interfaces.Metadata;
using Microsoft.Extensions.Logging;
@ -95,70 +96,78 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters, @@ -95,70 +96,78 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
await foreach ((TShow incoming, int totalShowCount) in showEntries.WithCancellation(cancellationToken))
{
if (cancellationToken.IsCancellationRequested)
using (ScanProfiler.Measure("Scan Show"))
{
return new ScanCanceled();
}
incomingItemIds.Add(MediaServerItemId(incoming));
decimal percentCompletion = Math.Clamp((decimal)incomingItemIds.Count / totalShowCount, 0, 1);
if (!await _scannerProxy.UpdateProgress(percentCompletion, cancellationToken))
{
return new ScanCanceled();
}
Either<BaseError, MediaItemScanResult<TShow>> maybeShow = await televisionRepository
.GetOrAdd(library, incoming, cancellationToken)
.BindT(existing => UpdateMetadata(connectionParameters, library, existing, incoming, deepScan));
if (maybeShow.IsLeft)
{
foreach (BaseError error in maybeShow.LeftToSeq())
if (cancellationToken.IsCancellationRequested)
{
_logger.LogWarning(
"Error processing show {Title}: {Error}",
incoming.ShowMetadata.Head().Title,
error.Value);
return new ScanCanceled();
}
continue;
}
incomingItemIds.Add(MediaServerItemId(incoming));
foreach (MediaItemScanResult<TShow> result in maybeShow.RightToSeq())
{
Either<BaseError, Unit> scanResult = await ScanSeasons(
televisionRepository,
library,
getLocalPath,
result.Item,
result.IsUpdated,
connectionParameters,
GetSeasonLibraryItems(library, connectionParameters, result.Item),
deepScan,
cancellationToken);
foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
decimal percentCompletion = Math.Clamp((decimal)incomingItemIds.Count / totalShowCount, 0, 1);
if (!await _scannerProxy.UpdateProgress(percentCompletion, cancellationToken))
{
return error;
return new ScanCanceled();
}
await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken);
Either<BaseError, MediaItemScanResult<TShow>> maybeShow = await televisionRepository
.GetOrAdd(library, incoming, cancellationToken)
.BindT(existing => UpdateMetadata(connectionParameters, library, existing, incoming, deepScan));
Option<int> flagResult = await televisionRepository.FlagNormal(library, result.Item, cancellationToken);
if (flagResult.IsSome)
if (maybeShow.IsLeft)
{
result.IsUpdated = true;
foreach (BaseError error in maybeShow.LeftToSeq())
{
_logger.LogWarning(
"Error processing show {Title}: {Error}",
incoming.ShowMetadata.Head().Title,
error.Value);
}
continue;
}
if (result.IsAdded || result.IsUpdated)
foreach (MediaItemScanResult<TShow> result in maybeShow.RightToSeq())
{
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken))
Either<BaseError, Unit> scanResult = await ScanSeasons(
televisionRepository,
library,
getLocalPath,
result.Item,
result.IsUpdated,
connectionParameters,
GetSeasonLibraryItems(library, connectionParameters, result.Item),
deepScan,
cancellationToken);
foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
{
_logger.LogWarning("Failed to reindex media items from scanner process");
return error;
}
await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken);
Option<int> flagResult = await televisionRepository.FlagNormal(
library,
result.Item,
cancellationToken);
if (flagResult.IsSome)
{
result.IsUpdated = true;
}
if (result.IsAdded || result.IsUpdated)
{
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken))
{
_logger.LogWarning("Failed to reindex media items from scanner process");
}
}
}
}
ScanProfiler.LogStatistics(s => _logger.LogInformation("{Profile}", s));
}
if (cleanupFileNotFoundItems)
@ -285,67 +294,73 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters, @@ -285,67 +294,73 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
await foreach ((TSeason incoming, int _) in seasonEntries.WithCancellation(cancellationToken))
{
incoming.ShowId = show.Id;
if (cancellationToken.IsCancellationRequested)
using (ScanProfiler.Measure("Scan Season"))
{
return new ScanCanceled();
}
incomingItemIds.Add(MediaServerItemId(incoming));
Either<BaseError, MediaItemScanResult<TSeason>> maybeSeason = await televisionRepository
.GetOrAdd(library, incoming, cancellationToken)
.BindT(existing => UpdateMetadata(connectionParameters, library, existing, incoming, deepScan));
incoming.ShowId = show.Id;
if (maybeSeason.IsLeft)
{
foreach (BaseError error in maybeSeason.LeftToSeq())
if (cancellationToken.IsCancellationRequested)
{
_logger.LogWarning(
"Error processing show {Title} season {SeasonNumber}: {Error}",
show.ShowMetadata.Head().Title,
incoming.SeasonNumber,
error.Value);
return new ScanCanceled();
}
continue;
}
incomingItemIds.Add(MediaServerItemId(incoming));
foreach (MediaItemScanResult<TSeason> result in maybeSeason.RightToSeq())
{
Either<BaseError, Unit> scanResult = await ScanEpisodes(
televisionRepository,
library,
getLocalPath,
show,
showIsUpdated,
result.Item,
connectionParameters,
GetEpisodeLibraryItems(library, connectionParameters, show, result.Item, result.IsAdded),
deepScan,
cancellationToken);
foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
Either<BaseError, MediaItemScanResult<TSeason>> maybeSeason = await televisionRepository
.GetOrAdd(library, incoming, cancellationToken)
.BindT(existing => UpdateMetadata(connectionParameters, library, existing, incoming, deepScan));
if (maybeSeason.IsLeft)
{
return error;
}
foreach (BaseError error in maybeSeason.LeftToSeq())
{
_logger.LogWarning(
"Error processing show {Title} season {SeasonNumber}: {Error}",
show.ShowMetadata.Head().Title,
incoming.SeasonNumber,
error.Value);
}
await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken);
continue;
}
Option<int> flagResult = await televisionRepository.FlagNormal(library, result.Item, cancellationToken);
if (flagResult.IsSome)
foreach (MediaItemScanResult<TSeason> result in maybeSeason.RightToSeq())
{
result.IsUpdated = true;
}
Either<BaseError, Unit> scanResult = await ScanEpisodes(
televisionRepository,
library,
getLocalPath,
show,
showIsUpdated,
result.Item,
connectionParameters,
GetEpisodeLibraryItems(library, connectionParameters, show, result.Item, result.IsAdded),
deepScan,
cancellationToken);
result.Item.Show = show;
foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
{
return error;
}
if (result.IsAdded || result.IsUpdated || showIsUpdated)
{
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken))
await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken);
Option<int> flagResult = await televisionRepository.FlagNormal(
library,
result.Item,
cancellationToken);
if (flagResult.IsSome)
{
_logger.LogWarning("Failed to reindex media items from scanner process");
result.IsUpdated = true;
}
result.Item.Show = show;
if (result.IsAdded || result.IsUpdated || showIsUpdated)
{
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken))
{
_logger.LogWarning("Failed to reindex media items from scanner process");
}
}
}
}
@ -379,129 +394,141 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters, @@ -379,129 +394,141 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
await foreach ((TEpisode incoming, int _) in episodeEntries.WithCancellation(cancellationToken))
{
if (cancellationToken.IsCancellationRequested)
{
return new ScanCanceled();
}
incomingItemIds.Add(MediaServerItemId(incoming));
string localPath = getLocalPath(incoming);
if (!await ShouldScanItem(
televisionRepository,
library,
show,
season,
existingEpisodes,
incoming,
localPath,
deepScan,
cancellationToken))
using (ScanProfiler.Measure("Scan Episode"))
{
continue;
}
incoming.SeasonId = season.Id;
if (cancellationToken.IsCancellationRequested)
{
return new ScanCanceled();
}
Either<BaseError, MediaItemScanResult<TEpisode>> maybeEpisode;
incomingItemIds.Add(MediaServerItemId(incoming));
if (ServerReturnsStatisticsWithMetadata)
{
maybeEpisode = await televisionRepository
.GetOrAdd(library, incoming, deepScan, cancellationToken)
.MapT(result =>
{
result.LocalPath = localPath;
return result;
})
.BindT(existing => UpdateMetadataAndStatistics(
connectionParameters,
library,
existing,
incoming,
deepScan,
cancellationToken))
.BindT(existing => UpdateChapters(existing, cancellationToken));
}
else
{
maybeEpisode = await televisionRepository
.GetOrAdd(library, incoming, deepScan, cancellationToken)
.MapT(result =>
{
result.LocalPath = localPath;
return result;
})
.BindT(existing => UpdateMetadata(
connectionParameters,
string localPath = getLocalPath(incoming);
if (!await ShouldScanItem(
televisionRepository,
library,
existing,
show,
season,
existingEpisodes,
incoming,
localPath,
deepScan,
None,
cancellationToken))
.BindT(existing => UpdateStatistics(
connectionParameters,
library,
existing,
incoming,
deepScan,
None))
.BindT(existing => UpdateSubtitles(existing, cancellationToken))
.BindT(existing => UpdateChapters(existing, cancellationToken));
}
if (maybeEpisode.IsLeft)
{
foreach (BaseError error in maybeEpisode.LeftToSeq())
{
_logger.LogWarning(
"Error processing episode {Title} s{SeasonNumber:00}e{EpisodeNumber:00}: {Error}",
show.ShowMetadata.Head().Title,
season.SeasonNumber,
incoming.EpisodeMetadata.Head().EpisodeNumber,
error.Value);
continue;
}
continue;
}
incoming.SeasonId = season.Id;
foreach (MediaItemScanResult<TEpisode> result in maybeEpisode.RightToSeq())
{
await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken);
Either<BaseError, MediaItemScanResult<TEpisode>> maybeEpisode;
if (_fileSystem.File.Exists(result.LocalPath))
if (ServerReturnsStatisticsWithMetadata)
{
Option<int> flagResult = await televisionRepository.FlagNormal(library, result.Item, cancellationToken);
if (flagResult.IsSome)
{
result.IsUpdated = true;
}
maybeEpisode = await televisionRepository
.GetOrAdd(library, incoming, deepScan, cancellationToken)
.MapT(result =>
{
result.LocalPath = localPath;
return result;
})
.BindT(existing => UpdateMetadataAndStatistics(
connectionParameters,
library,
existing,
incoming,
deepScan,
cancellationToken))
.BindT(existing => UpdateChapters(existing, cancellationToken));
}
else if (ServerSupportsRemoteStreaming)
else
{
Option<int> flagResult = await televisionRepository.FlagRemoteOnly(library, result.Item, cancellationToken);
if (flagResult.IsSome)
{
result.IsUpdated = true;
}
maybeEpisode = await televisionRepository
.GetOrAdd(library, incoming, deepScan, cancellationToken)
.MapT(result =>
{
result.LocalPath = localPath;
return result;
})
.BindT(existing => UpdateMetadata(
connectionParameters,
library,
existing,
incoming,
deepScan,
None,
cancellationToken))
.BindT(existing => UpdateStatistics(
connectionParameters,
library,
existing,
incoming,
deepScan,
None))
.BindT(existing => UpdateSubtitles(existing, cancellationToken))
.BindT(existing => UpdateChapters(existing, cancellationToken));
}
else
if (maybeEpisode.IsLeft)
{
Option<int> flagResult = await televisionRepository.FlagUnavailable(library, result.Item, cancellationToken);
if (flagResult.IsSome)
foreach (BaseError error in maybeEpisode.LeftToSeq())
{
result.IsUpdated = true;
_logger.LogWarning(
"Error processing episode {Title} s{SeasonNumber:00}e{EpisodeNumber:00}: {Error}",
show.ShowMetadata.Head().Title,
season.SeasonNumber,
incoming.EpisodeMetadata.Head().EpisodeNumber,
error.Value);
}
continue;
}
if (result.IsAdded || result.IsUpdated || showIsUpdated)
foreach (MediaItemScanResult<TEpisode> result in maybeEpisode.RightToSeq())
{
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken))
await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken);
if (_fileSystem.File.Exists(result.LocalPath))
{
_logger.LogWarning("Failed to reindex media items from scanner process");
Option<int> flagResult = await televisionRepository.FlagNormal(
library,
result.Item,
cancellationToken);
if (flagResult.IsSome)
{
result.IsUpdated = true;
}
}
else if (ServerSupportsRemoteStreaming)
{
Option<int> flagResult = await televisionRepository.FlagRemoteOnly(
library,
result.Item,
cancellationToken);
if (flagResult.IsSome)
{
result.IsUpdated = true;
}
}
else
{
Option<int> flagResult = await televisionRepository.FlagUnavailable(
library,
result.Item,
cancellationToken);
if (flagResult.IsSome)
{
result.IsUpdated = true;
}
}
if (result.IsAdded || result.IsUpdated || showIsUpdated)
{
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken))
{
_logger.LogWarning("Failed to reindex media items from scanner process");
}
}
}
}
}
@ -611,16 +638,19 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters, @@ -611,16 +638,19 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
TShow incoming,
bool deepScan)
{
foreach (ShowMetadata fullMetadata in await GetFullMetadata(
connectionParameters,
library,
result,
incoming,
deepScan))
using (ScanProfiler.Measure("Update Show Metadata"))
{
// TODO: move some of this code into this scanner
// will have to merge JF, Emby, Plex logic
return await UpdateMetadata(result, fullMetadata);
foreach (ShowMetadata fullMetadata in await GetFullMetadata(
connectionParameters,
library,
result,
incoming,
deepScan))
{
// TODO: move some of this code into this scanner
// will have to merge JF, Emby, Plex logic
return await UpdateMetadata(result, fullMetadata);
}
}
return result;
@ -633,16 +663,19 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters, @@ -633,16 +663,19 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
TSeason incoming,
bool deepScan)
{
foreach (SeasonMetadata fullMetadata in await GetFullMetadata(
connectionParameters,
library,
result,
incoming,
deepScan))
using (ScanProfiler.Measure("Update Season Metadata"))
{
// TODO: move some of this code into this scanner
// will have to merge JF, Emby, Plex logic
return await UpdateMetadata(result, fullMetadata);
foreach (SeasonMetadata fullMetadata in await GetFullMetadata(
connectionParameters,
library,
result,
incoming,
deepScan))
{
// TODO: move some of this code into this scanner
// will have to merge JF, Emby, Plex logic
return await UpdateMetadata(result, fullMetadata);
}
}
return result;
@ -714,19 +747,22 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters, @@ -714,19 +747,22 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
Option<EpisodeMetadata> maybeFullMetadata,
CancellationToken cancellationToken)
{
if (maybeFullMetadata.IsNone)
using (ScanProfiler.Measure("Update Episode Metadata"))
{
maybeFullMetadata = await GetFullMetadata(connectionParameters, library, result, incoming, deepScan);
}
if (maybeFullMetadata.IsNone)
{
maybeFullMetadata = await GetFullMetadata(connectionParameters, library, result, incoming, deepScan);
}
foreach (EpisodeMetadata fullMetadata in maybeFullMetadata)
{
// TODO: move some of this code into this scanner
// will have to merge JF, Emby, Plex logic
return await UpdateMetadata(result, fullMetadata, cancellationToken);
}
foreach (EpisodeMetadata fullMetadata in maybeFullMetadata)
{
// TODO: move some of this code into this scanner
// will have to merge JF, Emby, Plex logic
return await UpdateMetadata(result, fullMetadata, cancellationToken);
}
return result;
return result;
}
}
private async Task<Either<BaseError, MediaItemScanResult<TEpisode>>> UpdateStatistics(
@ -737,57 +773,60 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters, @@ -737,57 +773,60 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
bool deepScan,
Option<MediaVersion> maybeMediaVersion)
{
TEpisode existing = result.Item;
if (deepScan || result.IsAdded || MediaServerEtag(existing) != MediaServerEtag(incoming) ||
existing.MediaVersions.Head().Streams.Count == 0)
using (ScanProfiler.Measure("Update Episode Stats"))
{
// if (maybeMediaVersion.IsNone && _fileSystem.File.Exists(result.LocalPath))
// {
// _logger.LogDebug("Refreshing {Attribute} for {Path}", "Statistics", result.LocalPath);
// Either<BaseError, bool> refreshResult =
// await _localStatisticsProvider.RefreshStatistics(
// ffmpegPath,
// ffprobePath,
// existing,
// result.LocalPath);
//
// foreach (BaseError error in refreshResult.LeftToSeq())
// {
// _logger.LogWarning(
// "Unable to refresh {Attribute} for media item {Path}. Error: {Error}",
// "Statistics",
// result.LocalPath,
// error.Value);
// }
//
// foreach (bool _ in refreshResult.RightToSeq())
// {
// result.IsUpdated = true;
// }
// }
// else
// {
if (maybeMediaVersion.IsNone)
{
maybeMediaVersion = await GetMediaServerStatistics(
connectionParameters,
library,
result,
incoming);
}
TEpisode existing = result.Item;
foreach (MediaVersion mediaVersion in maybeMediaVersion)
if (deepScan || result.IsAdded || MediaServerEtag(existing) != MediaServerEtag(incoming) ||
existing.MediaVersions.Head().Streams.Count == 0)
{
if (await _metadataRepository.UpdateStatistics(result.Item, mediaVersion))
// if (maybeMediaVersion.IsNone && _fileSystem.File.Exists(result.LocalPath))
// {
// _logger.LogDebug("Refreshing {Attribute} for {Path}", "Statistics", result.LocalPath);
// Either<BaseError, bool> refreshResult =
// await _localStatisticsProvider.RefreshStatistics(
// ffmpegPath,
// ffprobePath,
// existing,
// result.LocalPath);
//
// foreach (BaseError error in refreshResult.LeftToSeq())
// {
// _logger.LogWarning(
// "Unable to refresh {Attribute} for media item {Path}. Error: {Error}",
// "Statistics",
// result.LocalPath,
// error.Value);
// }
//
// foreach (bool _ in refreshResult.RightToSeq())
// {
// result.IsUpdated = true;
// }
// }
// else
// {
if (maybeMediaVersion.IsNone)
{
result.IsUpdated = true;
maybeMediaVersion = await GetMediaServerStatistics(
connectionParameters,
library,
result,
incoming);
}
foreach (MediaVersion mediaVersion in maybeMediaVersion)
{
if (await _metadataRepository.UpdateStatistics(result.Item, mediaVersion))
{
result.IsUpdated = true;
}
}
// }
}
// }
}
return result;
return result;
}
}
private async Task<Either<BaseError, MediaItemScanResult<TEpisode>>> UpdateSubtitles(
@ -796,22 +835,25 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters, @@ -796,22 +835,25 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
{
try
{
MediaVersion version = existing.Item.GetHeadVersion();
Option<EpisodeMetadata> maybeMetadata = existing.Item.EpisodeMetadata.HeadOrNone();
foreach (EpisodeMetadata metadata in maybeMetadata)
using (ScanProfiler.Measure("Update Episode Subtitles"))
{
List<Subtitle> subtitles = version.Streams
.Filter(s => s.MediaStreamKind is MediaStreamKind.Subtitle or MediaStreamKind.ExternalSubtitle)
.Map(Subtitle.FromMediaStream)
.ToList();
if (await _metadataRepository.UpdateSubtitles(metadata, subtitles, cancellationToken))
MediaVersion version = existing.Item.GetHeadVersion();
Option<EpisodeMetadata> maybeMetadata = existing.Item.EpisodeMetadata.HeadOrNone();
foreach (EpisodeMetadata metadata in maybeMetadata)
{
return existing;
List<Subtitle> subtitles = version.Streams
.Filter(s => s.MediaStreamKind is MediaStreamKind.Subtitle or MediaStreamKind.ExternalSubtitle)
.Map(Subtitle.FromMediaStream)
.ToList();
if (await _metadataRepository.UpdateSubtitles(metadata, subtitles, cancellationToken))
{
return existing;
}
}
}
return BaseError.New("Failed to update media server subtitles");
return BaseError.New("Failed to update media server subtitles");
}
}
catch (Exception ex)
{
@ -825,18 +867,24 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters, @@ -825,18 +867,24 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
{
try
{
if (string.IsNullOrEmpty(existing.LocalPath))
using (ScanProfiler.Measure("Update Episode Chapters"))
{
// No local path available for external chapter file lookup
return existing;
}
if (string.IsNullOrEmpty(existing.LocalPath))
{
// No local path available for external chapter file lookup
return existing;
}
if (await _localChaptersProvider.UpdateChapters(existing.Item, Some(existing.LocalPath), cancellationToken))
{
existing.IsUpdated = true;
}
if (await _localChaptersProvider.UpdateChapters(
existing.Item,
Some(existing.LocalPath),
cancellationToken))
{
existing.IsUpdated = true;
}
return existing;
return existing;
}
}
catch (Exception ex)
{

Write Preview
Loading…
Cancel
Save