pimhwang
/
ErsatzTV
mirror of https://github.com/ErsatzTV/ErsatzTV.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

collect and print timings during jellyfin show library scans

pull/2748/head
Jason Dove 7 months ago
parent
474e647d6d
commit
45b573e782
No known key found for this signature in database
4 changed files with 547 additions and 410 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 187
      ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs
  2. 92
      ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs
  3. 58
      ErsatzTV.Infrastructure/ScanProfiler.cs
  4. 620
      ErsatzTV.Scanner/Core/Metadata/MediaServerTelevisionLibraryScanner.cs

187
ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs
Unescape View File

@ -86,40 +86,43 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository
JellyfinShow item, JellyfinShow item,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); using (ScanProfiler.Measure("DB Ins/Upd Show"))
Option<JellyfinShow> maybeExisting = await dbContext.JellyfinShows {
.TagWithCallSite() await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
.Include(m => m.LibraryPath) Option<JellyfinShow> maybeExisting = await dbContext.JellyfinShows
.ThenInclude(lp => lp.Library) .TagWithCallSite()
.Include(m => m.ShowMetadata) .Include(m => m.LibraryPath)
.ThenInclude(mm => mm.Genres) .ThenInclude(lp => lp.Library)
.Include(m => m.ShowMetadata) .Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Tags) .ThenInclude(mm => mm.Genres)
.Include(m => m.ShowMetadata) .Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Studios) .ThenInclude(mm => mm.Tags)
.Include(m => m.ShowMetadata) .Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Actors) .ThenInclude(mm => mm.Studios)
.Include(m => m.ShowMetadata) .Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Artwork) .ThenInclude(mm => mm.Actors)
.Include(m => m.ShowMetadata) .Include(m => m.ShowMetadata)
.ThenInclude(mm => mm.Guids) .ThenInclude(mm => mm.Artwork)
.Include(m => m.TraktListItems) .Include(m => m.ShowMetadata)
.ThenInclude(tli => tli.TraktList) .ThenInclude(mm => mm.Guids)
.SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken); .Include(m => m.TraktListItems)
.ThenInclude(tli => tli.TraktList)
foreach (JellyfinShow jellyfinShow in maybeExisting) .SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken);
{
var result = new MediaItemScanResult<JellyfinShow>(jellyfinShow) { IsAdded = false }; foreach (JellyfinShow jellyfinShow in maybeExisting)
if (jellyfinShow.Etag != item.Etag)
{ {
await UpdateShow(dbContext, jellyfinShow, item, cancellationToken); var result = new MediaItemScanResult<JellyfinShow>(jellyfinShow) { IsAdded = false };
result.IsUpdated = true; if (jellyfinShow.Etag != item.Etag)
{
await UpdateShow(dbContext, jellyfinShow, item, cancellationToken);
result.IsUpdated = true;
}
return result;
} }
return result; return await AddShow(dbContext, library, item, cancellationToken);
} }
return await AddShow(dbContext, library, item, cancellationToken);
} }
public async Task<Either<BaseError, MediaItemScanResult<JellyfinSeason>>> GetOrAdd( public async Task<Either<BaseError, MediaItemScanResult<JellyfinSeason>>> GetOrAdd(
@ -127,29 +130,32 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository
JellyfinSeason item, JellyfinSeason item,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); using (ScanProfiler.Measure("DB Ins/Upd Season"))
Option<JellyfinSeason> maybeExisting = await dbContext.JellyfinSeasons
.TagWithCallSite()
.Include(m => m.LibraryPath)
.Include(m => m.SeasonMetadata)
.ThenInclude(mm => mm.Artwork)
.Include(m => m.SeasonMetadata)
.ThenInclude(mm => mm.Guids)
.SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken);
foreach (JellyfinSeason jellyfinSeason in maybeExisting)
{ {
var result = new MediaItemScanResult<JellyfinSeason>(jellyfinSeason) { IsAdded = false }; await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
if (jellyfinSeason.Etag != item.Etag) Option<JellyfinSeason> maybeExisting = await dbContext.JellyfinSeasons
.TagWithCallSite()
.Include(m => m.LibraryPath)
.Include(m => m.SeasonMetadata)
.ThenInclude(mm => mm.Artwork)
.Include(m => m.SeasonMetadata)
.ThenInclude(mm => mm.Guids)
.SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken);
foreach (JellyfinSeason jellyfinSeason in maybeExisting)
{ {
await UpdateSeason(dbContext, jellyfinSeason, item, cancellationToken); var result = new MediaItemScanResult<JellyfinSeason>(jellyfinSeason) { IsAdded = false };
result.IsUpdated = true; if (jellyfinSeason.Etag != item.Etag)
{
await UpdateSeason(dbContext, jellyfinSeason, item, cancellationToken);
result.IsUpdated = true;
}
return result;
} }
return result; return await AddSeason(dbContext, library, item, cancellationToken);
} }
return await AddSeason(dbContext, library, item, cancellationToken);
} }
public async Task<Either<BaseError, MediaItemScanResult<JellyfinEpisode>>> GetOrAdd( public async Task<Either<BaseError, MediaItemScanResult<JellyfinEpisode>>> GetOrAdd(
@ -158,51 +164,54 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository
bool deepScan, bool deepScan,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); using (ScanProfiler.Measure("DB Ins/Upd Episode"))
Option<JellyfinEpisode> maybeExisting = await dbContext.JellyfinEpisodes {
.TagWithCallSite() await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
.Include(m => m.LibraryPath) Option<JellyfinEpisode> maybeExisting = await dbContext.JellyfinEpisodes
.ThenInclude(lp => lp.Library) .TagWithCallSite()
.Include(m => m.MediaVersions) .Include(m => m.LibraryPath)
.ThenInclude(mv => mv.MediaFiles) .ThenInclude(lp => lp.Library)
.Include(m => m.MediaVersions) .Include(m => m.MediaVersions)
.ThenInclude(mv => mv.Streams) .ThenInclude(mv => mv.MediaFiles)
.Include(m => m.MediaVersions) .Include(m => m.MediaVersions)
.ThenInclude(mv => mv.Chapters) .ThenInclude(mv => mv.Streams)
.Include(m => m.EpisodeMetadata) .Include(m => m.MediaVersions)
.ThenInclude(mm => mm.Artwork) .ThenInclude(mv => mv.Chapters)
.Include(m => m.EpisodeMetadata) .Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Guids) .ThenInclude(mm => mm.Artwork)
.Include(m => m.EpisodeMetadata) .Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Genres) .ThenInclude(mm => mm.Guids)
.Include(m => m.EpisodeMetadata) .Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Tags) .ThenInclude(mm => mm.Genres)
.Include(m => m.EpisodeMetadata) .Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Studios) .ThenInclude(mm => mm.Tags)
.Include(m => m.EpisodeMetadata) .Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Actors) .ThenInclude(mm => mm.Studios)
.Include(m => m.EpisodeMetadata) .Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Directors) .ThenInclude(mm => mm.Actors)
.Include(m => m.EpisodeMetadata) .Include(m => m.EpisodeMetadata)
.ThenInclude(mm => mm.Writers) .ThenInclude(mm => mm.Directors)
.Include(m => m.Season) .Include(m => m.EpisodeMetadata)
.Include(m => m.TraktListItems) .ThenInclude(mm => mm.Writers)
.ThenInclude(tli => tli.TraktList) .Include(m => m.Season)
.SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken); .Include(m => m.TraktListItems)
.ThenInclude(tli => tli.TraktList)
foreach (JellyfinEpisode jellyfinEpisode in maybeExisting) .SelectOneAsync(s => s.ItemId, s => s.ItemId == item.ItemId, cancellationToken);
{
var result = new MediaItemScanResult<JellyfinEpisode>(jellyfinEpisode) { IsAdded = false }; foreach (JellyfinEpisode jellyfinEpisode in maybeExisting)
if (jellyfinEpisode.Etag != item.Etag || deepScan)
{ {
await UpdateEpisode(dbContext, jellyfinEpisode, item, cancellationToken); var result = new MediaItemScanResult<JellyfinEpisode>(jellyfinEpisode) { IsAdded = false };
result.IsUpdated = true; if (jellyfinEpisode.Etag != item.Etag || deepScan)
{
await UpdateEpisode(dbContext, jellyfinEpisode, item, cancellationToken);
result.IsUpdated = true;
}
return result;
} }
return result; return await AddEpisode(dbContext, library, item, cancellationToken);
} }
return await AddEpisode(dbContext, library, item, cancellationToken);
} }
public async Task<Unit> SetEtag(JellyfinShow show, string etag, CancellationToken cancellationToken) public async Task<Unit> SetEtag(JellyfinShow show, string etag, CancellationToken cancellationToken)

92
ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs
Unescape View File

@ -81,6 +81,7 @@ public class JellyfinApiClient : IJellyfinApiClient
string apiKey, string apiKey,
JellyfinLibrary library) => JellyfinLibrary library) =>
GetPagedLibraryItems( GetPagedLibraryItems(
"JF Movies",
address, address,
library, library,
library.MediaSourceId, library.MediaSourceId,
@ -97,6 +98,7 @@ public class JellyfinApiClient : IJellyfinApiClient
string apiKey, string apiKey,
JellyfinLibrary library) => JellyfinLibrary library) =>
GetPagedLibraryItems( GetPagedLibraryItems(
"JF Shows",
address, address,
library, library,
library.MediaSourceId, library.MediaSourceId,
@ -114,6 +116,7 @@ public class JellyfinApiClient : IJellyfinApiClient
JellyfinLibrary library, JellyfinLibrary library,
string showId) => string showId) =>
GetPagedLibraryItems( GetPagedLibraryItems(
"JF Seasons",
address, address,
library, library,
library.MediaSourceId, library.MediaSourceId,
@ -131,6 +134,7 @@ public class JellyfinApiClient : IJellyfinApiClient
JellyfinLibrary library, JellyfinLibrary library,
string seasonId) => string seasonId) =>
GetPagedLibraryItems( GetPagedLibraryItems(
"JF Episodes*",
address, address,
library, library,
library.MediaSourceId, library.MediaSourceId,
@ -148,6 +152,7 @@ public class JellyfinApiClient : IJellyfinApiClient
JellyfinLibrary library, JellyfinLibrary library,
string seasonId) => string seasonId) =>
GetPagedLibraryItems( GetPagedLibraryItems(
"JF Episodes",
address, address,
library, library,
library.MediaSourceId, library.MediaSourceId,
@ -169,6 +174,7 @@ public class JellyfinApiClient : IJellyfinApiClient
if (_memoryCache.TryGetValue("jellyfin_collections_library_item_id", out string itemId)) if (_memoryCache.TryGetValue("jellyfin_collections_library_item_id", out string itemId))
{ {
return GetPagedLibraryItems( return GetPagedLibraryItems(
"JF Collections",
address, address,
None, None,
mediaSourceId, mediaSourceId,
@ -190,6 +196,7 @@ public class JellyfinApiClient : IJellyfinApiClient
int mediaSourceId, int mediaSourceId,
string collectionId) => string collectionId) =>
GetPagedLibraryItems( GetPagedLibraryItems(
"JF Collection Items",
address, address,
None, None,
mediaSourceId, mediaSourceId,
@ -209,10 +216,13 @@ public class JellyfinApiClient : IJellyfinApiClient
{ {
try try
{ {
IJellyfinApi service = ServiceForAddress(address); using (ScanProfiler.Measure("JF Playback Info"))
JellyfinPlaybackInfoResponse playbackInfo = await service.GetPlaybackInfo(apiKey, itemId); {
Option<MediaVersion> maybeVersion = ProjectToMediaVersion(playbackInfo); IJellyfinApi service = ServiceForAddress(address);
return maybeVersion.ToEither(() => BaseError.New("Unable to locate Jellyfin statistics")); JellyfinPlaybackInfoResponse playbackInfo = await service.GetPlaybackInfo(apiKey, itemId);
Option<MediaVersion> maybeVersion = ProjectToMediaVersion(playbackInfo);
return maybeVersion.ToEither(() => BaseError.New("Unable to locate Jellyfin statistics"));
}
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -229,21 +239,24 @@ public class JellyfinApiClient : IJellyfinApiClient
{ {
try try
{ {
IJellyfinApi service = ServiceForAddress(address); using (ScanProfiler.Measure("JF Single Show"))
JellyfinLibraryItemsResponse itemsResponse = await service.GetShowLibraryItems(
apiKey,
parentId: library.ItemId,
recursive: false,
startIndex: 0,
limit: 1,
ids: showId);
foreach (JellyfinLibraryItemResponse item in itemsResponse.Items)
{ {
return ProjectToShow(item); IJellyfinApi service = ServiceForAddress(address);
} JellyfinLibraryItemsResponse itemsResponse = await service.GetShowLibraryItems(
apiKey,
parentId: library.ItemId,
recursive: false,
startIndex: 0,
limit: 1,
ids: showId);
foreach (JellyfinLibraryItemResponse item in itemsResponse.Items)
{
return ProjectToShow(item);
}
return BaseError.New($"Unable to locate show with id {showId}"); return BaseError.New($"Unable to locate show with id {showId}");
}
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -310,21 +323,24 @@ public class JellyfinApiClient : IJellyfinApiClient
{ {
try try
{ {
IJellyfinApi service = ServiceForAddress(address); using (ScanProfiler.Measure("JF Single Episode"))
JellyfinLibraryItemsResponse itemsResponse = await service.GetEpisodeLibraryItems(
apiKey,
parentId: seasonId,
recursive: false,
startIndex: 0,
limit: 1,
ids: episodeId);
foreach (JellyfinLibraryItemResponse item in itemsResponse.Items)
{ {
return ProjectToEpisode(library, item); IJellyfinApi service = ServiceForAddress(address);
} JellyfinLibraryItemsResponse itemsResponse = await service.GetEpisodeLibraryItems(
apiKey,
parentId: seasonId,
recursive: false,
startIndex: 0,
limit: 1,
ids: episodeId);
foreach (JellyfinLibraryItemResponse item in itemsResponse.Items)
{
return ProjectToEpisode(library, item);
}
return BaseError.New($"Unable to locate episode with id {episodeId}"); return BaseError.New($"Unable to locate episode with id {episodeId}");
}
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -334,6 +350,7 @@ public class JellyfinApiClient : IJellyfinApiClient
} }
private async IAsyncEnumerable<Tuple<TItem, int>> GetPagedLibraryItems<TItem>( private async IAsyncEnumerable<Tuple<TItem, int>> GetPagedLibraryItems<TItem>(
string pageDescription,
string address, string address,
Option<JellyfinLibrary> maybeLibrary, Option<JellyfinLibrary> maybeLibrary,
int mediaSourceId, int mediaSourceId,
@ -348,11 +365,16 @@ public class JellyfinApiClient : IJellyfinApiClient
{ {
int skip = i * SystemEnvironment.JellyfinPageSize; int skip = i * SystemEnvironment.JellyfinPageSize;
JellyfinLibraryItemsResponse result = await getItems( JellyfinLibraryItemsResponse result;
service,
parentId, using (ScanProfiler.Measure(pageDescription))
skip, {
SystemEnvironment.JellyfinPageSize); result = await getItems(
service,
parentId,
skip,
SystemEnvironment.JellyfinPageSize);
}
// update page count // update page count
pages = Math.Min(pages, (result.TotalRecordCount - 1) / SystemEnvironment.JellyfinPageSize + 1); pages = Math.Min(pages, (result.TotalRecordCount - 1) / SystemEnvironment.JellyfinPageSize + 1);

58
ErsatzTV.Infrastructure/ScanProfiler.cs
Unescape View File

@ -0,0 +1,58 @@
using System.Collections.Concurrent;
using System.Diagnostics;
namespace ErsatzTV.Infrastructure;
public static class ScanProfiler
{
private static readonly ConcurrentDictionary<string, ConcurrentBag<long>> Measurements = new();
public static IDisposable Measure(string operationName)
{
return new TimerToken(operationName);
}
public static void Reset() => Measurements.Clear();
public static void LogStatistics(Action<string> logAction)
{
if (Measurements.IsEmpty)
{
return;
}
var sb = new System.Text.StringBuilder();
sb.AppendLine("Scan Performance Summary:");
sb.AppendLine(FormattableString.Invariant($"{"Operation",-25} | {"Count",-6} | {"Avg (ms)",-8} | {"Min",-6} | {"Max",-6} | {"P99",-6} | {"Total (s)",-8}"));
sb.AppendLine(new string('-', 85));
foreach (string key in Measurements.Keys.OrderBy(k => k))
{
var times = Measurements[key].ToList();
if (times.Count == 0) continue;
times.Sort();
double avg = times.Average();
long min = times.Min();
long max = times.Max();
long p99 = times[(int)(times.Count * 0.99)];
double totalSec = times.Sum() / 1000.0;
sb.AppendLine(FormattableString.Invariant($"{key,-25} | {times.Count,-6} | {avg,-8:F1} | {min,-6} | {max,-6} | {p99,-6} | {totalSec,-8:F2}"));
}
logAction(sb.ToString());
}
private readonly struct TimerToken(string name) : IDisposable
{
private readonly long _startTime = Stopwatch.GetTimestamp();
public void Dispose()
{
long elapsedMs = (long)Stopwatch.GetElapsedTime(_startTime).TotalMilliseconds;
ConcurrentBag<long> bag = Measurements.GetOrAdd(name, _ => []);
bag.Add(elapsedMs);
}
}
}

620
ErsatzTV.Scanner/Core/Metadata/MediaServerTelevisionLibraryScanner.cs
Unescape View File

@ -6,6 +6,7 @@ using ErsatzTV.Core.Errors;
using ErsatzTV.Core.Extensions; using ErsatzTV.Core.Extensions;
using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Core.Metadata; using ErsatzTV.Core.Metadata;
using ErsatzTV.Infrastructure;
using ErsatzTV.Scanner.Core.Interfaces; using ErsatzTV.Scanner.Core.Interfaces;
using ErsatzTV.Scanner.Core.Interfaces.Metadata; using ErsatzTV.Scanner.Core.Interfaces.Metadata;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
@ -95,70 +96,78 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
await foreach ((TShow incoming, int totalShowCount) in showEntries.WithCancellation(cancellationToken)) await foreach ((TShow incoming, int totalShowCount) in showEntries.WithCancellation(cancellationToken))
{ {
if (cancellationToken.IsCancellationRequested) using (ScanProfiler.Measure("Scan Show"))
{ {
return new ScanCanceled(); if (cancellationToken.IsCancellationRequested)
}
incomingItemIds.Add(MediaServerItemId(incoming));
decimal percentCompletion = Math.Clamp((decimal)incomingItemIds.Count / totalShowCount, 0, 1);
if (!await _scannerProxy.UpdateProgress(percentCompletion, cancellationToken))
{
return new ScanCanceled();
}
Either<BaseError, MediaItemScanResult<TShow>> maybeShow = await televisionRepository
.GetOrAdd(library, incoming, cancellationToken)
.BindT(existing => UpdateMetadata(connectionParameters, library, existing, incoming, deepScan));
if (maybeShow.IsLeft)
{
foreach (BaseError error in maybeShow.LeftToSeq())
{ {
_logger.LogWarning( return new ScanCanceled();
"Error processing show {Title}: {Error}",
incoming.ShowMetadata.Head().Title,
error.Value);
} }
continue; incomingItemIds.Add(MediaServerItemId(incoming));
}
foreach (MediaItemScanResult<TShow> result in maybeShow.RightToSeq()) decimal percentCompletion = Math.Clamp((decimal)incomingItemIds.Count / totalShowCount, 0, 1);
{ if (!await _scannerProxy.UpdateProgress(percentCompletion, cancellationToken))
Either<BaseError, Unit> scanResult = await ScanSeasons(
televisionRepository,
library,
getLocalPath,
result.Item,
result.IsUpdated,
connectionParameters,
GetSeasonLibraryItems(library, connectionParameters, result.Item),
deepScan,
cancellationToken);
foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
{ {
return error; return new ScanCanceled();
} }
await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken); Either<BaseError, MediaItemScanResult<TShow>> maybeShow = await televisionRepository
.GetOrAdd(library, incoming, cancellationToken)
.BindT(existing => UpdateMetadata(connectionParameters, library, existing, incoming, deepScan));
Option<int> flagResult = await televisionRepository.FlagNormal(library, result.Item, cancellationToken); if (maybeShow.IsLeft)
if (flagResult.IsSome)
{ {
result.IsUpdated = true; foreach (BaseError error in maybeShow.LeftToSeq())
{
_logger.LogWarning(
"Error processing show {Title}: {Error}",
incoming.ShowMetadata.Head().Title,
error.Value);
}
continue;
} }
if (result.IsAdded || result.IsUpdated) foreach (MediaItemScanResult<TShow> result in maybeShow.RightToSeq())
{ {
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken)) Either<BaseError, Unit> scanResult = await ScanSeasons(
televisionRepository,
library,
getLocalPath,
result.Item,
result.IsUpdated,
connectionParameters,
GetSeasonLibraryItems(library, connectionParameters, result.Item),
deepScan,
cancellationToken);
foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
{ {
_logger.LogWarning("Failed to reindex media items from scanner process"); return error;
}
await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken);
Option<int> flagResult = await televisionRepository.FlagNormal(
library,
result.Item,
cancellationToken);
if (flagResult.IsSome)
{
result.IsUpdated = true;
}
if (result.IsAdded || result.IsUpdated)
{
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken))
{
_logger.LogWarning("Failed to reindex media items from scanner process");
}
} }
} }
} }
ScanProfiler.LogStatistics(s => _logger.LogInformation("{Profile}", s));
} }
if (cleanupFileNotFoundItems) if (cleanupFileNotFoundItems)
@ -285,67 +294,73 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
await foreach ((TSeason incoming, int _) in seasonEntries.WithCancellation(cancellationToken)) await foreach ((TSeason incoming, int _) in seasonEntries.WithCancellation(cancellationToken))
{ {
incoming.ShowId = show.Id; using (ScanProfiler.Measure("Scan Season"))
if (cancellationToken.IsCancellationRequested)
{ {
return new ScanCanceled(); incoming.ShowId = show.Id;
}
incomingItemIds.Add(MediaServerItemId(incoming));
Either<BaseError, MediaItemScanResult<TSeason>> maybeSeason = await televisionRepository
.GetOrAdd(library, incoming, cancellationToken)
.BindT(existing => UpdateMetadata(connectionParameters, library, existing, incoming, deepScan));
if (maybeSeason.IsLeft) if (cancellationToken.IsCancellationRequested)
{
foreach (BaseError error in maybeSeason.LeftToSeq())
{ {
_logger.LogWarning( return new ScanCanceled();
"Error processing show {Title} season {SeasonNumber}: {Error}",
show.ShowMetadata.Head().Title,
incoming.SeasonNumber,
error.Value);
} }
continue; incomingItemIds.Add(MediaServerItemId(incoming));
}
foreach (MediaItemScanResult<TSeason> result in maybeSeason.RightToSeq()) Either<BaseError, MediaItemScanResult<TSeason>> maybeSeason = await televisionRepository
{ .GetOrAdd(library, incoming, cancellationToken)
Either<BaseError, Unit> scanResult = await ScanEpisodes( .BindT(existing => UpdateMetadata(connectionParameters, library, existing, incoming, deepScan));
televisionRepository,
library, if (maybeSeason.IsLeft)
getLocalPath,
show,
showIsUpdated,
result.Item,
connectionParameters,
GetEpisodeLibraryItems(library, connectionParameters, show, result.Item, result.IsAdded),
deepScan,
cancellationToken);
foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
{ {
return error; foreach (BaseError error in maybeSeason.LeftToSeq())
} {
_logger.LogWarning(
"Error processing show {Title} season {SeasonNumber}: {Error}",
show.ShowMetadata.Head().Title,
incoming.SeasonNumber,
error.Value);
}
await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken); continue;
}
Option<int> flagResult = await televisionRepository.FlagNormal(library, result.Item, cancellationToken); foreach (MediaItemScanResult<TSeason> result in maybeSeason.RightToSeq())
if (flagResult.IsSome)
{ {
result.IsUpdated = true; Either<BaseError, Unit> scanResult = await ScanEpisodes(
} televisionRepository,
library,
getLocalPath,
show,
showIsUpdated,
result.Item,
connectionParameters,
GetEpisodeLibraryItems(library, connectionParameters, show, result.Item, result.IsAdded),
deepScan,
cancellationToken);
result.Item.Show = show; foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
{
return error;
}
if (result.IsAdded || result.IsUpdated || showIsUpdated) await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken);
{
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken)) Option<int> flagResult = await televisionRepository.FlagNormal(
library,
result.Item,
cancellationToken);
if (flagResult.IsSome)
{ {
_logger.LogWarning("Failed to reindex media items from scanner process"); result.IsUpdated = true;
}
result.Item.Show = show;
if (result.IsAdded || result.IsUpdated || showIsUpdated)
{
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken))
{
_logger.LogWarning("Failed to reindex media items from scanner process");
}
} }
} }
} }
@ -379,129 +394,141 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
await foreach ((TEpisode incoming, int _) in episodeEntries.WithCancellation(cancellationToken)) await foreach ((TEpisode incoming, int _) in episodeEntries.WithCancellation(cancellationToken))
{ {
if (cancellationToken.IsCancellationRequested) using (ScanProfiler.Measure("Scan Episode"))
{
return new ScanCanceled();
}
incomingItemIds.Add(MediaServerItemId(incoming));
string localPath = getLocalPath(incoming);
if (!await ShouldScanItem(
televisionRepository,
library,
show,
season,
existingEpisodes,
incoming,
localPath,
deepScan,
cancellationToken))
{ {
continue; if (cancellationToken.IsCancellationRequested)
} {
return new ScanCanceled();
incoming.SeasonId = season.Id; }
Either<BaseError, MediaItemScanResult<TEpisode>> maybeEpisode; incomingItemIds.Add(MediaServerItemId(incoming));
if (ServerReturnsStatisticsWithMetadata) string localPath = getLocalPath(incoming);
{ if (!await ShouldScanItem(
maybeEpisode = await televisionRepository televisionRepository,
.GetOrAdd(library, incoming, deepScan, cancellationToken)
.MapT(result =>
{
result.LocalPath = localPath;
return result;
})
.BindT(existing => UpdateMetadataAndStatistics(
connectionParameters,
library,
existing,
incoming,
deepScan,
cancellationToken))
.BindT(existing => UpdateChapters(existing, cancellationToken));
}
else
{
maybeEpisode = await televisionRepository
.GetOrAdd(library, incoming, deepScan, cancellationToken)
.MapT(result =>
{
result.LocalPath = localPath;
return result;
})
.BindT(existing => UpdateMetadata(
connectionParameters,
library, library,
existing, show,
season,
existingEpisodes,
incoming, incoming,
localPath,
deepScan, deepScan,
None,
cancellationToken)) cancellationToken))
.BindT(existing => UpdateStatistics(
connectionParameters,
library,
existing,
incoming,
deepScan,
None))
.BindT(existing => UpdateSubtitles(existing, cancellationToken))
.BindT(existing => UpdateChapters(existing, cancellationToken));
}
if (maybeEpisode.IsLeft)
{
foreach (BaseError error in maybeEpisode.LeftToSeq())
{ {
_logger.LogWarning( continue;
"Error processing episode {Title} s{SeasonNumber:00}e{EpisodeNumber:00}: {Error}",
show.ShowMetadata.Head().Title,
season.SeasonNumber,
incoming.EpisodeMetadata.Head().EpisodeNumber,
error.Value);
} }
continue; incoming.SeasonId = season.Id;
}
foreach (MediaItemScanResult<TEpisode> result in maybeEpisode.RightToSeq()) Either<BaseError, MediaItemScanResult<TEpisode>> maybeEpisode;
{
await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken);
if (_fileSystem.File.Exists(result.LocalPath)) if (ServerReturnsStatisticsWithMetadata)
{ {
Option<int> flagResult = await televisionRepository.FlagNormal(library, result.Item, cancellationToken); maybeEpisode = await televisionRepository
if (flagResult.IsSome) .GetOrAdd(library, incoming, deepScan, cancellationToken)
{ .MapT(result =>
result.IsUpdated = true; {
} result.LocalPath = localPath;
return result;
})
.BindT(existing => UpdateMetadataAndStatistics(
connectionParameters,
library,
existing,
incoming,
deepScan,
cancellationToken))
.BindT(existing => UpdateChapters(existing, cancellationToken));
} }
else if (ServerSupportsRemoteStreaming) else
{ {
Option<int> flagResult = await televisionRepository.FlagRemoteOnly(library, result.Item, cancellationToken); maybeEpisode = await televisionRepository
if (flagResult.IsSome) .GetOrAdd(library, incoming, deepScan, cancellationToken)
{ .MapT(result =>
result.IsUpdated = true; {
} result.LocalPath = localPath;
return result;
})
.BindT(existing => UpdateMetadata(
connectionParameters,
library,
existing,
incoming,
deepScan,
None,
cancellationToken))
.BindT(existing => UpdateStatistics(
connectionParameters,
library,
existing,
incoming,
deepScan,
None))
.BindT(existing => UpdateSubtitles(existing, cancellationToken))
.BindT(existing => UpdateChapters(existing, cancellationToken));
} }
else
if (maybeEpisode.IsLeft)
{ {
Option<int> flagResult = await televisionRepository.FlagUnavailable(library, result.Item, cancellationToken); foreach (BaseError error in maybeEpisode.LeftToSeq())
if (flagResult.IsSome)
{ {
result.IsUpdated = true; _logger.LogWarning(
"Error processing episode {Title} s{SeasonNumber:00}e{EpisodeNumber:00}: {Error}",
show.ShowMetadata.Head().Title,
season.SeasonNumber,
incoming.EpisodeMetadata.Head().EpisodeNumber,
error.Value);
} }
continue;
} }
if (result.IsAdded || result.IsUpdated || showIsUpdated) foreach (MediaItemScanResult<TEpisode> result in maybeEpisode.RightToSeq())
{ {
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken)) await televisionRepository.SetEtag(result.Item, MediaServerEtag(incoming), cancellationToken);
if (_fileSystem.File.Exists(result.LocalPath))
{ {
_logger.LogWarning("Failed to reindex media items from scanner process"); Option<int> flagResult = await televisionRepository.FlagNormal(
library,
result.Item,
cancellationToken);
if (flagResult.IsSome)
{
result.IsUpdated = true;
}
}
else if (ServerSupportsRemoteStreaming)
{
Option<int> flagResult = await televisionRepository.FlagRemoteOnly(
library,
result.Item,
cancellationToken);
if (flagResult.IsSome)
{
result.IsUpdated = true;
}
}
else
{
Option<int> flagResult = await televisionRepository.FlagUnavailable(
library,
result.Item,
cancellationToken);
if (flagResult.IsSome)
{
result.IsUpdated = true;
}
} }
if (result.IsAdded || result.IsUpdated || showIsUpdated)
{
if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken))
{
_logger.LogWarning("Failed to reindex media items from scanner process");
}
}
} }
} }
} }
@ -611,16 +638,19 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
TShow incoming, TShow incoming,
bool deepScan) bool deepScan)
{ {
foreach (ShowMetadata fullMetadata in await GetFullMetadata( using (ScanProfiler.Measure("Update Show Metadata"))
connectionParameters,
library,
result,
incoming,
deepScan))
{ {
// TODO: move some of this code into this scanner foreach (ShowMetadata fullMetadata in await GetFullMetadata(
// will have to merge JF, Emby, Plex logic connectionParameters,
return await UpdateMetadata(result, fullMetadata); library,
result,
incoming,
deepScan))
{
// TODO: move some of this code into this scanner
// will have to merge JF, Emby, Plex logic
return await UpdateMetadata(result, fullMetadata);
}
} }
return result; return result;
@ -633,16 +663,19 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
TSeason incoming, TSeason incoming,
bool deepScan) bool deepScan)
{ {
foreach (SeasonMetadata fullMetadata in await GetFullMetadata( using (ScanProfiler.Measure("Update Season Metadata"))
connectionParameters,
library,
result,
incoming,
deepScan))
{ {
// TODO: move some of this code into this scanner foreach (SeasonMetadata fullMetadata in await GetFullMetadata(
// will have to merge JF, Emby, Plex logic connectionParameters,
return await UpdateMetadata(result, fullMetadata); library,
result,
incoming,
deepScan))
{
// TODO: move some of this code into this scanner
// will have to merge JF, Emby, Plex logic
return await UpdateMetadata(result, fullMetadata);
}
} }
return result; return result;
@ -714,19 +747,22 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
Option<EpisodeMetadata> maybeFullMetadata, Option<EpisodeMetadata> maybeFullMetadata,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
if (maybeFullMetadata.IsNone) using (ScanProfiler.Measure("Update Episode Metadata"))
{ {
maybeFullMetadata = await GetFullMetadata(connectionParameters, library, result, incoming, deepScan); if (maybeFullMetadata.IsNone)
} {
maybeFullMetadata = await GetFullMetadata(connectionParameters, library, result, incoming, deepScan);
}
foreach (EpisodeMetadata fullMetadata in maybeFullMetadata) foreach (EpisodeMetadata fullMetadata in maybeFullMetadata)
{ {
// TODO: move some of this code into this scanner // TODO: move some of this code into this scanner
// will have to merge JF, Emby, Plex logic // will have to merge JF, Emby, Plex logic
return await UpdateMetadata(result, fullMetadata, cancellationToken); return await UpdateMetadata(result, fullMetadata, cancellationToken);
} }
return result; return result;
}
} }
private async Task<Either<BaseError, MediaItemScanResult<TEpisode>>> UpdateStatistics( private async Task<Either<BaseError, MediaItemScanResult<TEpisode>>> UpdateStatistics(
@ -737,57 +773,60 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
bool deepScan, bool deepScan,
Option<MediaVersion> maybeMediaVersion) Option<MediaVersion> maybeMediaVersion)
{ {
TEpisode existing = result.Item; using (ScanProfiler.Measure("Update Episode Stats"))
if (deepScan || result.IsAdded || MediaServerEtag(existing) != MediaServerEtag(incoming) ||
existing.MediaVersions.Head().Streams.Count == 0)
{ {
// if (maybeMediaVersion.IsNone && _fileSystem.File.Exists(result.LocalPath)) TEpisode existing = result.Item;
// {
// _logger.LogDebug("Refreshing {Attribute} for {Path}", "Statistics", result.LocalPath);
// Either<BaseError, bool> refreshResult =
// await _localStatisticsProvider.RefreshStatistics(
// ffmpegPath,
// ffprobePath,
// existing,
// result.LocalPath);
//
// foreach (BaseError error in refreshResult.LeftToSeq())
// {
// _logger.LogWarning(
// "Unable to refresh {Attribute} for media item {Path}. Error: {Error}",
// "Statistics",
// result.LocalPath,
// error.Value);
// }
//
// foreach (bool _ in refreshResult.RightToSeq())
// {
// result.IsUpdated = true;
// }
// }
// else
// {
if (maybeMediaVersion.IsNone)
{
maybeMediaVersion = await GetMediaServerStatistics(
connectionParameters,
library,
result,
incoming);
}
foreach (MediaVersion mediaVersion in maybeMediaVersion) if (deepScan || result.IsAdded || MediaServerEtag(existing) != MediaServerEtag(incoming) ||
existing.MediaVersions.Head().Streams.Count == 0)
{ {
if (await _metadataRepository.UpdateStatistics(result.Item, mediaVersion)) // if (maybeMediaVersion.IsNone && _fileSystem.File.Exists(result.LocalPath))
// {
// _logger.LogDebug("Refreshing {Attribute} for {Path}", "Statistics", result.LocalPath);
// Either<BaseError, bool> refreshResult =
// await _localStatisticsProvider.RefreshStatistics(
// ffmpegPath,
// ffprobePath,
// existing,
// result.LocalPath);
//
// foreach (BaseError error in refreshResult.LeftToSeq())
// {
// _logger.LogWarning(
// "Unable to refresh {Attribute} for media item {Path}. Error: {Error}",
// "Statistics",
// result.LocalPath,
// error.Value);
// }
//
// foreach (bool _ in refreshResult.RightToSeq())
// {
// result.IsUpdated = true;
// }
// }
// else
// {
if (maybeMediaVersion.IsNone)
{ {
result.IsUpdated = true; maybeMediaVersion = await GetMediaServerStatistics(
connectionParameters,
library,
result,
incoming);
}
foreach (MediaVersion mediaVersion in maybeMediaVersion)
{
if (await _metadataRepository.UpdateStatistics(result.Item, mediaVersion))
{
result.IsUpdated = true;
}
} }
// }
} }
// }
}
return result; return result;
}
} }
private async Task<Either<BaseError, MediaItemScanResult<TEpisode>>> UpdateSubtitles( private async Task<Either<BaseError, MediaItemScanResult<TEpisode>>> UpdateSubtitles(
@ -796,22 +835,25 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
{ {
try try
{ {
MediaVersion version = existing.Item.GetHeadVersion(); using (ScanProfiler.Measure("Update Episode Subtitles"))
Option<EpisodeMetadata> maybeMetadata = existing.Item.EpisodeMetadata.HeadOrNone();
foreach (EpisodeMetadata metadata in maybeMetadata)
{ {
List<Subtitle> subtitles = version.Streams MediaVersion version = existing.Item.GetHeadVersion();
.Filter(s => s.MediaStreamKind is MediaStreamKind.Subtitle or MediaStreamKind.ExternalSubtitle) Option<EpisodeMetadata> maybeMetadata = existing.Item.EpisodeMetadata.HeadOrNone();
.Map(Subtitle.FromMediaStream) foreach (EpisodeMetadata metadata in maybeMetadata)
.ToList();
if (await _metadataRepository.UpdateSubtitles(metadata, subtitles, cancellationToken))
{ {
return existing; List<Subtitle> subtitles = version.Streams
.Filter(s => s.MediaStreamKind is MediaStreamKind.Subtitle or MediaStreamKind.ExternalSubtitle)
.Map(Subtitle.FromMediaStream)
.ToList();
if (await _metadataRepository.UpdateSubtitles(metadata, subtitles, cancellationToken))
{
return existing;
}
} }
}
return BaseError.New("Failed to update media server subtitles"); return BaseError.New("Failed to update media server subtitles");
}
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -825,18 +867,24 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
{ {
try try
{ {
if (string.IsNullOrEmpty(existing.LocalPath)) using (ScanProfiler.Measure("Update Episode Chapters"))
{ {
// No local path available for external chapter file lookup if (string.IsNullOrEmpty(existing.LocalPath))
return existing; {
} // No local path available for external chapter file lookup
return existing;
}
if (await _localChaptersProvider.UpdateChapters(existing.Item, Some(existing.LocalPath), cancellationToken)) if (await _localChaptersProvider.UpdateChapters(
{ existing.Item,
existing.IsUpdated = true; Some(existing.LocalPath),
} cancellationToken))
{
existing.IsUpdated = true;
}
return existing; return existing;
}
} }
catch (Exception ex) catch (Exception ex)
{ {

Write Preview
Loading…
Cancel
Save