add paging to media server show and collection calls (#827)

* add paging to media server show library calls

* add paging to media server season and episode library calls

* formatting

* add paging to media server collection calls

* add paging to media server collection item calls

* update changelog

CHANGELOG.md

@@ -13,7 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Enable QSV hardware acceleration for vaapi docker images
 
 ### Changed
-- Use paging to synchronize movies from Plex, Jellyfin and Emby
+- Use paging to synchronize all media from Plex, Jellyfin and Emby
   - This will reduce memory use and improve reliability of synchronizing large libraries
 - Disable low power mode for `h264_qsv` and `hevc_qsv` encoders
 

ErsatzTV.Application/Channels/Queries/GetChannelGuideHandler.cs

@@ -9,7 +9,9 @@ public class GetChannelGuideHandler : IRequestHandler<GetChannelGuide, ChannelGu
     private readonly IChannelRepository _channelRepository;
     private readonly RecyclableMemoryStreamManager _recyclableMemoryStreamManager;
 
-    public GetChannelGuideHandler(IChannelRepository channelRepository, RecyclableMemoryStreamManager recyclableMemoryStreamManager)
+    public GetChannelGuideHandler(
+        IChannelRepository channelRepository,
+        RecyclableMemoryStreamManager recyclableMemoryStreamManager)
     {
         _channelRepository = channelRepository;
         _recyclableMemoryStreamManager = recyclableMemoryStreamManager;

ErsatzTV.Application/Streaming/HlsSessionWorker.cs

@@ -381,7 +381,7 @@ public class HlsSessionWorker : IHlsSessionWorker
         try
         {
             long result = 0;
-            
+
             // the first process always starts at zero
             if (firstProcess)
             {

ErsatzTV.Core/Emby/EmbyCollectionScanner.cs

@@ -30,28 +30,22 @@ public class EmbyCollectionScanner : IEmbyCollectionScanner
 
     public async Task<Either<BaseError, Unit>> ScanCollections(string address, string apiKey)
     {
-        // get all collections from db (item id, etag)
+        try
-        List<EmbyCollection> existingCollections = await _embyCollectionRepository.GetCollections();
-
-        // get all collections from emby
-        Either<BaseError, List<EmbyCollection>> maybeIncomingCollections =
-            await _embyApiClient.GetCollectionLibraryItems(address, apiKey);
-
-        foreach (BaseError error in maybeIncomingCollections.LeftToSeq())
         {
-            _logger.LogWarning("Failed to get collections from Emby: {Error}", error.ToString());
+            var incomingItemIds = new List<string>();
-            return error;
-        }
 
-        foreach (List<EmbyCollection> incomingCollections in maybeIncomingCollections.RightToSeq())
+            // get all collections from db (item id, etag)
-        {
+            List<EmbyCollection> existingCollections = await _embyCollectionRepository.GetCollections();
-            // loop over collections
+
-            foreach (EmbyCollection collection in incomingCollections)
+            await foreach (EmbyCollection collection in _embyApiClient.GetCollectionLibraryItems(address, apiKey))
             {
+                incomingItemIds.Add(collection.ItemId);
+
                 Option<EmbyCollection> maybeExisting = existingCollections.Find(c => c.ItemId == collection.ItemId);
 
                 // skip if unchanged (etag)
-                if (await maybeExisting.Map(e => e.Etag ?? string.Empty).IfNoneAsync(string.Empty) == collection.Etag)
+                if (await maybeExisting.Map(e => e.Etag ?? string.Empty).IfNoneAsync(string.Empty) ==
+                    collection.Etag)
                 {
                     _logger.LogDebug("Emby collection {Name} is unchanged", collection.Name);
                     continue;
@@ -75,12 +69,16 @@ public class EmbyCollectionScanner : IEmbyCollectionScanner
             }
 
             // remove missing collections (and remove any lingering tags from those collections)
-            foreach (EmbyCollection collection in existingCollections
+            foreach (EmbyCollection collection in existingCollections.Filter(e => !incomingItemIds.Contains(e.ItemId)))
-                         .Filter(e => incomingCollections.All(i => i.ItemId != e.ItemId)))
             {
                 await _embyCollectionRepository.RemoveCollection(collection);
             }
         }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to get collections from Emby");
+            return BaseError.New(ex.Message);
+        }
 
         return Unit.Default;
     }
@@ -90,32 +88,31 @@ public class EmbyCollectionScanner : IEmbyCollectionScanner
         string apiKey,
         EmbyCollection collection)
     {
-        // get collection items from JF
+        try
-        Either<BaseError, List<MediaItem>> maybeItems =
-            await _embyApiClient.GetCollectionItems(address, apiKey, collection.ItemId);
-
-        foreach (BaseError error in maybeItems.LeftToSeq())
         {
-            _logger.LogWarning("Failed to get collection items from Emby: {Error}", error.ToString());
+            // get collection items from Emby
-            return;
+            IAsyncEnumerable<MediaItem> items = _embyApiClient.GetCollectionItems(address, apiKey, collection.ItemId);
-        }
 
-        List<int> removedIds = await _embyCollectionRepository.RemoveAllTags(collection);
+            List<int> removedIds = await _embyCollectionRepository.RemoveAllTags(collection);
 
-        var embyItems = maybeItems.RightToSeq().Flatten().ToList();
+            // sync tags on items
-        _logger.LogDebug("Emby collection {Name} contains {Count} items", collection.Name, embyItems.Count);
+            var addedIds = new List<int>();
+            await foreach (MediaItem item in items)
+            {
+                addedIds.Add(await _embyCollectionRepository.AddTag(item, collection));
+            }
 
-        // sync tags on items
+            _logger.LogDebug("Emby collection {Name} contains {Count} items", collection.Name, addedIds.Count);
-        var addedIds = new List<int>();
-        foreach (MediaItem item in embyItems)
-        {
-            addedIds.Add(await _embyCollectionRepository.AddTag(item, collection));
-        }
 
-        var changedIds = removedIds.Except(addedIds).ToList();
+            var changedIds = removedIds.Except(addedIds).ToList();
-        changedIds.AddRange(addedIds.Except(removedIds));
+            changedIds.AddRange(addedIds.Except(removedIds));
 
-        await _searchIndex.RebuildItems(_searchRepository, changedIds);
+            await _searchIndex.RebuildItems(_searchRepository, changedIds);
-        _searchIndex.Commit();
+            _searchIndex.Commit();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to synchronize Emby collection {Name}", collection.Name);
+        }
     }
 }

ErsatzTV.Core/Emby/EmbyItemType.cs

@@ -3,4 +3,9 @@
 public static class EmbyItemType
 {
     public static readonly string Movie = "Movie";
+    public static readonly string Show = "Series";
+    public static readonly string Season = "Season";
+    public static readonly string Episode = "Episode";
+    public static readonly string Collection = "BoxSet";
+    public static readonly string CollectionItems = "Movie,Series,Season,Episode";
 }

ErsatzTV.Core/Emby/EmbyMovieLibraryScanner.cs

@@ -85,7 +85,7 @@ public class EmbyMovieLibraryScanner :
         _embyApiClient.GetLibraryItemCount(
             connectionParameters.Address,
             connectionParameters.ApiKey,
-            library,
+            library.ItemId,
             EmbyItemType.Movie);
 
     protected override IAsyncEnumerable<EmbyMovie> GetMovieLibraryItems(

ErsatzTV.Core/Emby/EmbyTelevisionLibraryScanner.cs

@@ -76,13 +76,19 @@ public class EmbyTelevisionLibraryScanner : MediaServerTelevisionLibraryScanner<
             cancellationToken);
     }
 
-    protected override Task<Either<BaseError, List<EmbyShow>>> GetShowLibraryItems(
+    protected override Task<Either<BaseError, int>> CountShowLibraryItems(
         EmbyConnectionParameters connectionParameters,
-        EmbyLibrary library) =>
+        EmbyLibrary library)
-        _embyApiClient.GetShowLibraryItems(
+        => _embyApiClient.GetLibraryItemCount(
             connectionParameters.Address,
             connectionParameters.ApiKey,
-            library.ItemId);
+            library.ItemId,
+            EmbyItemType.Show);
+
+    protected override IAsyncEnumerable<EmbyShow> GetShowLibraryItems(
+        EmbyConnectionParameters connectionParameters,
+        EmbyLibrary library) =>
+        _embyApiClient.GetShowLibraryItems(connectionParameters.Address, connectionParameters.ApiKey, library);
 
     protected override string MediaServerItemId(EmbyShow show) => show.ItemId;
     protected override string MediaServerItemId(EmbySeason season) => season.ItemId;
@@ -92,23 +98,46 @@ public class EmbyTelevisionLibraryScanner : MediaServerTelevisionLibraryScanner<
     protected override string MediaServerEtag(EmbySeason season) => season.Etag;
     protected override string MediaServerEtag(EmbyEpisode episode) => episode.Etag;
 
-    protected override Task<Either<BaseError, List<EmbySeason>>> GetSeasonLibraryItems(
+    protected override Task<Either<BaseError, int>> CountSeasonLibraryItems(
+        EmbyConnectionParameters connectionParameters,
+        EmbyLibrary library,
+        EmbyShow show) =>
+        _embyApiClient.GetLibraryItemCount(
+            connectionParameters.Address,
+            connectionParameters.ApiKey,
+            show.ItemId,
+            EmbyItemType.Season);
+
+    protected override IAsyncEnumerable<EmbySeason> GetSeasonLibraryItems(
         EmbyLibrary library,
         EmbyConnectionParameters connectionParameters,
         EmbyShow show) =>
         _embyApiClient.GetSeasonLibraryItems(
             connectionParameters.Address,
             connectionParameters.ApiKey,
+            library,
             show.ItemId);
 
-    protected override Task<Either<BaseError, List<EmbyEpisode>>> GetEpisodeLibraryItems(
+    protected override Task<Either<BaseError, int>> CountEpisodeLibraryItems(
+        EmbyConnectionParameters connectionParameters,
+        EmbyLibrary library,
+        EmbySeason season) =>
+        _embyApiClient.GetLibraryItemCount(
+            connectionParameters.Address,
+            connectionParameters.ApiKey,
+            season.ItemId,
+            EmbyItemType.Episode);
+
+    protected override IAsyncEnumerable<EmbyEpisode> GetEpisodeLibraryItems(
         EmbyLibrary library,
         EmbyConnectionParameters connectionParameters,
+        EmbyShow show,
         EmbySeason season) =>
         _embyApiClient.GetEpisodeLibraryItems(
             connectionParameters.Address,
             connectionParameters.ApiKey,
             library,
+            show.ItemId,
             season.ItemId);
 
     protected override Task<Option<ShowMetadata>> GetFullMetadata(

ErsatzTV.Core/Interfaces/Emby/IEmbyApiClient.cs

@@ -8,39 +8,30 @@ public interface IEmbyApiClient
     Task<Either<BaseError, EmbyServerInformation>> GetServerInformation(string address, string apiKey);
     Task<Either<BaseError, List<EmbyLibrary>>> GetLibraries(string address, string apiKey);
 
-    IAsyncEnumerable<EmbyMovie> GetMovieLibraryItems(
+    IAsyncEnumerable<EmbyMovie> GetMovieLibraryItems(string address, string apiKey, EmbyLibrary library);
-        string address,
-        string apiKey,
-        EmbyLibrary library);
 
-    Task<Either<BaseError, List<EmbyShow>>> GetShowLibraryItems(
+    IAsyncEnumerable<EmbyShow> GetShowLibraryItems(string address, string apiKey, EmbyLibrary library);
-        string address,
-        string apiKey,
-        string libraryId);
 
-    Task<Either<BaseError, List<EmbySeason>>> GetSeasonLibraryItems(
+    IAsyncEnumerable<EmbySeason> GetSeasonLibraryItems(
         string address,
         string apiKey,
+        EmbyLibrary library,
         string showId);
 
-    Task<Either<BaseError, List<EmbyEpisode>>> GetEpisodeLibraryItems(
+    IAsyncEnumerable<EmbyEpisode> GetEpisodeLibraryItems(
         string address,
         string apiKey,
         EmbyLibrary library,
+        string showId,
         string seasonId);
 
-    Task<Either<BaseError, List<EmbyCollection>>> GetCollectionLibraryItems(
+    IAsyncEnumerable<EmbyCollection> GetCollectionLibraryItems(string address, string apiKey);
-        string address,
-        string apiKey);
 
-    Task<Either<BaseError, List<MediaItem>>> GetCollectionItems(
+    IAsyncEnumerable<MediaItem> GetCollectionItems(string address, string apiKey, string collectionId);
-        string address,
-        string apiKey,
-        string collectionId);
 
     Task<Either<BaseError, int>> GetLibraryItemCount(
         string address,
         string apiKey,
-        EmbyLibrary library,
+        string parentId,
         string includeItemTypes);
 }

ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs

@@ -9,35 +9,25 @@ public interface IJellyfinApiClient
     Task<Either<BaseError, List<JellyfinLibrary>>> GetLibraries(string address, string apiKey);
     Task<Either<BaseError, string>> GetAdminUserId(string address, string apiKey);
 
-    IAsyncEnumerable<JellyfinMovie> GetMovieLibraryItems(
+    IAsyncEnumerable<JellyfinMovie> GetMovieLibraryItems(string address, string apiKey, JellyfinLibrary library);
-        string address,
-        string apiKey,
-        JellyfinLibrary library);
 
-    Task<Either<BaseError, List<JellyfinShow>>> GetShowLibraryItems(
+    IAsyncEnumerable<JellyfinShow> GetShowLibraryItems(string address, string apiKey, JellyfinLibrary library);
-        string address,
-        string apiKey,
-        int mediaSourceId,
-        string libraryId);
 
-    Task<Either<BaseError, List<JellyfinSeason>>> GetSeasonLibraryItems(
+    IAsyncEnumerable<JellyfinSeason> GetSeasonLibraryItems(
         string address,
         string apiKey,
-        int mediaSourceId,
+        JellyfinLibrary library,
         string showId);
 
-    Task<Either<BaseError, List<JellyfinEpisode>>> GetEpisodeLibraryItems(
+    IAsyncEnumerable<JellyfinEpisode> GetEpisodeLibraryItems(
         string address,
         string apiKey,
         JellyfinLibrary library,
         string seasonId);
 
-    Task<Either<BaseError, List<JellyfinCollection>>> GetCollectionLibraryItems(
+    IAsyncEnumerable<JellyfinCollection> GetCollectionLibraryItems(string address, string apiKey, int mediaSourceId);
-        string address,
-        string apiKey,
-        int mediaSourceId);
 
-    Task<Either<BaseError, List<MediaItem>>> GetCollectionItems(
+    IAsyncEnumerable<MediaItem> GetCollectionItems(
         string address,
         string apiKey,
         int mediaSourceId,
@@ -47,5 +37,7 @@ public interface IJellyfinApiClient
         string address,
         string apiKey,
         JellyfinLibrary library,
-        string includeItemTypes);
+        string parentId,
+        string includeItemTypes,
+        bool excludeFolders);
 }

ErsatzTV.Core/Interfaces/Plex/IPlexServerApiClient.cs

@@ -18,18 +18,28 @@ public interface IPlexServerApiClient
         PlexConnection connection,
         PlexServerAuthToken token);
 
-    Task<Either<BaseError, List<PlexShow>>> GetShowLibraryContents(
+    IAsyncEnumerable<PlexShow> GetShowLibraryContents(
         PlexLibrary library,
         PlexConnection connection,
         PlexServerAuthToken token);
 
-    Task<Either<BaseError, List<PlexSeason>>> GetShowSeasons(
+    Task<Either<BaseError, int>> CountShowSeasons(
+        PlexShow show,
+        PlexConnection connection,
+        PlexServerAuthToken token);
+
+    IAsyncEnumerable<PlexSeason> GetShowSeasons(
         PlexLibrary library,
         PlexShow show,
         PlexConnection connection,
         PlexServerAuthToken token);
 
-    Task<Either<BaseError, List<PlexEpisode>>> GetSeasonEpisodes(
+    Task<Either<BaseError, int>> CountSeasonEpisodes(
+        PlexSeason season,
+        PlexConnection connection,
+        PlexServerAuthToken token);
+
+    IAsyncEnumerable<PlexEpisode> GetSeasonEpisodes(
         PlexLibrary library,
         PlexSeason season,
         PlexConnection connection,

ErsatzTV.Core/Jellyfin/JellyfinCollectionScanner.cs

@@ -30,24 +30,21 @@ public class JellyfinCollectionScanner : IJellyfinCollectionScanner
 
     public async Task<Either<BaseError, Unit>> ScanCollections(string address, string apiKey, int mediaSourceId)
     {
-        // get all collections from db (item id, etag)
+        try
-        List<JellyfinCollection> existingCollections = await _jellyfinCollectionRepository.GetCollections();
-
-        // get all collections from jellyfin
-        Either<BaseError, List<JellyfinCollection>> maybeIncomingCollections =
-            await _jellyfinApiClient.GetCollectionLibraryItems(address, apiKey, mediaSourceId);
-
-        foreach (BaseError error in maybeIncomingCollections.LeftToSeq())
         {
-            _logger.LogWarning("Failed to get collections from Jellyfin: {Error}", error.ToString());
+            var incomingItemIds = new List<string>();
-            return error;
+
-        }
+            // get all collections from db (item id, etag)
+            List<JellyfinCollection> existingCollections = await _jellyfinCollectionRepository.GetCollections();
 
-        foreach (List<JellyfinCollection> incomingCollections in maybeIncomingCollections.RightToSeq())
-        {
             // loop over collections
-            foreach (JellyfinCollection collection in incomingCollections)
+            await foreach (JellyfinCollection collection in _jellyfinApiClient.GetCollectionLibraryItems(
+                               address,
+                               apiKey,
+                               mediaSourceId))
             {
+                incomingItemIds.Add(collection.ItemId);
+
                 Option<JellyfinCollection> maybeExisting = existingCollections.Find(c => c.ItemId == collection.ItemId);
 
                 // skip if unchanged (etag)
@@ -75,12 +72,17 @@ public class JellyfinCollectionScanner : IJellyfinCollectionScanner
             }
 
             // remove missing collections (and remove any lingering tags from those collections)
-            foreach (JellyfinCollection collection in existingCollections
+            foreach (JellyfinCollection collection in existingCollections.Filter(
-                         .Filter(e => incomingCollections.All(i => i.ItemId != e.ItemId)))
+                         e => !incomingItemIds.Contains(e.ItemId)))
             {
                 await _jellyfinCollectionRepository.RemoveCollection(collection);
             }
         }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to get collections from Jellyfin");
+            return BaseError.New(ex.Message);
+        }
 
         return Unit.Default;
     }
@@ -91,32 +93,35 @@ public class JellyfinCollectionScanner : IJellyfinCollectionScanner
         int mediaSourceId,
         JellyfinCollection collection)
     {
-        // get collection items from JF
+        try
-        Either<BaseError, List<MediaItem>> maybeItems =
-            await _jellyfinApiClient.GetCollectionItems(address, apiKey, mediaSourceId, collection.ItemId);
-
-        foreach (BaseError error in maybeItems.LeftToSeq())
         {
-            _logger.LogWarning("Failed to get collection items from Jellyfin: {Error}", error.ToString());
+            // get collection items from JF
-            return;
+            IAsyncEnumerable<MediaItem> items = _jellyfinApiClient.GetCollectionItems(
-        }
+                address,
+                apiKey,
+                mediaSourceId,
+                collection.ItemId);
+
+            List<int> removedIds = await _jellyfinCollectionRepository.RemoveAllTags(collection);
+
+            // sync tags on items
+            var addedIds = new List<int>();
+            await foreach (MediaItem item in items)
+            {
+                addedIds.Add(await _jellyfinCollectionRepository.AddTag(item, collection));
+            }
 
-        List<int> removedIds = await _jellyfinCollectionRepository.RemoveAllTags(collection);
+            _logger.LogDebug("Jellyfin collection {Name} contains {Count} items", collection.Name, addedIds.Count);
 
-        var jellyfinItems = maybeItems.RightToSeq().Flatten().ToList();
+            var changedIds = removedIds.Except(addedIds).ToList();
-        _logger.LogDebug("Jellyfin collection {Name} contains {Count} items", collection.Name, jellyfinItems.Count);
+            changedIds.AddRange(addedIds.Except(removedIds));
 
-        // sync tags on items
+            await _searchIndex.RebuildItems(_searchRepository, changedIds);
-        var addedIds = new List<int>();
+            _searchIndex.Commit();
-        foreach (MediaItem item in jellyfinItems)
+        }
+        catch (Exception ex)
         {
-            addedIds.Add(await _jellyfinCollectionRepository.AddTag(item, collection));
+            _logger.LogWarning(ex, "Failed to synchronize Jellyfin collection {Name}", collection.Name);
         }
-
-        var changedIds = removedIds.Except(addedIds).ToList();
-        changedIds.AddRange(addedIds.Except(removedIds));
-
-        await _searchIndex.RebuildItems(_searchRepository, changedIds);
-        _searchIndex.Commit();
     }
 }

ErsatzTV.Core/Jellyfin/JellyfinItemType.cs

@@ -3,4 +3,9 @@
 public static class JellyfinItemType
 {
     public static readonly string Movie = "Movie";
+    public static readonly string Show = "Series";
+    public static readonly string Season = "Season";
+    public static readonly string Episode = "Episode";
+    public static readonly string Collection = "BoxSet";
+    public static readonly string CollectionItems = "Movie,Series,Season,Episode";
 }

ErsatzTV.Core/Jellyfin/JellyfinMovieLibraryScanner.cs

@@ -87,7 +87,9 @@ public class JellyfinMovieLibraryScanner :
             connectionParameters.Address,
             connectionParameters.ApiKey,
             library,
-            JellyfinItemType.Movie);
+            library.ItemId,
+            JellyfinItemType.Movie,
+            true);
 
     protected override IAsyncEnumerable<JellyfinMovie> GetMovieLibraryItems(
         JellyfinConnectionParameters connectionParameters,

ErsatzTV.Core/Jellyfin/JellyfinTelevisionLibraryScanner.cs

@@ -77,14 +77,21 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan
             cancellationToken);
     }
 
-    protected override Task<Either<BaseError, List<JellyfinShow>>> GetShowLibraryItems(
+    protected override Task<Either<BaseError, int>> CountShowLibraryItems(
         JellyfinConnectionParameters connectionParameters,
-        JellyfinLibrary library) =>
+        JellyfinLibrary library)
-        _jellyfinApiClient.GetShowLibraryItems(
+        => _jellyfinApiClient.GetLibraryItemCount(
             connectionParameters.Address,
             connectionParameters.ApiKey,
-            library.MediaSourceId,
+            library,
-            library.ItemId);
+            library.ItemId,
+            JellyfinItemType.Show,
+            false);
+
+    protected override IAsyncEnumerable<JellyfinShow> GetShowLibraryItems(
+        JellyfinConnectionParameters connectionParameters,
+        JellyfinLibrary library) =>
+        _jellyfinApiClient.GetShowLibraryItems(connectionParameters.Address, connectionParameters.ApiKey, library);
 
     protected override string MediaServerItemId(JellyfinShow show) => show.ItemId;
     protected override string MediaServerItemId(JellyfinSeason season) => season.ItemId;
@@ -94,19 +101,44 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan
     protected override string MediaServerEtag(JellyfinSeason season) => season.Etag;
     protected override string MediaServerEtag(JellyfinEpisode episode) => episode.Etag;
 
-    protected override Task<Either<BaseError, List<JellyfinSeason>>> GetSeasonLibraryItems(
+    protected override Task<Either<BaseError, int>> CountSeasonLibraryItems(
+        JellyfinConnectionParameters connectionParameters,
+        JellyfinLibrary library,
+        JellyfinShow show) =>
+        _jellyfinApiClient.GetLibraryItemCount(
+            connectionParameters.Address,
+            connectionParameters.ApiKey,
+            library,
+            show.ItemId,
+            JellyfinItemType.Season,
+            false);
+
+    protected override IAsyncEnumerable<JellyfinSeason> GetSeasonLibraryItems(
         JellyfinLibrary library,
         JellyfinConnectionParameters connectionParameters,
         JellyfinShow show) =>
         _jellyfinApiClient.GetSeasonLibraryItems(
             connectionParameters.Address,
             connectionParameters.ApiKey,
-            library.MediaSourceId,
+            library,
             show.ItemId);
 
-    protected override Task<Either<BaseError, List<JellyfinEpisode>>> GetEpisodeLibraryItems(
+    protected override Task<Either<BaseError, int>> CountEpisodeLibraryItems(
+        JellyfinConnectionParameters connectionParameters,
+        JellyfinLibrary library,
+        JellyfinSeason season) =>
+        _jellyfinApiClient.GetLibraryItemCount(
+            connectionParameters.Address,
+            connectionParameters.ApiKey,
+            library,
+            season.ItemId,
+            JellyfinItemType.Episode,
+            true);
+
+    protected override IAsyncEnumerable<JellyfinEpisode> GetEpisodeLibraryItems(
         JellyfinLibrary library,
         JellyfinConnectionParameters connectionParameters,
+        JellyfinShow _,
         JellyfinSeason season) =>
         _jellyfinApiClient.GetEpisodeLibraryItems(
             connectionParameters.Address,

ErsatzTV.Core/Metadata/MediaServerMovieLibraryScanner.cs

@@ -59,19 +59,23 @@ public abstract class MediaServerMovieLibraryScanner<TConnectionParameters, TLib
                 return error;
             }
 
-            int count = await maybeCount.RightToSeq().HeadOrNone().IfNoneAsync(1);
+            foreach (int count in maybeCount.RightToSeq())
-
+            {
-            return await ScanLibrary(
+                return await ScanLibrary(
-                movieRepository,
+                    movieRepository,
-                connectionParameters,
+                    connectionParameters,
-                library,
+                    library,
-                getLocalPath,
+                    getLocalPath,
-                ffmpegPath,
+                    ffmpegPath,
-                ffprobePath,
+                    ffprobePath,
-                GetMovieLibraryItems(connectionParameters, library),
+                    GetMovieLibraryItems(connectionParameters, library),
-                count,
+                    count,
-                deepScan,
+                    deepScan,
-                cancellationToken);
+                    cancellationToken);
+            }
+
+            // this won't happen
+            return Unit.Default;
         }
         catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
         {

ErsatzTV.Core/Metadata/MediaServerTelevisionLibraryScanner.cs

@@ -56,23 +56,31 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
     {
         try
         {
-            Either<BaseError, List<TShow>> entries = await GetShowLibraryItems(connectionParameters, library);
+            Either<BaseError, int> maybeCount = await CountShowLibraryItems(connectionParameters, library);
-
+            foreach (BaseError error in maybeCount.LeftToSeq())
-            foreach (BaseError error in entries.LeftToSeq())
             {
                 return error;
             }
 
-            return await ScanLibrary(
+            foreach (int count in maybeCount.RightToSeq())
-                televisionRepository,
+            {
-                connectionParameters,
+                _logger.LogDebug("Library {Library} contains {Count} shows", library.Name, count);
-                library,
+
-                getLocalPath,
+                return await ScanLibrary(
-                ffmpegPath,
+                    televisionRepository,
-                ffprobePath,
+                    connectionParameters,
-                entries.RightToSeq().Flatten().ToList(),
+                    library,
-                deepScan,
+                    getLocalPath,
-                cancellationToken);
+                    ffmpegPath,
+                    ffprobePath,
+                    GetShowLibraryItems(connectionParameters, library),
+                    count,
+                    deepScan,
+                    cancellationToken);
+            }
+
+            // this won't happen
+            return Unit.Default;
         }
         catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
         {
@@ -84,7 +92,11 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
         }
     }
 
-    protected abstract Task<Either<BaseError, List<TShow>>> GetShowLibraryItems(
+    protected abstract Task<Either<BaseError, int>> CountShowLibraryItems(
+        TConnectionParameters connectionParameters,
+        TLibrary library);
+
+    protected abstract IAsyncEnumerable<TShow> GetShowLibraryItems(
         TConnectionParameters connectionParameters,
         TLibrary library);
 
@@ -102,21 +114,24 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
         Func<TEpisode, string> getLocalPath,
         string ffmpegPath,
         string ffprobePath,
-        List<TShow> showEntries,
+        IAsyncEnumerable<TShow> showEntries,
+        int totalShowCount,
         bool deepScan,
         CancellationToken cancellationToken)
     {
+        var incomingItemIds = new List<string>();
         List<TEtag> existingShows = await televisionRepository.GetExistingShows(library);
 
-        var sortedShows = showEntries.OrderBy(s => s.ShowMetadata.Head().SortTitle).ToList();
+        await foreach (TShow incoming in showEntries.WithCancellation(cancellationToken))
-        foreach (TShow incoming in showEntries)
         {
             if (cancellationToken.IsCancellationRequested)
             {
                 return new ScanCanceled();
             }
 
-            decimal percentCompletion = (decimal)sortedShows.IndexOf(incoming) / sortedShows.Count;
+            incomingItemIds.Add(MediaServerItemId(incoming));
+
+            decimal percentCompletion = Math.Clamp((decimal)incomingItemIds.Count / totalShowCount, 0, 1);
             await _mediator.Publish(new LibraryScanProgress(library.Id, percentCompletion), cancellationToken);
 
             Either<BaseError, MediaItemScanResult<TShow>> maybeShow = await televisionRepository
@@ -138,16 +153,23 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
 
             foreach (MediaItemScanResult<TShow> result in maybeShow.RightToSeq())
             {
-                Either<BaseError, List<TSeason>> entries = await GetSeasonLibraryItems(
+                Either<BaseError, int> maybeCount = await CountSeasonLibraryItems(
-                    library,
                     connectionParameters,
+                    library,
                     result.Item);
-
+                foreach (BaseError error in maybeCount.LeftToSeq())
-                foreach (BaseError error in entries.LeftToSeq())
                 {
                     return error;
                 }
 
+                foreach (int count in maybeCount.RightToSeq())
+                {
+                    _logger.LogDebug(
+                        "Show {Title} contains {Count} seasons",
+                        result.Item.ShowMetadata.Head().Title,
+                        count);
+                }
+
                 Either<BaseError, Unit> scanResult = await ScanSeasons(
                     televisionRepository,
                     library,
@@ -156,7 +178,7 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
                     connectionParameters,
                     ffmpegPath,
                     ffprobePath,
-                    entries.RightToSeq().Flatten().ToList(),
+                    GetSeasonLibraryItems(library, connectionParameters, result.Item),
                     deepScan,
                     cancellationToken);
 
@@ -175,8 +197,7 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
         }
 
         // trash shows that are no longer present on the media server
-        var fileNotFoundItemIds = existingShows.Map(s => s.MediaServerItemId)
+        var fileNotFoundItemIds = existingShows.Map(s => s.MediaServerItemId).Except(incomingItemIds).ToList();
-            .Except(showEntries.Map(MediaServerItemId)).ToList();
         List<int> ids = await televisionRepository.FlagFileNotFoundShows(library, fileNotFoundItemIds);
         await _searchIndex.RebuildItems(_searchRepository, ids);
 
@@ -185,14 +206,25 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
         return Unit.Default;
     }
 
-    protected abstract Task<Either<BaseError, List<TSeason>>> GetSeasonLibraryItems(
+    protected abstract Task<Either<BaseError, int>> CountSeasonLibraryItems(
+        TConnectionParameters connectionParameters,
+        TLibrary library,
+        TShow show);
+
+    protected abstract IAsyncEnumerable<TSeason> GetSeasonLibraryItems(
         TLibrary library,
         TConnectionParameters connectionParameters,
         TShow show);
 
-    protected abstract Task<Either<BaseError, List<TEpisode>>> GetEpisodeLibraryItems(
+    protected abstract Task<Either<BaseError, int>> CountEpisodeLibraryItems(
+        TConnectionParameters connectionParameters,
+        TLibrary library,
+        TSeason season);
+
+    protected abstract IAsyncEnumerable<TEpisode> GetEpisodeLibraryItems(
         TLibrary library,
         TConnectionParameters connectionParameters,
+        TShow show,
         TSeason season);
 
     protected abstract Task<Option<ShowMetadata>> GetFullMetadata(
@@ -236,14 +268,14 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
         TConnectionParameters connectionParameters,
         string ffmpegPath,
         string ffprobePath,
-        List<TSeason> seasonEntries,
+        IAsyncEnumerable<TSeason> seasonEntries,
         bool deepScan,
         CancellationToken cancellationToken)
     {
+        var incomingItemIds = new List<string>();
         List<TEtag> existingSeasons = await televisionRepository.GetExistingSeasons(library, show);
 
-        var sortedSeasons = seasonEntries.OrderBy(s => s.SeasonNumber).ToList();
+        await foreach (TSeason incoming in seasonEntries.WithCancellation(cancellationToken))
-        foreach (TSeason incoming in sortedSeasons)
         {
             incoming.ShowId = show.Id;
 
@@ -252,6 +284,8 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
                 return new ScanCanceled();
             }
 
+            incomingItemIds.Add(MediaServerItemId(incoming));
+
             Either<BaseError, MediaItemScanResult<TSeason>> maybeSeason = await televisionRepository
                 .GetOrAdd(library, incoming)
                 .BindT(existing => UpdateMetadata(connectionParameters, library, existing, incoming, deepScan));
@@ -272,16 +306,24 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
 
             foreach (MediaItemScanResult<TSeason> result in maybeSeason.RightToSeq())
             {
-                Either<BaseError, List<TEpisode>> entries = await GetEpisodeLibraryItems(
+                Either<BaseError, int> maybeCount = await CountEpisodeLibraryItems(
-                    library,
                     connectionParameters,
+                    library,
                     result.Item);
-
+                foreach (BaseError error in maybeCount.LeftToSeq())
-                foreach (BaseError error in entries.LeftToSeq())
                 {
                     return error;
                 }
 
+                foreach (int count in maybeCount.RightToSeq())
+                {
+                    _logger.LogDebug(
+                        "Show {Title} season {Season} contains {Count} episodes",
+                        show.ShowMetadata.Head().Title,
+                        result.Item.SeasonNumber,
+                        count);
+                }
+
                 Either<BaseError, Unit> scanResult = await ScanEpisodes(
                     televisionRepository,
                     library,
@@ -291,7 +333,7 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
                     connectionParameters,
                     ffmpegPath,
                     ffprobePath,
-                    entries.RightToSeq().Flatten().ToList(),
+                    GetEpisodeLibraryItems(library, connectionParameters, show, result.Item),
                     deepScan,
                     cancellationToken);
 
@@ -312,8 +354,7 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
         }
 
         // trash seasons that are no longer present on the media server
-        var fileNotFoundItemIds = existingSeasons.Map(s => s.MediaServerItemId)
+        var fileNotFoundItemIds = existingSeasons.Map(s => s.MediaServerItemId).Except(incomingItemIds).ToList();
-            .Except(seasonEntries.Map(MediaServerItemId)).ToList();
         List<int> ids = await televisionRepository.FlagFileNotFoundSeasons(library, fileNotFoundItemIds);
         await _searchIndex.RebuildItems(_searchRepository, ids);
 
@@ -329,20 +370,22 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
         TConnectionParameters connectionParameters,
         string ffmpegPath,
         string ffprobePath,
-        List<TEpisode> episodeEntries,
+        IAsyncEnumerable<TEpisode> episodeEntries,
         bool deepScan,
         CancellationToken cancellationToken)
     {
+        var incomingItemIds = new List<string>();
         List<TEtag> existingEpisodes = await televisionRepository.GetExistingEpisodes(library, season);
 
-        var sortedEpisodes = episodeEntries.OrderBy(s => s.EpisodeMetadata.Head().EpisodeNumber).ToList();
+        await foreach (TEpisode incoming in episodeEntries.WithCancellation(cancellationToken))
-        foreach (TEpisode incoming in sortedEpisodes)
         {
             if (cancellationToken.IsCancellationRequested)
             {
                 return new ScanCanceled();
             }
 
+            incomingItemIds.Add(MediaServerItemId(incoming));
+
             string localPath = getLocalPath(incoming);
             if (await ShouldScanItem(
                     televisionRepository,
@@ -414,8 +457,7 @@ public abstract class MediaServerTelevisionLibraryScanner<TConnectionParameters,
         }
 
         // trash episodes that are no longer present on the media server
-        var fileNotFoundItemIds = existingEpisodes.Map(m => m.MediaServerItemId)
+        var fileNotFoundItemIds = existingEpisodes.Map(m => m.MediaServerItemId).Except(incomingItemIds).ToList();
-            .Except(episodeEntries.Map(MediaServerItemId)).ToList();
         List<int> ids = await televisionRepository.FlagFileNotFoundEpisodes(library, fileNotFoundItemIds);
         await _searchIndex.RebuildItems(_searchRepository, ids);
 

ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs

@@ -137,7 +137,15 @@ public class PlexTelevisionLibraryScanner :
     //     }
     // }
 
-    protected override Task<Either<BaseError, List<PlexShow>>> GetShowLibraryItems(
+    protected override Task<Either<BaseError, int>> CountShowLibraryItems(
+        PlexConnectionParameters connectionParameters,
+        PlexLibrary library) =>
+        _plexServerApiClient.GetLibraryItemCount(
+            library,
+            connectionParameters.Connection,
+            connectionParameters.Token);
+
+    protected override IAsyncEnumerable<PlexShow> GetShowLibraryItems(
         PlexConnectionParameters connectionParameters,
         PlexLibrary library) =>
         _plexServerApiClient.GetShowLibraryContents(
@@ -145,7 +153,16 @@ public class PlexTelevisionLibraryScanner :
             connectionParameters.Connection,
             connectionParameters.Token);
 
-    protected override Task<Either<BaseError, List<PlexSeason>>> GetSeasonLibraryItems(
+    protected override Task<Either<BaseError, int>> CountSeasonLibraryItems(
+        PlexConnectionParameters connectionParameters,
+        PlexLibrary library,
+        PlexShow show) =>
+        _plexServerApiClient.CountShowSeasons(
+            show,
+            connectionParameters.Connection,
+            connectionParameters.Token);
+
+    protected override IAsyncEnumerable<PlexSeason> GetSeasonLibraryItems(
         PlexLibrary library,
         PlexConnectionParameters connectionParameters,
         PlexShow show) =>
@@ -155,9 +172,19 @@ public class PlexTelevisionLibraryScanner :
             connectionParameters.Connection,
             connectionParameters.Token);
 
-    protected override Task<Either<BaseError, List<PlexEpisode>>> GetEpisodeLibraryItems(
+    protected override Task<Either<BaseError, int>> CountEpisodeLibraryItems(
+        PlexConnectionParameters connectionParameters,
+        PlexLibrary library,
+        PlexSeason season) =>
+        _plexServerApiClient.CountSeasonEpisodes(
+            season,
+            connectionParameters.Connection,
+            connectionParameters.Token);
+
+    protected override IAsyncEnumerable<PlexEpisode> GetEpisodeLibraryItems(
         PlexLibrary library,
         PlexConnectionParameters connectionParameters,
+        PlexShow _,
         PlexSeason season) =>
         _plexServerApiClient.GetSeasonEpisodes(
             library,

ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs

@@ -75,7 +75,8 @@ public class PipelineGeneratorTests
         result.PipelineSteps.Should().Contain(ps => ps is EncoderLibx265);
 
         string command = PrintCommand(videoInputFile, audioInputFile, None, None, result);
-        command.Should().Be("-threads 1 -nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -re -i /tmp/whatever.mkv -map 0:1 -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:a aac -ac 2 -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -f mpegts -mpegts_flags +initial_discontinuity pipe:1");
+        command.Should().Be(
+            "-threads 1 -nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -re -i /tmp/whatever.mkv -map 0:1 -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:a aac -ac 2 -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -f mpegts -mpegts_flags +initial_discontinuity pipe:1");
     }
 
     [Test]

ErsatzTV.Infrastructure/AsyncEnumerable.cs

@@ -0,0 +1,26 @@
+namespace ErsatzTV.Infrastructure;
+
+public static class AsyncEnumerable
+{
+    /// <summary>
+    ///     Creates an <see cref="IAsyncEnumerable{T}" /> which yields no results, similar to
+    ///     <see cref="Enumerable.Empty{TResult}" />.
+    /// </summary>
+    public static IAsyncEnumerable<T> Empty<T>() => EmptyAsyncEnumerator<T>.Instance;
+
+    private class EmptyAsyncEnumerator<T> : IAsyncEnumerator<T>, IAsyncEnumerable<T>
+    {
+        public static readonly EmptyAsyncEnumerator<T> Instance = new();
+
+        public IAsyncEnumerator<T> GetAsyncEnumerator(CancellationToken cancellationToken = default)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            return this;
+        }
+
+        public T Current => default;
+        public ValueTask DisposeAsync() => default;
+
+        public ValueTask<bool> MoveNextAsync() => new(false);
+    }
+}

ErsatzTV.Infrastructure/Emby/EmbyApiClient.cs

@@ -71,158 +71,160 @@ public class EmbyApiClient : IEmbyApiClient
         }
     }
 
-    public async IAsyncEnumerable<EmbyMovie> GetMovieLibraryItems(string address, string apiKey, EmbyLibrary library)
+    public IAsyncEnumerable<EmbyMovie> GetMovieLibraryItems(string address, string apiKey, EmbyLibrary library)
+        => GetPagedLibraryContents(
+            address,
+            apiKey,
+            library,
+            library.ItemId,
+            EmbyItemType.Movie,
+            (service, itemId, skip, pageSize) => service.GetMovieLibraryItems(
+                apiKey,
+                itemId,
+                startIndex: skip,
+                limit: pageSize),
+            (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToMovie(lib, item)).Flatten());
+
+    public IAsyncEnumerable<EmbyShow> GetShowLibraryItems(string address, string apiKey, EmbyLibrary library)
+        => GetPagedLibraryContents(
+            address,
+            apiKey,
+            library,
+            library.ItemId,
+            EmbyItemType.Show,
+            (service, itemId, skip, pageSize) => service.GetShowLibraryItems(
+                apiKey,
+                itemId,
+                startIndex: skip,
+                limit: pageSize),
+            (_, item) => ProjectToShow(item));
+
+    public IAsyncEnumerable<EmbySeason> GetSeasonLibraryItems(
+        string address,
+        string apiKey,
+        EmbyLibrary library,
+        string showId) => GetPagedLibraryContents(
+        address,
+        apiKey,
+        library,
+        showId,
+        EmbyItemType.Season,
+        (service, itemId, skip, pageSize) => service.GetSeasonLibraryItems(
+            apiKey,
+            itemId,
+            startIndex: skip,
+            limit: pageSize),
+        (_, item) => ProjectToSeason(item));
+
+    public IAsyncEnumerable<EmbyEpisode> GetEpisodeLibraryItems(
+        string address,
+        string apiKey,
+        EmbyLibrary library,
+        string showId,
+        string seasonId) => GetPagedLibraryContents(
+        address,
+        apiKey,
+        library,
+        seasonId,
+        EmbyItemType.Episode,
+        (service, _, skip, pageSize) => service.GetEpisodeLibraryItems(
+            apiKey,
+            showId,
+            seasonId,
+            startIndex: skip,
+            limit: pageSize),
+        (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToEpisode(lib, item)).Flatten());
+
+    public IAsyncEnumerable<EmbyCollection> GetCollectionLibraryItems(string address, string apiKey)
     {
-        IEmbyApi service = RestService.For<IEmbyApi>(address);
+        // TODO: should we enumerate collection libraries here?
-        int size = await service
-            .GetLibraryStats(apiKey, library.ItemId, EmbyItemType.Movie)
-            .Map(r => r.TotalRecordCount);
-
-        const int PAGE_SIZE = 10;
-
-        int pages = (size - 1) / PAGE_SIZE + 1;
 
-        for (var i = 0; i < pages; i++)
+        if (_memoryCache.TryGetValue("emby_collections_library_item_id", out string itemId))
         {
-            int skip = i * PAGE_SIZE;
+            return GetPagedLibraryContents(
-
+                address,
-            Task<IEnumerable<EmbyMovie>> result = service
+                apiKey,
-                .GetMovieLibraryItems(apiKey, library.ItemId, startIndex: skip, limit: PAGE_SIZE)
+                None,
-                .Map(items => items.Items.Map(item => ProjectToMovie(library, item)).Somes());
+                itemId,
-
+                EmbyItemType.Collection,
-            foreach (EmbyMovie movie in await result)
+                (service, _, skip, pageSize) => service.GetCollectionLibraryItems(
-            {
+                    apiKey,
-                yield return movie;
+                    itemId,
-            }
+                    startIndex: skip,
+                    limit: pageSize),
+                (_, item) => ProjectToCollection(item));
         }
+
+        return AsyncEnumerable.Empty<EmbyCollection>();
     }
 
-    public async Task<Either<BaseError, List<EmbyShow>>> GetShowLibraryItems(
+    public IAsyncEnumerable<MediaItem> GetCollectionItems(
         string address,
         string apiKey,
-        string libraryId)
+        string collectionId) =>
-    {
+        GetPagedLibraryContents(
-        try
+            address,
-        {
+            apiKey,
-            IEmbyApi service = RestService.For<IEmbyApi>(address);
+            None,
-            EmbyLibraryItemsResponse items = await service.GetShowLibraryItems(apiKey, libraryId);
+            collectionId,
-            return items.Items
+            EmbyItemType.CollectionItems,
-                .Map(ProjectToShow)
+            (service, _, skip, pageSize) => service.GetCollectionItems(
-                .Somes()
+                apiKey,
-                .ToList();
+                collectionId,
-        }
+                startIndex: skip,
-        catch (Exception ex)
+                limit: pageSize),
-        {
+            (_, item) => ProjectToCollectionMediaItem(item));
-            _logger.LogError(ex, "Error getting emby show library items");
-            return BaseError.New(ex.Message);
-        }
-    }
 
-    public async Task<Either<BaseError, List<EmbySeason>>> GetSeasonLibraryItems(
+    public async Task<Either<BaseError, int>> GetLibraryItemCount(
         string address,
         string apiKey,
-        string showId)
+        string parentId,
+        string includeItemTypes)
     {
         try
         {
             IEmbyApi service = RestService.For<IEmbyApi>(address);
-            EmbyLibraryItemsResponse items = await service.GetSeasonLibraryItems(apiKey, showId);
+            EmbyLibraryItemsResponse items = await service.GetLibraryStats(apiKey, parentId, includeItemTypes);
-            return items.Items
+            return items.TotalRecordCount;
-                .Map(ProjectToSeason)
-                .Somes()
-                .ToList();
         }
         catch (Exception ex)
         {
-            _logger.LogError(ex, "Error getting emby show library items");
+            _logger.LogError(ex, "Error getting Emby library item count");
             return BaseError.New(ex.Message);
         }
     }
 
-    public async Task<Either<BaseError, List<EmbyEpisode>>> GetEpisodeLibraryItems(
+    private static async IAsyncEnumerable<TItem> GetPagedLibraryContents<TItem>(
         string address,
         string apiKey,
-        EmbyLibrary library,
+        Option<EmbyLibrary> maybeLibrary,
-        string seasonId)
+        string parentId,
+        string itemType,
+        Func<IEmbyApi, string, int, int, Task<EmbyLibraryItemsResponse>> getItems,
+        Func<Option<EmbyLibrary>, EmbyLibraryItemResponse, Option<TItem>> mapper)
     {
-        try
+        IEmbyApi service = RestService.For<IEmbyApi>(address);
-        {
+        int size = await service
-            IEmbyApi service = RestService.For<IEmbyApi>(address);
+            .GetLibraryStats(apiKey, parentId, itemType)
-            EmbyLibraryItemsResponse items = await service.GetEpisodeLibraryItems(apiKey, seasonId);
+            .Map(r => r.TotalRecordCount);
-            return items.Items
-                .Map(i => ProjectToEpisode(library, i))
-                .Somes()
-                .ToList();
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError(ex, "Error getting emby episode library items");
-            return BaseError.New(ex.Message);
-        }
-    }
 
-    public async Task<Either<BaseError, List<EmbyCollection>>> GetCollectionLibraryItems(string address, string apiKey)
+        const int PAGE_SIZE = 10;
-    {
-        try
-        {
-            // TODO: should we enumerate collection libraries here?
 
-            if (_memoryCache.TryGetValue("emby_collections_library_item_id", out string itemId))
+        int pages = (size - 1) / PAGE_SIZE + 1;
-            {
-                IEmbyApi service = RestService.For<IEmbyApi>(address);
-                EmbyLibraryItemsResponse items = await service.GetCollectionLibraryItems(apiKey, itemId);
-                return items.Items
-                    .Map(ProjectToCollection)
-                    .Somes()
-                    .ToList();
-            }
 
-            return BaseError.New("Emby collection item id is not available");
+        for (var i = 0; i < pages; i++)
-        }
-        catch (Exception ex)
         {
-            _logger.LogError(ex, "Error getting Emby collection library items");
+            int skip = i * PAGE_SIZE;
-            return BaseError.New(ex.Message);
-        }
-    }
 
-    public async Task<Either<BaseError, List<MediaItem>>> GetCollectionItems(
+            Task<IEnumerable<TItem>> result = getItems(service, parentId, skip, PAGE_SIZE)
-        string address,
+                .Map(items => items.Items.Map(item => mapper(maybeLibrary, item)).Somes());
-        string apiKey,
-        string collectionId)
-    {
-        try
-        {
-            IEmbyApi service = RestService.For<IEmbyApi>(address);
-            EmbyLibraryItemsResponse items = await service.GetCollectionItems(apiKey, collectionId);
-            return items.Items
-                .Map(ProjectToCollectionMediaItem)
-                .Somes()
-                .ToList();
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError(ex, "Error getting Emby collection items");
-            return BaseError.New(ex.Message);
-        }
-    }
 
-    public async Task<Either<BaseError, int>> GetLibraryItemCount(
+#pragma warning disable VSTHRD003
-        string address,
+            foreach (TItem item in await result)
-        string apiKey,
+#pragma warning restore VSTHRD003
-        EmbyLibrary library,
+            {
-        string includeItemTypes)
+                yield return item;
-    {
+            }
-        try
-        {
-            IEmbyApi service = RestService.For<IEmbyApi>(address);
-            EmbyLibraryItemsResponse items = await service.GetLibraryStats(apiKey, library.ItemId, includeItemTypes);
-            return items.TotalRecordCount;
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError(ex, "Error getting Emby library item count");
-            return BaseError.New(ex.Message);
         }
     }
 

ErsatzTV.Infrastructure/Emby/IEmbyApi.cs

@@ -32,7 +32,7 @@ public interface IEmbyApi
         [Query]
         int limit = 0);
 
-    [Get("/Items")]
+    [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<EmbyLibraryItemsResponse> GetMovieLibraryItems(
         [Header("X-Emby-Token")]
         string apiKey,
@@ -50,7 +50,7 @@ public interface IEmbyApi
         [Query]
         int limit = 0);
 
-    [Get("/Items")]
+    [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<EmbyLibraryItemsResponse> GetShowLibraryItems(
         [Header("X-Emby-Token")]
         string apiKey,
@@ -62,40 +62,40 @@ public interface IEmbyApi
         [Query]
         string includeItemTypes = "Series",
         [Query]
-        bool recursive = true);
+        bool recursive = true,
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
 
-    [Get("/Items")]
+    [Get("/Shows/{parentId}/Seasons?sortOrder=Ascending&sortBy=SortName")]
     public Task<EmbyLibraryItemsResponse> GetSeasonLibraryItems(
         [Header("X-Emby-Token")]
         string apiKey,
-        [Query]
         string parentId,
         [Query]
         string fields = "Path,DateCreated,Etag,Taglines,ProviderIds",
         [Query]
-        string includeItemTypes = "Season",
+        int startIndex = 0,
-        [Query]
-        string excludeLocationTypes = "Virtual",
         [Query]
-        bool recursive = true);
+        int limit = 0);
 
-    [Get("/Items")]
+    [Get("/Shows/{showId}/Episodes?sortOrder=Ascending&sortBy=SortName")]
     public Task<EmbyLibraryItemsResponse> GetEpisodeLibraryItems(
         [Header("X-Emby-Token")]
         string apiKey,
+        string showId,
         [Query]
-        string parentId,
+        string seasonId,
         [Query]
         string fields =
             "Path,DateCreated,Etag,Overview,ProductionYear,PremiereDate,MediaSources,LocationType,ProviderIds,People",
         [Query]
-        string includeItemTypes = "Episode",
+        int startIndex = 0,
-        [Query]
-        string excludeLocationTypes = "Virtual",
         [Query]
-        bool recursive = true);
+        int limit = 0);
 
-    [Get("/Items")]
+    [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<EmbyLibraryItemsResponse> GetCollectionLibraryItems(
         [Header("X-Emby-Token")]
         string apiKey,
@@ -106,9 +106,13 @@ public interface IEmbyApi
         [Query]
         string includeItemTypes = "BoxSet",
         [Query]
-        bool recursive = true);
+        bool recursive = true,
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
 
-    [Get("/Items")]
+    [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<EmbyLibraryItemsResponse> GetCollectionItems(
         [Header("X-Emby-Token")]
         string apiKey,
@@ -121,5 +125,9 @@ public interface IEmbyApi
         [Query]
         string excludeLocationTypes = "Virtual",
         [Query]
-        bool recursive = true);
+        bool recursive = true,
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
 }

ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs

@@ -63,7 +63,7 @@ public interface IJellyfinApi
         [Query]
         int limit = 0);
 
-    [Get("/Items")]
+    [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<JellyfinLibraryItemsResponse> GetShowLibraryItems(
         [Header("X-Emby-Token")]
         string apiKey,
@@ -77,9 +77,13 @@ public interface IJellyfinApi
         [Query]
         string includeItemTypes = "Series",
         [Query]
-        bool recursive = true);
+        bool recursive = true,
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
 
-    [Get("/Items")]
+    [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<JellyfinLibraryItemsResponse> GetSeasonLibraryItems(
         [Header("X-Emby-Token")]
         string apiKey,
@@ -92,9 +96,13 @@ public interface IJellyfinApi
         [Query]
         string includeItemTypes = "Season",
         [Query]
-        bool recursive = true);
+        bool recursive = true,
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
 
-    [Get("/Items")]
+    [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<JellyfinLibraryItemsResponse> GetEpisodeLibraryItems(
         [Header("X-Emby-Token")]
         string apiKey,
@@ -107,9 +115,13 @@ public interface IJellyfinApi
         [Query]
         string includeItemTypes = "Episode",
         [Query]
-        bool recursive = true);
+        bool recursive = true,
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
 
-    [Get("/Items")]
+    [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<JellyfinLibraryItemsResponse> GetCollectionLibraryItems(
         [Header("X-Emby-Token")]
         string apiKey,
@@ -122,9 +134,13 @@ public interface IJellyfinApi
         [Query]
         string includeItemTypes = "BoxSet",
         [Query]
-        bool recursive = true);
+        bool recursive = true,
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
 
-    [Get("/Items")]
+    [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<JellyfinLibraryItemsResponse> GetCollectionItems(
         [Header("X-Emby-Token")]
         string apiKey,
@@ -137,5 +153,9 @@ public interface IJellyfinApi
         [Query]
         string includeItemTypes = "Movie,Series,Season,Episode",
         [Query]
-        bool recursive = true);
+        bool recursive = true,
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
 }

ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs

@@ -91,188 +91,139 @@ public class JellyfinApiClient : IJellyfinApiClient
         }
     }
 
-    public async IAsyncEnumerable<JellyfinMovie> GetMovieLibraryItems(
+    public IAsyncEnumerable<JellyfinMovie> GetMovieLibraryItems(
         string address,
         string apiKey,
-        JellyfinLibrary library)
+        JellyfinLibrary library) =>
-    {
+        GetPagedLibraryItems(
-        if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{library.MediaSourceId}", out string userId))
+            address,
-        {
+            apiKey,
-            IJellyfinApi service = RestService.For<IJellyfinApi>(address);
+            library,
-            int size = await service
+            library.MediaSourceId,
-                .GetLibraryStats(apiKey, userId, library.ItemId, JellyfinItemType.Movie)
+            library.ItemId,
-                .Map(r => r.TotalRecordCount);
+            JellyfinItemType.Movie,
-
+            (service, userId, itemId, skip, pageSize) => service.GetMovieLibraryItems(
-            const int PAGE_SIZE = 10;
+                apiKey,
-
+                userId,
-            int pages = (size - 1) / PAGE_SIZE + 1;
+                itemId,
-
+                startIndex: skip,
-            for (var i = 0; i < pages; i++)
+                limit: pageSize),
-            {
+            (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToMovie(lib, item)).Flatten());
-                int skip = i * PAGE_SIZE;
+
-
+    public IAsyncEnumerable<JellyfinShow> GetShowLibraryItems(
-                Task<IEnumerable<JellyfinMovie>> result = service
-                    .GetMovieLibraryItems(apiKey, userId, library.ItemId, startIndex: skip, limit: PAGE_SIZE)
-                    .Map(items => items.Items.Map(item => ProjectToMovie(library, item)).Somes());
-
-                foreach (JellyfinMovie movie in await result)
-                {
-                    yield return movie;
-                }
-            }
-        }
-    }
-
-    public async Task<Either<BaseError, List<JellyfinShow>>> GetShowLibraryItems(
         string address,
         string apiKey,
-        int mediaSourceId,
+        JellyfinLibrary library) =>
-        string libraryId)
+        GetPagedLibraryItems(
-    {
+            address,
-        try
+            apiKey,
-        {
+            library,
-            if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{mediaSourceId}", out string userId))
+            library.MediaSourceId,
-            {
+            library.ItemId,
-                IJellyfinApi service = RestService.For<IJellyfinApi>(address);
+            JellyfinItemType.Show,
-                JellyfinLibraryItemsResponse items = await service.GetShowLibraryItems(apiKey, userId, libraryId);
+            (service, userId, itemId, skip, pageSize) => service.GetShowLibraryItems(
-                return items.Items
+                apiKey,
-                    .Map(ProjectToShow)
+                userId,
-                    .Somes()
+                itemId,
-                    .ToList();
+                startIndex: skip,
-            }
+                limit: pageSize),
-
+            (_, item) => ProjectToShow(item));
-            return BaseError.New("Jellyfin admin user id is not available");
+
-        }
+    public IAsyncEnumerable<JellyfinSeason> GetSeasonLibraryItems(
-        catch (Exception ex)
-        {
-            _logger.LogError(ex, "Error getting jellyfin show library items");
-            return BaseError.New(ex.Message);
-        }
-    }
-
-    public async Task<Either<BaseError, List<JellyfinSeason>>> GetSeasonLibraryItems(
         string address,
         string apiKey,
-        int mediaSourceId,
+        JellyfinLibrary library,
-        string showId)
+        string showId) =>
-    {
+        GetPagedLibraryItems(
-        try
+            address,
-        {
+            apiKey,
-            if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{mediaSourceId}", out string userId))
+            library,
-            {
+            library.MediaSourceId,
-                IJellyfinApi service = RestService.For<IJellyfinApi>(address);
+            showId,
-                JellyfinLibraryItemsResponse items = await service.GetSeasonLibraryItems(apiKey, userId, showId);
+            JellyfinItemType.Season,
-                return items.Items
+            (service, userId, _, skip, pageSize) => service.GetSeasonLibraryItems(
-                    .Map(ProjectToSeason)
+                apiKey,
-                    .Somes()
+                userId,
-                    .ToList();
+                showId,
-            }
+                startIndex: skip,
-
+                limit: pageSize),
-            return BaseError.New("Jellyfin admin user id is not available");
+            (_, item) => ProjectToSeason(item));
-        }
+
-        catch (Exception ex)
+    public IAsyncEnumerable<JellyfinEpisode> GetEpisodeLibraryItems(
-        {
-            _logger.LogError(ex, "Error getting jellyfin show library items");
-            return BaseError.New(ex.Message);
-        }
-    }
-
-    public async Task<Either<BaseError, List<JellyfinEpisode>>> GetEpisodeLibraryItems(
         string address,
         string apiKey,
         JellyfinLibrary library,
-        string seasonId)
+        string seasonId) =>
-    {
+        GetPagedLibraryItems(
-        try
+            address,
-        {
+            apiKey,
-            if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{library.MediaSourceId}", out string userId))
+            library,
-            {
+            library.MediaSourceId,
-                IJellyfinApi service = RestService.For<IJellyfinApi>(address);
+            seasonId,
-                JellyfinLibraryItemsResponse items = await service.GetEpisodeLibraryItems(apiKey, userId, seasonId);
+            JellyfinItemType.Episode,
-                return items.Items
+            (service, userId, _, skip, pageSize) => service.GetEpisodeLibraryItems(
-                    .Map(i => ProjectToEpisode(library, i))
+                apiKey,
-                    .Somes()
+                userId,
-                    .ToList();
+                seasonId,
-            }
+                startIndex: skip,
-
+                limit: pageSize),
-            return BaseError.New("Jellyfin admin user id is not available");
+            (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToEpisode(lib, item)).Flatten());
-        }
+
-        catch (Exception ex)
+    public IAsyncEnumerable<JellyfinCollection> GetCollectionLibraryItems(
-        {
-            _logger.LogError(ex, "Error getting jellyfin episode library items");
-            return BaseError.New(ex.Message);
-        }
-    }
-
-    public async Task<Either<BaseError, List<JellyfinCollection>>> GetCollectionLibraryItems(
         string address,
         string apiKey,
         int mediaSourceId)
     {
-        try
+        // TODO: should we enumerate collection libraries here?
-        {
+
-            if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{mediaSourceId}", out string userId))
+        if (_memoryCache.TryGetValue("jellyfin_collections_library_item_id", out string itemId))
-            {
+        {
-                // TODO: should we enumerate collection libraries here?
+            return GetPagedLibraryItems(
-
+                address,
-                if (_memoryCache.TryGetValue("jellyfin_collections_library_item_id", out string itemId))
+                apiKey,
-                {
+                None,
-                    IJellyfinApi service = RestService.For<IJellyfinApi>(address);
+                mediaSourceId,
-                    JellyfinLibraryItemsResponse items =
+                itemId,
-                        await service.GetCollectionLibraryItems(apiKey, userId, itemId);
+                JellyfinItemType.Collection,
-                    return items.Items
+                (service, userId, _, skip, pageSize) => service.GetCollectionLibraryItems(
-                        .Map(ProjectToCollection)
+                    apiKey,
-                        .Somes()
+                    userId,
-                        .ToList();
+                    itemId,
-                }
+                    startIndex: skip,
-
+                    limit: pageSize),
-                return BaseError.New("Jellyfin collection item id is not available");
+                (_, item) => ProjectToCollection(item));
-            }
-
-            return BaseError.New("Jellyfin admin user id is not available");
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError(ex, "Error getting jellyfin collection library items");
-            return BaseError.New(ex.Message);
         }
+
+        return AsyncEnumerable.Empty<JellyfinCollection>();
     }
 
-    public async Task<Either<BaseError, List<MediaItem>>> GetCollectionItems(
+    public IAsyncEnumerable<MediaItem> GetCollectionItems(
         string address,
         string apiKey,
         int mediaSourceId,
-        string collectionId)
+        string collectionId) =>
-    {
+        GetPagedLibraryItems(
-        try
+            address,
-        {
+            apiKey,
-            if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{mediaSourceId}", out string userId))
+            None,
-            {
+            mediaSourceId,
-                IJellyfinApi service = RestService.For<IJellyfinApi>(address);
+            collectionId,
-                JellyfinLibraryItemsResponse items = await service.GetCollectionItems(
+            JellyfinItemType.CollectionItems,
-                    apiKey,
+            (service, userId, _, skip, pageSize) => service.GetCollectionItems(
-                    userId,
+                apiKey,
-                    collectionId);
+                userId,
-                return items.Items
+                collectionId,
-                    .Map(ProjectToCollectionMediaItem)
+                startIndex: skip,
-                    .Somes()
+                limit: pageSize),
-                    .ToList();
+            (_, item) => ProjectToCollectionMediaItem(item));
-            }
-
-            return BaseError.New("Jellyfin admin user id is not available");
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError(ex, "Error getting jellyfin collection items");
-            return BaseError.New(ex.Message);
-        }
-    }
 
     public async Task<Either<BaseError, int>> GetLibraryItemCount(
         string address,
         string apiKey,
         JellyfinLibrary library,
-        string includeItemTypes)
+        string parentId,
+        string includeItemTypes,
+        bool excludeFolders)
     {
         try
         {
@@ -282,8 +233,9 @@ public class JellyfinApiClient : IJellyfinApiClient
                 JellyfinLibraryItemsResponse items = await service.GetLibraryStats(
                     apiKey,
                     userId,
-                    library.ItemId,
+                    parentId,
-                    includeItemTypes);
+                    includeItemTypes,
+                    filters: excludeFolders ? "IsNotFolder" : null);
                 return items.TotalRecordCount;
             }
 
@@ -296,6 +248,42 @@ public class JellyfinApiClient : IJellyfinApiClient
         }
     }
 
+    private async IAsyncEnumerable<TItem> GetPagedLibraryItems<TItem>(
+        string address,
+        string apiKey,
+        Option<JellyfinLibrary> maybeLibrary,
+        int mediaSourceId,
+        string parentId,
+        string itemType,
+        Func<IJellyfinApi, string, string, int, int, Task<JellyfinLibraryItemsResponse>> getItems,
+        Func<Option<JellyfinLibrary>, JellyfinLibraryItemResponse, Option<TItem>> mapper)
+    {
+        if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{mediaSourceId}", out string userId))
+        {
+            IJellyfinApi service = RestService.For<IJellyfinApi>(address);
+            int size = await service
+                .GetLibraryStats(apiKey, userId, parentId, itemType)
+                .Map(r => r.TotalRecordCount);
+
+            const int PAGE_SIZE = 10;
+
+            int pages = (size - 1) / PAGE_SIZE + 1;
+
+            for (var i = 0; i < pages; i++)
+            {
+                int skip = i * PAGE_SIZE;
+
+                Task<IEnumerable<TItem>> result = getItems(service, userId, parentId, skip, PAGE_SIZE)
+                    .Map(items => items.Items.Map(item => mapper(maybeLibrary, item)).Somes());
+
+                foreach (TItem item in await result)
+                {
+                    yield return item;
+                }
+            }
+        }
+    }
+
     private Option<MediaItem> ProjectToCollectionMediaItem(JellyfinLibraryItemResponse item)
     {
         try

ErsatzTV.Infrastructure/Plex/IPlexServerApi.cs

@@ -24,14 +24,6 @@ public interface IPlexServerApi
         [Query] [AliasAs("X-Plex-Token")]
         string token);
 
-    [Get("/library/sections/{key}/all")]
-    [Headers("Accept: application/json")]
-    public Task<PlexMediaContainerResponse<PlexMediaContainerMetadataContent<PlexMetadataResponse>>>
-        GetLibrarySectionContents(
-            string key,
-            [Query] [AliasAs("X-Plex-Token")]
-            string token);
-
     [Get("/library/sections/{key}/all")]
     [Headers("Accept: application/json")]
     public Task<PlexMediaContainerResponse<PlexMediaContainerMetadataContent<PlexMetadataResponse>>>
@@ -60,19 +52,41 @@ public interface IPlexServerApi
             [Query] [AliasAs("X-Plex-Token")]
             string token);
 
+    [Get("/library/metadata/{key}/children?X-Plex-Container-Start=0&X-Plex-Container-Size=0")]
+    [Headers("Accept: text/xml")]
+    public Task<PlexXmlMediaContainerStatsResponse> CountShowChildren(
+        string key,
+        [Query] [AliasAs("X-Plex-Token")]
+        string token);
+
     [Get("/library/metadata/{key}/children")]
     [Headers("Accept: text/xml")]
     public Task<PlexXmlSeasonsMetadataResponseContainer>
         GetShowChildren(
             string key,
+            [Query] [AliasAs("X-Plex-Container-Start")]
+            int skip,
+            [Query] [AliasAs("X-Plex-Container-Size")]
+            int take,
             [Query] [AliasAs("X-Plex-Token")]
             string token);
 
+    [Get("/library/metadata/{key}/children?X-Plex-Container-Start=0&X-Plex-Container-Size=0")]
+    [Headers("Accept: text/xml")]
+    public Task<PlexXmlMediaContainerStatsResponse> CountSeasonChildren(
+        string key,
+        [Query] [AliasAs("X-Plex-Token")]
+        string token);
+
     [Get("/library/metadata/{key}/children")]
     [Headers("Accept: text/xml")]
     public Task<PlexXmlEpisodesMetadataResponseContainer>
         GetSeasonChildren(
             string key,
+            [Query] [AliasAs("X-Plex-Container-Start")]
+            int skip,
+            [Query] [AliasAs("X-Plex-Container-Size")]
+            int take,
             [Query] [AliasAs("X-Plex-Token")]
             string token);
 }

ErsatzTV.Infrastructure/Plex/PlexEtag.cs

@@ -8,10 +8,8 @@ public class PlexEtag
 {
     private readonly RecyclableMemoryStreamManager _recyclableMemoryStreamManager;
 
-    public PlexEtag(RecyclableMemoryStreamManager recyclableMemoryStreamManager)
+    public PlexEtag(RecyclableMemoryStreamManager recyclableMemoryStreamManager) =>
-    {
         _recyclableMemoryStreamManager = recyclableMemoryStreamManager;
-    }
 
     public string ForMovie(PlexMetadataResponse response)
     {

ErsatzTV.Infrastructure/Plex/PlexServerApiClient.cs

@@ -74,67 +74,58 @@ public class PlexServerApiClient : IPlexServerApiClient
         }
     }
 
-    public async IAsyncEnumerable<PlexMovie> GetMovieLibraryContents(
+    public IAsyncEnumerable<PlexMovie> GetMovieLibraryContents(
         PlexLibrary library,
         PlexConnection connection,
         PlexServerAuthToken token)
     {
-        IPlexServerApi xmlService = XmlServiceFor(connection.Uri);
+        Task<PlexXmlMediaContainerStatsResponse> CountItems(IPlexServerApi service)
-        int size = await xmlService.GetLibrarySection(library.Key, token.AuthToken).Map(r => r.TotalSize);
-
-        const int PAGE_SIZE = 10;
-
-        IPlexServerApi service = RestService.For<IPlexServerApi>(connection.Uri);
-        int pages = (size - 1) / PAGE_SIZE + 1;
-
-        for (var i = 0; i < pages; i++)
         {
-            int skip = i * PAGE_SIZE;
+            return service.GetLibrarySection(library.Key, token.AuthToken);
+        }
 
-            Task<IEnumerable<PlexMovie>> result = service
+        Task<IEnumerable<PlexMovie>> GetItems(IPlexServerApi _, IPlexServerApi jsonService, int skip, int pageSize)
-                .GetLibrarySectionContents(library.Key, skip, PAGE_SIZE, token.AuthToken)
+        {
+            return jsonService
+                .GetLibrarySectionContents(library.Key, skip, pageSize, token.AuthToken)
                 .Map(r => r.MediaContainer.Metadata.Filter(m => m.Media.Count > 0 && m.Media[0].Part.Count > 0))
                 .Map(list => list.Map(metadata => ProjectToMovie(metadata, library.MediaSourceId)));
-
-            foreach (PlexMovie movie in await result)
-            {
-                yield return movie;
-            }
         }
+
+        return GetPagedLibraryContents(connection, CountItems, GetItems);
     }
 
-    public async Task<Either<BaseError, List<PlexShow>>> GetShowLibraryContents(
+    public IAsyncEnumerable<PlexShow> GetShowLibraryContents(
         PlexLibrary library,
         PlexConnection connection,
         PlexServerAuthToken token)
     {
-        try
+        Task<PlexXmlMediaContainerStatsResponse> CountItems(IPlexServerApi service)
         {
-            IPlexServerApi service = RestService.For<IPlexServerApi>(connection.Uri);
+            return service.GetLibrarySection(library.Key, token.AuthToken);
-            return await service.GetLibrarySectionContents(library.Key, token.AuthToken)
-                .Map(r => r.MediaContainer.Metadata)
-                .Map(
-                    list => (list ?? new List<PlexMetadataResponse>())
-                        .Map(metadata => ProjectToShow(metadata, library.MediaSourceId)).ToList());
         }
-        catch (Exception ex)
+
+        Task<IEnumerable<PlexShow>> GetItems(IPlexServerApi _, IPlexServerApi jsonService, int skip, int pageSize)
         {
-            return BaseError.New(ex.ToString());
+            return jsonService
+                .GetLibrarySectionContents(library.Key, skip, pageSize, token.AuthToken)
+                .Map(r => r.MediaContainer.Metadata)
+                .Map(list => list.Map(metadata => ProjectToShow(metadata, library.MediaSourceId)));
         }
+
+        return GetPagedLibraryContents(connection, CountItems, GetItems);
     }
 
-    public async Task<Either<BaseError, List<PlexSeason>>> GetShowSeasons(
+    public async Task<Either<BaseError, int>> CountShowSeasons(
-        PlexLibrary library,
         PlexShow show,
         PlexConnection connection,
         PlexServerAuthToken token)
     {
         try
         {
+            string showMetadataKey = show.Key.Split("/").Reverse().Skip(1).Head();
             IPlexServerApi service = XmlServiceFor(connection.Uri);
-            return await service.GetShowChildren(show.Key.Split("/").Reverse().Skip(1).Head(), token.AuthToken)
+            return await service.CountShowChildren(showMetadataKey, token.AuthToken).Map(r => r.TotalSize);
-                .Map(r => r.Metadata.Filter(m => !m.Key.Contains("allLeaves")))
-                .Map(list => list.Map(metadata => ProjectToSeason(metadata, library.MediaSourceId)).ToList());
         }
         catch (Exception ex)
         {
@@ -142,19 +133,39 @@ public class PlexServerApiClient : IPlexServerApiClient
         }
     }
 
-    public async Task<Either<BaseError, List<PlexEpisode>>> GetSeasonEpisodes(
+    public IAsyncEnumerable<PlexSeason> GetShowSeasons(
         PlexLibrary library,
+        PlexShow show,
+        PlexConnection connection,
+        PlexServerAuthToken token)
+    {
+        string showMetadataKey = show.Key.Split("/").Reverse().Skip(1).Head();
+
+        Task<PlexXmlMediaContainerStatsResponse> CountItems(IPlexServerApi service)
+        {
+            return service.CountShowChildren(showMetadataKey, token.AuthToken);
+        }
+
+        Task<IEnumerable<PlexSeason>> GetItems(IPlexServerApi xmlService, IPlexServerApi _, int skip, int pageSize)
+        {
+            return xmlService.GetShowChildren(showMetadataKey, skip, pageSize, token.AuthToken)
+                .Map(r => r.Metadata.Filter(m => !m.Key.Contains("allLeaves")))
+                .Map(list => list.Map(metadata => ProjectToSeason(metadata, library.MediaSourceId)));
+        }
+
+        return GetPagedLibraryContents(connection, CountItems, GetItems);
+    }
+
+    public async Task<Either<BaseError, int>> CountSeasonEpisodes(
         PlexSeason season,
         PlexConnection connection,
         PlexServerAuthToken token)
     {
         try
         {
+            string seasonMetadataKey = season.Key.Split("/").Reverse().Skip(1).Head();
             IPlexServerApi service = XmlServiceFor(connection.Uri);
-            return await service.GetSeasonChildren(season.Key.Split("/").Reverse().Skip(1).Head(), token.AuthToken)
+            return await service.CountSeasonChildren(seasonMetadataKey, token.AuthToken).Map(r => r.TotalSize);
-                .Map(r => r.Metadata.Filter(m => m.Media.Count > 0 && m.Media[0].Part.Count > 0))
-                .Map(list => list.Map(metadata => ProjectToEpisode(metadata, library.MediaSourceId)))
-                .Map(ProcessMultiEpisodeFiles);
         }
         catch (Exception ex)
         {
@@ -162,6 +173,29 @@ public class PlexServerApiClient : IPlexServerApiClient
         }
     }
 
+    public IAsyncEnumerable<PlexEpisode> GetSeasonEpisodes(
+        PlexLibrary library,
+        PlexSeason season,
+        PlexConnection connection,
+        PlexServerAuthToken token)
+    {
+        string seasonMetadataKey = season.Key.Split("/").Reverse().Skip(1).Head();
+
+        Task<PlexXmlMediaContainerStatsResponse> CountItems(IPlexServerApi service)
+        {
+            return service.CountSeasonChildren(seasonMetadataKey, token.AuthToken);
+        }
+
+        Task<IEnumerable<PlexEpisode>> GetItems(IPlexServerApi xmlService, IPlexServerApi _, int skip, int pageSize)
+        {
+            return xmlService.GetSeasonChildren(seasonMetadataKey, skip, pageSize, token.AuthToken)
+                .Map(r => r.Metadata.Filter(m => m.Media.Count > 0 && m.Media[0].Part.Count > 0))
+                .Map(list => list.Map(metadata => ProjectToEpisode(metadata, library.MediaSourceId)));
+        }
+
+        return GetPagedLibraryContents(connection, CountItems, GetItems);
+    }
+
     public async Task<Either<BaseError, MovieMetadata>> GetMovieMetadata(
         PlexLibrary library,
         string key,
@@ -279,7 +313,34 @@ public class PlexServerApiClient : IPlexServerApiClient
         }
     }
 
+    private async IAsyncEnumerable<TItem> GetPagedLibraryContents<TItem>(
+        PlexConnection connection,
+        Func<IPlexServerApi, Task<PlexXmlMediaContainerStatsResponse>> countItems,
+        Func<IPlexServerApi, IPlexServerApi, int, int, Task<IEnumerable<TItem>>> getItems)
+    {
+        IPlexServerApi xmlService = XmlServiceFor(connection.Uri);
+
+        int size = await countItems(xmlService).Map(r => r.TotalSize);
+
+        const int PAGE_SIZE = 10;
+
+        IPlexServerApi jsonService = RestService.For<IPlexServerApi>(connection.Uri);
+        int pages = (size - 1) / PAGE_SIZE + 1;
+
+        for (var i = 0; i < pages; i++)
+        {
+            int skip = i * PAGE_SIZE;
+
+            Task<IEnumerable<TItem>> result = getItems(xmlService, jsonService, skip, PAGE_SIZE);
+
+            foreach (TItem item in await result)
+            {
+                yield return item;
+            }
+        }
+    }
 
+    // TODO: fix this with the addition of paging
     private List<PlexEpisode> ProcessMultiEpisodeFiles(IEnumerable<PlexEpisode> episodes)
     {
         // add all metadata from duplicate paths to first entry with given path