page media server movie libraries

CHANGELOG.md

@@ -12,6 +12,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Added
 - Enable QSV hardware acceleration for vaapi docker images
 
+### Changed
+- Use paging to synchronize movies from Plex, Jellyfin and Emby
+  - This will reduce memory use and improve reliability of synchronizing large libraries
+
 ## [0.5.8-beta] - 2022-05-20
 ### Fixed
 - Fix error display with `HLS Segmenter` and `MPEG-TS` streaming modes

ErsatzTV.Core/Emby/EmbyItemType.cs

@@ -0,0 +1,6 @@
+namespace ErsatzTV.Core.Emby;
+
+public static class EmbyItemType
+{
+    public static readonly string Movie = "Movie";
+}

ErsatzTV.Core/Emby/EmbyMovieLibraryScanner.cs

@@ -79,7 +79,16 @@ public class EmbyMovieLibraryScanner :
     protected override string MediaServerItemId(EmbyMovie movie) => movie.ItemId;
     protected override string MediaServerEtag(EmbyMovie movie) => movie.Etag;
 
-    protected override Task<Either<BaseError, List<EmbyMovie>>> GetMovieLibraryItems(
+    protected override Task<Either<BaseError, int>> CountMovieLibraryItems(
+        EmbyConnectionParameters connectionParameters,
+        EmbyLibrary library) =>
+        _embyApiClient.GetLibraryItemCount(
+            connectionParameters.Address,
+            connectionParameters.ApiKey,
+            library,
+            EmbyItemType.Movie);
+
+    protected override IAsyncEnumerable<EmbyMovie> GetMovieLibraryItems(
         EmbyConnectionParameters connectionParameters,
         EmbyLibrary library) =>
         _embyApiClient.GetMovieLibraryItems(

ErsatzTV.Core/Interfaces/Emby/IEmbyApiClient.cs

@@ -8,7 +8,7 @@ public interface IEmbyApiClient
     Task<Either<BaseError, EmbyServerInformation>> GetServerInformation(string address, string apiKey);
     Task<Either<BaseError, List<EmbyLibrary>>> GetLibraries(string address, string apiKey);
 
-    Task<Either<BaseError, List<EmbyMovie>>> GetMovieLibraryItems(
+    IAsyncEnumerable<EmbyMovie> GetMovieLibraryItems(
         string address,
         string apiKey,
         EmbyLibrary library);
@@ -37,4 +37,10 @@ public interface IEmbyApiClient
         string address,
         string apiKey,
         string collectionId);
+
+    Task<Either<BaseError, int>> GetLibraryItemCount(
+        string address,
+        string apiKey,
+        EmbyLibrary library,
+        string includeItemTypes);
 }

ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs

@@ -9,7 +9,7 @@ public interface IJellyfinApiClient
     Task<Either<BaseError, List<JellyfinLibrary>>> GetLibraries(string address, string apiKey);
     Task<Either<BaseError, string>> GetAdminUserId(string address, string apiKey);
 
-    Task<Either<BaseError, List<JellyfinMovie>>> GetMovieLibraryItems(
+    IAsyncEnumerable<JellyfinMovie> GetMovieLibraryItems(
         string address,
         string apiKey,
         JellyfinLibrary library);
@@ -42,4 +42,10 @@ public interface IJellyfinApiClient
         string apiKey,
         int mediaSourceId,
         string collectionId);
+
+    Task<Either<BaseError, int>> GetLibraryItemCount(
+        string address,
+        string apiKey,
+        JellyfinLibrary library,
+        string includeItemTypes);
 }

ErsatzTV.Core/Interfaces/Plex/IPlexServerApiClient.cs

@@ -13,7 +13,7 @@ public interface IPlexServerApiClient
         PlexConnection connection,
         PlexServerAuthToken token);
 
-    Task<Either<BaseError, List<PlexMovie>>> GetMovieLibraryContents(
+    IAsyncEnumerable<PlexMovie> GetMovieLibraryContents(
         PlexLibrary library,
         PlexConnection connection,
         PlexServerAuthToken token);
@@ -58,4 +58,9 @@ public interface IPlexServerApiClient
         string key,
         PlexConnection connection,
         PlexServerAuthToken token);
+
+    Task<Either<BaseError, int>> GetLibraryItemCount(
+        PlexLibrary library,
+        PlexConnection connection,
+        PlexServerAuthToken token);
 }

ErsatzTV.Core/Jellyfin/JellyfinItemType.cs

@@ -0,0 +1,6 @@
+namespace ErsatzTV.Core.Jellyfin;
+
+public static class JellyfinItemType
+{
+    public static readonly string Movie = "Movie";
+}

ErsatzTV.Core/Jellyfin/JellyfinMovieLibraryScanner.cs

@@ -80,7 +80,16 @@ public class JellyfinMovieLibraryScanner :
 
     protected override string MediaServerEtag(JellyfinMovie movie) => movie.Etag;
 
-    protected override Task<Either<BaseError, List<JellyfinMovie>>> GetMovieLibraryItems(
+    protected override Task<Either<BaseError, int>> CountMovieLibraryItems(
+        JellyfinConnectionParameters connectionParameters,
+        JellyfinLibrary library) =>
+        _jellyfinApiClient.GetLibraryItemCount(
+            connectionParameters.Address,
+            connectionParameters.ApiKey,
+            library,
+            JellyfinItemType.Movie);
+
+    protected override IAsyncEnumerable<JellyfinMovie> GetMovieLibraryItems(
         JellyfinConnectionParameters connectionParameters,
         JellyfinLibrary library) =>
         _jellyfinApiClient.GetMovieLibraryItems(

ErsatzTV.Core/Metadata/MediaServerMovieLibraryScanner.cs

@@ -53,13 +53,14 @@ public abstract class MediaServerMovieLibraryScanner<TConnectionParameters, TLib
     {
         try
         {
-            Either<BaseError, List<TMovie>> entries = await GetMovieLibraryItems(connectionParameters, library);
+            Either<BaseError, int> maybeCount = await CountMovieLibraryItems(connectionParameters, library);
-
+            foreach (BaseError error in maybeCount.LeftToSeq())
-            foreach (BaseError error in entries.LeftToSeq())
             {
                 return error;
             }
 
+            int count = await maybeCount.RightToSeq().HeadOrNone().IfNoneAsync(1);
+
             return await ScanLibrary(
                 movieRepository,
                 connectionParameters,
@@ -67,7 +68,8 @@ public abstract class MediaServerMovieLibraryScanner<TConnectionParameters, TLib
                 getLocalPath,
                 ffmpegPath,
                 ffprobePath,
-                entries.RightToSeq().Flatten().ToList(),
+                GetMovieLibraryItems(connectionParameters, library),
+                count,
                 deepScan,
                 cancellationToken);
         }
@@ -88,21 +90,24 @@ public abstract class MediaServerMovieLibraryScanner<TConnectionParameters, TLib
         Func<TMovie, string> getLocalPath,
         string ffmpegPath,
         string ffprobePath,
-        List<TMovie> movieEntries,
+        IAsyncEnumerable<TMovie> movieEntries,
+        int totalMovieCount,
         bool deepScan,
         CancellationToken cancellationToken)
     {
+        var incomingItemIds = new List<string>();
         List<TEtag> existingMovies = await movieRepository.GetExistingMovies(library);
 
-        var sortedMovies = movieEntries.OrderBy(m => m.MovieMetadata.Head().SortTitle).ToList();
+        await foreach (TMovie incoming in movieEntries.WithCancellation(cancellationToken))
-        foreach (TMovie incoming in sortedMovies)
         {
             if (cancellationToken.IsCancellationRequested)
             {
                 return new ScanCanceled();
             }
 
-            decimal percentCompletion = (decimal)sortedMovies.IndexOf(incoming) / sortedMovies.Count;
+            incomingItemIds.Add(MediaServerItemId(incoming));
+
+            decimal percentCompletion = Math.Clamp((decimal)incomingItemIds.Count / totalMovieCount, 0, 1);
             await _mediator.Publish(new LibraryScanProgress(library.Id, percentCompletion), cancellationToken);
 
             string localPath = getLocalPath(incoming);
@@ -165,8 +170,7 @@ public abstract class MediaServerMovieLibraryScanner<TConnectionParameters, TLib
         }
 
         // trash movies that are no longer present on the media server
-        var fileNotFoundItemIds = existingMovies.Map(m => m.MediaServerItemId)
+        var fileNotFoundItemIds = existingMovies.Map(m => m.MediaServerItemId).Except(incomingItemIds).ToList();
-            .Except(movieEntries.Map(MediaServerItemId)).ToList();
         List<int> ids = await movieRepository.FlagFileNotFound(library, fileNotFoundItemIds);
         await _searchIndex.RebuildItems(_searchRepository, ids);
 
@@ -178,7 +182,11 @@ public abstract class MediaServerMovieLibraryScanner<TConnectionParameters, TLib
     protected abstract string MediaServerItemId(TMovie movie);
     protected abstract string MediaServerEtag(TMovie movie);
 
-    protected abstract Task<Either<BaseError, List<TMovie>>> GetMovieLibraryItems(
+    protected abstract Task<Either<BaseError, int>> CountMovieLibraryItems(
+        TConnectionParameters connectionParameters,
+        TLibrary library);
+
+    protected abstract IAsyncEnumerable<TMovie> GetMovieLibraryItems(
         TConnectionParameters connectionParameters,
         TLibrary library);
 

ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs

@@ -89,7 +89,15 @@ public class PlexMovieLibraryScanner :
 
     protected override string MediaServerEtag(PlexMovie movie) => movie.Etag;
 
-    protected override Task<Either<BaseError, List<PlexMovie>>> GetMovieLibraryItems(
+    protected override Task<Either<BaseError, int>> CountMovieLibraryItems(
+        PlexConnectionParameters connectionParameters,
+        PlexLibrary library)
+        => _plexServerApiClient.GetLibraryItemCount(
+            library,
+            connectionParameters.Connection,
+            connectionParameters.Token);
+
+    protected override IAsyncEnumerable<PlexMovie> GetMovieLibraryItems(
         PlexConnectionParameters connectionParameters,
         PlexLibrary library) =>
         _plexServerApiClient.GetMovieLibraryContents(

ErsatzTV.Infrastructure/Emby/EmbyApiClient.cs

@@ -71,24 +71,29 @@ public class EmbyApiClient : IEmbyApiClient
         }
     }
 
-    public async Task<Either<BaseError, List<EmbyMovie>>> GetMovieLibraryItems(
+    public async IAsyncEnumerable<EmbyMovie> GetMovieLibraryItems(string address, string apiKey, EmbyLibrary library)
-        string address,
-        string apiKey,
-        EmbyLibrary library)
     {
-        try
+        IEmbyApi service = RestService.For<IEmbyApi>(address);
-        {
+        int size = await service
-            IEmbyApi service = RestService.For<IEmbyApi>(address);
+            .GetLibraryStats(apiKey, library.ItemId, EmbyItemType.Movie)
-            EmbyLibraryItemsResponse items = await service.GetMovieLibraryItems(apiKey, library.ItemId);
+            .Map(r => r.TotalRecordCount);
-            return items.Items
+
-                .Map(i => ProjectToMovie(library, i))
+        const int PAGE_SIZE = 10;
-                .Somes()
+
-                .ToList();
+        int pages = (size - 1) / PAGE_SIZE + 1;
-        }
+
-        catch (Exception ex)
+        for (var i = 0; i < pages; i++)
         {
-            _logger.LogError(ex, "Error getting emby movie library items");
+            int skip = i * PAGE_SIZE;
-            return BaseError.New(ex.Message);
+
+            Task<IEnumerable<EmbyMovie>> result = service
+                .GetMovieLibraryItems(apiKey, library.ItemId, startIndex: skip, limit: PAGE_SIZE)
+                .Map(items => items.Items.Map(item => ProjectToMovie(library, item)).Somes());
+
+            foreach (EmbyMovie movie in await result)
+            {
+                yield return movie;
+            }
         }
     }
 
@@ -202,6 +207,25 @@ public class EmbyApiClient : IEmbyApiClient
         }
     }
 
+    public async Task<Either<BaseError, int>> GetLibraryItemCount(
+        string address,
+        string apiKey,
+        EmbyLibrary library,
+        string includeItemTypes)
+    {
+        try
+        {
+            IEmbyApi service = RestService.For<IEmbyApi>(address);
+            EmbyLibraryItemsResponse items = await service.GetLibraryStats(apiKey, library.ItemId, includeItemTypes);
+            return items.TotalRecordCount;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error getting Emby library item count");
+            return BaseError.New(ex.Message);
+        }
+    }
+
     private Option<EmbyCollection> ProjectToCollection(EmbyLibraryItemResponse item)
     {
         try

ErsatzTV.Infrastructure/Emby/IEmbyApi.cs

@@ -17,6 +17,21 @@ public interface IEmbyApi
         [Header("X-Emby-Token")]
         string apiKey);
 
+    [Get("/Items")]
+    public Task<EmbyLibraryItemsResponse> GetLibraryStats(
+        [Header("X-Emby-Token")]
+        string apiKey,
+        [Query]
+        string parentId,
+        [Query]
+        string includeItemTypes,
+        [Query]
+        bool recursive = true,
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
+
     [Get("/Items")]
     public Task<EmbyLibraryItemsResponse> GetMovieLibraryItems(
         [Header("X-Emby-Token")]
@@ -29,7 +44,11 @@ public interface IEmbyApi
         [Query]
         string includeItemTypes = "Movie",
         [Query]
-        bool recursive = true);
+        bool recursive = true,
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
 
     [Get("/Items")]
     public Task<EmbyLibraryItemsResponse> GetShowLibraryItems(

ErsatzTV.Infrastructure/Emby/Models/EmbyLibraryItemsResponse.cs

@@ -3,4 +3,5 @@
 public class EmbyLibraryItemsResponse
 {
     public List<EmbyLibraryItemResponse> Items { get; set; }
+    public int TotalRecordCount { get; set; }
 }

ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs

@@ -23,6 +23,25 @@ public interface IJellyfinApi
         string apiKey);
 
     [Get("/Items")]
+    public Task<JellyfinLibraryItemsResponse> GetLibraryStats(
+        [Header("X-Emby-Token")]
+        string apiKey,
+        [Query]
+        string userId,
+        [Query]
+        string parentId,
+        [Query]
+        string includeItemTypes,
+        [Query]
+        bool recursive = true,
+        [Query]
+        string filters = "IsNotFolder",
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
+
+    [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<JellyfinLibraryItemsResponse> GetMovieLibraryItems(
         [Header("X-Emby-Token")]
         string apiKey,
@@ -38,7 +57,11 @@ public interface IJellyfinApi
         [Query]
         bool recursive = true,
         [Query]
-        string filters = "IsNotFolder");
+        string filters = "IsNotFolder",
+        [Query]
+        int startIndex = 0,
+        [Query]
+        int limit = 0);
 
     [Get("/Items")]
     public Task<JellyfinLibraryItemsResponse> GetShowLibraryItems(

ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs

@@ -91,29 +91,35 @@ public class JellyfinApiClient : IJellyfinApiClient
         }
     }
 
-    public async Task<Either<BaseError, List<JellyfinMovie>>> GetMovieLibraryItems(
+    public async IAsyncEnumerable<JellyfinMovie> GetMovieLibraryItems(
         string address,
         string apiKey,
         JellyfinLibrary library)
     {
-        try
+        if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{library.MediaSourceId}", out string userId))
         {
-            if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{library.MediaSourceId}", out string userId))
+            IJellyfinApi service = RestService.For<IJellyfinApi>(address);
+            int size = await service
+                .GetLibraryStats(apiKey, userId, library.ItemId, JellyfinItemType.Movie)
+                .Map(r => r.TotalRecordCount);
+
+            const int PAGE_SIZE = 10;
+
+            int pages = (size - 1) / PAGE_SIZE + 1;
+
+            for (var i = 0; i < pages; i++)
             {
-                IJellyfinApi service = RestService.For<IJellyfinApi>(address);
+                int skip = i * PAGE_SIZE;
-                JellyfinLibraryItemsResponse items = await service.GetMovieLibraryItems(apiKey, userId, library.ItemId);
-                return items.Items
-                    .Map(i => ProjectToMovie(library, i))
-                    .Somes()
-                    .ToList();
-            }
 
-            return BaseError.New("Jellyfin admin user id is not available");
+                Task<IEnumerable<JellyfinMovie>> result = service
-        }
+                    .GetMovieLibraryItems(apiKey, userId, library.ItemId, startIndex: skip, limit: PAGE_SIZE)
-        catch (Exception ex)
+                    .Map(items => items.Items.Map(item => ProjectToMovie(library, item)).Somes());
-        {
+
-            _logger.LogError(ex, "Error getting jellyfin movie library items");
+                foreach (JellyfinMovie movie in await result)
-            return BaseError.New(ex.Message);
+                {
+                    yield return movie;
+                }
+            }
         }
     }
 
@@ -262,6 +268,34 @@ public class JellyfinApiClient : IJellyfinApiClient
         }
     }
 
+    public async Task<Either<BaseError, int>> GetLibraryItemCount(
+        string address,
+        string apiKey,
+        JellyfinLibrary library,
+        string includeItemTypes)
+    {
+        try
+        {
+            if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{library.MediaSourceId}", out string userId))
+            {
+                IJellyfinApi service = RestService.For<IJellyfinApi>(address);
+                JellyfinLibraryItemsResponse items = await service.GetLibraryStats(
+                    apiKey,
+                    userId,
+                    library.ItemId,
+                    includeItemTypes);
+                return items.TotalRecordCount;
+            }
+
+            return BaseError.New("Jellyfin admin user id is not available");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error getting jellyfin library item count");
+            return BaseError.New(ex.Message);
+        }
+    }
+
     private Option<MediaItem> ProjectToCollectionMediaItem(JellyfinLibraryItemResponse item)
     {
         try

ErsatzTV.Infrastructure/Jellyfin/Models/JellyfinLibraryItemsResponse.cs

@@ -3,4 +3,5 @@
 public class JellyfinLibraryItemsResponse
 {
     public List<JellyfinLibraryItemResponse> Items { get; set; }
+    public int TotalRecordCount { get; set; }
 }

ErsatzTV.Infrastructure/Plex/IPlexServerApi.cs

@@ -17,11 +17,30 @@ public interface IPlexServerApi
         [Query] [AliasAs("X-Plex-Token")]
         string token);
 
+    [Get("/library/sections/{key}/all?X-Plex-Container-Start=0&X-Plex-Container-Size=0")]
+    [Headers("Accept: text/xml")]
+    public Task<PlexXmlMediaContainerStatsResponse> GetLibrarySection(
+        string key,
+        [Query] [AliasAs("X-Plex-Token")]
+        string token);
+
+    [Get("/library/sections/{key}/all")]
+    [Headers("Accept: application/json")]
+    public Task<PlexMediaContainerResponse<PlexMediaContainerMetadataContent<PlexMetadataResponse>>>
+        GetLibrarySectionContents(
+            string key,
+            [Query] [AliasAs("X-Plex-Token")]
+            string token);
+
     [Get("/library/sections/{key}/all")]
     [Headers("Accept: application/json")]
     public Task<PlexMediaContainerResponse<PlexMediaContainerMetadataContent<PlexMetadataResponse>>>
         GetLibrarySectionContents(
             string key,
+            [Query] [AliasAs("X-Plex-Container-Start")]
+            int skip,
+            [Query] [AliasAs("X-Plex-Container-Size")]
+            int take,
             [Query] [AliasAs("X-Plex-Token")]
             string token);
 

ErsatzTV.Infrastructure/Plex/Models/PlexMediaContainerResponse.cs

@@ -17,6 +17,13 @@ public class PlexMediaContainerMetadataContent<T>
     public List<T> Metadata { get; set; }
 }
 
+[XmlRoot("MediaContainer", Namespace = null)]
+public class PlexXmlMediaContainerStatsResponse
+{
+    [XmlAttribute("totalSize")]
+    public int TotalSize { get; set; }
+}
+
 [XmlRoot("MediaContainer", Namespace = null)]
 public class PlexXmlVideoMetadataResponseContainer
 {

ErsatzTV.Infrastructure/Plex/PlexServerApiClient.cs

@@ -26,9 +26,7 @@ public class PlexServerApiClient : IPlexServerApiClient
         _logger = logger;
     }
 
-    public async Task<bool> Ping(
+    public async Task<bool> Ping(PlexConnection connection, PlexServerAuthToken token)
-        PlexConnection connection,
-        PlexServerAuthToken token)
     {
         try
         {
@@ -76,21 +74,32 @@ public class PlexServerApiClient : IPlexServerApiClient
         }
     }
 
-    public async Task<Either<BaseError, List<PlexMovie>>> GetMovieLibraryContents(
+    public async IAsyncEnumerable<PlexMovie> GetMovieLibraryContents(
         PlexLibrary library,
         PlexConnection connection,
         PlexServerAuthToken token)
     {
-        try
+        IPlexServerApi xmlService = XmlServiceFor(connection.Uri);
+        int size = await xmlService.GetLibrarySection(library.Key, token.AuthToken).Map(r => r.TotalSize);
+
+        const int PAGE_SIZE = 10;
+
+        IPlexServerApi service = RestService.For<IPlexServerApi>(connection.Uri);
+        int pages = (size - 1) / PAGE_SIZE + 1;
+
+        for (var i = 0; i < pages; i++)
         {
-            IPlexServerApi service = RestService.For<IPlexServerApi>(connection.Uri);
+            int skip = i * PAGE_SIZE;
-            return await service.GetLibrarySectionContents(library.Key, token.AuthToken)
+
+            Task<IEnumerable<PlexMovie>> result = service
+                .GetLibrarySectionContents(library.Key, skip, PAGE_SIZE, token.AuthToken)
                 .Map(r => r.MediaContainer.Metadata.Filter(m => m.Media.Count > 0 && m.Media[0].Part.Count > 0))
-                .Map(list => list.Map(metadata => ProjectToMovie(metadata, library.MediaSourceId)).ToList());
+                .Map(list => list.Map(metadata => ProjectToMovie(metadata, library.MediaSourceId)));
-        }
+
-        catch (Exception ex)
+            foreach (PlexMovie movie in await result)
-        {
+            {
-            return BaseError.New(ex.ToString());
+                yield return movie;
+            }
         }
     }
 
@@ -254,6 +263,23 @@ public class PlexServerApiClient : IPlexServerApiClient
         }
     }
 
+    public async Task<Either<BaseError, int>> GetLibraryItemCount(
+        PlexLibrary library,
+        PlexConnection connection,
+        PlexServerAuthToken token)
+    {
+        try
+        {
+            IPlexServerApi service = XmlServiceFor(connection.Uri);
+            return await service.GetLibrarySection(library.Key, token.AuthToken).Map(r => r.TotalSize);
+        }
+        catch (Exception ex)
+        {
+            return BaseError.New(ex.ToString());
+        }
+    }
+
+
     private List<PlexEpisode> ProcessMultiEpisodeFiles(IEnumerable<PlexEpisode> episodes)
     {
         // add all metadata from duplicate paths to first entry with given path