add plex deep scan mode and sync labels (#749)

CHANGELOG.md

@@ -4,12 +4,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
+### Fixed
+- Fix unlocking libraries when scanning fails for any reason
+
 ### Added
 - Add support for burning in embedded text subtitles
 - Add support for burning in external text subtitles
   - **This requires a one-time full library scan, which may take a long time with large libraries.**
 - Sync Plex collections as tags on movies, shows, seasons and episodes
   - This allows smart collections that use queries like `tag:"Plex Collection Name"`
+- Sync Plex labels as tags on movies and shows
+  - This allows smart collections that use queries like `tag:"Plex Label Name"`
+- Add `Deep Scan` button for Plex libraries
+  - This scanning mode is *slow* but is required to detect some changes like labels
 
 ### Changed
 - Improve the speed and change detection of the Plex library scanners

ErsatzTV.Application/Plex/Commands/SynchronizePlexLibraryById.cs

@@ -6,16 +6,16 @@ public interface ISynchronizePlexLibraryById : IRequest<Either<BaseError, string
 {
     int PlexLibraryId { get; }
     bool ForceScan { get; }
+    bool DeepScan { get; }
 }
 
-public record SynchronizePlexLibraryByIdIfNeeded
-    (int PlexLibraryId) : ISynchronizePlexLibraryById
+public record SynchronizePlexLibraryByIdIfNeeded(int PlexLibraryId) : ISynchronizePlexLibraryById
 {
     public bool ForceScan => false;
+    public bool DeepScan => false;
 }
 
-public record ForceSynchronizePlexLibraryById
-    (int PlexLibraryId) : ISynchronizePlexLibraryById
+public record ForceSynchronizePlexLibraryById(int PlexLibraryId, bool DeepScan) : ISynchronizePlexLibraryById
 {
     public bool ForceScan => true;
 }

ErsatzTV.Application/Plex/Commands/SynchronizePlexLibraryByIdHandler.cs

@@ -71,7 +71,8 @@ public class
                             parameters.ConnectionParameters.PlexServerAuthToken,
                             parameters.Library,
                             parameters.FFmpegPath,
-                            parameters.FFprobePath);
+                            parameters.FFprobePath,
+                            parameters.DeepScan);
                         break;
                     case LibraryMediaKind.Shows:
                         await _plexTelevisionLibraryScanner.ScanLibrary(
@@ -79,7 +80,8 @@ public class
                             parameters.ConnectionParameters.PlexServerAuthToken,
                             parameters.Library,
                             parameters.FFmpegPath,
-                            parameters.FFprobePath);
+                            parameters.FFprobePath,
+                            parameters.DeepScan);
                         break;
                 }
 
@@ -112,7 +114,8 @@ public class
                     request.ForceScan,
                     libraryRefreshInterval,
                     ffmpegPath,
-                    ffprobePath
+                    ffprobePath,
+                    request.DeepScan
                 ));
 
     private Task<Validation<BaseError, ConnectionParameters>> ValidateConnection(
@@ -176,7 +179,8 @@ public class
         bool ForceScan,
         int LibraryRefreshInterval,
         string FFmpegPath,
-        string FFprobePath);
+        string FFprobePath,
+        bool DeepScan);
 
     private record ConnectionParameters(PlexMediaSource PlexMediaSource, PlexConnection ActiveConnection)
     {

ErsatzTV.Core/Interfaces/Plex/IPlexMovieLibraryScanner.cs

@@ -10,5 +10,6 @@ public interface IPlexMovieLibraryScanner
         PlexServerAuthToken token,
         PlexLibrary library,
         string ffmpegPath,
-        string ffprobePath);
+        string ffprobePath,
+        bool deepScan);
 }

ErsatzTV.Core/Interfaces/Plex/IPlexTelevisionLibraryScanner.cs

@@ -10,5 +10,6 @@ public interface IPlexTelevisionLibraryScanner
         PlexServerAuthToken token,
         PlexLibrary library,
         string ffmpegPath,
-        string ffprobePath);
+        string ffprobePath,
+        bool deepScan);
 }

ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs

@@ -58,7 +58,8 @@ public class PlexMovieLibraryScanner : PlexLibraryScanner, IPlexMovieLibraryScan
         PlexServerAuthToken token,
         PlexLibrary library,
         string ffmpegPath,
-        string ffprobePath)
+        string ffprobePath,
+        bool deepScan)
     {
         List<PlexItemEtag> existingMovies = await _movieRepository.GetExistingPlexMovies(library);
 
@@ -96,14 +97,21 @@ public class PlexMovieLibraryScanner : PlexLibraryScanner, IPlexMovieLibraryScan
                     decimal percentCompletion = (decimal)validMovies.IndexOf(incoming) / validMovies.Count;
                     await _mediator.Publish(new LibraryScanProgress(library.Id, percentCompletion));
 
-                    Option<PlexItemEtag> maybeExisting = existingMovies.Find(ie => ie.Key == incoming.Key);
-                    if (await maybeExisting.Map(e => e.Etag ?? string.Empty).IfNoneAsync(string.Empty) == incoming.Etag)
+                    // deep scan will pull every movie from the plex api
+                    if (!deepScan)
                     {
-                        // _logger.LogDebug("NOOP: etag has not changed for plex movie with key {Key}", incoming.Key);
-                        continue;
-                    }
+                        Option<PlexItemEtag> maybeExisting = existingMovies.Find(ie => ie.Key == incoming.Key);
+                        if (await maybeExisting.Map(e => e.Etag ?? string.Empty).IfNoneAsync(string.Empty) ==
+                            incoming.Etag)
+                        {
+                            // _logger.LogDebug("NOOP: etag has not changed for plex movie with key {Key}", incoming.Key);
+                            continue;
+                        }
 
-                    _logger.LogDebug("UPDATE: Etag has changed for movie {Movie}", incoming.MovieMetadata.Head().Title);
+                        _logger.LogDebug(
+                            "UPDATE: Etag has changed for movie {Movie}",
+                            incoming.MovieMetadata.Head().Title);
+                    }
 
                     // TODO: figure out how to rebuild playlists
                     Either<BaseError, MediaItemScanResult<PlexMovie>> maybeMovie = await _movieRepository
@@ -171,55 +179,58 @@ public class PlexMovieLibraryScanner : PlexLibraryScanner, IPlexMovieLibraryScan
         MediaVersion existingVersion = existing.MediaVersions.Head();
         MediaVersion incomingVersion = incoming.MediaVersions.Head();
 
-        foreach (MediaFile incomingFile in incomingVersion.MediaFiles.HeadOrNone())
+        if (existing.Etag != incoming.Etag)
         {
-            foreach (MediaFile existingFile in existingVersion.MediaFiles.HeadOrNone())
+            foreach (MediaFile incomingFile in incomingVersion.MediaFiles.HeadOrNone())
             {
-                if (incomingFile.Path != existingFile.Path)
+                foreach (MediaFile existingFile in existingVersion.MediaFiles.HeadOrNone())
                 {
-                    _logger.LogDebug(
-                        "Plex movie has moved from {OldPath} to {NewPath}",
-                        existingFile.Path,
-                        incomingFile.Path);
+                    if (incomingFile.Path != existingFile.Path)
+                    {
+                        _logger.LogDebug(
+                            "Plex movie has moved from {OldPath} to {NewPath}",
+                            existingFile.Path,
+                            incomingFile.Path);
 
-                    existingFile.Path = incomingFile.Path;
+                        existingFile.Path = incomingFile.Path;
 
-                    await _movieRepository.UpdatePath(existingFile.Id, incomingFile.Path);
+                        await _movieRepository.UpdatePath(existingFile.Id, incomingFile.Path);
+                    }
                 }
             }
-        }
 
-        string localPath = _plexPathReplacementService.GetReplacementPlexPath(
-            pathReplacements,
-            incoming.MediaVersions.Head().MediaFiles.Head().Path,
-            false);
+            string localPath = _plexPathReplacementService.GetReplacementPlexPath(
+                pathReplacements,
+                incoming.MediaVersions.Head().MediaFiles.Head().Path,
+                false);
 
-        _logger.LogDebug("Refreshing {Attribute} for {Path}", "Statistics", localPath);
-        Either<BaseError, bool> refreshResult =
-            await _localStatisticsProvider.RefreshStatistics(ffmpegPath, ffprobePath, existing, localPath);
+            _logger.LogDebug("Refreshing {Attribute} for {Path}", "Statistics", localPath);
+            Either<BaseError, bool> refreshResult =
+                await _localStatisticsProvider.RefreshStatistics(ffmpegPath, ffprobePath, existing, localPath);
 
-        await refreshResult.Match(
-            async _ =>
-            {
-                foreach (MediaItem updated in await _searchRepository.GetItemToIndex(incoming.Id))
+            await refreshResult.Match(
+                async _ =>
                 {
-                    await _searchIndex.UpdateItems(
-                        _searchRepository,
-                        new List<MediaItem> { updated });
-                }
-
-                await _metadataRepository.UpdatePlexStatistics(existingVersion.Id, incomingVersion);
-            },
-            error =>
-            {
-                _logger.LogWarning(
-                    "Unable to refresh {Attribute} for media item {Path}. Error: {Error}",
-                    "Statistics",
-                    localPath,
-                    error.Value);
+                    foreach (MediaItem updated in await _searchRepository.GetItemToIndex(incoming.Id))
+                    {
+                        await _searchIndex.UpdateItems(
+                            _searchRepository,
+                            new List<MediaItem> { updated });
+                    }
 
-                return Task.CompletedTask;
-            });
+                    await _metadataRepository.UpdatePlexStatistics(existingVersion.Id, incomingVersion);
+                },
+                error =>
+                {
+                    _logger.LogWarning(
+                        "Unable to refresh {Attribute} for media item {Path}. Error: {Error}",
+                        "Statistics",
+                        localPath,
+                        error.Value);
+
+                    return Task.CompletedTask;
+                });
+        }
 
         return result;
     }

ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs

@@ -59,7 +59,8 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
         PlexServerAuthToken token,
         PlexLibrary library,
         string ffmpegPath,
-        string ffprobePath)
+        string ffprobePath,
+        bool deepScan)
     {
         List<PlexPathReplacement> pathReplacements = await _mediaSourceRepository
             .GetPlexPathReplacements(library.MediaSourceId);
@@ -80,7 +81,7 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
                     // TODO: figure out how to rebuild playlists
                     Either<BaseError, MediaItemScanResult<PlexShow>> maybeShow = await _televisionRepository
                         .GetOrAddPlexShow(library, incoming)
-                        .BindT(existing => UpdateMetadata(existing, incoming, library, connection, token))
+                        .BindT(existing => UpdateMetadata(existing, incoming, library, connection, token, deepScan))
                         .BindT(existing => UpdateArtwork(existing, incoming));
 
                     await maybeShow.Match(
@@ -93,7 +94,8 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
                                 connection,
                                 token,
                                 ffmpegPath,
-                                ffprobePath);
+                                ffprobePath,
+                                deepScan);
 
                             await _televisionRepository.SetPlexEtag(result.Item, incoming.Etag);
 
@@ -144,12 +146,13 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
         PlexShow incoming,
         PlexLibrary library,
         PlexConnection connection,
-        PlexServerAuthToken token)
+        PlexServerAuthToken token,
+        bool deepScan)
     {
         PlexShow existing = result.Item;
         ShowMetadata existingMetadata = existing.ShowMetadata.Head();
 
-        if (existing.Etag != incoming.Etag)
+        if (existing.Etag != incoming.Etag || deepScan)
         {
             Either<BaseError, ShowMetadata> maybeMetadata =
                 await _plexServerApiClient.GetShowMetadata(
@@ -322,7 +325,8 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
         PlexConnection connection,
         PlexServerAuthToken token,
         string ffmpegPath,
-        string ffprobePath)
+        string ffprobePath,
+        bool deepScan)
     {
         Either<BaseError, List<PlexSeason>> entries = await _plexServerApiClient.GetShowSeasons(
             library,
@@ -352,7 +356,8 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
                                 connection,
                                 token,
                                 ffmpegPath,
-                                ffprobePath);
+                                ffprobePath,
+                                deepScan);
 
                             await _televisionRepository.SetPlexEtag(season, incoming.Etag);
 
@@ -441,7 +446,8 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
         PlexConnection connection,
         PlexServerAuthToken token,
         string ffmpegPath,
-        string ffprobePath)
+        string ffprobePath,
+        bool deepScan)
     {
         List<PlexItemEtag> existingEpisodes = await _televisionRepository.GetExistingPlexEpisodes(library, season);
 
@@ -476,16 +482,20 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
 
                 foreach (PlexEpisode incoming in validEpisodes)
                 {
-                    Option<PlexItemEtag> maybeExisting = existingEpisodes.Find(ie => ie.Key == incoming.Key);
-                    if (await maybeExisting.Map(e => e.Etag ?? string.Empty).IfNoneAsync(string.Empty) == incoming.Etag)
+                    if (!deepScan)
                     {
-                        // _logger.LogDebug("NOOP: etag has not changed for plex episode with key {Key}", incoming.Key);
-                        continue;
-                    }
+                        Option<PlexItemEtag> maybeExisting = existingEpisodes.Find(ie => ie.Key == incoming.Key);
+                        if (await maybeExisting.Map(e => e.Etag ?? string.Empty).IfNoneAsync(string.Empty) ==
+                            incoming.Etag)
+                        {
+                            // _logger.LogDebug("NOOP: etag has not changed for plex episode with key {Key}", incoming.Key);
+                            continue;
+                        }
 
-                    // _logger.LogDebug(
-                    //     "UPDATE: Etag has changed for episode {Episode}",
-                    //     $"s{season.SeasonNumber}e{incoming.EpisodeMetadata.Head().EpisodeNumber}");
+                        // _logger.LogDebug(
+                        //     "UPDATE: Etag has changed for episode {Episode}",
+                        //     $"s{season.SeasonNumber}e{incoming.EpisodeMetadata.Head().EpisodeNumber}");
+                    }
 
                     incoming.SeasonId = season.Id;
 
@@ -502,7 +512,8 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
                                 connection,
                                 token,
                                 ffmpegPath,
-                                ffprobePath))
+                                ffprobePath,
+                                deepScan))
                         .BindT(existing => UpdateSubtitles(pathReplacements, existing, incoming))
                         .BindT(existing => UpdateArtwork(existing, incoming));
 
@@ -589,40 +600,49 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
         PlexConnection connection,
         PlexServerAuthToken token,
         string ffmpegPath,
-        string ffprobePath)
+        string ffprobePath,
+        bool deepScan)
     {
         PlexEpisode existing = result.Item;
         MediaVersion existingVersion = existing.MediaVersions.Head();
         MediaVersion incomingVersion = incoming.MediaVersions.Head();
 
-        foreach (MediaFile incomingFile in incomingVersion.MediaFiles.HeadOrNone())
+        if (existing.Etag != incoming.Etag || deepScan)
         {
-            foreach (MediaFile existingFile in existingVersion.MediaFiles.HeadOrNone())
+            foreach (MediaFile incomingFile in incomingVersion.MediaFiles.HeadOrNone())
             {
-                if (incomingFile.Path != existingFile.Path)
+                foreach (MediaFile existingFile in existingVersion.MediaFiles.HeadOrNone())
                 {
-                    _logger.LogDebug(
-                        "Plex episode has moved from {OldPath} to {NewPath}",
-                        existingFile.Path,
-                        incomingFile.Path);
+                    if (incomingFile.Path != existingFile.Path)
+                    {
+                        _logger.LogDebug(
+                            "Plex episode has moved from {OldPath} to {NewPath}",
+                            existingFile.Path,
+                            incomingFile.Path);
 
-                    existingFile.Path = incomingFile.Path;
+                        existingFile.Path = incomingFile.Path;
 
-                    await _televisionRepository.UpdatePath(existingFile.Id, incomingFile.Path);
+                        await _televisionRepository.UpdatePath(existingFile.Id, incomingFile.Path);
+                    }
                 }
             }
-        }
 
-        if (existing.Etag != incoming.Etag)
-        {
+            Either<BaseError, bool> refreshResult = true;
+
             string localPath = _plexPathReplacementService.GetReplacementPlexPath(
                 pathReplacements,
                 incoming.MediaVersions.Head().MediaFiles.Head().Path,
                 false);
 
-            _logger.LogDebug("Refreshing {Attribute} for {Path}", "Statistics", localPath);
-            Either<BaseError, bool> refreshResult =
-                await _localStatisticsProvider.RefreshStatistics(ffmpegPath, ffprobePath, existing, localPath);
+            if (existing.Etag != incoming.Etag)
+            {
+                _logger.LogDebug("Refreshing {Attribute} for {Path}", "Statistics", localPath);
+                refreshResult = await _localStatisticsProvider.RefreshStatistics(
+                    ffmpegPath,
+                    ffprobePath,
+                    existing,
+                    localPath);
+            }
 
             await refreshResult.Match(
                 async _ =>
@@ -715,15 +735,12 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
         {
             PlexEpisode existing = result.Item;
 
-            if (existing.Etag != incoming.Etag)
-            {
-                string localPath = _plexPathReplacementService.GetReplacementPlexPath(
-                    pathReplacements,
-                    incoming.MediaVersions.Head().MediaFiles.Head().Path,
-                    false);
+            string localPath = _plexPathReplacementService.GetReplacementPlexPath(
+                pathReplacements,
+                incoming.MediaVersions.Head().MediaFiles.Head().Path,
+                false);
 
-                await _localSubtitlesProvider.UpdateSubtitles(existing, localPath, false);
-            }
+            await _localSubtitlesProvider.UpdateSubtitles(existing, localPath, false);
 
             return result;
         }
@@ -745,11 +762,8 @@ public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionL
             if (maybeExistingMetadata.IsSome)
             {
                 EpisodeMetadata existingMetadata = maybeExistingMetadata.ValueUnsafe();
-                if (existing.Etag != incoming.Etag)
-                {
-                    await UpdateArtworkIfNeeded(existingMetadata, incomingMetadata, ArtworkKind.Thumbnail);
-                    await _metadataRepository.MarkAsUpdated(existingMetadata, incomingMetadata.DateUpdated);
-                }
+                await UpdateArtworkIfNeeded(existingMetadata, incomingMetadata, ArtworkKind.Thumbnail);
+                await _metadataRepository.MarkAsUpdated(existingMetadata, incomingMetadata.DateUpdated);
             }
         }
 

ErsatzTV.Infrastructure/Plex/Models/PlexLabelResponse.cs

@@ -0,0 +1,12 @@
+using System.Xml.Serialization;
+
+namespace ErsatzTV.Infrastructure.Plex.Models;
+
+public class PlexLabelResponse
+{
+    [XmlAttribute("id")]
+    public int Id { get; set; }
+
+    [XmlAttribute("tag")]
+    public string Tag { get; set; }
+}

ErsatzTV.Infrastructure/Plex/Models/PlexMetadataResponse.cs

@@ -61,6 +61,9 @@ public class PlexMetadataResponse
     [XmlElement("Genre")]
     public List<PlexGenreResponse> Genre { get; set; }
 
+    [XmlElement("Label")]
+    public List<PlexLabelResponse> Label { get; set; }
+
     [XmlElement("Role")]
     public List<PlexRoleResponse> Role { get; set; }
 

ErsatzTV.Infrastructure/Plex/PlexEtag.cs

@@ -40,6 +40,13 @@ public static class PlexEtag
             bw.Write(genre.Tag);
         }
 
+        // label ids
+        foreach (PlexLabelResponse label in Optional(response.Label).Flatten())
+        {
+            bw.Write((byte)FieldKey.LabelTag);
+            bw.Write(label.Tag);
+        }
+
         // director ids
         foreach (PlexDirectorResponse director in Optional(response.Director).Flatten())
         {
@@ -108,6 +115,13 @@ public static class PlexEtag
             bw.Write(role.Tag);
         }
 
+        // label ids
+        foreach (PlexLabelResponse label in Optional(response.Label).Flatten())
+        {
+            bw.Write((byte)FieldKey.LabelTag);
+            bw.Write(label.Tag);
+        }
+
         ms.Position = 0;
         byte[] hash = SHA1.Create().ComputeHash(ms);
         return BitConverter.ToString(hash).Replace("-", string.Empty);
@@ -237,6 +251,7 @@ public static class PlexEtag
         WriterTag = 12,
         CollectionTag = 13,
         RoleTag = 14,
+        LabelTag = 15,
 
         Thumb = 20,
         Art = 21

ErsatzTV.Infrastructure/Plex/PlexServerApiClient.cs

@@ -402,6 +402,11 @@ public class PlexServerApiClient : IPlexServerApiClient
             metadata.Tags.Add(new Tag { Name = collection.Tag });
         }
 
+        foreach (PlexLabelResponse label in Optional(response.Label).Flatten())
+        {
+            metadata.Tags.Add(new Tag { Name = label.Tag });
+        }
+
         if (!string.IsNullOrWhiteSpace(response.Studio))
         {
             metadata.Studios.Add(new Studio { Name = response.Studio });
@@ -584,6 +589,11 @@ public class PlexServerApiClient : IPlexServerApiClient
             metadata.Tags.Add(new Tag { Name = collection.Tag });
         }
 
+        foreach (PlexLabelResponse label in Optional(response.Label).Flatten())
+        {
+            metadata.Tags.Add(new Tag { Name = label.Tag });
+        }
+
         if (DateTime.TryParse(response.OriginallyAvailableAt, out DateTime releaseDate))
         {
             metadata.ReleaseDate = releaseDate;

ErsatzTV/Pages/Libraries.razor

@@ -55,7 +55,19 @@
                     }
                     else
                     {
-                        <div style="width: 48px"></div>
+                        if (context is PlexLibraryViewModel)
+                        {
+                            <MudTooltip Text="Deep Scan Library">
+                                <MudIconButton Icon="@Icons.Material.Filled.FindReplace"
+                                               Disabled="@_locker.IsLibraryLocked(context.Id)"
+                                               OnClick="@(_ => ScanLibrary(context, true))">
+                                </MudIconButton>
+                            </MudTooltip>
+                        }
+                        else
+                        {
+                            <div style="width: 48px"></div>
+                        }
                         <MudTooltip Text="Scan Library">
                             <MudIconButton Icon="@Icons.Material.Filled.Refresh"
                                            Disabled="@_locker.IsLibraryLocked(context.Id)"
@@ -94,7 +106,7 @@
         _progressByLibrary = _libraries.ToDictionary(vm => vm.Id, _ => 0);
     }
 
-    private async Task ScanLibrary(LibraryViewModel library)
+    private async Task ScanLibrary(LibraryViewModel library, bool deepScan = false)
     {
         if (_locker.LockLibrary(library.Id))
         {
@@ -104,7 +116,7 @@
                     await _workerChannel.WriteAsync(new ForceScanLocalLibrary(library.Id), _cts.Token);
                     break;
                 case PlexLibraryViewModel:
-                    await _plexWorkerChannel.WriteAsync(new ForceSynchronizePlexLibraryById(library.Id), _cts.Token);
+                    await _plexWorkerChannel.WriteAsync(new ForceSynchronizePlexLibraryById(library.Id, deepScan), _cts.Token);
                     break;
                 case JellyfinLibraryViewModel:
                     await _jellyfinWorkerChannel.WriteAsync(new ForceSynchronizeJellyfinLibraryById(library.Id), _cts.Token);