batch search index updates (#1591)

CHANGELOG.md

@@ -21,6 +21,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
   - A deep scan can be used to fix all movies, otherwise any future updates made to JF movies will correctly sync to ETV
 - Automatically generate JWT tokens to allow channel previews of protected streams 
 
+### Changed
+- Log search index updates under scanner category at debug level, to indicate a potential cause for the UI being out of date
+- Batch search index updates to keep pace with library scans
+  - Previously, search index updates would slowly process over minutes/hours after library scans completed
+  - Search index updates should now complete at the same time as library scans
+
 ## [0.8.5-beta] - 2024-01-30
 ### Added
 - Respect browser's `Accept-Language` header for date time display

ErsatzTV.Application/Libraries/Commands/CallLibraryScannerHandler.cs

@@ -25,6 +25,9 @@ public abstract class CallLibraryScannerHandler<TRequest>
     private readonly IDbContextFactory<TvContext> _dbContextFactory;
     private readonly IMediator _mediator;
     private readonly IRuntimeInfo _runtimeInfo;
+    private readonly List<int> _toReindex = [];
+    private readonly List<int> _toRemove = [];
+    private readonly int _batchSize = 100;
     private string _libraryName;
 
     protected CallLibraryScannerHandler(
@@ -65,6 +68,18 @@ public abstract class CallLibraryScannerHandler<TRequest>
             {
                 return BaseError.New($"ErsatzTV.Scanner exited with code {process.ExitCode}");
             }
+
+            if (_toReindex.Count > 0)
+            {
+                await _channel.WriteAsync(new ReindexMediaItems(_toReindex.ToArray()), cancellationToken);
+                _toReindex.Clear();
+            }
+                    
+            if (_toRemove.Count > 0)
+            {
+                await _channel.WriteAsync(new RemoveMediaItems(_toReindex.ToArray()), cancellationToken);
+                _toRemove.Clear();
+            }
         }
         catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
         {
@@ -122,25 +137,27 @@ public abstract class CallLibraryScannerHandler<TRequest>
                         _libraryName = progressUpdate.LibraryName;
                     }
 
-                    if (progressUpdate.PercentComplete is not null)
+                    _toReindex.AddRange(progressUpdate.ItemsToReindex);
+                    if (_toReindex.Count >= _batchSize)
                     {
-                        var progress = new LibraryScanProgress(
-                            progressUpdate.LibraryId,
-                            progressUpdate.PercentComplete.Value);
-
-                        await _mediator.Publish(progress);
+                        await _channel.WriteAsync(new ReindexMediaItems(_toReindex.ToArray()));
+                        _toReindex.Clear();
                     }
                     
-                    if (progressUpdate.ItemsToReindex.Length > 0)
+                    _toRemove.AddRange(progressUpdate.ItemsToRemove);
+                    if (_toRemove.Count >= _batchSize)
                     {
-                        var reindex = new ReindexMediaItems(progressUpdate.ItemsToReindex);
-                        await _channel.WriteAsync(reindex);
+                        await _channel.WriteAsync(new RemoveMediaItems(_toReindex.ToArray()));
+                        _toRemove.Clear();
                     }
 
-                    if (progressUpdate.ItemsToRemove.Length > 0)
+                    if (progressUpdate.PercentComplete is not null)
                     {
-                        var remove = new RemoveMediaItems(progressUpdate.ItemsToRemove);
-                        await _channel.WriteAsync(remove);
+                        var progress = new LibraryScanProgress(
+                            progressUpdate.LibraryId,
+                            progressUpdate.PercentComplete.Value);
+
+                        await _mediator.Publish(progress);
                     }
                 }
             }

ErsatzTV/Program.cs

@@ -66,6 +66,7 @@ public class Program
 
             // scanning
             .MinimumLevel.Override("ErsatzTV.Services.ScannerService", LoggingLevelSwitches.ScanningLevelSwitch)
+            .MinimumLevel.Override("ErsatzTV.Services.SearchIndexService", LoggingLevelSwitches.ScanningLevelSwitch)
             .MinimumLevel.Override("ErsatzTV.Scanner", LoggingLevelSwitches.ScanningLevelSwitch)
 
             // scheduling

ErsatzTV/Services/SearchIndexService.cs

@@ -9,13 +9,13 @@ namespace ErsatzTV.Services;
 public class SearchIndexService : BackgroundService
 {
     private readonly ChannelReader<ISearchIndexBackgroundServiceRequest> _channel;
-    private readonly ILogger<WorkerService> _logger;
+    private readonly ILogger<SearchIndexService> _logger;
     private readonly IServiceScopeFactory _serviceScopeFactory;
 
     public SearchIndexService(
         ChannelReader<ISearchIndexBackgroundServiceRequest> channel,
         IServiceScopeFactory serviceScopeFactory,
-        ILogger<WorkerService> logger)
+        ILogger<SearchIndexService> logger)
     {
         _channel = channel;
         _serviceScopeFactory = serviceScopeFactory;
@@ -40,9 +40,11 @@ public class SearchIndexService : BackgroundService
                     switch (request)
                     {
                         case ReindexMediaItems reindexMediaItems:
+                            _logger.LogDebug("Reindexing media items: {MediaItemIds}", reindexMediaItems.MediaItemIds);
                             await mediator.Send(reindexMediaItems, stoppingToken);
                             break;
                         case RemoveMediaItems removeMediaItems:
+                            _logger.LogDebug("Removing media items: {MediaItemIds}", removeMediaItems.MediaItemIds);
                             await mediator.Send(removeMediaItems, stoppingToken);
                             break;
                     }