timeout playout builds after 2 minutes (#1269)

* add cancellation token support to playout builds and collection enumerators

* fix playout bug with shuffle in order

* update changelog

CHANGELOG.md

@@ -9,6 +9,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix fallback filler looping by forcing software mode for this content
   - Other content will still use hardware acceleration as configured
   - Hardware-accelerated fallback filler may be re-enabled in the future
+- Fix playout building when shuffle in order is used with a single media item
+
+### Changed
+- Timeout playout builds after 2 minutes; this should prevent playout bugs from blocking other functionality
 
 ## [0.7.8-beta] - 2023-04-29
 ### Added

ErsatzTV.Application/Playouts/Commands/BuildPlayoutHandler.cs

@@ -40,19 +40,25 @@ public class BuildPlayoutHandler : IRequestHandler<BuildPlayout, Either<BaseErro
     {
         await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
         Validation<BaseError, Playout> validation = await Validate(dbContext, request);
-        return await validation.Apply(playout => ApplyUpdateRequest(dbContext, request, playout));
+        return await validation.Match(
+            playout => ApplyUpdateRequest(dbContext, request, playout, cancellationToken),
+            error => Task.FromResult<Either<BaseError, Unit>>(error.Join()));
     }
 
-    private async Task<Unit> ApplyUpdateRequest(TvContext dbContext, BuildPlayout request, Playout playout)
+    private async Task<Either<BaseError, Unit>> ApplyUpdateRequest(
+        TvContext dbContext,
+        BuildPlayout request,
+        Playout playout,
+        CancellationToken cancellationToken)
     {
         try
         {
-            await _playoutBuilder.Build(playout, request.Mode);
+            await _playoutBuilder.Build(playout, request.Mode, cancellationToken);
 
             // let any active segmenter processes know that the playout has been modified
             // and therefore the segmenter may need to seek into the next item instead of
             // starting at the beginning (if already working ahead)
-            bool hasChanges = await dbContext.SaveChangesAsync() > 0;
+            bool hasChanges = await dbContext.SaveChangesAsync(cancellationToken) > 0;
             if (request.Mode != PlayoutBuildMode.Continue && hasChanges)
             {
                 _ffmpegSegmenterService.PlayoutUpdated(playout.Channel.Number);
@@ -71,15 +77,23 @@ public class BuildPlayoutHandler : IRequestHandler<BuildPlayout, Either<BaseErro
                 string fileName = Path.Combine(FileSystemLayout.ChannelGuideCacheFolder, $"{channelNumber}.xml");
                 if (hasChanges || !File.Exists(fileName))
                 {
-                    await _workerChannel.WriteAsync(new RefreshChannelData(channelNumber));
+                    await _workerChannel.WriteAsync(new RefreshChannelData(channelNumber), cancellationToken);
                 }
             }
 
-            await _workerChannel.WriteAsync(new ExtractEmbeddedSubtitles(playout.Id));
+            await _workerChannel.WriteAsync(new ExtractEmbeddedSubtitles(playout.Id), cancellationToken);
+        }
+        catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
+        {
+            _client.Notify(ex);
+            return BaseError.New(
+                $"Timeout building playout for channel {playout.Channel.Name}; this may be a bug!");
         }
         catch (Exception ex)
         {
             _client.Notify(ex);
+            return BaseError.New(
+                $"Unexpected error building playout for channel {playout.Channel.Name}: {ex.Message}");
         }
 
         return Unit.Default;

ErsatzTV.Core.Tests/Scheduling/ChronologicalContentTests.cs

@@ -8,6 +8,14 @@ namespace ErsatzTV.Core.Tests.Scheduling;
 [TestFixture]
 public class ChronologicalContentTests
 {
+    private CancellationToken _cancellationToken;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+    }
+    
     [Test]
     public void Episodes_Should_Sort_By_Aired()
     {

ErsatzTV.Core.Tests/Scheduling/CustomOrderContentTests.cs

@@ -7,6 +7,14 @@ namespace ErsatzTV.Core.Tests.Scheduling;
 
 public class CustomOrderContentTests
 {
+    private CancellationToken _cancellationToken;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+    }
+
     [Test]
     public void MediaItems_Should_Sort_By_CustomOrder()
     {

ErsatzTV.Core.Tests/Scheduling/PlayoutBuilderTests.cs

@@ -32,6 +32,14 @@ public class PlayoutBuilderTests
         _logger = loggerFactory?.CreateLogger<PlayoutBuilder>();
     }
 
+    private CancellationToken _cancellationToken;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+    }
+
     [TestFixture]
     public class NewPlayout : PlayoutBuilderTests
     {
@@ -46,7 +54,7 @@ public class PlayoutBuilderTests
 
             (PlayoutBuilder builder, Playout playout) = TestDataFloodForItems(mediaItems, PlaybackOrder.Random);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, _cancellationToken);
 
             result.Items.Should().BeEmpty();
         }
@@ -64,7 +72,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(1);
             result.Items.Head().MediaItemId.Should().Be(2);
@@ -94,7 +102,7 @@ public class PlayoutBuilderTests
             (PlayoutBuilder builder, Playout playout) =
                 TestDataFloodForItems(mediaItems, PlaybackOrder.Random, configRepo);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, _cancellationToken);
 
             configRepo.Verify();
 
@@ -123,7 +131,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             configRepo.Verify();
 
@@ -155,7 +163,7 @@ public class PlayoutBuilderTests
             (PlayoutBuilder builder, Playout playout) =
                 TestDataFloodForItems(mediaItems, PlaybackOrder.Random, configRepo);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, _cancellationToken);
 
             configRepo.Verify();
 
@@ -184,7 +192,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             configRepo.Verify();
 
@@ -217,7 +225,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(1);
             result.Items.Head().StartOffset.TimeOfDay.Should().Be(TimeSpan.Zero);
@@ -247,7 +255,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(1);
             result.Items.Head().StartOffset.TimeOfDay.Should().Be(TimeSpan.Zero);
@@ -268,7 +276,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(1);
             result.Items.Head().StartOffset.TimeOfDay.Should().Be(TimeSpan.Zero);
@@ -289,7 +297,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(1);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(2);
             result.Items[0].StartOffset.TimeOfDay.Should().Be(TimeSpan.Zero);
@@ -313,7 +321,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(4);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(4);
             result.Items[0].StartOffset.TimeOfDay.Should().Be(TimeSpan.Zero);
@@ -342,7 +350,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(1);
             result.Items.Head().MediaItemId.Should().Be(1);
@@ -355,7 +363,12 @@ public class PlayoutBuilderTests
             DateTimeOffset start2 = HoursAfterMidnight(1);
             DateTimeOffset finish2 = start2 + TimeSpan.FromHours(6);
 
-            Playout result2 = await builder.Build(playout, PlayoutBuildMode.Continue, start2, finish2);
+            Playout result2 = await builder.Build(
+                playout,
+                PlayoutBuildMode.Continue,
+                start2,
+                finish2,
+                _cancellationToken);
 
             result2.Items.Count.Should().Be(2);
             result2.Items.Last().StartOffset.TimeOfDay.Should().Be(TimeSpan.FromHours(6));
@@ -380,7 +393,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(1);
             result.Items.Head().MediaItemId.Should().Be(1);
@@ -392,7 +405,12 @@ public class PlayoutBuilderTests
             DateTimeOffset start2 = HoursAfterMidnight(1);
             DateTimeOffset finish2 = start2 + TimeSpan.FromHours(12);
 
-            Playout result2 = await builder.Build(playout, PlayoutBuildMode.Continue, start2, finish2);
+            Playout result2 = await builder.Build(
+                playout,
+                PlayoutBuildMode.Continue,
+                start2,
+                finish2,
+                _cancellationToken);
 
             result2.Items.Count.Should().Be(3);
             result2.Items[1].StartOffset.TimeOfDay.Should().Be(TimeSpan.FromHours(6));
@@ -422,7 +440,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(6);
             result.Anchor.NextStartOffset.Should().Be(DateTime.Today.AddHours(6));
@@ -435,7 +453,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start2 = HoursAfterMidnight(0);
             DateTimeOffset finish2 = start2 + TimeSpan.FromHours(6);
 
-            Playout result2 = await builder.Build(playout, PlayoutBuildMode.Reset, start2, finish2);
+            Playout result2 = await builder.Build(playout, PlayoutBuildMode.Reset, start2, finish2, _cancellationToken);
 
             result2.Items.Count.Should().Be(6);
             result2.Anchor.NextStartOffset.Should().Be(DateTime.Today.AddHours(6));
@@ -462,7 +480,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(6);
             result.ProgramScheduleAnchors.Count.Should().Be(1);
@@ -475,7 +493,12 @@ public class PlayoutBuilderTests
             DateTimeOffset start2 = HoursAfterMidnight(0);
             DateTimeOffset finish2 = start2 + TimeSpan.FromHours(6);
 
-            Playout result2 = await builder.Build(playout, PlayoutBuildMode.Continue, start2, finish2);
+            Playout result2 = await builder.Build(
+                playout,
+                PlayoutBuildMode.Continue,
+                start2,
+                finish2,
+                _cancellationToken);
 
             int secondSeedValue = result2.ProgramScheduleAnchors.Head().EnumeratorState.Seed;
 
@@ -564,7 +587,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(5);
             result.Items[0].StartOffset.TimeOfDay.Should().Be(TimeSpan.Zero);
@@ -661,7 +684,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(30);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(28);
             result.Items[0].StartOffset.TimeOfDay.Should().Be(TimeSpan.Zero);
@@ -806,7 +829,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(7);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(6);
 
@@ -909,7 +932,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(24);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(6);
 
@@ -1021,7 +1044,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(32);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(5);
 
@@ -1126,7 +1149,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(7);
 
@@ -1235,7 +1258,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(6);
 
@@ -1349,7 +1372,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(5);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(4);
 
@@ -1452,7 +1475,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(5);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(5);
 
@@ -1566,7 +1589,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(5);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(4);
 
@@ -1691,7 +1714,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(12);
 
@@ -1808,7 +1831,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(1);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(2);
 
@@ -1885,7 +1908,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(6);
 
@@ -1921,7 +1944,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromDays(2);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(8);
             result.Items[0].MediaItemId.Should().Be(1);
@@ -1980,7 +2003,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(6);
             result.Anchor.NextStartOffset.Should().Be(DateTime.Today.AddHours(6));
@@ -1993,7 +2016,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start2 = HoursAfterMidnight(0);
             DateTimeOffset finish2 = start2 + TimeSpan.FromHours(6);
 
-            Playout result2 = await builder.Build(playout, PlayoutBuildMode.Reset, start2, finish2);
+            Playout result2 = await builder.Build(playout, PlayoutBuildMode.Reset, start2, finish2, _cancellationToken);
 
             result2.Items.Count.Should().Be(6);
             result2.Anchor.NextStartOffset.Should().Be(DateTime.Today.AddHours(6));
@@ -2098,7 +2121,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(24);
             DateTimeOffset finish = start + TimeSpan.FromDays(1);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Refresh, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Refresh, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(4);
             result.Items[0].MediaItemId.Should().Be(2);
@@ -2135,7 +2158,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(1);
             result.Items.Head().MediaItemId.Should().Be(1);
@@ -2148,7 +2171,12 @@ public class PlayoutBuilderTests
             DateTimeOffset start2 = HoursAfterMidnight(1);
             DateTimeOffset finish2 = start2 + TimeSpan.FromHours(6);
 
-            Playout result2 = await builder.Build(playout, PlayoutBuildMode.Continue, start2, finish2);
+            Playout result2 = await builder.Build(
+                playout,
+                PlayoutBuildMode.Continue,
+                start2,
+                finish2,
+                _cancellationToken);
 
             result2.Items.Count.Should().Be(2);
             result2.Items.Last().StartOffset.TimeOfDay.Should().Be(TimeSpan.FromHours(6));
@@ -2173,7 +2201,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(1);
             result.Items.Head().MediaItemId.Should().Be(1);
@@ -2185,7 +2213,12 @@ public class PlayoutBuilderTests
             DateTimeOffset start2 = HoursAfterMidnight(1);
             DateTimeOffset finish2 = start2 + TimeSpan.FromHours(12);
 
-            Playout result2 = await builder.Build(playout, PlayoutBuildMode.Continue, start2, finish2);
+            Playout result2 = await builder.Build(
+                playout,
+                PlayoutBuildMode.Continue,
+                start2,
+                finish2,
+                _cancellationToken);
 
             result2.Items.Count.Should().Be(3);
             result2.Items[1].StartOffset.TimeOfDay.Should().Be(TimeSpan.FromHours(6));
@@ -2212,7 +2245,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromDays(1);
 
-            await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             playout.Items.Count.Should().Be(4);
             playout.Items.Map(i => i.MediaItemId).ToList().Should().Equal(1, 2, 1, 2);
@@ -2249,7 +2282,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start2 = HoursAfterMidnight(1);
             DateTimeOffset finish2 = start2 + TimeSpan.FromDays(1);
 
-            await builder.Build(playout, PlayoutBuildMode.Continue, start2, finish2);
+            await builder.Build(playout, PlayoutBuildMode.Continue, start2, finish2, _cancellationToken);
 
             playout.Items.Count.Should().Be(5);
             playout.Items[0].StartOffset.TimeOfDay.Should().Be(TimeSpan.FromHours(0));
@@ -2272,7 +2305,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start3 = HoursAfterMidnight(2);
             DateTimeOffset finish3 = start3 + TimeSpan.FromDays(1);
 
-            await builder.Build(playout, PlayoutBuildMode.Continue, start3, finish3);
+            await builder.Build(playout, PlayoutBuildMode.Continue, start3, finish3, _cancellationToken);
 
             playout.Items.Count.Should().Be(5);
             playout.Items[0].StartOffset.TimeOfDay.Should().Be(TimeSpan.FromHours(0));
@@ -2306,7 +2339,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(6);
             result.ProgramScheduleAnchors.Count.Should().Be(1);
@@ -2318,7 +2351,12 @@ public class PlayoutBuilderTests
             DateTimeOffset start2 = HoursAfterMidnight(0);
             DateTimeOffset finish2 = start2 + TimeSpan.FromHours(6);
 
-            Playout result2 = await builder.Build(result, PlayoutBuildMode.Continue, start2, finish2);
+            Playout result2 = await builder.Build(
+                result,
+                PlayoutBuildMode.Continue,
+                start2,
+                finish2,
+                _cancellationToken);
 
             int secondSeedValue = result2.ProgramScheduleAnchors.Head().EnumeratorState.Seed;
 
@@ -2340,7 +2378,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0).AddSeconds(5);
             DateTimeOffset finish = start + TimeSpan.FromDays(2);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(53);
             result.ProgramScheduleAnchors.Count.Should().Be(2);
@@ -2368,7 +2406,12 @@ public class PlayoutBuilderTests
             DateTimeOffset start2 = start.AddHours(1);
             DateTimeOffset finish2 = start2 + TimeSpan.FromDays(2);
 
-            Playout result2 = await builder.Build(result, PlayoutBuildMode.Continue, start2, finish2);
+            Playout result2 = await builder.Build(
+                result,
+                PlayoutBuildMode.Continue,
+                start2,
+                finish2,
+                _cancellationToken);
 
             PlayoutProgramScheduleAnchor continueAnchor =
                 result2.ProgramScheduleAnchors.First(x => x.AnchorDate is null);
@@ -2393,7 +2436,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(6);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(6);
             result.ProgramScheduleAnchors.Count.Should().Be(2);
@@ -2406,7 +2449,12 @@ public class PlayoutBuilderTests
             DateTimeOffset start2 = HoursAfterMidnight(0);
             DateTimeOffset finish2 = start2 + TimeSpan.FromHours(6);
 
-            Playout result2 = await builder.Build(result, PlayoutBuildMode.Continue, start2, finish2);
+            Playout result2 = await builder.Build(
+                result,
+                PlayoutBuildMode.Continue,
+                start2,
+                finish2,
+                _cancellationToken);
 
             primaryAnchor = result2.ProgramScheduleAnchors.First(a => a.SmartCollectionId == 1);
             int secondSeedValue = primaryAnchor.EnumeratorState.Seed;
@@ -2430,7 +2478,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0).AddSeconds(5);
             DateTimeOffset finish = start + TimeSpan.FromDays(2);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(53);
             result.ProgramScheduleAnchors.Count.Should().Be(4);
@@ -2450,7 +2498,12 @@ public class PlayoutBuilderTests
                 DateTimeOffset start2 = start.AddHours(i);
                 DateTimeOffset finish2 = start2 + TimeSpan.FromDays(2);
 
-                Playout result2 = await builder.Build(result, PlayoutBuildMode.Continue, start2, finish2);
+                Playout result2 = await builder.Build(
+                    result,
+                    PlayoutBuildMode.Continue,
+                    start2,
+                    finish2,
+                    _cancellationToken);
 
                 PlayoutProgramScheduleAnchor continueAnchor =
                     result2.ProgramScheduleAnchors
@@ -2554,7 +2607,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(32);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(5);
 
@@ -2666,7 +2719,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(5);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(4);
 
@@ -2778,7 +2831,7 @@ public class PlayoutBuilderTests
             DateTimeOffset start = HoursAfterMidnight(0);
             DateTimeOffset finish = start + TimeSpan.FromHours(5);
 
-            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish);
+            Playout result = await builder.Build(playout, PlayoutBuildMode.Continue, start, finish, _cancellationToken);
 
             result.Items.Count.Should().Be(4);
 

ErsatzTV.Core.Tests/Scheduling/PlayoutModeSchedulerBaseTests.cs

@@ -13,8 +13,18 @@ namespace ErsatzTV.Core.Tests.Scheduling;
 [TestFixture]
 public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
 {
+    private CancellationToken _cancellationToken;
+    private PlayoutModeSchedulerBase<ProgramScheduleItem> _scheduler;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+        _scheduler = new TestScheduler();
+    }
+
     [TestFixture]
-    public class CalculateEndTimeWithFiller
+    public class CalculateEndTimeWithFiller : PlayoutModeSchedulerBaseTests
     {
         [Test]
         public void Should_Not_Touch_Enumerator()
@@ -197,7 +207,7 @@ public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
     }
 
     [TestFixture]
-    public class AddFiller
+    public class AddFiller : PlayoutModeSchedulerBaseTests
     {
         [Test]
         public void Should_Not_Crash_Mid_Roll_Zero_Chapters()
@@ -224,13 +234,14 @@ public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
 
             PlayoutBuilderState startState = StartState(scheduleItemsEnumerator);
 
-            List<PlayoutItem> playoutItems = Scheduler()
+            List<PlayoutItem> playoutItems = _scheduler
                 .AddFiller(
                     startState,
                     CollectionEnumerators(scheduleItem, enumerator),
                     scheduleItem,
                     new PlayoutItem(),
-                    new List<MediaChapter>());
+                    new List<MediaChapter>(),
+                    _cancellationToken);
 
             playoutItems.Count.Should().Be(1);
         }
@@ -275,13 +286,14 @@ public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
             // too lazy to make another enumerator for the filler that we don't want
             enumerators.Add(CollectionKey.ForFillerPreset(scheduleItem.MidRollFiller), enumerator);
 
-            List<PlayoutItem> playoutItems = Scheduler()
+            List<PlayoutItem> playoutItems = _scheduler
                 .AddFiller(
                     startState,
                     enumerators,
                     scheduleItem,
                     new PlayoutItem(),
-                    new List<MediaChapter> { new() });
+                    new List<MediaChapter> { new() },
+                    _cancellationToken);
 
             playoutItems.Count.Should().Be(1);
         }
@@ -331,7 +343,7 @@ public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
 
             enumerators.Add(CollectionKey.ForFillerPreset(scheduleItem.MidRollFiller), fillerEnumerator);
 
-            List<PlayoutItem> playoutItems = Scheduler()
+            List<PlayoutItem> playoutItems = _scheduler
                 .AddFiller(
                     startState,
                     enumerators,
@@ -346,7 +358,8 @@ public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
                     {
                         new() { StartTime = TimeSpan.Zero, EndTime = TimeSpan.FromMinutes(6) },
                         new() { StartTime = TimeSpan.FromMinutes(6), EndTime = TimeSpan.FromMinutes(60) }
-                    });
+                    },
+                    _cancellationToken);
 
             playoutItems.Count.Should().Be(3);
             playoutItems[0].MediaItemId.Should().Be(1);
@@ -421,7 +434,7 @@ public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
             enumerators.Add(CollectionKey.ForFillerPreset(scheduleItem.MidRollFiller), midRollFillerEnumerator);
             enumerators.Add(CollectionKey.ForFillerPreset(scheduleItem.PostRollFiller), postRollFillerEnumerator);
 
-            List<PlayoutItem> playoutItems = Scheduler()
+            List<PlayoutItem> playoutItems = _scheduler
                 .AddFiller(
                     startState,
                     enumerators,
@@ -436,7 +449,8 @@ public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
                     {
                         new() { StartTime = TimeSpan.Zero, EndTime = TimeSpan.FromMinutes(6) },
                         new() { StartTime = TimeSpan.FromMinutes(6), EndTime = TimeSpan.FromMinutes(45) }
-                    });
+                    },
+                    _cancellationToken);
 
             playoutItems.Count.Should().Be(5);
             
@@ -525,7 +539,7 @@ public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
             enumerators.Add(CollectionKey.ForFillerPreset(scheduleItem.MidRollFiller), midRollFillerEnumerator);
             enumerators.Add(CollectionKey.ForFillerPreset(scheduleItem.PostRollFiller), postRollFillerEnumerator);
 
-            List<PlayoutItem> playoutItems = Scheduler()
+            List<PlayoutItem> playoutItems = _scheduler
                 .AddFiller(
                     startState,
                     enumerators,
@@ -540,7 +554,8 @@ public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
                     {
                         new() { StartTime = TimeSpan.Zero, EndTime = TimeSpan.FromMinutes(6) },
                         new() { StartTime = TimeSpan.FromMinutes(6), EndTime = TimeSpan.FromMinutes(45) }
-                    });
+                    },
+                    _cancellationToken);
 
             playoutItems.Count.Should().Be(5);
             
@@ -606,8 +621,6 @@ public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
             }
         };
 
-    private static PlayoutModeSchedulerBase<ProgramScheduleItem> Scheduler() =>
-        new TestScheduler();
 
     private class TestScheduler : PlayoutModeSchedulerBase<ProgramScheduleItem>
     {
@@ -632,7 +645,8 @@ public class PlayoutModeSchedulerBaseTests : SchedulerTestBase
             Dictionary<CollectionKey, IMediaCollectionEnumerator> collectionEnumerators,
             ProgramScheduleItem scheduleItem,
             ProgramScheduleItem nextScheduleItem,
-            DateTimeOffset hardStop) =>
+            DateTimeOffset hardStop,
+            CancellationToken cancellationToken) =>
             throw new NotSupportedException();
     }
 }

ErsatzTV.Core.Tests/Scheduling/PlayoutModeSchedulerDurationTests.cs

@@ -11,6 +11,14 @@ namespace ErsatzTV.Core.Tests.Scheduling;
 [TestFixture]
 public class PlayoutModeSchedulerDurationTests : SchedulerTestBase
 {
+    private CancellationToken _cancellationToken;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+    }
+
     [Test]
     public void Should_Fill_Exact_Duration()
     {
@@ -45,7 +53,8 @@ public class PlayoutModeSchedulerDurationTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -117,7 +126,8 @@ public class PlayoutModeSchedulerDurationTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -188,7 +198,8 @@ public class PlayoutModeSchedulerDurationTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.Add(new TimeSpan(2, 45, 0)));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -256,7 +267,8 @@ public class PlayoutModeSchedulerDurationTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         // duration block should end after exact duration, with gap
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
@@ -338,7 +350,8 @@ public class PlayoutModeSchedulerDurationTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.FallbackFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -424,7 +437,8 @@ public class PlayoutModeSchedulerDurationTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.TailFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -522,7 +536,8 @@ public class PlayoutModeSchedulerDurationTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.TailFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(startState.CurrentTime.Add(new TimeSpan(2, 57, 0)));
@@ -637,7 +652,8 @@ public class PlayoutModeSchedulerDurationTests : SchedulerTestBase
                 enumerator3),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -738,7 +754,8 @@ public class PlayoutModeSchedulerDurationTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutItems.Should().BeEmpty();
 

ErsatzTV.Core.Tests/Scheduling/PlayoutModeSchedulerFloodTests.cs

@@ -11,6 +11,14 @@ namespace ErsatzTV.Core.Tests.Scheduling;
 [TestFixture]
 public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
 {
+    private CancellationToken _cancellationToken;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+    }
+
     [Test]
     public void Should_Fill_Exactly_To_Next_Schedule_Item()
     {
@@ -51,7 +59,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -125,7 +134,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             scheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(6));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -221,7 +231,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -305,7 +316,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.PostRollFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -392,7 +404,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.Add(new TimeSpan(2, 45, 0)));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -473,7 +486,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.TailFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -570,7 +584,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.FallbackFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -657,7 +672,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.TailFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.Add(new TimeSpan(2, 57, 0)));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -770,7 +786,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
                 enumerator3),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -881,7 +898,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
                 enumerator2),
             scheduleItem,
             NextScheduleItem,
-            hardStop);
+            hardStop,
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(2));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -986,7 +1004,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
                 enumerator2),
             scheduleItem,
             NextScheduleItem,
-            hardStop);
+            hardStop,
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(2));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -1099,7 +1118,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
                 enumerator3),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -1172,7 +1192,8 @@ public class PlayoutModeSchedulerFloodTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutItems.Should().BeEmpty();
 

ErsatzTV.Core.Tests/Scheduling/PlayoutModeSchedulerMultipleTests.cs

@@ -11,6 +11,14 @@ namespace ErsatzTV.Core.Tests.Scheduling;
 [TestFixture]
 public class PlayoutModeSchedulerMultipleTests : SchedulerTestBase
 {
+    private CancellationToken _cancellationToken;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+    }
+
     [Test]
     public void Should_Fill_Exactly_To_Next_Schedule_Item()
     {
@@ -52,7 +60,8 @@ public class PlayoutModeSchedulerMultipleTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -126,7 +135,8 @@ public class PlayoutModeSchedulerMultipleTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.Add(new TimeSpan(2, 45, 0)));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -208,7 +218,8 @@ public class PlayoutModeSchedulerMultipleTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.TailFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -306,7 +317,8 @@ public class PlayoutModeSchedulerMultipleTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.FallbackFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -394,7 +406,8 @@ public class PlayoutModeSchedulerMultipleTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.TailFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.Add(new TimeSpan(2, 57, 0)));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -509,7 +522,8 @@ public class PlayoutModeSchedulerMultipleTests : SchedulerTestBase
                 enumerator3),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -630,7 +644,8 @@ public class PlayoutModeSchedulerMultipleTests : SchedulerTestBase
                 enumerator3),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -709,7 +724,8 @@ public class PlayoutModeSchedulerMultipleTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutItems.Should().BeEmpty();
 

ErsatzTV.Core.Tests/Scheduling/PlayoutModeSchedulerOneTests.cs

@@ -11,6 +11,14 @@ namespace ErsatzTV.Core.Tests.Scheduling;
 [TestFixture]
 public class PlayoutModeSchedulerOneTests : SchedulerTestBase
 {
+    private CancellationToken _cancellationToken;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+    }
+
     [Test]
     public void Should_Have_Gap_With_No_Tail_No_Fallback()
     {
@@ -45,7 +53,8 @@ public class PlayoutModeSchedulerOneTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(1));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -127,7 +136,8 @@ public class PlayoutModeSchedulerOneTests : SchedulerTestBase
                 enumerator3),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(1));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -194,7 +204,8 @@ public class PlayoutModeSchedulerOneTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.TailFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -275,7 +286,8 @@ public class PlayoutModeSchedulerOneTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.FallbackFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -346,7 +358,8 @@ public class PlayoutModeSchedulerOneTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator1, scheduleItem.TailFiller, enumerator2),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.Add(new TimeSpan(2, 57, 0)));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -443,7 +456,8 @@ public class PlayoutModeSchedulerOneTests : SchedulerTestBase
                 enumerator3),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -546,7 +560,8 @@ public class PlayoutModeSchedulerOneTests : SchedulerTestBase
                 enumerator3),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -631,7 +646,8 @@ public class PlayoutModeSchedulerOneTests : SchedulerTestBase
                 enumerator3),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -730,7 +746,8 @@ public class PlayoutModeSchedulerOneTests : SchedulerTestBase
                 enumerator3),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutBuilderState.CurrentTime.Should().Be(startState.CurrentTime.AddHours(3));
         playoutItems.Last().FinishOffset.Should().Be(playoutBuilderState.CurrentTime);
@@ -808,7 +825,8 @@ public class PlayoutModeSchedulerOneTests : SchedulerTestBase
             CollectionEnumerators(scheduleItem, enumerator),
             scheduleItem,
             NextScheduleItem,
-            HardStop(scheduleItemsEnumerator));
+            HardStop(scheduleItemsEnumerator),
+            _cancellationToken);
 
         playoutItems.Should().BeEmpty();
 

ErsatzTV.Core.Tests/Scheduling/RandomizedContentTests.cs

@@ -17,6 +17,14 @@ public class RandomizedContentTests
         1, 8, 2, 1, 1, 5, 5, 5, 3, 5, 8, 10, 4, 8, 7, 3, 3, 4, 4, 9, 2, 8, 8, 10, 8, 4, 3, 10, 7, 8, 9, 9
     };
 
+    private CancellationToken _cancellationToken;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+    }
+
     [Test]
     public void Episodes_Should_Randomize()
     {

ErsatzTV.Core.Tests/Scheduling/ScheduleIntegrationTests.cs

@@ -30,6 +30,8 @@ namespace ErsatzTV.Core.Tests.Scheduling;
 [Explicit]
 public class ScheduleIntegrationTests
 {
+    private CancellationToken _cancellationToken;
+
     public ScheduleIntegrationTests()
     {
         Log.Logger = new LoggerConfiguration()
@@ -40,6 +42,12 @@ public class ScheduleIntegrationTests
             .Destructure.UsingAttributes()
             .CreateLogger();
     }
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromMinutes(1)).Token;
+    }
 
     [Test]
     public async Task TestExistingData()
@@ -122,38 +130,48 @@ public class ScheduleIntegrationTests
             provider.GetRequiredService<ILogger<PlayoutBuilder>>());
 
         {
-            await using TvContext context = await factory.CreateDbContextAsync();
+            await using TvContext context = await factory.CreateDbContextAsync(_cancellationToken);
 
             Option<Playout> maybePlayout = await GetPlayout(context, PLAYOUT_ID);
             Playout playout = maybePlayout.ValueUnsafe();
 
-            await builder.Build(playout, PlayoutBuildMode.Reset, start, finish);
+            await builder.Build(playout, PlayoutBuildMode.Reset, start, finish, _cancellationToken);
 
-            await context.SaveChangesAsync();
+            await context.SaveChangesAsync(_cancellationToken);
         }
 
         for (var i = 1; i <= (24 * 1); i++)
         {
-            await using TvContext context = await factory.CreateDbContextAsync();
+            await using TvContext context = await factory.CreateDbContextAsync(_cancellationToken);
             
             Option<Playout> maybePlayout = await GetPlayout(context, PLAYOUT_ID);
             Playout playout = maybePlayout.ValueUnsafe();
 
-            await builder.Build(playout, PlayoutBuildMode.Continue, start.AddHours(i), finish.AddHours(i));
+            await builder.Build(
+                playout,
+                PlayoutBuildMode.Continue,
+                start.AddHours(i),
+                finish.AddHours(i),
+                _cancellationToken);
 
-            await context.SaveChangesAsync();
+            await context.SaveChangesAsync(_cancellationToken);
         }
         
         for (var i = 25; i <= 26; i++)
         {
-            await using TvContext context = await factory.CreateDbContextAsync();
+            await using TvContext context = await factory.CreateDbContextAsync(_cancellationToken);
             
             Option<Playout> maybePlayout = await GetPlayout(context, PLAYOUT_ID);
             Playout playout = maybePlayout.ValueUnsafe();
 
-            await builder.Build(playout, PlayoutBuildMode.Continue, start.AddHours(i), finish.AddHours(i));
+            await builder.Build(
+                playout,
+                PlayoutBuildMode.Continue,
+                start.AddHours(i),
+                finish.AddHours(i),
+                _cancellationToken);
 
-            await context.SaveChangesAsync();
+            await context.SaveChangesAsync(_cancellationToken);
         }
     }
 
@@ -216,8 +234,8 @@ public class ScheduleIntegrationTests
             MediaSource = new LocalMediaSource()
         };
 
-        await dbContext.Libraries.AddAsync(library);
-        await dbContext.SaveChangesAsync();
+        await dbContext.Libraries.AddAsync(library, _cancellationToken);
+        await dbContext.SaveChangesAsync(_cancellationToken);
         
         var movies = new List<Movie>();
         for (var i = 1; i < 25; i++)
@@ -243,8 +261,8 @@ public class ScheduleIntegrationTests
             movies.Add(movie);
         }
 
-        await dbContext.Movies.AddRangeAsync(movies);
-        await dbContext.SaveChangesAsync();
+        await dbContext.Movies.AddRangeAsync(movies, _cancellationToken);
+        await dbContext.SaveChangesAsync(_cancellationToken);
 
         var collection = new Collection
         {
@@ -252,8 +270,8 @@ public class ScheduleIntegrationTests
             MediaItems = movies.Cast<MediaItem>().ToList()
         };
 
-        await dbContext.Collections.AddAsync(collection);
-        await dbContext.SaveChangesAsync();
+        await dbContext.Collections.AddAsync(collection, _cancellationToken);
+        await dbContext.SaveChangesAsync(_cancellationToken);
 
         var scheduleItems = new List<ProgramScheduleItem>
         {
@@ -283,14 +301,19 @@ public class ScheduleIntegrationTests
 
         for (var i = 0; i <= (24 * 4); i++)
         {
-            await using TvContext context = await factory.CreateDbContextAsync();
+            await using TvContext context = await factory.CreateDbContextAsync(_cancellationToken);
             
             Option<Playout> maybePlayout = await GetPlayout(context, playoutId);
             Playout playout = maybePlayout.ValueUnsafe();
 
-            await builder.Build(playout, PlayoutBuildMode.Continue, start.AddHours(i), finish.AddHours(i));
+            await builder.Build(
+                playout,
+                PlayoutBuildMode.Continue,
+                start.AddHours(i),
+                finish.AddHours(i),
+                _cancellationToken);
 
-            await context.SaveChangesAsync();
+            await context.SaveChangesAsync(_cancellationToken);
         }
     }
 

ErsatzTV.Core.Tests/Scheduling/SeasonEpisodeContentTests.cs

@@ -8,6 +8,14 @@ namespace ErsatzTV.Core.Tests.Scheduling;
 [TestFixture]
 public class SeasonEpisodeContentTests
 {
+    private CancellationToken _cancellationToken;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+    }
+
     [Test]
     public void Episodes_Should_Sort_By_EpisodeNumber()
     {

ErsatzTV.Core.Tests/Scheduling/ShuffledContentTests.cs

@@ -11,6 +11,14 @@ public class ShuffledContentTests
     // this seed will produce (shuffle) 1-10 in order
     private const int MagicSeed = 670596;
 
+    private CancellationToken _cancellationToken;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+    }
+
     [Test]
     public void Episodes_Should_Not_Duplicate_When_Reshuffling()
     {
@@ -20,7 +28,7 @@ public class ShuffledContentTests
         var state = new CollectionEnumeratorState { Seed = 8 };
 
         var groupedMediaItems = contents.Map(mi => new GroupedMediaItem(mi, null)).ToList();
-        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state);
+        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state, _cancellationToken);
 
         var list = new List<int>();
         for (var i = 1; i <= 1000; i++)
@@ -48,7 +56,7 @@ public class ShuffledContentTests
         var state = new CollectionEnumeratorState();
 
         var groupedMediaItems = contents.Map(mi => new GroupedMediaItem(mi, null)).ToList();
-        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state);
+        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state, _cancellationToken);
 
         var list = new List<int>();
         for (var i = 1; i <= 10; i++)
@@ -69,7 +77,7 @@ public class ShuffledContentTests
         var state = new CollectionEnumeratorState();
 
         var groupedMediaItems = contents.Map(mi => new GroupedMediaItem(mi, null)).ToList();
-        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state);
+        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state, _cancellationToken);
 
         var list = new List<int>();
         for (var i = 1; i <= 10; i++)
@@ -90,7 +98,7 @@ public class ShuffledContentTests
         var state = new CollectionEnumeratorState();
 
         var groupedMediaItems = contents.Map(mi => new GroupedMediaItem(mi, null)).ToList();
-        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state);
+        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state, _cancellationToken);
 
         for (var i = 0; i < 10; i++)
         {
@@ -106,7 +114,7 @@ public class ShuffledContentTests
         var state = new CollectionEnumeratorState { Index = 5, Seed = MagicSeed };
 
         var groupedMediaItems = contents.Map(mi => new GroupedMediaItem(mi, null)).ToList();
-        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state);
+        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state, _cancellationToken);
 
         for (var i = 6; i <= 10; i++)
         {
@@ -125,7 +133,7 @@ public class ShuffledContentTests
         var state = new CollectionEnumeratorState { Index = 10, Seed = MagicSeed };
 
         var groupedMediaItems = contents.Map(mi => new GroupedMediaItem(mi, null)).ToList();
-        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state);
+        var shuffledContent = new ShuffledMediaCollectionEnumerator(groupedMediaItems, state, _cancellationToken);
 
         shuffledContent.State.Index.Should().Be(0);
         shuffledContent.State.Seed.Should().NotBe(MagicSeed);

ErsatzTV.Core.Tests/Scheduling/ShuffledMediaCollectionEnumeratorTests.cs

@@ -15,12 +15,20 @@ public class ShuffledMediaCollectionEnumeratorTests
         new GroupedMediaItem(new MediaItem { Id = 2 }, new List<MediaItem>()),
         new GroupedMediaItem(new MediaItem { Id = 3 }, new List<MediaItem>())
     };
+    
+    private CancellationToken _cancellationToken;
+    
+    [SetUp]
+    public void SetUp()
+    {
+        _cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+    }
 
     [Test]
     public void Peek_Zero_Should_Match_Current()
     {
         var state = new CollectionEnumeratorState { Index = 0, Seed = 0 };
-        var enumerator = new ShuffledMediaCollectionEnumerator(_mediaItems, state);
+        var enumerator = new ShuffledMediaCollectionEnumerator(_mediaItems, state, _cancellationToken);
 
         Option<MediaItem> peek = enumerator.Peek(0);
         Option<MediaItem> current = enumerator.Current;
@@ -35,7 +43,7 @@ public class ShuffledMediaCollectionEnumeratorTests
     public void Peek_One_Should_Match_Next()
     {
         var state = new CollectionEnumeratorState { Index = 0, Seed = 0 };
-        var enumerator = new ShuffledMediaCollectionEnumerator(_mediaItems, state);
+        var enumerator = new ShuffledMediaCollectionEnumerator(_mediaItems, state, _cancellationToken);
 
         Option<MediaItem> peek = enumerator.Peek(1);
 
@@ -52,7 +60,7 @@ public class ShuffledMediaCollectionEnumeratorTests
     public void Peek_Two_Should_Match_NextNext()
     {
         var state = new CollectionEnumeratorState { Index = 0, Seed = 0 };
-        var enumerator = new ShuffledMediaCollectionEnumerator(_mediaItems, state);
+        var enumerator = new ShuffledMediaCollectionEnumerator(_mediaItems, state, _cancellationToken);
 
         Option<MediaItem> peek = enumerator.Peek(2);
 
@@ -70,7 +78,7 @@ public class ShuffledMediaCollectionEnumeratorTests
     public void Peek_Three_Should_Match_NextNextNext()
     {
         var state = new CollectionEnumeratorState { Index = 0, Seed = 0 };
-        var enumerator = new ShuffledMediaCollectionEnumerator(_mediaItems, state);
+        var enumerator = new ShuffledMediaCollectionEnumerator(_mediaItems, state, _cancellationToken);
 
         Option<MediaItem> peek = enumerator.Peek(3);
 

ErsatzTV.Core/Interfaces/Scheduling/IMultiEpisodeShuffleCollectionEnumeratorFactory.cs

@@ -7,5 +7,6 @@ public interface IMultiEpisodeShuffleCollectionEnumeratorFactory
     IMediaCollectionEnumerator Create(
         string jsScriptPath,
         IList<MediaItem> mediaItems,
-        CollectionEnumeratorState state);
+        CollectionEnumeratorState state,
+        CancellationToken cancellationToken);
 }

ErsatzTV.Core/Interfaces/Scheduling/IPlayoutBuilder.cs

@@ -5,5 +5,5 @@ namespace ErsatzTV.Core.Interfaces.Scheduling;
 
 public interface IPlayoutBuilder
 {
-    Task<Playout> Build(Playout playout, PlayoutBuildMode mode);
+    Task<Playout> Build(Playout playout, PlayoutBuildMode mode, CancellationToken cancellationToken);
 }

ErsatzTV.Core/Interfaces/Scheduling/IPlayoutModeScheduler.cs

@@ -10,5 +10,6 @@ public interface IPlayoutModeScheduler<in T> where T : ProgramScheduleItem
         Dictionary<CollectionKey, IMediaCollectionEnumerator> collectionEnumerators,
         T scheduleItem,
         ProgramScheduleItem nextScheduleItem,
-        DateTimeOffset hardStop);
+        DateTimeOffset hardStop,
+        CancellationToken cancellationToken);
 }

ErsatzTV.Core/Scheduling/PlayoutBuilder.cs

@@ -40,7 +40,7 @@ public class PlayoutBuilder : IPlayoutBuilder
         _logger = logger;
     }
 
-    public async Task<Playout> Build(Playout playout, PlayoutBuildMode mode)
+    public async Task<Playout> Build(Playout playout, PlayoutBuildMode mode, CancellationToken cancellationToken)
     {
         foreach (PlayoutParameters parameters in await Validate(playout))
         {
@@ -50,35 +50,43 @@ public class PlayoutBuilder : IPlayoutBuilder
             //     return await Build(playout, mode, parameters with { Start = parameters.Start.AddDays(-2) });
             // }
 
-            return await Build(playout, mode, parameters);
+            return await Build(playout, mode, parameters, cancellationToken);
         }
 
         return playout;
     }
 
-    private Task<Playout> Build(Playout playout, PlayoutBuildMode mode, PlayoutParameters parameters) =>
+    private Task<Playout> Build(
+        Playout playout,
+        PlayoutBuildMode mode,
+        PlayoutParameters parameters,
+        CancellationToken cancellationToken) =>
         mode switch
         {
-            PlayoutBuildMode.Refresh => RefreshPlayout(playout, parameters),
-            PlayoutBuildMode.Reset => ResetPlayout(playout, parameters),
-            _ => ContinuePlayout(playout, parameters)
+            PlayoutBuildMode.Refresh => RefreshPlayout(playout, parameters, cancellationToken),
+            PlayoutBuildMode.Reset => ResetPlayout(playout, parameters, cancellationToken),
+            _ => ContinuePlayout(playout, parameters, cancellationToken)
         };
 
     internal async Task<Playout> Build(
         Playout playout,
         PlayoutBuildMode mode,
         DateTimeOffset start,
-        DateTimeOffset finish)
+        DateTimeOffset finish,
+        CancellationToken cancellationToken)
     {
         foreach (PlayoutParameters parameters in await Validate(playout))
         {
-            return await Build(playout, mode, parameters with { Start = start, Finish = finish });
+            return await Build(playout, mode, parameters with { Start = start, Finish = finish }, cancellationToken);
         }
 
         return playout;
     }
 
-    private async Task<Playout> RefreshPlayout(Playout playout, PlayoutParameters parameters)
+    private async Task<Playout> RefreshPlayout(
+        Playout playout,
+        PlayoutParameters parameters,
+        CancellationToken cancellationToken)
     {
         _logger.LogDebug(
             "Refreshing playout {PlayoutId} for channel {ChannelNumber} - {ChannelName}",
@@ -183,10 +191,14 @@ public class PlayoutBuilder : IPlayoutBuilder
             parameters.Start,
             parameters.Finish,
             parameters.CollectionMediaItems,
-            false);
+            false,
+            cancellationToken);
     }
 
-    private async Task<Playout> ResetPlayout(Playout playout, PlayoutParameters parameters)
+    private async Task<Playout> ResetPlayout(
+        Playout playout,
+        PlayoutParameters parameters,
+        CancellationToken cancellationToken)
     {
         _logger.LogDebug(
             "Resetting playout {PlayoutId} for channel {ChannelNumber} - {ChannelName}",
@@ -203,12 +215,16 @@ public class PlayoutBuilder : IPlayoutBuilder
             parameters.Start,
             parameters.Finish,
             parameters.CollectionMediaItems,
-            true);
+            true,
+            cancellationToken);
 
         return playout;
     }
 
-    private async Task<Playout> ContinuePlayout(Playout playout, PlayoutParameters parameters)
+    private async Task<Playout> ContinuePlayout(
+        Playout playout,
+        PlayoutParameters parameters,
+        CancellationToken cancellationToken)
     {
         _logger.LogDebug(
             "Building playout {PlayoutId} for channel {ChannelNumber} - {ChannelName}",
@@ -227,7 +243,8 @@ public class PlayoutBuilder : IPlayoutBuilder
             parameters.Start,
             parameters.Finish,
             parameters.CollectionMediaItems,
-            false);
+            false,
+            cancellationToken);
 
         return playout;
     }
@@ -289,7 +306,8 @@ public class PlayoutBuilder : IPlayoutBuilder
         DateTimeOffset playoutStart,
         DateTimeOffset playoutFinish,
         Map<CollectionKey, List<MediaItem>> collectionMediaItems,
-        bool randomStartPoint)
+        bool randomStartPoint,
+        CancellationToken cancellationToken)
     {
         DateTimeOffset trimBefore = playoutStart.AddHours(-4);
         DateTimeOffset trimAfter = playoutFinish;
@@ -308,7 +326,14 @@ public class PlayoutBuilder : IPlayoutBuilder
         while (finish < playoutFinish)
         {
             _logger.LogDebug("Building playout from {Start} to {Finish}", start, finish);
-            playout = await BuildPlayoutItems(playout, start, finish, collectionMediaItems, true, randomStartPoint);
+            playout = await BuildPlayoutItems(
+                playout,
+                start,
+                finish,
+                collectionMediaItems,
+                true,
+                randomStartPoint,
+                cancellationToken);
 
             // only randomize once (at the start of the playout)
             randomStartPoint = false;
@@ -327,7 +352,8 @@ public class PlayoutBuilder : IPlayoutBuilder
                 playoutFinish,
                 collectionMediaItems,
                 false,
-                randomStartPoint);
+                randomStartPoint,
+                cancellationToken);
         }
 
         // remove old items
@@ -359,7 +385,8 @@ public class PlayoutBuilder : IPlayoutBuilder
         DateTimeOffset playoutFinish,
         Map<CollectionKey, List<MediaItem>> collectionMediaItems,
         bool saveAnchorDate,
-        bool randomStartPoint)
+        bool randomStartPoint,
+        CancellationToken cancellationToken)
     {
         ProgramSchedule activeSchedule = PlayoutScheduleSelector.GetProgramScheduleFor(
             playout.ProgramSchedule,
@@ -391,7 +418,8 @@ public class PlayoutBuilder : IPlayoutBuilder
                     collectionKey,
                     mediaItems,
                     playbackOrder,
-                    randomStartPoint);
+                    randomStartPoint,
+                    cancellationToken);
             collectionEnumerators.Add(collectionKey, enumerator);
         }
 
@@ -443,7 +471,7 @@ public class PlayoutBuilder : IPlayoutBuilder
         var timeCount = new Dictionary<DateTimeOffset, int>();
         
         // loop until we're done filling the desired amount of time
-        while (playoutBuilderState.CurrentTime < playoutFinish)
+        while (playoutBuilderState.CurrentTime < playoutFinish && !cancellationToken.IsCancellationRequested)
         {
             if (timeCount.TryGetValue(playoutBuilderState.CurrentTime, out int count))
             {
@@ -477,25 +505,29 @@ public class PlayoutBuilder : IPlayoutBuilder
                     collectionEnumerators,
                     multiple,
                     nextScheduleItem,
-                    playoutFinish),
+                    playoutFinish,
+                    cancellationToken),
                 ProgramScheduleItemDuration duration => schedulerDuration.Schedule(
                     playoutBuilderState,
                     collectionEnumerators,
                     duration,
                     nextScheduleItem,
-                    playoutFinish),
+                    playoutFinish,
+                    cancellationToken),
                 ProgramScheduleItemFlood flood => schedulerFlood.Schedule(
                     playoutBuilderState,
                     collectionEnumerators,
                     flood,
                     nextScheduleItem,
-                    playoutFinish),
+                    playoutFinish,
+                    cancellationToken),
                 ProgramScheduleItemOne one => schedulerOne.Schedule(
                     playoutBuilderState,
                     collectionEnumerators,
                     one,
                     nextScheduleItem,
-                    playoutFinish),
+                    playoutFinish,
+                    cancellationToken),
                 _ => throw new ArgumentOutOfRangeException(nameof(scheduleItem))
             };
 
@@ -715,7 +747,8 @@ public class PlayoutBuilder : IPlayoutBuilder
         CollectionKey collectionKey,
         List<MediaItem> mediaItems,
         PlaybackOrder playbackOrder,
-        bool randomStartPoint)
+        bool randomStartPoint,
+        CancellationToken cancellationToken)
     {
         Option<PlayoutProgramScheduleAnchor> maybeAnchor = playout.ProgramScheduleAnchors
             .OrderByDescending(a => a.AnchorDate ?? DateTime.MaxValue)
@@ -789,7 +822,8 @@ public class PlayoutBuilder : IPlayoutBuilder
                 return new ShuffleInOrderCollectionEnumerator(
                     await GetCollectionItemsForShuffleInOrder(collectionKey),
                     state,
-                    activeSchedule.RandomStartPoint);
+                    activeSchedule.RandomStartPoint,
+                    cancellationToken);
             case PlaybackOrder.MultiEpisodeShuffle when
                 collectionKey.CollectionType == ProgramScheduleItemCollectionType.TelevisionShow &&
                 collectionKey.MediaItemId.HasValue:
@@ -808,7 +842,7 @@ public class PlayoutBuilder : IPlayoutBuilder
                             _logger.LogDebug("Found JS Script at {Path}", jsScriptPath);
                             try
                             {
-                                return _multiEpisodeFactory.Create(jsScriptPath, mediaItems, state);
+                                return _multiEpisodeFactory.Create(jsScriptPath, mediaItems, state, cancellationToken);
                             }
                             catch (Exception ex)
                             {
@@ -828,7 +862,8 @@ public class PlayoutBuilder : IPlayoutBuilder
             case PlaybackOrder.Shuffle:
                 return new ShuffledMediaCollectionEnumerator(
                     await GetGroupedMediaItemsForShuffle(activeSchedule, mediaItems, collectionKey),
-                    state);
+                    state,
+                    cancellationToken);
             default:
                 // TODO: handle this error case differently?
                 return new RandomizedMediaCollectionEnumerator(mediaItems, state);

ErsatzTV.Core/Scheduling/PlayoutModeSchedulerBase.cs

@@ -18,13 +18,13 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
         Dictionary<CollectionKey, IMediaCollectionEnumerator> collectionEnumerators,
         T scheduleItem,
         ProgramScheduleItem nextScheduleItem,
-        DateTimeOffset hardStop);
+        DateTimeOffset hardStop,
+        CancellationToken cancellationToken);
 
     public static DateTimeOffset GetFillerStartTimeAfter(
         PlayoutBuilderState state,
         ProgramScheduleItem scheduleItem,
-        DateTimeOffset hardStop
-    )
+        DateTimeOffset hardStop)
     {
         DateTimeOffset startTime = GetStartTimeAfter(state, scheduleItem);
         
@@ -74,7 +74,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
         Dictionary<CollectionKey, IMediaCollectionEnumerator> collectionEnumerators,
         ProgramScheduleItem scheduleItem,
         List<PlayoutItem> playoutItems,
-        DateTimeOffset nextItemStart)
+        DateTimeOffset nextItemStart,
+        CancellationToken cancellationToken)
     {
         var newItems = new List<PlayoutItem>(playoutItems);
         PlayoutBuilderState nextState = playoutBuilderState;
@@ -131,7 +132,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
         Dictionary<CollectionKey, IMediaCollectionEnumerator> collectionEnumerators,
         ProgramScheduleItem scheduleItem,
         List<PlayoutItem> playoutItems,
-        DateTimeOffset nextItemStart)
+        DateTimeOffset nextItemStart,
+        CancellationToken cancellationToken)
     {
         var newItems = new List<PlayoutItem>(playoutItems);
         PlayoutBuilderState nextState = playoutBuilderState;
@@ -328,7 +330,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
         Dictionary<CollectionKey, IMediaCollectionEnumerator> enumerators,
         ProgramScheduleItem scheduleItem,
         PlayoutItem playoutItem,
-        List<MediaChapter> chapters)
+        List<MediaChapter> chapters,
+        CancellationToken cancellationToken)
     {
         var result = new List<PlayoutItem>();
 
@@ -363,7 +366,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                             e1,
                             filler.Duration.Value,
                             FillerKind.PreRoll,
-                            filler.AllowWatermarks));
+                            filler.AllowWatermarks,
+                            cancellationToken));
                     break;
                 case FillerMode.Count when filler.Count.HasValue:
                     IMediaCollectionEnumerator e2 = enumerators[CollectionKey.ForFillerPreset(filler)];
@@ -373,7 +377,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                             e2,
                             filler.Count.Value,
                             FillerKind.PreRoll,
-                            filler.AllowWatermarks));
+                            filler.AllowWatermarks,
+                            cancellationToken));
                     break;
             }
         }
@@ -402,7 +407,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                                         e1,
                                         filler.Duration.Value,
                                         FillerKind.MidRoll,
-                                        filler.AllowWatermarks));
+                                        filler.AllowWatermarks,
+                                        cancellationToken));
                             }
                         }
 
@@ -420,7 +426,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                                         e2,
                                         filler.Count.Value,
                                         FillerKind.MidRoll,
-                                        filler.AllowWatermarks));
+                                        filler.AllowWatermarks,
+                                        cancellationToken));
                             }
                         }
 
@@ -442,7 +449,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                             e1,
                             filler.Duration.Value,
                             FillerKind.PostRoll,
-                            filler.AllowWatermarks));
+                            filler.AllowWatermarks,
+                            cancellationToken));
                     break;
                 case FillerMode.Count when filler.Count.HasValue:
                     IMediaCollectionEnumerator e2 = enumerators[CollectionKey.ForFillerPreset(filler)];
@@ -452,7 +460,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                             e2,
                             filler.Count.Value,
                             FillerKind.PostRoll,
-                            filler.AllowWatermarks));
+                            filler.AllowWatermarks,
+                            cancellationToken));
                     break;
             }
         }
@@ -508,7 +517,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                             pre1,
                             remainingToFill,
                             FillerKind.PreRoll,
-                            padFiller.AllowWatermarks));
+                            padFiller.AllowWatermarks,
+                            cancellationToken));
                     totalDuration =
                         TimeSpan.FromMilliseconds(result.Sum(pi => (pi.Finish - pi.Start).TotalMilliseconds));
                     remainingToFill = targetTime - totalDuration - playoutItem.StartOffset;
@@ -520,7 +530,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                                 playoutBuilderState,
                                 enumerators,
                                 scheduleItem,
-                                remainingToFill));
+                                remainingToFill,
+                                cancellationToken));
                     }
 
                     break;
@@ -532,7 +543,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                             mid1,
                             remainingToFill,
                             FillerKind.MidRoll,
-                            padFiller.AllowWatermarks));
+                            padFiller.AllowWatermarks,
+                            cancellationToken));
                     TimeSpan average = effectiveChapters.Count <= 1
                         ? remainingToFill
                         : remainingToFill / (effectiveChapters.Count - 1);
@@ -569,7 +581,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                                         playoutBuilderState,
                                         enumerators,
                                         scheduleItem,
-                                        i < effectiveChapters.Count - 1 ? maxThisBreak : leftOverall);
+                                        i < effectiveChapters.Count - 1 ? maxThisBreak : leftOverall,
+                                        cancellationToken);
 
                                     foreach (PlayoutItem fallback in maybeFallback)
                                     {
@@ -593,7 +606,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                             post1,
                             remainingToFill,
                             FillerKind.PostRoll,
-                            padFiller.AllowWatermarks));
+                            padFiller.AllowWatermarks,
+                            cancellationToken));
                     totalDuration =
                         TimeSpan.FromMilliseconds(result.Sum(pi => (pi.Finish - pi.Start).TotalMilliseconds));
                     remainingToFill = targetTime - totalDuration - playoutItem.StartOffset;
@@ -604,7 +618,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
                                 playoutBuilderState,
                                 enumerators,
                                 scheduleItem,
-                                remainingToFill));
+                                remainingToFill,
+                                cancellationToken));
                     }
 
                     break;
@@ -630,7 +645,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
         IMediaCollectionEnumerator enumerator,
         int count,
         FillerKind fillerKind,
-        bool allowWatermarks)
+        bool allowWatermarks,
+        CancellationToken cancellationToken)
     {
         var result = new List<PlayoutItem>();
 
@@ -665,7 +681,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
         IMediaCollectionEnumerator enumerator,
         TimeSpan duration,
         FillerKind fillerKind,
-        bool allowWatermarks)
+        bool allowWatermarks,
+        CancellationToken cancellationToken)
     {
         var result = new List<PlayoutItem>();
 
@@ -736,7 +753,8 @@ public abstract class PlayoutModeSchedulerBase<T> : IPlayoutModeScheduler<T> whe
         PlayoutBuilderState playoutBuilderState,
         Dictionary<CollectionKey, IMediaCollectionEnumerator> enumerators,
         ProgramScheduleItem scheduleItem,
-        TimeSpan duration)
+        TimeSpan duration,
+        CancellationToken cancellationToken)
     {
         if (scheduleItem.FallbackFiller != null)
         {

ErsatzTV.Core/Scheduling/PlayoutModeSchedulerDuration.cs

@@ -17,7 +17,8 @@ public class PlayoutModeSchedulerDuration : PlayoutModeSchedulerBase<ProgramSche
         Dictionary<CollectionKey, IMediaCollectionEnumerator> collectionEnumerators,
         ProgramScheduleItemDuration scheduleItem,
         ProgramScheduleItem nextScheduleItem,
-        DateTimeOffset hardStop)
+        DateTimeOffset hardStop,
+        CancellationToken cancellationToken)
     {
         var playoutItems = new List<PlayoutItem>();
 
@@ -101,7 +102,13 @@ public class PlayoutModeSchedulerDuration : PlayoutModeSchedulerBase<ProgramSche
             {
                 // LogScheduledItem(scheduleItem, mediaItem, itemStartTime);
                 playoutItems.AddRange(
-                    AddFiller(nextState, collectionEnumerators, scheduleItem, playoutItem, itemChapters));
+                    AddFiller(
+                        nextState,
+                        collectionEnumerators,
+                        scheduleItem,
+                        playoutItem,
+                        itemChapters,
+                        cancellationToken));
 
                 nextState = nextState with
                 {
@@ -163,7 +170,8 @@ public class PlayoutModeSchedulerDuration : PlayoutModeSchedulerBase<ProgramSche
                             collectionEnumerators,
                             scheduleItem,
                             playoutItems,
-                            nextItemStart);
+                            nextItemStart,
+                            cancellationToken);
                     }
 
                     if (scheduleItem.FallbackFiller != null)
@@ -173,7 +181,8 @@ public class PlayoutModeSchedulerDuration : PlayoutModeSchedulerBase<ProgramSche
                             collectionEnumerators,
                             scheduleItem,
                             playoutItems,
-                            nextItemStart);
+                            nextItemStart,
+                            cancellationToken);
                     }
 
                     nextState = nextState with { CurrentTime = nextItemStart };
@@ -186,7 +195,8 @@ public class PlayoutModeSchedulerDuration : PlayoutModeSchedulerBase<ProgramSche
                             collectionEnumerators,
                             scheduleItem,
                             playoutItems,
-                            nextItemStart);
+                            nextItemStart,
+                            cancellationToken);
                     }
 
                     nextState = nextState with { CurrentTime = nextItemStart };

ErsatzTV.Core/Scheduling/PlayoutModeSchedulerFlood.cs

@@ -18,7 +18,8 @@ public class PlayoutModeSchedulerFlood : PlayoutModeSchedulerBase<ProgramSchedul
         Dictionary<CollectionKey, IMediaCollectionEnumerator> collectionEnumerators,
         ProgramScheduleItemFlood scheduleItem,
         ProgramScheduleItem nextScheduleItem,
-        DateTimeOffset hardStop)
+        DateTimeOffset hardStop,
+        CancellationToken cancellationToken)
     {
         var playoutItems = new List<PlayoutItem>();
 
@@ -88,7 +89,13 @@ public class PlayoutModeSchedulerFlood : PlayoutModeSchedulerBase<ProgramSchedul
             if (willFinishInTime)
             {
                 playoutItems.AddRange(
-                    AddFiller(nextState, collectionEnumerators, scheduleItem, playoutItem, itemChapters));
+                    AddFiller(
+                        nextState,
+                        collectionEnumerators,
+                        scheduleItem,
+                        playoutItem,
+                        itemChapters,
+                        cancellationToken));
                 // LogScheduledItem(scheduleItem, mediaItem, itemStartTime);
 
                 if (playoutItems.Count > 0)
@@ -150,7 +157,8 @@ public class PlayoutModeSchedulerFlood : PlayoutModeSchedulerBase<ProgramSchedul
                 collectionEnumerators,
                 scheduleItem,
                 playoutItems,
-                peekItemStart);
+                peekItemStart,
+                cancellationToken);
         }
 
         if (scheduleItem.FallbackFiller != null)
@@ -160,7 +168,8 @@ public class PlayoutModeSchedulerFlood : PlayoutModeSchedulerBase<ProgramSchedul
                 collectionEnumerators,
                 scheduleItem,
                 playoutItems,
-                peekItemStart);
+                peekItemStart,
+                cancellationToken);
         }
 
         nextState = nextState with { NextGuideGroup = nextState.IncrementGuideGroup };

ErsatzTV.Core/Scheduling/PlayoutModeSchedulerMultiple.cs

@@ -19,7 +19,8 @@ public class PlayoutModeSchedulerMultiple : PlayoutModeSchedulerBase<ProgramSche
         Dictionary<CollectionKey, IMediaCollectionEnumerator> collectionEnumerators,
         ProgramScheduleItemMultiple scheduleItem,
         ProgramScheduleItem nextScheduleItem,
-        DateTimeOffset hardStop)
+        DateTimeOffset hardStop,
+        CancellationToken cancellationToken)
     {
         var playoutItems = new List<PlayoutItem>();
 
@@ -84,7 +85,13 @@ public class PlayoutModeSchedulerMultiple : PlayoutModeSchedulerBase<ProgramSche
                 itemChapters);
 
             playoutItems.AddRange(
-                AddFiller(nextState, collectionEnumerators, scheduleItem, playoutItem, itemChapters));
+                AddFiller(
+                    nextState,
+                    collectionEnumerators,
+                    scheduleItem,
+                    playoutItem,
+                    itemChapters,
+                    cancellationToken));
 
             nextState = nextState with
             {
@@ -128,7 +135,8 @@ public class PlayoutModeSchedulerMultiple : PlayoutModeSchedulerBase<ProgramSche
                 collectionEnumerators,
                 scheduleItem,
                 playoutItems,
-                nextItemStart);
+                nextItemStart,
+                cancellationToken);
         }
 
         if (scheduleItem.FallbackFiller != null)
@@ -138,7 +146,8 @@ public class PlayoutModeSchedulerMultiple : PlayoutModeSchedulerBase<ProgramSche
                 collectionEnumerators,
                 scheduleItem,
                 playoutItems,
-                nextItemStart);
+                nextItemStart,
+                cancellationToken);
         }
 
         nextState = nextState with { NextGuideGroup = nextState.IncrementGuideGroup };

ErsatzTV.Core/Scheduling/PlayoutModeSchedulerOne.cs

@@ -16,7 +16,8 @@ public class PlayoutModeSchedulerOne : PlayoutModeSchedulerBase<ProgramScheduleI
         Dictionary<CollectionKey, IMediaCollectionEnumerator> collectionEnumerators,
         ProgramScheduleItemOne scheduleItem,
         ProgramScheduleItem nextScheduleItem,
-        DateTimeOffset hardStop)
+        DateTimeOffset hardStop,
+        CancellationToken cancellationToken)
     {
         IMediaCollectionEnumerator contentEnumerator =
             collectionEnumerators[CollectionKey.ForScheduleItem(scheduleItem)];
@@ -64,7 +65,8 @@ public class PlayoutModeSchedulerOne : PlayoutModeSchedulerBase<ProgramScheduleI
                 collectionEnumerators,
                 scheduleItem,
                 playoutItem,
-                itemChapters);
+                itemChapters,
+                cancellationToken);
 
             PlayoutBuilderState nextState = playoutBuilderState with
             {
@@ -90,7 +92,8 @@ public class PlayoutModeSchedulerOne : PlayoutModeSchedulerBase<ProgramScheduleI
                     collectionEnumerators,
                     scheduleItem,
                     playoutItems,
-                    nextItemStart);
+                    nextItemStart,
+                    cancellationToken);
             }
 
             if (scheduleItem.FallbackFiller != null)
@@ -100,7 +103,8 @@ public class PlayoutModeSchedulerOne : PlayoutModeSchedulerBase<ProgramScheduleI
                     collectionEnumerators,
                     scheduleItem,
                     playoutItems,
-                    nextItemStart);
+                    nextItemStart,
+                    cancellationToken);
             }
 
             nextState = nextState with { NextGuideGroup = nextState.IncrementGuideGroup };

ErsatzTV.Core/Scheduling/ShuffleInOrderCollectionEnumerator.cs

@@ -8,16 +8,19 @@ public class ShuffleInOrderCollectionEnumerator : IMediaCollectionEnumerator
     private readonly IList<CollectionWithItems> _collections;
     private readonly int _mediaItemCount;
     private readonly bool _randomStartPoint;
+    private readonly CancellationToken _cancellationToken;
     private Random _random;
     private IList<MediaItem> _shuffled;
 
     public ShuffleInOrderCollectionEnumerator(
         IList<CollectionWithItems> collections,
         CollectionEnumeratorState state,
-        bool randomStartPoint)
+        bool randomStartPoint,
+        CancellationToken cancellationToken)
     {
         _collections = collections;
         _randomStartPoint = randomStartPoint;
+        _cancellationToken = cancellationToken;
         _mediaItemCount = collections.Sum(c => c.MediaItems.Count);
 
         if (state.Index >= _mediaItemCount)
@@ -52,7 +55,8 @@ public class ShuffleInOrderCollectionEnumerator : IMediaCollectionEnumerator
                 State.Seed = _random.Next();
                 _random = new Random(State.Seed);
                 _shuffled = Shuffle(_collections, _random);
-            } while (_collections.Count > 1 && Current.Map(x => x.Id) == tail.Map(x => x.Id));
+            } while (!_cancellationToken.IsCancellationRequested && _collections.Count > 1 &&
+                     Current.Map(x => x.Id) == tail.Map(x => x.Id));
         }
         else
         {

ErsatzTV.Core/Scheduling/ShuffledMediaCollectionEnumerator.cs

@@ -7,15 +7,18 @@ public class ShuffledMediaCollectionEnumerator : IMediaCollectionEnumerator
 {
     private readonly int _mediaItemCount;
     private readonly IList<GroupedMediaItem> _mediaItems;
+    private readonly CancellationToken _cancellationToken;
     private CloneableRandom _random;
     private IList<MediaItem> _shuffled;
 
     public ShuffledMediaCollectionEnumerator(
         IList<GroupedMediaItem> mediaItems,
-        CollectionEnumeratorState state)
+        CollectionEnumeratorState state,
+        CancellationToken cancellationToken)
     {
         _mediaItemCount = mediaItems.Sum(i => 1 + Optional(i.Additional).Flatten().Count());
         _mediaItems = mediaItems;
+        _cancellationToken = cancellationToken;
 
         if (state.Index >= _mediaItems.Count)
         {
@@ -49,7 +52,8 @@ public class ShuffledMediaCollectionEnumerator : IMediaCollectionEnumerator
                 State.Seed = _random.Next();
                 _random = new CloneableRandom(State.Seed);
                 _shuffled = Shuffle(_mediaItems, _random);
-            } while (_mediaItems.Count > 1 && Current.Map(x => x.Id) == tail.Map(x => x.Id));
+            } while (!_cancellationToken.IsCancellationRequested && _mediaItems.Count > 1 &&
+                     Current.Map(x => x.Id) == tail.Map(x => x.Id));
         }
         else
         {

ErsatzTV.Infrastructure/Data/Repositories/MediaCollectionRepository.cs

@@ -437,7 +437,7 @@ public class MediaCollectionRepository : IMediaCollectionRepository
                 PlaybackOrder.Chronological,
                 false));
 
-        return result;
+        return result.Filter(c => c.MediaItems.Any()).ToList();
     }
 
     private async Task<List<Movie>> GetMovieItems(TvContext dbContext, int collectionId)

ErsatzTV.Infrastructure/Scheduling/MultiEpisodeShuffleCollectionEnumerator.cs

@@ -9,6 +9,7 @@ namespace ErsatzTV.Infrastructure.Scheduling;
 public class MultiEpisodeShuffleCollectionEnumerator : IMediaCollectionEnumerator
 {
     private readonly ILogger _logger;
+    private readonly CancellationToken _cancellationToken;
     private readonly int _mediaItemCount;
     private readonly Dictionary<int, List<MediaItem>> _mediaItemGroups;
     private readonly List<MediaItem> _ungrouped;
@@ -20,9 +21,11 @@ public class MultiEpisodeShuffleCollectionEnumerator : IMediaCollectionEnumerato
         CollectionEnumeratorState state,
         IScriptEngine scriptEngine,
         string scriptFile,
-        ILogger logger)
+        ILogger logger,
+        CancellationToken cancellationToken)
     {
         _logger = logger;
+        _cancellationToken = cancellationToken;
 
         scriptEngine.Load(scriptFile);
 
@@ -93,7 +96,8 @@ public class MultiEpisodeShuffleCollectionEnumerator : IMediaCollectionEnumerato
                 State.Seed = _random.Next();
                 _random = new CloneableRandom(State.Seed);
                 _shuffled = Shuffle(_random);
-            } while (_mediaItemCount > 1 && Current.Map(x => x.Id) == tail.Map(x => x.Id));
+            } while (!_cancellationToken.IsCancellationRequested && _mediaItemCount > 1 &&
+                     Current.Map(x => x.Id) == tail.Map(x => x.Id));
         }
         else
         {

ErsatzTV.Infrastructure/Scheduling/MultiEpisodeShuffleCollectionEnumeratorFactory.cs

@@ -22,6 +22,13 @@ public class MultiEpisodeShuffleCollectionEnumeratorFactory
     public IMediaCollectionEnumerator Create(
         string jsScriptPath,
         IList<MediaItem> mediaItems,
-        CollectionEnumeratorState state) =>
-        new MultiEpisodeShuffleCollectionEnumerator(mediaItems, state, _scriptEngine, jsScriptPath, _logger);
+        CollectionEnumeratorState state,
+        CancellationToken cancellationToken) =>
+        new MultiEpisodeShuffleCollectionEnumerator(
+            mediaItems,
+            state,
+            _scriptEngine,
+            jsScriptPath,
+            _logger,
+            cancellationToken);
 }

ErsatzTV/Pages/MultiCollectionEditor.razor

@@ -218,7 +218,7 @@
 
     private void RemoveCollection(MultiCollectionItemEditViewModel item) => _model.Items.Remove(item);
 
-    private void AddCollection()
+    private async Task AddCollection()
     {
         if (_selectedCollection != null && _model.Items.All(i => i.Collection != _selectedCollection))
         {
@@ -229,11 +229,11 @@
             });
 
             _selectedCollection = null;
-            _collectionSelect.Reset();
+            await _collectionSelect.ResetAsync();
         }
     }
 
-    private void AddSmartCollection()
+    private async Task AddSmartCollection()
     {
         if (_selectedSmartCollection != null && _model.Items.OfType<MultiCollectionSmartItemEditViewModel>().All(i => i.SmartCollection != _selectedSmartCollection))
         {
@@ -244,7 +244,7 @@
             });
 
             _selectedSmartCollection = null;
-            _smartCollectionSelect.Reset();
+            await _smartCollectionSelect.ResetAsync();
         }
     }
 

ErsatzTV/Services/WorkerService.cs

@@ -49,62 +49,46 @@ public class WorkerService : BackgroundService
                     switch (request)
                     {
                         case RefreshChannelList refreshChannelList:
-                            _logger.LogDebug("WorkerService - RefreshChannelList START");
                             await mediator.Send(refreshChannelList, cancellationToken);
-                            _logger.LogDebug("WorkerService - RefreshChannelList FINISH");
                             break;
                         case RefreshChannelData refreshChannelData:
-                            _logger.LogDebug("WorkerService - RefreshChannelData START");
                             await mediator.Send(refreshChannelData, cancellationToken);
-                            _logger.LogDebug("WorkerService - RefreshChannelData FINISH");
                             break;
                         case BuildPlayout buildPlayout:
-                            _logger.LogDebug("WorkerService - BuildPlayout START");
+                            var cts = new CancellationTokenSource(TimeSpan.FromMinutes(2));
+                            var linkedTokenSource =
+                                CancellationTokenSource.CreateLinkedTokenSource(cts.Token, cancellationToken);
+
                             Either<BaseError, Unit> buildPlayoutResult = await mediator.Send(
                                 buildPlayout,
-                                cancellationToken);
+                                linkedTokenSource.Token);
                             buildPlayoutResult.BiIter(
                                 _ => _logger.LogDebug("Built playout {PlayoutId}", buildPlayout.PlayoutId),
                                 error => _logger.LogWarning(
                                     "Unable to build playout {PlayoutId}: {Error}",
                                     buildPlayout.PlayoutId,
                                     error.Value));
-                            _logger.LogDebug("WorkerService - BuildPlayout FINISH");
                             break;
                         case DeleteOrphanedArtwork deleteOrphanedArtwork:
-                            _logger.LogDebug("WorkerService - DeleteOrphanedArtwork START");
                             await mediator.Send(deleteOrphanedArtwork, cancellationToken);
-                            _logger.LogDebug("WorkerService - DeleteOrphanedArtwork FINISH");
                             break;
                         case DeleteOrphanedSubtitles deleteOrphanedSubtitles:
-                            _logger.LogDebug("WorkerService - DeleteOrphanedSubtitles START");
                             await mediator.Send(deleteOrphanedSubtitles, cancellationToken);
-                            _logger.LogDebug("WorkerService - DeleteOrphanedSubtitles FINISH");
                             break;
                         case AddTraktList addTraktList:
-                            _logger.LogDebug("WorkerService - AddTraktList START");
                             await mediator.Send(addTraktList, cancellationToken);
-                            _logger.LogDebug("WorkerService - AddTraktList FINISH");
                             break;
                         case DeleteTraktList deleteTraktList:
-                            _logger.LogDebug("WorkerService - DeleteTraktList START");
                             await mediator.Send(deleteTraktList, cancellationToken);
-                            _logger.LogDebug("WorkerService - DeleteTraktList FINISH");
                             break;
                         case MatchTraktListItems matchTraktListItems:
-                            _logger.LogDebug("WorkerService - MatchTraktListItems START");
                             await mediator.Send(matchTraktListItems, cancellationToken);
-                            _logger.LogDebug("WorkerService - MatchTraktListItems FINISH");
                             break;
                         case ExtractEmbeddedSubtitles extractEmbeddedSubtitles:
-                            _logger.LogDebug("WorkerService - ExtractEmbeddedSubtitles START");
                             await mediator.Send(extractEmbeddedSubtitles, cancellationToken);
-                            _logger.LogDebug("WorkerService - ExtractEmbeddedSubtitles FINISH");
                             break;
                         case ReleaseMemory aggressivelyReleaseMemory:
-                            _logger.LogDebug("WorkerService - ReleaseMemory START");
                             await mediator.Send(aggressivelyReleaseMemory, cancellationToken);
-                            _logger.LogDebug("WorkerService - ReleaseMemory FINISH");
                             break;
                     }
                 }