fix: checkpoint classic schedules every day

CHANGELOG.md

@@ -17,6 +17,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Always randomize start points for all collections the first time they are used
   - Previously, only collections scheduled during the first day of a playout build had their start points randomized
 - Allow multiple ETV instances when multiple config folders are used
+- Maintain collection progress when refreshing a classic playout
+  - Classic playouts save a checkpoint for each day they build, and refresh rewinds to the checkpoint for the current day
+  - Checkpoints were only saved when a build happened to stop on a day boundary, so playouts with long items (movies, long blocks) were often missing the checkpoint for the current day, and refreshing those restarted every collection from the beginning
+  - Checkpoints are now saved whenever a build crosses midnight
 - Fix green line sometimes seen with NVIDIA and AMD/VAAPI encoding
   - Both bugs were in ffmpeg, and ETV's patched ffmpeg 8.1.2 is required for the fixes
 

ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/ContinuePlayoutTests.cs

@@ -166,7 +166,9 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase
         }
 
         playout.Anchor.NextStartOffset.ShouldBe(finish);
-        playout.ProgramScheduleAnchors.Count.ShouldBe(1);
+
+        // the continue anchor, plus a checkpoint for the midnight this build ended on
+        playout.ProgramScheduleAnchors.Count.ShouldBe(2);
         playout.ProgramScheduleAnchors.Head().EnumeratorState.Index.ShouldBe(0);
 
         PlayoutProgramScheduleAnchor headAnchor = playout.ProgramScheduleAnchors.Head();
@@ -216,7 +218,8 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase
 
         playout.Anchor.NextStartOffset.ShouldBe(start + TimeSpan.FromHours(30));
 
-        playout.ProgramScheduleAnchors.Count.ShouldBe(2);
+        // continue anchor, detractor checkpoint, and the checkpoint for the midnight the first build crossed
+        playout.ProgramScheduleAnchors.Count.ShouldBe(3);
         playout.ProgramScheduleAnchors.Head().EnumeratorState.Index.ShouldBe(1);
 
         // continue 1h later
@@ -240,7 +243,8 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase
 
         playout.Anchor.NextStartOffset.ShouldBe(start + TimeSpan.FromHours(30));
 
-        playout.ProgramScheduleAnchors.Count.ShouldBe(2);
+        // continue anchor, detractor checkpoint, and the checkpoint for the midnight the first build crossed
+        playout.ProgramScheduleAnchors.Count.ShouldBe(3);
         playout.ProgramScheduleAnchors.Head().EnumeratorState.Index.ShouldBe(1);
     }
 
@@ -329,9 +333,10 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase
             result.AddedItems.Count.ShouldBe(53);
         }
 
-        playout.ProgramScheduleAnchors.Count.ShouldBe(2);
+        // a checkpoint for each of the two midnights the build crossed, plus the continue anchor
+        playout.ProgramScheduleAnchors.Count.ShouldBe(3);
+        playout.ProgramScheduleAnchors.Count(x => x.AnchorDate is not null).ShouldBe(2);
 
-        playout.ProgramScheduleAnchors.All(x => x.AnchorDate is not null).ShouldBeTrue();
         PlayoutProgramScheduleAnchor lastCheckpoint = playout.ProgramScheduleAnchors
             .OrderByDescending(a => a.AnchorDate ?? DateTime.MinValue)
             .First();
@@ -339,11 +344,14 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase
         lastCheckpoint.EnumeratorState.Index.ShouldBe(3);
 
         // we need to mess up the ordering to trigger the problematic behavior
-        // this simulates the way the rows are loaded with EF
+        // this simulates the way the rows are loaded with EF, with only the checkpoints kept
-        PlayoutProgramScheduleAnchor oldest = playout.ProgramScheduleAnchors.OrderByDescending(a => a.AnchorDate)
+        var checkpoints = playout.ProgramScheduleAnchors
-            .Last();
+            .Filter(a => a.AnchorDate is not null)
-        PlayoutProgramScheduleAnchor newest = playout.ProgramScheduleAnchors.OrderByDescending(a => a.AnchorDate)
+            .OrderBy(a => a.AnchorDate)
-            .First();
+            .ToList();
+
+        PlayoutProgramScheduleAnchor oldest = checkpoints.Head();
+        PlayoutProgramScheduleAnchor newest = checkpoints.Last();
 
         playout.ProgramScheduleAnchors =
         [
@@ -460,9 +468,10 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase
             result.AddedItems.Count.ShouldBe(53);
         }
 
-        playout.ProgramScheduleAnchors.Count.ShouldBe(4);
+        // two collections, each with a checkpoint for the two midnights the build crossed
-
+        // plus a continue anchor
-        playout.ProgramScheduleAnchors.All(x => x.AnchorDate is not null).ShouldBeTrue();
+        playout.ProgramScheduleAnchors.Count.ShouldBe(6);
+        playout.ProgramScheduleAnchors.Count(x => x.AnchorDate is not null).ShouldBe(4);
         PlayoutProgramScheduleAnchor lastCheckpoint = playout.ProgramScheduleAnchors
             .Filter(psa => psa.SmartCollectionId == 1)
             .OrderByDescending(a => a.AnchorDate ?? DateTime.MinValue)

ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/MultiDayBuildTests.cs

@@ -20,6 +20,7 @@ namespace ErsatzTV.Core.Tests.Scheduling.ClassicScheduling;
 public class MultiDayBuildTests : PlayoutBuilderTestBase
 {
     private const int DaysToBuild = 2;
+    private const int WeekOfDaysToBuild = 7;
 
     [Test]
     public async Task Continue_Should_Keep_A_Checkpoint_For_The_Current_Day()
@@ -115,6 +116,58 @@ public class MultiDayBuildTests : PlayoutBuilderTestBase
             $"[tz {TimeZoneInfo.Local.Id}] refresh reset collection progress (was {before}, now {after})");
     }
 
+    [Test]
+    public async Task Continue_Should_Keep_A_Checkpoint_For_Every_Day_In_The_Build_Window()
+    {
+        // shaped like the playouts that were missing checkpoints in a real user database: a week
+        // long build window, shuffled schedule items, and blocks of five ~100 minute items
+        (PlayoutBuilder builder, Playout playout, PlayoutReferenceData referenceData) = MultipleTestData(
+            TimeSpan.FromMinutes(100),
+            multipleCount: "5",
+            itemCount: 300,
+            daysToBuild: WeekOfDaysToBuild,
+            shuffleScheduleItems: true,
+            scheduleItemCount: 3);
+
+        DateTimeOffset start = LocalTime(20);
+        var missing = new List<string>();
+
+        for (var hour = 0; hour < 24 * 21; hour++)
+        {
+            DateTimeOffset now = start.AddHours(hour);
+
+            Either<BaseError, PlayoutBuildResult> result = await builder.Build(
+                now,
+                playout,
+                referenceData,
+                PlayoutBuildMode.Continue,
+                CancellationToken);
+
+            result.IsRight.ShouldBeTrue($"build failed at {now:yyyy-MM-dd HH:mm}");
+
+            // every day the playout covers needs a checkpoint, so that a refresh on any of them has
+            // somewhere to rewind to. the build's own first day is the one exception, and days
+            // already in the past are pruned
+            DateTime firstExpected = now.Date > start.Date ? now.Date : start.Date.AddDays(1);
+            DateTime lastExpected = now.AddDays(WeekOfDaysToBuild).Date;
+
+            for (DateTime date = firstExpected; date <= lastExpected; date = date.AddDays(1))
+            {
+                bool hasCheckpoint = playout.ProgramScheduleAnchors.Any(a =>
+                    a.AnchorDateOffset.HasValue && a.AnchorDateOffset.Value.Date == date);
+
+                if (!hasCheckpoint)
+                {
+                    missing.Add($"{date:yyyy-MM-dd} (at {now:MM-dd HH:mm})");
+                }
+            }
+        }
+
+        missing.ShouldBeEmpty(
+            $"[tz {TimeZoneInfo.Local.Id}] {missing.Count} day(s) in the build window had no checkpoint, "
+            + $"first: {missing.FirstOrDefault()}");
+    }
+
     private static int HeadIndex(Playout playout) =>
         playout.ProgramScheduleAnchors
             .Filter(a => a.AnchorDate is null)
@@ -125,7 +178,10 @@ public class MultiDayBuildTests : PlayoutBuilderTestBase
     private (PlayoutBuilder, Playout, PlayoutReferenceData) MultipleTestData(
         TimeSpan itemDuration,
         string multipleCount,
-        int itemCount = 5)
+        int itemCount = 5,
+        int daysToBuild = DaysToBuild,
+        bool shuffleScheduleItems = false,
+        int scheduleItemCount = 1)
     {
         var collection = new Collection
         {
@@ -139,9 +195,14 @@ public class MultiDayBuildTests : PlayoutBuilderTestBase
         var collectionRepo = new FakeMediaCollectionRepository(Map((collection.Id, collection.MediaItems.ToList())));
 
         IConfigElementRepository configRepo = Substitute.For<IConfigElementRepository>();
+
+        // ConfigElementKey has no value equality and hands out a new instance on every access, so
+        // the key has to be matched on its string, not with Arg.Is(theKey)
         configRepo
-            .GetValue<int>(Arg.Is(ConfigElementKey.PlayoutDaysToBuild), Arg.Any<CancellationToken>())
+            .GetValue<int>(
-            .Returns(Some(DaysToBuild));
+                Arg.Is<ConfigElementKey>(k => k.Key == ConfigElementKey.PlayoutDaysToBuild.Key),
+                Arg.Any<CancellationToken>())
+            .Returns(Some(daysToBuild));
 
         var televisionRepo = new FakeTelevisionRepository();
         IArtistRepository artistRepo = Substitute.For<IArtistRepository>();
@@ -159,26 +220,29 @@ public class MultiDayBuildTests : PlayoutBuilderTestBase
             rerunHelper,
             Logger);
 
-        var items = new List<ProgramScheduleItem>
+        var items = Enumerable.Range(1, scheduleItemCount)
-        {
+            .Map(i => (ProgramScheduleItem)new ProgramScheduleItemMultiple
-            new ProgramScheduleItemMultiple
             {
-                Id = 1,
+                Id = i,
-                Index = 1,
+                Index = i,
                 CollectionType = CollectionType.Collection,
                 Collection = collection,
                 CollectionId = collection.Id,
                 StartTime = null,
                 PlaybackOrder = PlaybackOrder.Chronological,
                 Count = multipleCount
-            }
+            })
-        };
+            .ToList();
 
         var playout = new Playout
         {
             Id = 1,
             ScheduleKind = PlayoutScheduleKind.Classic,
-            ProgramSchedule = new ProgramSchedule { Items = items },
+            ProgramSchedule = new ProgramSchedule
+            {
+                Items = items,
+                ShuffleScheduleItems = shuffleScheduleItems
+            },
             Channel = new Channel(Guid.Empty) { Id = 1, Name = "Test Channel" },
             Items = [],
             ProgramScheduleAnchors = [],

ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/NewPlayoutTests.cs

@@ -2119,9 +2119,15 @@ public class NewPlayoutTests : PlayoutBuilderTestBase
             result.AddedItems[7].FinishOffset.ShouldBe(start + TimeSpan.FromHours(48));
         }
 
-        playout.ProgramScheduleAnchors.Count.ShouldBe(2);
+        // a checkpoint for each midnight the two day build crossed, plus the continue anchor
-        playout.ProgramScheduleAnchors.Count(a => a.EnumeratorState.Index == 4 % 3).ShouldBe(1);
+        var checkpoints = playout.ProgramScheduleAnchors.Filter(a => a.AnchorDate is not null).ToList();
-        playout.ProgramScheduleAnchors.Count(a => a.EnumeratorState.Index == 8 % 3).ShouldBe(1);
+        checkpoints.Count.ShouldBe(2);
+        checkpoints.Count(a => a.EnumeratorState.Index == 4 % 3).ShouldBe(1);
+        checkpoints.Count(a => a.EnumeratorState.Index == 8 % 3).ShouldBe(1);
+
+        PlayoutProgramScheduleAnchor continueAnchor =
+            playout.ProgramScheduleAnchors.Single(a => a.AnchorDate is null);
+        continueAnchor.EnumeratorState.Index.ShouldBe(8 % 3);
 
         int seed = playout.ProgramScheduleAnchors[0].EnumeratorState.Seed;
         playout.ProgramScheduleAnchors.All(a => a.EnumeratorState.Seed == seed).ShouldBeTrue();

ErsatzTV.Core/Scheduling/PlayoutBuilder.cs

@@ -457,7 +457,8 @@ public class PlayoutBuilder : IPlayoutBuilder
             return result;
         }
 
-        // build one whole day at a time, saving a checkpoint anchor at each day boundary
+        // build one whole day at a time; this is also how alternate schedules switch per day,
+        // since the active schedule is selected once per pass
         while (finish < playoutFinish)
         {
             if (cancellationToken.IsCancellationRequested)
@@ -466,14 +467,13 @@ public class PlayoutBuilder : IPlayoutBuilder
             }
 
             _logger.LogDebug("Building playout from {Start} to {Finish}", start, finish);
-            Either<BaseError, PlayoutBuildResult> buildResult = await BuildPlayoutItems(
+            Either<BaseError, PlayoutBuildResult> buildResult = await BuildPlayoutItemsForPass(
                 playout,
                 referenceData,
                 result,
                 start,
                 finish,
                 collectionMediaItems,
-                true,
                 cancellationToken);
 
             foreach (BaseError error in buildResult.LeftToSeq())
@@ -497,17 +497,15 @@ public class PlayoutBuilder : IPlayoutBuilder
 
         if (start < playoutFinish)
         {
-            // build the remaining partial day; this writes "continue" anchors (null anchor date)
+            // build the remaining partial day
-            // instead of a checkpoint, since we aren't stopping on a day boundary
             _logger.LogDebug("Building final playout from {Start} to {Finish}", start, playoutFinish);
-            Either<BaseError, PlayoutBuildResult> buildResult = await BuildPlayoutItems(
+            Either<BaseError, PlayoutBuildResult> buildResult = await BuildPlayoutItemsForPass(
                 playout,
                 referenceData,
                 result,
                 start,
                 playoutFinish,
                 collectionMediaItems,
-                false,
                 cancellationToken);
 
             foreach (BaseError error in buildResult.LeftToSeq())
@@ -551,14 +549,13 @@ public class PlayoutBuilder : IPlayoutBuilder
         return result;
     }
 
-    private async Task<Either<BaseError, PlayoutBuildResult>> BuildPlayoutItems(
+    private async Task<Either<BaseError, PlayoutBuildResult>> BuildPlayoutItemsForPass(
         Playout playout,
         PlayoutReferenceData referenceData,
         PlayoutBuildResult result,
         DateTimeOffset playoutStart,
         DateTimeOffset playoutFinish,
         Map<CollectionKey, List<MediaItem>> collectionMediaItems,
-        bool saveAnchorDate,
         CancellationToken cancellationToken)
     {
         var random = new Random(playout.Seed);
@@ -904,6 +901,15 @@ public class PlayoutBuilder : IPlayoutBuilder
 
             result.AddedItems.AddRange(playoutItems);
 
+            // save a checkpoint anchor whenever scheduling crosses a local midnight; this has to
+            // happen here rather than at the end of a pass, because pass boundaries only
+            // aspirationally line up with midnights - the final partial pass never ends on one, and
+            // a long item can carry a whole-day pass well past its own boundary
+            if (nextState.CurrentTime.ToLocalTime().Date > playoutBuilderState.CurrentTime.ToLocalTime().Date)
+            {
+                SaveCheckpointAnchors(playout, collectionEnumerators, nextState.CurrentTime);
+            }
+
             playoutBuilderState = nextState;
         }
 
@@ -1000,7 +1006,7 @@ public class PlayoutBuilder : IPlayoutBuilder
         }
 
         // build program schedule anchors
-        playout.ProgramScheduleAnchors = BuildProgramScheduleAnchors(playout, collectionEnumerators, saveAnchorDate);
+        playout.ProgramScheduleAnchors = BuildProgramScheduleAnchors(playout, collectionEnumerators);
 
         // build fill group indices
         playout.FillGroupIndices = BuildFillGroupIndices(playout, scheduleItemsFillGroupEnumerators);
@@ -1195,26 +1201,82 @@ public class PlayoutBuilder : IPlayoutBuilder
             }
         });
 
-    private static List<PlayoutProgramScheduleAnchor> BuildProgramScheduleAnchors(
+    private static bool AnchorMatchesCollectionKey(PlayoutProgramScheduleAnchor anchor, CollectionKey collectionKey) =>
+        anchor.CollectionType == collectionKey.CollectionType
+        && anchor.CollectionId == collectionKey.CollectionId
+        && anchor.MediaItemId == collectionKey.MediaItemId
+        && anchor.FakeCollectionKey == collectionKey.FakeCollectionKey
+        && anchor.SmartCollectionId == collectionKey.SmartCollectionId
+        && anchor.RerunCollectionId == collectionKey.RerunCollectionId
+        && anchor.MultiCollectionId == collectionKey.MultiCollectionId
+        && anchor.PlaylistId == collectionKey.PlaylistId
+        && anchor.SearchQuery == collectionKey.SearchQuery;
+
+    /// <summary>
+    ///     Records the collection progress as of <paramref name="checkpointTime" />, which is the first
+    ///     resumable point at or after a local midnight. Refresh rewinds to the checkpoint dated today,
+    ///     so every local day of the playout needs one.
+    /// </summary>
+    private static void SaveCheckpointAnchors(
         Playout playout,
         Dictionary<CollectionKey, IMediaCollectionEnumerator> collectionEnumerators,
-        bool saveAnchorDate)
+        DateTimeOffset checkpointTime)
+    {
+        // anchor dates are compared as local dates, the same way AnchorDateOffset exposes them
+        DateTime checkpointDate = checkpointTime.ToLocalTime().Date;
+
+        foreach ((CollectionKey collectionKey, IMediaCollectionEnumerator enumerator) in collectionEnumerators)
+        {
+            // one checkpoint per collection key per local date; rebuilding a day that already has a
+            // checkpoint replaces it, since the new state is the one that matches the new items
+            Option<PlayoutProgramScheduleAnchor> maybeExisting = playout.ProgramScheduleAnchors.FirstOrDefault(a =>
+                AnchorMatchesCollectionKey(a, collectionKey)
+                && a.AnchorDateOffset.HasValue
+                && a.AnchorDateOffset.Value.Date == checkpointDate);
+
+            // the enumerator keeps mutating its own state as the build continues, so the checkpoint
+            // needs a copy of it, frozen at this instant
+            CollectionEnumeratorState state = enumerator.State.Clone();
+
+            foreach (PlayoutProgramScheduleAnchor existing in maybeExisting)
+            {
+                existing.AnchorDate = checkpointTime.UtcDateTime;
+                existing.EnumeratorState = state;
+            }
+
+            if (maybeExisting.IsNone)
+            {
+                playout.ProgramScheduleAnchors.Add(
+                    new PlayoutProgramScheduleAnchor
+                    {
+                        Playout = playout,
+                        PlayoutId = playout.Id,
+                        CollectionType = collectionKey.CollectionType,
+                        CollectionId = collectionKey.CollectionId,
+                        MultiCollectionId = collectionKey.MultiCollectionId,
+                        SmartCollectionId = collectionKey.SmartCollectionId,
+                        RerunCollectionId = collectionKey.RerunCollectionId,
+                        MediaItemId = collectionKey.MediaItemId,
+                        PlaylistId = collectionKey.PlaylistId,
+                        SearchQuery = collectionKey.SearchQuery,
+                        FakeCollectionKey = collectionKey.FakeCollectionKey,
+                        AnchorDate = checkpointTime.UtcDateTime,
+                        EnumeratorState = state
+                    });
+            }
+        }
+    }
+
+    private static List<PlayoutProgramScheduleAnchor> BuildProgramScheduleAnchors(
+        Playout playout,
+        Dictionary<CollectionKey, IMediaCollectionEnumerator> collectionEnumerators)
     {
         var result = new List<PlayoutProgramScheduleAnchor>();
 
         foreach (CollectionKey collectionKey in collectionEnumerators.Keys)
         {
             Option<PlayoutProgramScheduleAnchor> maybeExisting = playout.ProgramScheduleAnchors.FirstOrDefault(a =>
-                a.CollectionType == collectionKey.CollectionType
+                AnchorMatchesCollectionKey(a, collectionKey) && a.AnchorDate is null);
-                && a.CollectionId == collectionKey.CollectionId
-                && a.MediaItemId == collectionKey.MediaItemId
-                && a.FakeCollectionKey == collectionKey.FakeCollectionKey
-                && a.SmartCollectionId == collectionKey.SmartCollectionId
-                && a.RerunCollectionId == collectionKey.RerunCollectionId
-                && a.MultiCollectionId == collectionKey.MultiCollectionId
-                && a.PlaylistId == collectionKey.PlaylistId
-                && a.SearchQuery == collectionKey.SearchQuery
-                && a.AnchorDate is null);
 
             var maybeEnumeratorState = collectionEnumerators.ToDictionary(e => e.Key, e => e.Value.State);
 
@@ -1240,14 +1302,10 @@ public class PlayoutBuilder : IPlayoutBuilder
                     EnumeratorState = maybeEnumeratorState[collectionKey]
                 });
 
-            if (saveAnchorDate)
-            {
-                scheduleAnchor.AnchorDate = playout.Anchor?.NextStart;
-            }
-
             result.Add(scheduleAnchor);
         }
 
+        // checkpoints are written by SaveCheckpointAnchors during the build; keep them all
         foreach (PlayoutProgramScheduleAnchor checkpointAnchor in playout.ProgramScheduleAnchors.Where(a =>
                      a.AnchorDate is not null))
         {