From aa40afe2d118ce2f76576d6d6da9ba469ed8e13e Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Sat, 25 Jul 2026 11:21:49 -0500 Subject: [PATCH] fix: checkpoint classic schedules every day --- CHANGELOG.md | 4 + .../ClassicScheduling/ContinuePlayoutTests.cs | 35 ++++-- .../ClassicScheduling/MultiDayBuildTests.cs | 86 ++++++++++++-- .../ClassicScheduling/NewPlayoutTests.cs | 12 +- ErsatzTV.Core/Scheduling/PlayoutBuilder.cs | 112 +++++++++++++----- 5 files changed, 195 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6be7ad65..3b2e7da62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Always randomize start points for all collections the first time they are used - Previously, only collections scheduled during the first day of a playout build had their start points randomized - Allow multiple ETV instances when multiple config folders are used +- Maintain collection progress when refreshing a classic playout + - Classic playouts save a checkpoint for each day they build, and refresh rewinds to the checkpoint for the current day + - Checkpoints were only saved when a build happened to stop on a day boundary, so playouts with long items (movies, long blocks) were often missing the checkpoint for the current day, and refreshing those restarted every collection from the beginning + - Checkpoints are now saved whenever a build crosses midnight - Fix green line sometimes seen with NVIDIA and AMD/VAAPI encoding - Both bugs were in ffmpeg, and ETV's patched ffmpeg 8.1.2 is required for the fixes diff --git a/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/ContinuePlayoutTests.cs b/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/ContinuePlayoutTests.cs index c3d9048b1..62c961230 100644 --- a/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/ContinuePlayoutTests.cs +++ b/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/ContinuePlayoutTests.cs @@ -166,7 +166,9 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase } playout.Anchor.NextStartOffset.ShouldBe(finish); - playout.ProgramScheduleAnchors.Count.ShouldBe(1); + + // the continue anchor, plus a checkpoint for the midnight this build ended on + playout.ProgramScheduleAnchors.Count.ShouldBe(2); playout.ProgramScheduleAnchors.Head().EnumeratorState.Index.ShouldBe(0); PlayoutProgramScheduleAnchor headAnchor = playout.ProgramScheduleAnchors.Head(); @@ -216,7 +218,8 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase playout.Anchor.NextStartOffset.ShouldBe(start + TimeSpan.FromHours(30)); - playout.ProgramScheduleAnchors.Count.ShouldBe(2); + // continue anchor, detractor checkpoint, and the checkpoint for the midnight the first build crossed + playout.ProgramScheduleAnchors.Count.ShouldBe(3); playout.ProgramScheduleAnchors.Head().EnumeratorState.Index.ShouldBe(1); // continue 1h later @@ -240,7 +243,8 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase playout.Anchor.NextStartOffset.ShouldBe(start + TimeSpan.FromHours(30)); - playout.ProgramScheduleAnchors.Count.ShouldBe(2); + // continue anchor, detractor checkpoint, and the checkpoint for the midnight the first build crossed + playout.ProgramScheduleAnchors.Count.ShouldBe(3); playout.ProgramScheduleAnchors.Head().EnumeratorState.Index.ShouldBe(1); } @@ -329,9 +333,10 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase result.AddedItems.Count.ShouldBe(53); } - playout.ProgramScheduleAnchors.Count.ShouldBe(2); + // a checkpoint for each of the two midnights the build crossed, plus the continue anchor + playout.ProgramScheduleAnchors.Count.ShouldBe(3); + playout.ProgramScheduleAnchors.Count(x => x.AnchorDate is not null).ShouldBe(2); - playout.ProgramScheduleAnchors.All(x => x.AnchorDate is not null).ShouldBeTrue(); PlayoutProgramScheduleAnchor lastCheckpoint = playout.ProgramScheduleAnchors .OrderByDescending(a => a.AnchorDate ?? DateTime.MinValue) .First(); @@ -339,11 +344,14 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase lastCheckpoint.EnumeratorState.Index.ShouldBe(3); // we need to mess up the ordering to trigger the problematic behavior - // this simulates the way the rows are loaded with EF - PlayoutProgramScheduleAnchor oldest = playout.ProgramScheduleAnchors.OrderByDescending(a => a.AnchorDate) - .Last(); - PlayoutProgramScheduleAnchor newest = playout.ProgramScheduleAnchors.OrderByDescending(a => a.AnchorDate) - .First(); + // this simulates the way the rows are loaded with EF, with only the checkpoints kept + var checkpoints = playout.ProgramScheduleAnchors + .Filter(a => a.AnchorDate is not null) + .OrderBy(a => a.AnchorDate) + .ToList(); + + PlayoutProgramScheduleAnchor oldest = checkpoints.Head(); + PlayoutProgramScheduleAnchor newest = checkpoints.Last(); playout.ProgramScheduleAnchors = [ @@ -460,9 +468,10 @@ public class ContinuePlayoutTests : PlayoutBuilderTestBase result.AddedItems.Count.ShouldBe(53); } - playout.ProgramScheduleAnchors.Count.ShouldBe(4); - - playout.ProgramScheduleAnchors.All(x => x.AnchorDate is not null).ShouldBeTrue(); + // two collections, each with a checkpoint for the two midnights the build crossed + // plus a continue anchor + playout.ProgramScheduleAnchors.Count.ShouldBe(6); + playout.ProgramScheduleAnchors.Count(x => x.AnchorDate is not null).ShouldBe(4); PlayoutProgramScheduleAnchor lastCheckpoint = playout.ProgramScheduleAnchors .Filter(psa => psa.SmartCollectionId == 1) .OrderByDescending(a => a.AnchorDate ?? DateTime.MinValue) diff --git a/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/MultiDayBuildTests.cs b/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/MultiDayBuildTests.cs index db7fc160b..b7142c411 100644 --- a/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/MultiDayBuildTests.cs +++ b/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/MultiDayBuildTests.cs @@ -20,6 +20,7 @@ namespace ErsatzTV.Core.Tests.Scheduling.ClassicScheduling; public class MultiDayBuildTests : PlayoutBuilderTestBase { private const int DaysToBuild = 2; + private const int WeekOfDaysToBuild = 7; [Test] public async Task Continue_Should_Keep_A_Checkpoint_For_The_Current_Day() @@ -115,6 +116,58 @@ public class MultiDayBuildTests : PlayoutBuilderTestBase $"[tz {TimeZoneInfo.Local.Id}] refresh reset collection progress (was {before}, now {after})"); } + [Test] + public async Task Continue_Should_Keep_A_Checkpoint_For_Every_Day_In_The_Build_Window() + { + // shaped like the playouts that were missing checkpoints in a real user database: a week + // long build window, shuffled schedule items, and blocks of five ~100 minute items + (PlayoutBuilder builder, Playout playout, PlayoutReferenceData referenceData) = MultipleTestData( + TimeSpan.FromMinutes(100), + multipleCount: "5", + itemCount: 300, + daysToBuild: WeekOfDaysToBuild, + shuffleScheduleItems: true, + scheduleItemCount: 3); + + DateTimeOffset start = LocalTime(20); + var missing = new List(); + + for (var hour = 0; hour < 24 * 21; hour++) + { + DateTimeOffset now = start.AddHours(hour); + + Either result = await builder.Build( + now, + playout, + referenceData, + PlayoutBuildMode.Continue, + CancellationToken); + + result.IsRight.ShouldBeTrue($"build failed at {now:yyyy-MM-dd HH:mm}"); + + // every day the playout covers needs a checkpoint, so that a refresh on any of them has + // somewhere to rewind to. the build's own first day is the one exception, and days + // already in the past are pruned + DateTime firstExpected = now.Date > start.Date ? now.Date : start.Date.AddDays(1); + DateTime lastExpected = now.AddDays(WeekOfDaysToBuild).Date; + + for (DateTime date = firstExpected; date <= lastExpected; date = date.AddDays(1)) + { + bool hasCheckpoint = playout.ProgramScheduleAnchors.Any(a => + a.AnchorDateOffset.HasValue && a.AnchorDateOffset.Value.Date == date); + + if (!hasCheckpoint) + { + missing.Add($"{date:yyyy-MM-dd} (at {now:MM-dd HH:mm})"); + } + } + } + + missing.ShouldBeEmpty( + $"[tz {TimeZoneInfo.Local.Id}] {missing.Count} day(s) in the build window had no checkpoint, " + + $"first: {missing.FirstOrDefault()}"); + } + private static int HeadIndex(Playout playout) => playout.ProgramScheduleAnchors .Filter(a => a.AnchorDate is null) @@ -125,7 +178,10 @@ public class MultiDayBuildTests : PlayoutBuilderTestBase private (PlayoutBuilder, Playout, PlayoutReferenceData) MultipleTestData( TimeSpan itemDuration, string multipleCount, - int itemCount = 5) + int itemCount = 5, + int daysToBuild = DaysToBuild, + bool shuffleScheduleItems = false, + int scheduleItemCount = 1) { var collection = new Collection { @@ -139,9 +195,14 @@ public class MultiDayBuildTests : PlayoutBuilderTestBase var collectionRepo = new FakeMediaCollectionRepository(Map((collection.Id, collection.MediaItems.ToList()))); IConfigElementRepository configRepo = Substitute.For(); + + // ConfigElementKey has no value equality and hands out a new instance on every access, so + // the key has to be matched on its string, not with Arg.Is(theKey) configRepo - .GetValue(Arg.Is(ConfigElementKey.PlayoutDaysToBuild), Arg.Any()) - .Returns(Some(DaysToBuild)); + .GetValue( + Arg.Is(k => k.Key == ConfigElementKey.PlayoutDaysToBuild.Key), + Arg.Any()) + .Returns(Some(daysToBuild)); var televisionRepo = new FakeTelevisionRepository(); IArtistRepository artistRepo = Substitute.For(); @@ -159,26 +220,29 @@ public class MultiDayBuildTests : PlayoutBuilderTestBase rerunHelper, Logger); - var items = new List - { - new ProgramScheduleItemMultiple + var items = Enumerable.Range(1, scheduleItemCount) + .Map(i => (ProgramScheduleItem)new ProgramScheduleItemMultiple { - Id = 1, - Index = 1, + Id = i, + Index = i, CollectionType = CollectionType.Collection, Collection = collection, CollectionId = collection.Id, StartTime = null, PlaybackOrder = PlaybackOrder.Chronological, Count = multipleCount - } - }; + }) + .ToList(); var playout = new Playout { Id = 1, ScheduleKind = PlayoutScheduleKind.Classic, - ProgramSchedule = new ProgramSchedule { Items = items }, + ProgramSchedule = new ProgramSchedule + { + Items = items, + ShuffleScheduleItems = shuffleScheduleItems + }, Channel = new Channel(Guid.Empty) { Id = 1, Name = "Test Channel" }, Items = [], ProgramScheduleAnchors = [], diff --git a/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/NewPlayoutTests.cs b/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/NewPlayoutTests.cs index bf3a4d225..78b553382 100644 --- a/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/NewPlayoutTests.cs +++ b/ErsatzTV.Core.Tests/Scheduling/ClassicScheduling/NewPlayoutTests.cs @@ -2119,9 +2119,15 @@ public class NewPlayoutTests : PlayoutBuilderTestBase result.AddedItems[7].FinishOffset.ShouldBe(start + TimeSpan.FromHours(48)); } - playout.ProgramScheduleAnchors.Count.ShouldBe(2); - playout.ProgramScheduleAnchors.Count(a => a.EnumeratorState.Index == 4 % 3).ShouldBe(1); - playout.ProgramScheduleAnchors.Count(a => a.EnumeratorState.Index == 8 % 3).ShouldBe(1); + // a checkpoint for each midnight the two day build crossed, plus the continue anchor + var checkpoints = playout.ProgramScheduleAnchors.Filter(a => a.AnchorDate is not null).ToList(); + checkpoints.Count.ShouldBe(2); + checkpoints.Count(a => a.EnumeratorState.Index == 4 % 3).ShouldBe(1); + checkpoints.Count(a => a.EnumeratorState.Index == 8 % 3).ShouldBe(1); + + PlayoutProgramScheduleAnchor continueAnchor = + playout.ProgramScheduleAnchors.Single(a => a.AnchorDate is null); + continueAnchor.EnumeratorState.Index.ShouldBe(8 % 3); int seed = playout.ProgramScheduleAnchors[0].EnumeratorState.Seed; playout.ProgramScheduleAnchors.All(a => a.EnumeratorState.Seed == seed).ShouldBeTrue(); diff --git a/ErsatzTV.Core/Scheduling/PlayoutBuilder.cs b/ErsatzTV.Core/Scheduling/PlayoutBuilder.cs index 15b62d580..10c5c202b 100644 --- a/ErsatzTV.Core/Scheduling/PlayoutBuilder.cs +++ b/ErsatzTV.Core/Scheduling/PlayoutBuilder.cs @@ -457,7 +457,8 @@ public class PlayoutBuilder : IPlayoutBuilder return result; } - // build one whole day at a time, saving a checkpoint anchor at each day boundary + // build one whole day at a time; this is also how alternate schedules switch per day, + // since the active schedule is selected once per pass while (finish < playoutFinish) { if (cancellationToken.IsCancellationRequested) @@ -466,14 +467,13 @@ public class PlayoutBuilder : IPlayoutBuilder } _logger.LogDebug("Building playout from {Start} to {Finish}", start, finish); - Either buildResult = await BuildPlayoutItems( + Either buildResult = await BuildPlayoutItemsForPass( playout, referenceData, result, start, finish, collectionMediaItems, - true, cancellationToken); foreach (BaseError error in buildResult.LeftToSeq()) @@ -497,17 +497,15 @@ public class PlayoutBuilder : IPlayoutBuilder if (start < playoutFinish) { - // build the remaining partial day; this writes "continue" anchors (null anchor date) - // instead of a checkpoint, since we aren't stopping on a day boundary + // build the remaining partial day _logger.LogDebug("Building final playout from {Start} to {Finish}", start, playoutFinish); - Either buildResult = await BuildPlayoutItems( + Either buildResult = await BuildPlayoutItemsForPass( playout, referenceData, result, start, playoutFinish, collectionMediaItems, - false, cancellationToken); foreach (BaseError error in buildResult.LeftToSeq()) @@ -551,14 +549,13 @@ public class PlayoutBuilder : IPlayoutBuilder return result; } - private async Task> BuildPlayoutItems( + private async Task> BuildPlayoutItemsForPass( Playout playout, PlayoutReferenceData referenceData, PlayoutBuildResult result, DateTimeOffset playoutStart, DateTimeOffset playoutFinish, Map> collectionMediaItems, - bool saveAnchorDate, CancellationToken cancellationToken) { var random = new Random(playout.Seed); @@ -904,6 +901,15 @@ public class PlayoutBuilder : IPlayoutBuilder result.AddedItems.AddRange(playoutItems); + // save a checkpoint anchor whenever scheduling crosses a local midnight; this has to + // happen here rather than at the end of a pass, because pass boundaries only + // aspirationally line up with midnights - the final partial pass never ends on one, and + // a long item can carry a whole-day pass well past its own boundary + if (nextState.CurrentTime.ToLocalTime().Date > playoutBuilderState.CurrentTime.ToLocalTime().Date) + { + SaveCheckpointAnchors(playout, collectionEnumerators, nextState.CurrentTime); + } + playoutBuilderState = nextState; } @@ -1000,7 +1006,7 @@ public class PlayoutBuilder : IPlayoutBuilder } // build program schedule anchors - playout.ProgramScheduleAnchors = BuildProgramScheduleAnchors(playout, collectionEnumerators, saveAnchorDate); + playout.ProgramScheduleAnchors = BuildProgramScheduleAnchors(playout, collectionEnumerators); // build fill group indices playout.FillGroupIndices = BuildFillGroupIndices(playout, scheduleItemsFillGroupEnumerators); @@ -1195,26 +1201,82 @@ public class PlayoutBuilder : IPlayoutBuilder } }); - private static List BuildProgramScheduleAnchors( + private static bool AnchorMatchesCollectionKey(PlayoutProgramScheduleAnchor anchor, CollectionKey collectionKey) => + anchor.CollectionType == collectionKey.CollectionType + && anchor.CollectionId == collectionKey.CollectionId + && anchor.MediaItemId == collectionKey.MediaItemId + && anchor.FakeCollectionKey == collectionKey.FakeCollectionKey + && anchor.SmartCollectionId == collectionKey.SmartCollectionId + && anchor.RerunCollectionId == collectionKey.RerunCollectionId + && anchor.MultiCollectionId == collectionKey.MultiCollectionId + && anchor.PlaylistId == collectionKey.PlaylistId + && anchor.SearchQuery == collectionKey.SearchQuery; + + /// + /// Records the collection progress as of , which is the first + /// resumable point at or after a local midnight. Refresh rewinds to the checkpoint dated today, + /// so every local day of the playout needs one. + /// + private static void SaveCheckpointAnchors( Playout playout, Dictionary collectionEnumerators, - bool saveAnchorDate) + DateTimeOffset checkpointTime) + { + // anchor dates are compared as local dates, the same way AnchorDateOffset exposes them + DateTime checkpointDate = checkpointTime.ToLocalTime().Date; + + foreach ((CollectionKey collectionKey, IMediaCollectionEnumerator enumerator) in collectionEnumerators) + { + // one checkpoint per collection key per local date; rebuilding a day that already has a + // checkpoint replaces it, since the new state is the one that matches the new items + Option maybeExisting = playout.ProgramScheduleAnchors.FirstOrDefault(a => + AnchorMatchesCollectionKey(a, collectionKey) + && a.AnchorDateOffset.HasValue + && a.AnchorDateOffset.Value.Date == checkpointDate); + + // the enumerator keeps mutating its own state as the build continues, so the checkpoint + // needs a copy of it, frozen at this instant + CollectionEnumeratorState state = enumerator.State.Clone(); + + foreach (PlayoutProgramScheduleAnchor existing in maybeExisting) + { + existing.AnchorDate = checkpointTime.UtcDateTime; + existing.EnumeratorState = state; + } + + if (maybeExisting.IsNone) + { + playout.ProgramScheduleAnchors.Add( + new PlayoutProgramScheduleAnchor + { + Playout = playout, + PlayoutId = playout.Id, + CollectionType = collectionKey.CollectionType, + CollectionId = collectionKey.CollectionId, + MultiCollectionId = collectionKey.MultiCollectionId, + SmartCollectionId = collectionKey.SmartCollectionId, + RerunCollectionId = collectionKey.RerunCollectionId, + MediaItemId = collectionKey.MediaItemId, + PlaylistId = collectionKey.PlaylistId, + SearchQuery = collectionKey.SearchQuery, + FakeCollectionKey = collectionKey.FakeCollectionKey, + AnchorDate = checkpointTime.UtcDateTime, + EnumeratorState = state + }); + } + } + } + + private static List BuildProgramScheduleAnchors( + Playout playout, + Dictionary collectionEnumerators) { var result = new List(); foreach (CollectionKey collectionKey in collectionEnumerators.Keys) { Option maybeExisting = playout.ProgramScheduleAnchors.FirstOrDefault(a => - a.CollectionType == collectionKey.CollectionType - && a.CollectionId == collectionKey.CollectionId - && a.MediaItemId == collectionKey.MediaItemId - && a.FakeCollectionKey == collectionKey.FakeCollectionKey - && a.SmartCollectionId == collectionKey.SmartCollectionId - && a.RerunCollectionId == collectionKey.RerunCollectionId - && a.MultiCollectionId == collectionKey.MultiCollectionId - && a.PlaylistId == collectionKey.PlaylistId - && a.SearchQuery == collectionKey.SearchQuery - && a.AnchorDate is null); + AnchorMatchesCollectionKey(a, collectionKey) && a.AnchorDate is null); var maybeEnumeratorState = collectionEnumerators.ToDictionary(e => e.Key, e => e.Value.State); @@ -1240,14 +1302,10 @@ public class PlayoutBuilder : IPlayoutBuilder EnumeratorState = maybeEnumeratorState[collectionKey] }); - if (saveAnchorDate) - { - scheduleAnchor.AnchorDate = playout.Anchor?.NextStart; - } - result.Add(scheduleAnchor); } + // checkpoints are written by SaveCheckpointAnchors during the build; keep them all foreach (PlayoutProgramScheduleAnchor checkpointAnchor in playout.ProgramScheduleAnchors.Where(a => a.AnchorDate is not null)) {