use super shuffle in block playouts (#1572)

ErsatzTV.Application/Playouts/Commands/CreateBlockPlayoutHandler.cs

@@ -41,7 +41,8 @@ public class CreateBlockPlayoutHandler(
             (channel, playoutType) => new Playout
             {
                 ChannelId = channel.Id,
-                ProgramSchedulePlayoutType = playoutType
+                ProgramSchedulePlayoutType = playoutType,
+                Seed = new Random().Next()
             });
 
     private static Task<Validation<BaseError, Channel>> ValidateChannel(

ErsatzTV.Application/Scheduling/Commands/EraseBlockPlayoutHistoryHandler.cs

@@ -13,17 +13,22 @@ public class EraseBlockPlayoutHistoryHandler(IDbContextFactory<TvContext> dbCont
         await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken);
 
         Option<Playout> maybePlayout = await dbContext.Playouts
-            .Include(p => p.Items)
-            .Include(p => p.PlayoutHistory)
             .Filter(p => p.ProgramSchedulePlayoutType == ProgramSchedulePlayoutType.Block)
             .SelectOneAsync(p => p.Id, p => p.Id == request.PlayoutId);
 
         foreach (Playout playout in maybePlayout)
         {
-            playout.Items.Clear();
-            playout.PlayoutHistory.Clear();
-
+            int nextSeed = new Random().Next();
+            playout.Seed = nextSeed;
             await dbContext.SaveChangesAsync(cancellationToken);
+
+            await dbContext.Database.ExecuteSqlAsync(
+                $"DELETE FROM PlayoutItem WHERE PlayoutId = {playout.Id}",
+                cancellationToken);
+
+            await dbContext.Database.ExecuteSqlAsync(
+                $"DELETE FROM PlayoutHistory WHERE PlayoutId = {playout.Id}",
+                cancellationToken);
         }
     }
 }

ErsatzTV.Core/Domain/Playout.cs

@@ -18,5 +18,6 @@ public class Playout
     public List<PlayoutScheduleItemFillGroupIndex> FillGroupIndices { get; set; }
     public ICollection<PlayoutTemplate> Templates { get; set; }
     public ICollection<PlayoutHistory> PlayoutHistory { get; set; }
+    public int Seed { get; set; }
     public TimeSpan? DailyRebuildTime { get; set; }
 }

ErsatzTV.Core/Domain/Scheduling/PlayoutHistory.cs

@@ -10,7 +10,7 @@ public class PlayoutHistory
     public int BlockId { get; set; }
     public Block Block { get; set; }
     public PlaybackOrder PlaybackOrder { get; set; }
-    public int Seed { get; set; }
+    public int Index { get; set; }
     
     // something that uniquely identifies the collection within the block 
     public string Key { get; set; }

ErsatzTV.Core/Scheduling/BlockScheduling/BlockPlayoutBuilder.cs

@@ -179,7 +179,7 @@ public class BlockPlayoutBuilder(
                         PlayoutId = playout.Id,
                         BlockId = blockItem.BlockId,
                         PlaybackOrder = blockItem.PlaybackOrder,
-                        Seed = enumerator.State.Seed,
+                        Index = enumerator.State.Index,
                         When = currentTime.UtcDateTime,
                         Key = historyKey,
                         Details = HistoryDetails.ForMediaItem(mediaItem)
@@ -287,20 +287,8 @@ public class BlockPlayoutBuilder(
 
             IEnumerable<PlayoutHistory> toDelete = group
                 .Filter(h => h.When < start.UtcDateTime)
-                .OrderByDescending(h => h.When);
-
-            // chronological and season, episode only need to keep most recent entry
-            if (group.Count > 0 && group[0].PlaybackOrder is PlaybackOrder.Chronological or PlaybackOrder.SeasonEpisode)
-            {
-                toDelete = toDelete.Tail();
-            }
-
-            // shuffle needs to keep all entries with current seed
-            if (group.Count > 0 && group[0].PlaybackOrder is PlaybackOrder.Shuffle)
-            {
-                int currentSeed = group[0].Seed;
-                toDelete = toDelete.Filter(h => h.Seed != currentSeed);
-            }
+                .OrderByDescending(h => h.When)
+                .Tail();
 
             foreach (PlayoutHistory delete in toDelete)
             {

ErsatzTV.Core/Scheduling/BlockScheduling/BlockPlayoutEnumerator.cs

@@ -86,49 +86,18 @@ public static class BlockPlayoutEnumerator
         BlockItem blockItem,
         string historyKey)
     {
-        // need a new shuffled media collection enumerator that can "hide" items for one iteration, then include all items again
-        // maybe take a "masked items" hash set, then clear it after shuffling
-        
         DateTime historyTime = currentTime.UtcDateTime;
-        var maskedMediaItemIds = new System.Collections.Generic.HashSet<int>();
-        List<PlayoutHistory> history = playout.PlayoutHistory
+        Option<PlayoutHistory> maybeHistory = playout.PlayoutHistory
             .Filter(h => h.BlockId == blockItem.BlockId)
             .Filter(h => h.Key == historyKey)
             .Filter(h => h.When < historyTime)
             .OrderByDescending(h => h.When)
-            .ToList();
-
-        if (history.Count > 0)
-        {
-            int currentSeed = history[0].Seed;
-            history = history.Filter(h => h.Seed == currentSeed).ToList();
-        }
-
-        var knownMediaIds = collectionItems.Map(ci => ci.Id).ToImmutableHashSet();
-        foreach (PlayoutHistory h in history)
-        {
-            HistoryDetails.Details details = JsonConvert.DeserializeObject<HistoryDetails.Details>(h.Details);
-            foreach (int mediaItemId in Optional(details.MediaItemId))
-            {
-                if (knownMediaIds.Contains(mediaItemId))
-                {
-                    maskedMediaItemIds.Add(mediaItemId);
-                }
-            }
-        }
-
-        var state = new CollectionEnumeratorState { Seed = new Random().Next(), Index = 0 };
-
-        // keep the current seed if one exists 
-        if (maskedMediaItemIds.Count > 0 && maskedMediaItemIds.Count < collectionItems.Count && history.Count > 0)
-        {
-            state.Seed = history[0].Seed;
-        }
+            .HeadOrNone();
 
-        // if everything is masked, nothing is masked
-        if (maskedMediaItemIds.Count == collectionItems.Count)
+        var state = new CollectionEnumeratorState { Seed = playout.Id, Index = 0 };
+        foreach (PlayoutHistory h in maybeHistory)
         {
-            maskedMediaItemIds.Clear();
+            state.Index = h.Index + 1;
         }
 
         // TODO: fix multi-collection groups, keep multi-part episodes together
@@ -136,17 +105,8 @@ public static class BlockPlayoutEnumerator
             .Map(mi => new GroupedMediaItem(mi, null))
             .ToList();
 
-        Serilog.Log.Logger.Debug(
-            "scheduling {X} media items with {Y} masked",
-            mediaItems.Count,
-            maskedMediaItemIds.Count);
-
         // it shouldn't matter which order the remaining items are shuffled in,
         // as long as already-played items are not included
-        return new MaskedShuffledMediaCollectionEnumerator(
-            mediaItems,
-            maskedMediaItemIds,
-            state,
-            CancellationToken.None);
+        return new BlockPlayoutShuffledMediaCollectionEnumerator(mediaItems, state);
     }
 }

ErsatzTV.Core/Scheduling/BlockScheduling/BlockPlayoutShuffledMediaCollectionEnumerator.cs

@@ -0,0 +1,67 @@
+using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.Extensions;
+using ErsatzTV.Core.Interfaces.Scheduling;
+
+namespace ErsatzTV.Core.Scheduling.BlockScheduling;
+
+public class BlockPlayoutShuffledMediaCollectionEnumerator : IMediaCollectionEnumerator
+{
+    private readonly IList<GroupedMediaItem> _mediaItems;
+    private readonly Lazy<Option<TimeSpan>> _lazyMinimumDuration;
+    private readonly int _mediaItemCount;
+    private IList<MediaItem> _shuffled;
+
+    public BlockPlayoutShuffledMediaCollectionEnumerator(
+        IList<GroupedMediaItem> mediaItems,
+        CollectionEnumeratorState state)
+    {
+        _mediaItems = mediaItems;
+        _mediaItemCount = _mediaItems.Sum(i => 1 + Optional(i.Additional).Flatten().Count());
+
+        State = state;
+
+        _shuffled = Shuffle(_mediaItems);
+        _lazyMinimumDuration =
+            new Lazy<Option<TimeSpan>>(
+                () => _shuffled.Bind(i => i.GetNonZeroDuration()).OrderBy(identity).HeadOrNone());
+    }
+
+    public void ResetState(CollectionEnumeratorState state)
+    {
+        // only re-shuffle if needed
+        if (State.Seed != state.Seed || State.Index != state.Index)
+        {
+            State.Seed = state.Seed;
+            State.Index = state.Index;
+
+            _shuffled = Shuffle(_mediaItems);
+        }
+    }
+
+    public CollectionEnumeratorState State { get; }
+
+    public Option<MediaItem> Current => _shuffled.Any() ? _shuffled[State.Index % _mediaItemCount] : None;
+
+    public void MoveNext()
+    {
+        State.Index++;
+    }
+
+    public Option<TimeSpan> MinimumDuration => _lazyMinimumDuration.Value;
+
+    public int Count => _shuffled.Count;
+
+    private IList<MediaItem> Shuffle(IList<GroupedMediaItem> list)
+    {
+        var copy = new GroupedMediaItem[list.Count];
+
+        var superShuffle = new SuperShuffle();
+        for (var i = 0; i < list.Count; i++)
+        {
+            int toSelect = superShuffle.Shuffle(i, State.Seed + (State.Index / list.Count), list.Count);
+            copy[i] = list[toSelect];
+        }
+
+        return GroupedMediaItem.FlattenGroups(copy, _mediaItemCount);
+    }
+}

ErsatzTV.Core/Scheduling/MaskedShuffledMediaCollectionEnumerator.cs

@@ -1,106 +0,0 @@
-using ErsatzTV.Core.Domain;
-using ErsatzTV.Core.Extensions;
-using ErsatzTV.Core.Interfaces.Scheduling;
-
-namespace ErsatzTV.Core.Scheduling;
-
-public class MaskedShuffledMediaCollectionEnumerator : IMediaCollectionEnumerator
-{
-    private readonly CancellationToken _cancellationToken;
-    private readonly Lazy<Option<TimeSpan>> _lazyMinimumDuration;
-    private readonly int _mediaItemCount;
-    private readonly IList<GroupedMediaItem> _mediaItems;
-    private CloneableRandom _random;
-    private IList<MediaItem> _shuffled;
-
-    public MaskedShuffledMediaCollectionEnumerator(
-        IList<GroupedMediaItem> mediaItems,
-        IReadOnlySet<int> maskedMediaItemIds,
-        CollectionEnumeratorState state,
-        CancellationToken cancellationToken)
-    {
-        _mediaItemCount = mediaItems.Sum(i => 1 + Optional(i.Additional).Flatten().Count());
-        _mediaItems = mediaItems;
-        _cancellationToken = cancellationToken;
-
-        if (state.Index >= _mediaItems.Count)
-        {
-            state.Index = 0;
-            state.Seed = new Random(state.Seed).Next();
-        }
-
-        _random = new CloneableRandom(state.Seed);
-
-        // remove masked items from initial shuffle
-        var filtered = _mediaItems.Filter(mi => !maskedMediaItemIds.Contains(mi.First.Id)).ToList();
-        foreach (GroupedMediaItem group in filtered)
-        {
-            group.Additional.RemoveAll(mi => maskedMediaItemIds.Contains(mi.Id));
-        }
-        
-        _shuffled = Shuffle(filtered, _random);
-        _lazyMinimumDuration =
-            new Lazy<Option<TimeSpan>>(
-                () => _shuffled.Bind(i => i.GetNonZeroDuration()).OrderBy(identity).HeadOrNone());
-
-        State = state;
-    }
-
-    public void ResetState(CollectionEnumeratorState state)
-    {
-        // only re-shuffle if needed
-        if (State.Seed != state.Seed)
-        {
-            _random = new CloneableRandom(state.Seed);
-            _shuffled = Shuffle(_mediaItems, _random);
-        }
-
-        State.Index = state.Index;
-    }
-
-    public CollectionEnumeratorState State { get; }
-
-    public Option<MediaItem> Current => _shuffled.Any() ? _shuffled[State.Index % _mediaItemCount] : None;
-
-    public void MoveNext()
-    {
-        if ((State.Index + 1) % _mediaItemCount == 0)
-        {
-            Option<MediaItem> tail = Current;
-
-            State.Index = 0;
-            do
-            {
-                State.Seed = _random.Next();
-                _random = new CloneableRandom(State.Seed);
-                _shuffled = Shuffle(_mediaItems, _random);
-            } while (!_cancellationToken.IsCancellationRequested && _mediaItems.Count > 1 &&
-                     Current.Map(x => x.Id) == tail.Map(x => x.Id));
-        }
-        else
-        {
-            State.Index++;
-        }
-
-        State.Index %= _mediaItemCount;
-    }
-
-    public Option<TimeSpan> MinimumDuration => _lazyMinimumDuration.Value;
-
-    public int Count => _shuffled.Count;
-
-    private IList<MediaItem> Shuffle(IEnumerable<GroupedMediaItem> list, CloneableRandom random)
-    {
-        GroupedMediaItem[] copy = list.ToArray();
-
-        int n = copy.Length;
-        while (n > 1)
-        {
-            n--;
-            int k = random.Next(n + 1);
-            (copy[k], copy[n]) = (copy[n], copy[k]);
-        }
-
-        return GroupedMediaItem.FlattenGroups(copy, _mediaItemCount);
-    }
-}

ErsatzTV.Core/Scheduling/SuperShuffle.cs

@@ -0,0 +1,65 @@
+// adapted from https://github.com/RondeSC/Super_Shuffle
+
+namespace ErsatzTV.Core.Scheduling;
+
+public class SuperShuffle
+{
+    private int userSID = 1;
+    private int si, r1, r2, r3, r4;
+    private int randR, halfN, rx, rkey;
+    
+    public int Shuffle(int inx, int shuffleId, int listSize) {
+        int si, hi, offset;
+        int halfSize = listSize / 2;  // for 1/2 the processing range
+
+        shuffleId += 131 * (inx / listSize);  // have inx overflows supported
+        si = (inx % listSize);
+        if (si < halfSize) {
+            hi = si;
+            offset = 0;
+        } else {
+            hi = listSize - 1 - si;
+            offset = halfSize;
+            halfSize = listSize - halfSize;
+            shuffleId++;
+        }
+
+        hi = MillerShuffleE(hi, shuffleId, halfSize);     // use any STD MSA() shuffle (aka: a PRIG function)
+        si = MillerShuffleE(hi + offset, userSID, listSize);  // indexing into the baseline shuffle
+        return(si);
+    }
+
+    private int MillerShuffleE(int inx, int shuffleId, int listSize)
+    {
+        const int p1 = 24317, p2 = 32141, p3 = 63629; // good for shuffling >60,000 indexes
+
+        shuffleId += 131 * (inx / listSize); // have inx overflow effect the mix
+        si = (inx + shuffleId) % listSize; // cut the deck
+
+        randR = shuffleId; //local randomizer
+        r1 = randR % p3;
+        r2 = randR % p1; // Now, per Chinese remainder theorem, (r1,r2,r3) will be a unique set
+        r3 = randR % p2;
+        r4 = randR % 2749;
+        halfN = listSize / 2 + 1;
+        rx = (randR / listSize) % listSize + 1;
+        rkey = (randR / listSize / listSize) % listSize + 1;
+
+        // perform the conditional multi-faceted mathematical mixing (on avg 2 5/6 shuffle ops done + 2 simple Xors)
+        if (si % 3 == 0) si = ((si / 3 * p1 + r1) % ((listSize + 2) / 3)) * 3; // spin multiples of 3 
+        if (si <= halfN)
+        {
+            si = (si + r3) % (halfN + 1);
+            si = halfN - si;
+        } // improves large permu distro
+
+        if (si % 2 == 0) si = ((si / 2 * p2 + r2) % ((listSize + 1) / 2)) * 2; // spin multiples of 2 
+        if (si < halfN) si = (si * p3 + r3) % halfN;
+
+        if ((si ^ rx) < listSize) si ^= rx; // flip some bits with Xor
+        si = (si * p3 + r4) % listSize; // a relatively prime gears churning operation
+        if ((si ^ rkey) < listSize) si ^= rkey;
+
+        return si;
+    }
+}

ErsatzTV.Infrastructure.MySql/Migrations/20240124212712_Add_PlayoutSeed.cs

@@ -0,0 +1,39 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ErsatzTV.Infrastructure.MySql.Migrations
+{
+    /// <inheritdoc />
+    public partial class Add_PlayoutSeed : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.RenameColumn(
+                name: "Seed",
+                table: "PlayoutHistory",
+                newName: "Index");
+
+            migrationBuilder.AddColumn<int>(
+                name: "Seed",
+                table: "Playout",
+                type: "int",
+                nullable: false,
+                defaultValue: 0);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "Seed",
+                table: "Playout");
+
+            migrationBuilder.RenameColumn(
+                name: "Index",
+                table: "PlayoutHistory",
+                newName: "Seed");
+        }
+    }
+}

ErsatzTV.Infrastructure.MySql/Migrations/TvContextModelSnapshot.cs

@@ -1422,6 +1422,9 @@ namespace ErsatzTV.Infrastructure.MySql.Migrations
                     b.Property<int>("ProgramSchedulePlayoutType")
                         .HasColumnType("int");
 
+                    b.Property<int>("Seed")
+                        .HasColumnType("int");
+
                     b.HasKey("Id");
 
                     b.HasIndex("ChannelId");
@@ -1934,6 +1937,9 @@ namespace ErsatzTV.Infrastructure.MySql.Migrations
                     b.Property<string>("Details")
                         .HasColumnType("longtext");
 
+                    b.Property<int>("Index")
+                        .HasColumnType("int");
+
                     b.Property<string>("Key")
                         .HasColumnType("longtext");
 
@@ -1943,9 +1949,6 @@ namespace ErsatzTV.Infrastructure.MySql.Migrations
                     b.Property<int>("PlayoutId")
                         .HasColumnType("int");
 
-                    b.Property<int>("Seed")
-                        .HasColumnType("int");
-
                     b.Property<DateTime>("When")
                         .HasColumnType("datetime(6)");
 

ErsatzTV.Infrastructure.Sqlite/Migrations/20240124204743_Add_PlayoutSeed.cs

@@ -0,0 +1,39 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ErsatzTV.Infrastructure.Sqlite.Migrations
+{
+    /// <inheritdoc />
+    public partial class Add_PlayoutSeed : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.RenameColumn(
+                name: "Seed",
+                table: "PlayoutHistory",
+                newName: "Index");
+
+            migrationBuilder.AddColumn<int>(
+                name: "Seed",
+                table: "Playout",
+                type: "INTEGER",
+                nullable: false,
+                defaultValue: 0);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "Seed",
+                table: "Playout");
+
+            migrationBuilder.RenameColumn(
+                name: "Index",
+                table: "PlayoutHistory",
+                newName: "Seed");
+        }
+    }
+}

ErsatzTV.Infrastructure.Sqlite/Migrations/TvContextModelSnapshot.cs

@@ -1420,6 +1420,9 @@ namespace ErsatzTV.Infrastructure.Sqlite.Migrations
                     b.Property<int>("ProgramSchedulePlayoutType")
                         .HasColumnType("INTEGER");
 
+                    b.Property<int>("Seed")
+                        .HasColumnType("INTEGER");
+
                     b.HasKey("Id");
 
                     b.HasIndex("ChannelId");
@@ -1932,6 +1935,9 @@ namespace ErsatzTV.Infrastructure.Sqlite.Migrations
                     b.Property<string>("Details")
                         .HasColumnType("TEXT");
 
+                    b.Property<int>("Index")
+                        .HasColumnType("INTEGER");
+
                     b.Property<string>("Key")
                         .HasColumnType("TEXT");
 
@@ -1941,9 +1947,6 @@ namespace ErsatzTV.Infrastructure.Sqlite.Migrations
                     b.Property<int>("PlayoutId")
                         .HasColumnType("INTEGER");
 
-                    b.Property<int>("Seed")
-                        .HasColumnType("INTEGER");
-
                     b.Property<DateTime>("When")
                         .HasColumnType("TEXT");