rework fallback filler (#2719)

* fallback fixes

* use hardware encoding for fallback filler

* rework fallback filler

* fixes

CHANGELOG.md

@@ -54,10 +54,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix playback of certain BT.2020 content
 - Use playlist item count when using a playlist as filler (instead of a fixed count of 1 for each playlist item)
 - NVIDIA: fix stream failure with certain content that should decode in hardware but falls back to software
+- Fix stream failure when configured fallback filler collection is empty
+- Fix high CPU when errors are displayed; errors will now work ahead before throttling to realtime, similar to primary content
 
 ### Changed
 - No longer round framerate to nearest integer when normalizing framerate
 - Allow playlists to have no items included in EPG
+- Change how fallback filler works
+  - Items will no longer loop; instead, a sequence of random items will be selected from the collection
+  - Items may still be cut as needed
+  - Hardware acceleration will now be used
+  - Items can "work ahead" (transcode faster than realtime) when less than 3 minutes in duration
 
 ## [25.9.0] - 2025-11-29
 ### Added

ErsatzTV.Application/Streaming/HlsSessionWorker.cs

@@ -488,6 +488,11 @@ public class HlsSessionWorker : IHlsSessionWorker
 
             foreach (PlayoutItemProcessModel processModel in result.RightAsEnumerable())
             {
+                if (!realtime && !processModel.IsWorkingAhead)
+                {
+                    _logger.LogDebug("HLS session throttling (NOT working ahead) based on playout item");
+                }
+
                 await TrimAndDelete(cancellationToken);
 
                 // increment discontinuity sequence and store with segment key (generated at)

ErsatzTV.Application/Streaming/PlayoutItemProcessModel.cs

@@ -13,7 +13,8 @@ public class PlayoutItemProcessModel
         bool isComplete,
         Option<long> segmentKey,
         Option<int> mediaItemId,
-        Option<TimeSpan> playoutOffset)
+        Option<TimeSpan> playoutOffset,
+        bool isWorkingAhead)
     {
         Process = process;
         GraphicsEngineContext = graphicsEngineContext;
@@ -22,6 +23,7 @@ public class PlayoutItemProcessModel
         IsComplete = isComplete;
         SegmentKey = segmentKey;
         MediaItemId = mediaItemId;
+        IsWorkingAhead = isWorkingAhead;
 
         // undo the offset applied in FFmpegProcessHandler
         // so we don't continually walk backward/forward in time by the offset amount
@@ -49,4 +51,6 @@ public class PlayoutItemProcessModel
     public Option<long> SegmentKey { get; init; }
 
     public Option<int> MediaItemId { get; init; }
+
+    public bool IsWorkingAhead { get; init; }
 }

ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumberHandler.cs

@@ -46,6 +46,7 @@ public class GetConcatProcessByChannelNumberHandler : FFmpegProcessHandler<GetCo
             true,
             Option<long>.None,
             Option<int>.None,
-            Option<TimeSpan>.None);
+            Option<TimeSpan>.None,
+            false);
     }
 }

ErsatzTV.Application/Streaming/Queries/GetErrorProcessHandler.cs

@@ -42,6 +42,7 @@ public class GetErrorProcessHandler(
             true,
             request.Now.ToUnixTimeSeconds(),
             Option<int>.None,
-            Optional(channel.PlayoutOffset));
+            Optional(channel.PlayoutOffset),
+            !request.HlsRealtime);
     }
 }

ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs

@@ -27,6 +27,8 @@ namespace ErsatzTV.Application.Streaming;
 
 public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<GetPlayoutItemProcessByChannelNumber>
 {
+    private static readonly Random FallbackRandom = new();
+
     private readonly IArtistRepository _artistRepository;
     private readonly IEmbyPathReplacementService _embyPathReplacementService;
     private readonly IExternalJsonPlayoutItemProvider _externalJsonPlayoutItemProvider;
@@ -284,9 +286,20 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
 
             bool isComplete = true;
 
+            bool effectiveRealtime = request.HlsRealtime;
+
+            // only work ahead on fallback filler up to 3 minutes in duration
+            // since we always transcode a full fallback filler item
+            if (!effectiveRealtime &&
+                playoutItemWithPath.PlayoutItem.FillerKind is FillerKind.Fallback &&
+                duration > TimeSpan.FromMinutes(3))
+            {
+                effectiveRealtime = true;
+            }
+
             TimeSpan limit = TimeSpan.Zero;
 
-            if (!request.HlsRealtime)
+            if (!effectiveRealtime)
             {
                 // if we are working ahead, limit to 44s (multiple of segment size)
                 limit = TimeSpan.FromSeconds(44);
@@ -319,7 +332,7 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                     now,
                     duration,
                     $"DEBUG_NO_SYNC:\n{Mapper.GetDisplayTitle(playoutItemWithPath.PlayoutItem.MediaItem, Option<string>.None)}\nFrom: {start} To: {finish}",
-                    request.HlsRealtime,
+                    effectiveRealtime,
                     request.PtsOffset,
                     channel.FFmpegProfile.VaapiDisplay,
                     channel.FFmpegProfile.VaapiDriver,
@@ -334,7 +347,8 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                     true,
                     effectiveNow.ToUnixTimeSeconds(),
                     Option<int>.None,
-                    Optional(channel.PlayoutOffset));
+                    Optional(channel.PlayoutOffset),
+                    !effectiveRealtime);
             }
 
             MediaVersion version = playoutItemWithPath.PlayoutItem.MediaItem.GetHeadVersion();
@@ -443,7 +457,7 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                 channel.FFmpegProfile.VaapiDriver,
                 channel.FFmpegProfile.VaapiDevice,
                 Optional(channel.FFmpegProfile.QsvExtraHardwareFrames),
-                hlsRealtime: request.HlsRealtime,
+                effectiveRealtime,
                 playoutItemWithPath.PlayoutItem.MediaItem is RemoteStream { IsLive: true }
                     ? StreamInputKind.Live
                     : StreamInputKind.Vod,
@@ -465,7 +479,8 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                 isComplete,
                 effectiveNow.ToUnixTimeSeconds(),
                 playoutItemResult.MediaItemId,
-                Optional(channel.PlayoutOffset));
+                Optional(channel.PlayoutOffset),
+                !effectiveRealtime);
 
             return Right<BaseError, PlayoutItemProcessModel>(result);
         }
@@ -482,6 +497,13 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
 
             Option<TimeSpan> maybeDuration = maybeNextStart.Map(s => s - now);
 
+            // limit working ahead on errors to 1 minute
+            if (!request.HlsRealtime && maybeDuration.IfNone(TimeSpan.FromMinutes(2)) > TimeSpan.FromMinutes(1))
+            {
+                maybeNextStart = now.AddMinutes(1);
+                maybeDuration = TimeSpan.FromMinutes(1);
+            }
+
             DateTimeOffset finish = maybeNextStart.Match(s => s, () => now);
 
             if (request.IsTroubleshooting)
@@ -522,7 +544,8 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                         true,
                         now.ToUnixTimeSeconds(),
                         Option<int>.None,
-                        Optional(channel.PlayoutOffset));
+                        Optional(channel.PlayoutOffset),
+                        !request.HlsRealtime);
                 case PlayoutItemDoesNotExistOnDisk:
                     Command doesNotExistProcess = await _ffmpegProcessService.ForError(
                         ffmpegPath,
@@ -545,7 +568,8 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                         true,
                         now.ToUnixTimeSeconds(),
                         Option<int>.None,
-                        Optional(channel.PlayoutOffset));
+                        Optional(channel.PlayoutOffset),
+                        !request.HlsRealtime);
                 default:
                     Command errorProcess = await _ffmpegProcessService.ForError(
                         ffmpegPath,
@@ -568,7 +592,8 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                         true,
                         now.ToUnixTimeSeconds(),
                         Option<int>.None,
-                        Optional(channel.PlayoutOffset));
+                        Optional(channel.PlayoutOffset),
+                        !request.HlsRealtime);
             }
         }
 
@@ -711,8 +736,14 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                 collectionKey,
                 cancellationToken);
 
-            // TODO: shuffle? does it really matter since we loop anyway
-            MediaItem item = items[new Random().Next(items.Count)];
+            // ignore the fallback filler preset if it has no items
+            if (items.Count == 0)
+            {
+                break;
+            }
+
+            // get a random item
+            MediaItem item = items[FallbackRandom.Next(items.Count)];
 
             Option<TimeSpan> maybeDuration = await dbContext.PlayoutItems
                 .Filter(pi => pi.Playout.ChannelId == (channel.MirrorSourceChannelId ?? channel.Id))
@@ -734,15 +765,14 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                 .Filter(ms => ms.MediaVersionId == version.Id)
                 .ToListAsync(cancellationToken);
 
-            DateTimeOffset finish = maybeDuration.Match(
-                // next playout item exists
-                // loop until it starts
-                now.Add,
-                // no next playout item exists
-                // loop for 5 minutes if less than 30s, otherwise play full item
-                () => version.Duration < TimeSpan.FromSeconds(30)
-                    ? now.AddMinutes(5)
-                    : now.Add(version.Duration));
+            // always play min(duration to next item, version.Duration)
+            TimeSpan duration = maybeDuration.IfNone(version.Duration);
+            if (version.Duration < duration)
+            {
+                duration = version.Duration;
+            }
+
+            DateTimeOffset finish = now.Add(duration);
 
             var playoutItem = new PlayoutItem
             {
@@ -752,7 +782,7 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                 Finish = finish.UtcDateTime,
                 FillerKind = FillerKind.Fallback,
                 InPoint = TimeSpan.Zero,
-                OutPoint = version.Duration,
+                OutPoint = duration,
                 DisableWatermarks = !fallbackPreset.AllowWatermarks,
                 Watermarks = [],
                 PlayoutItemWatermarks = [],

ErsatzTV.Application/Streaming/Queries/GetWrappedProcessByChannelNumberHandler.cs

@@ -48,6 +48,7 @@ public class GetWrappedProcessByChannelNumberHandler : FFmpegProcessHandler<GetW
             true,
             Option<long>.None,
             Option<int>.None,
-            Option<TimeSpan>.None);
+            Option<TimeSpan>.None,
+            false);
     }
 }

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -404,7 +404,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             graphicsElementContexts.AddRange(watermarks.Map(wm => new WatermarkElementContext(wm)));
         }
 
-        HardwareAccelerationMode hwAccel = GetHardwareAccelerationMode(playbackSettings, fillerKind);
+        HardwareAccelerationMode hwAccel = GetHardwareAccelerationMode(playbackSettings);
 
         string videoFormat = GetVideoFormat(playbackSettings);
         Option<string> maybeVideoProfile = GetVideoProfile(videoFormat, channel.FFmpegProfile.VideoProfile);
@@ -499,7 +499,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
         var desiredState = new FrameState(
             playbackSettings.RealtimeOutput,
-            fillerKind == FillerKind.Fallback,
+            InfiniteLoop: false,
             videoFormat,
             maybeVideoProfile,
             maybeVideoPreset,
@@ -700,7 +700,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
         var desiredState = new FrameState(
             playbackSettings.RealtimeOutput,
-            false,
+            InfiniteLoop: false,
             videoFormat,
             GetVideoProfile(videoFormat, channel.FFmpegProfile.VideoProfile),
             VideoPreset.Unset,
@@ -789,7 +789,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         var videoInputFile = new VideoInputFile(videoPath, new List<VideoStream> { ffmpegVideoStream });
 
         // TODO: ignore accel if this already failed once
-        HardwareAccelerationMode hwAccel = GetHardwareAccelerationMode(playbackSettings, FillerKind.None);
+        HardwareAccelerationMode hwAccel = GetHardwareAccelerationMode(playbackSettings);
         _logger.LogDebug("HW accel mode: {HwAccel}", hwAccel);
 
         var ffmpegState = new FFmpegState(
@@ -1187,12 +1187,10 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             .ForAccelAndFormat(hardwareAccelerationMode, videoFormat, bitDepth)
             .Find(p => string.Equals(p, videoPreset, StringComparison.OrdinalIgnoreCase));
 
-    private static HardwareAccelerationMode GetHardwareAccelerationMode(
-        FFmpegPlaybackSettings playbackSettings,
-        FillerKind fillerKind) =>
+    private static HardwareAccelerationMode GetHardwareAccelerationMode(FFmpegPlaybackSettings playbackSettings) =>
         playbackSettings.HardwareAcceleration switch
         {
-            _ when fillerKind == FillerKind.Fallback => HardwareAccelerationMode.None,
+            //_ when fillerKind == FillerKind.Fallback => HardwareAccelerationMode.None,
             HardwareAccelerationKind.Nvenc => HardwareAccelerationMode.Nvenc,
             HardwareAccelerationKind.Qsv => HardwareAccelerationMode.Qsv,
             HardwareAccelerationKind.Vaapi => HardwareAccelerationMode.Vaapi,