feat: support media server items in next streaming engine (#2865)

* fix next playback of items with no audio

* support media server items through next engine

ErsatzTV.Application/Playouts/Commands/SyncNextPlayoutHandler.cs

@@ -6,18 +6,24 @@ using CliWrap;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Extensions;
+using ErsatzTV.Core.Interfaces.Emby;
+using ErsatzTV.Core.Interfaces.Jellyfin;
 using ErsatzTV.Core.Interfaces.Metadata;
-using ErsatzTV.Core.Next;
+using ErsatzTV.Core.Interfaces.Plex;
 using ErsatzTV.Infrastructure.Data;
 using ErsatzTV.Infrastructure.Extensions;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
+using PlayoutItem = ErsatzTV.Core.Domain.PlayoutItem;
 
 namespace ErsatzTV.Application.Playouts;
 
 public partial class SyncNextPlayoutHandler(
     IFileSystem fileSystem,
     ILocalFileSystem localFileSystem,
+    IPlexPathReplacementService plexPathReplacementService,
+    IJellyfinPathReplacementService jellyfinPathReplacementService,
+    IEmbyPathReplacementService embyPathReplacementService,
     IDbContextFactory<TvContext> dbContextFactory,
     ILogger<SyncNextPlayoutHandler> logger)
     : IRequestHandler<SyncNextPlayout>
@@ -112,27 +118,37 @@ public partial class SyncNextPlayoutHandler(
             playoutOffset = channel.PlayoutOffset ?? TimeSpan.Zero;
         }
 
-        List<int> localLibraryIds = await dbContext.LocalLibraries
-            .AsNoTracking()
-            .Map(l => l.Id)
-            .ToListAsync(cancellationToken);
-
         List<PlayoutItem> playoutItems = await dbContext.PlayoutItems
             .AsNoTracking()
             .Where(i => i.Playout.Channel.Number == (mirrorChannelNumber ?? channelNumber))
-            .Where(i => localLibraryIds.Contains(i.MediaItem.LibraryPath.LibraryId))
+            .Include(i => i.MediaItem)
+            .ThenInclude(mi => mi.LibraryPath)
+            .ThenInclude(lp => lp.Library)
             .Include(i => i.MediaItem)
             .ThenInclude(i => (i as Episode).MediaVersions)
             .ThenInclude(mv => mv.MediaFiles)
             .Include(i => i.MediaItem)
+            .ThenInclude(i => (i as Episode).MediaVersions)
+            .ThenInclude(mv => mv.Streams)
+            .Include(i => i.MediaItem)
             .ThenInclude(i => (i as Movie).MediaVersions)
             .ThenInclude(mv => mv.MediaFiles)
             .Include(i => i.MediaItem)
+            .ThenInclude(i => (i as Movie).MediaVersions)
+            .ThenInclude(mv => mv.Streams)
+            .Include(i => i.MediaItem)
             .ThenInclude(i => (i as OtherVideo).MediaVersions)
             .ThenInclude(mv => mv.MediaFiles)
             .Include(i => i.MediaItem)
+            .ThenInclude(i => (i as OtherVideo).MediaVersions)
+            .ThenInclude(mv => mv.Streams)
+            .Include(i => i.MediaItem)
             .ThenInclude(i => (i as MusicVideo).MediaVersions)
             .ThenInclude(mv => mv.MediaFiles)
+            .Include(i => i.MediaItem)
+            .ThenInclude(i => (i as MusicVideo).MediaVersions)
+            .ThenInclude(mv => mv.Streams)
+            .AsSplitQuery()
             .ToListAsync(cancellationToken);
 
         logger.LogDebug("Located {Count} local playout items", playoutItems.Count);
@@ -156,31 +172,132 @@ public partial class SyncNextPlayoutHandler(
                     continue;
                 }
 
-                string path = playoutItem.MediaItem.GetHeadVersion().MediaFiles.Head().Path;
+                MediaVersion headVersion = playoutItem.MediaItem.GetHeadVersion();
 
                 playoutItem.Start += playoutOffset;
                 playoutItem.Finish += playoutOffset;
 
-                var nextPlayoutItem = new ItemElement
+                var nextPlayoutItem = new Core.Next.PlayoutItem
                 {
                     Id = playoutItem.Id.ToString(CultureInfo.InvariantCulture),
-                    Start = playoutItem.StartOffset.ToString("O"),
+                    Start = playoutItem.StartOffset,
-                    Finish = playoutItem.FinishOffset.ToString("O"),
+                    Finish = playoutItem.FinishOffset
-                    Source = new ItemSource
-                    {
-                        SourceType = SourceType.Local,
-                        Path = path,
-                    }
                 };
 
+                Option<Core.Next.Source> maybeSource = await SourceForItem(playoutItem, cancellationToken);
+                if (maybeSource.IsNone)
+                {
+                    continue;
+                }
+
+                foreach (Core.Next.Source source in maybeSource)
+                {
+                    nextPlayoutItem.Source = source;
+                }
+
+                // if no audio streams, use lavfi to insert silence
+                if (headVersion.Streams.All(s => s.MediaStreamKind is not MediaStreamKind.Audio))
+                {
+                    var videoSource = nextPlayoutItem.Source;
+
+                    nextPlayoutItem.Source = null;
+                    nextPlayoutItem.Tracks = new Core.Next.PlayoutItemTracks
+                    {
+                        Audio = new Core.Next.TrackSelection
+                        {
+                            Source =
+                                new Core.Next.Source
+                                {
+                                    SourceType = Core.Next.SourceType.Lavfi,
+                                    Params = "anullsrc=channel_layout=stereo:sample_rate=48000"
+                                }
+                        },
+                        Video = new Core.Next.TrackSelection
+                        {
+                            Source = new Core.Next.Source
+                            {
+                                SourceType = videoSource.SourceType,
+                                Path = videoSource.Path,
+                            }
+                        }
+                    };
+                }
+
                 playout.Items.Add(nextPlayoutItem);
             }
 
-            await fileSystem.File.WriteAllTextAsync(fileName, playout.ToJson(), cancellationToken);
+            await fileSystem.File.WriteAllTextAsync(fileName, Core.Next.Serialize.ToJson(playout), cancellationToken);
+        }
+    }
+
+    private async Task<Option<Core.Next.Source>> SourceForItem(
+        PlayoutItem playoutItem,
+        CancellationToken cancellationToken)
+    {
+        string path = await playoutItem.MediaItem.GetLocalPath(
+            plexPathReplacementService,
+            jellyfinPathReplacementService,
+            embyPathReplacementService,
+            cancellationToken);
+
+        // check filesystem first
+        if (fileSystem.File.Exists(path))
+        {
+            return new Core.Next.Source
+            {
+                SourceType = Core.Next.SourceType.Local,
+                Path = path,
+            };
+        }
+
+        MediaFile file = playoutItem.MediaItem.GetHeadVersion().MediaFiles.Head();
+        int mediaSourceId = playoutItem.MediaItem.LibraryPath.Library.MediaSourceId;
+        if (file is PlexMediaFile pmf)
+        {
+            return new Core.Next.Source
+            {
+                SourceType = Core.Next.SourceType.Http,
+                Uri = $"http://localhost:{Settings.StreamingPort}/media/plex/{mediaSourceId}/{pmf.Key}"
+            };
+        }
+
+        Option<string> jellyfinItemId = playoutItem.MediaItem switch
+        {
+            JellyfinEpisode e => e.ItemId,
+            JellyfinMovie m => m.ItemId,
+            _ => None
+        };
+
+        foreach (string itemId in jellyfinItemId)
+        {
+            return new Core.Next.Source
+            {
+                SourceType = Core.Next.SourceType.Http,
+                Uri = $"http://localhost:{Settings.StreamingPort}/media/jellyfin/{itemId}"
+            };
+        }
+
+        // attempt to remotely stream emby
+        Option<string> embyItemId = playoutItem.MediaItem switch
+        {
+            EmbyEpisode e => e.ItemId,
+            EmbyMovie m => m.ItemId,
+            _ => None
+        };
+
+        foreach (string itemId in embyItemId)
+        {
+            return new Core.Next.Source
+            {
+                SourceType = Core.Next.SourceType.Http,
+                Uri = $"http://localhost:{Settings.StreamingPort}/media/emby/{itemId}"
+            };
         }
+
+        return Option<Core.Next.Source>.None;
     }
 
-    public void CleanOldVersions(
+    private void CleanOldVersions(
         string playoutRoot,
         string currentLinkPath,
         int keepVersions = 2,

ErsatzTV.Core/Next/Playout.cs

@@ -18,171 +18,189 @@ namespace ErsatzTV.Core.Next
     /// <summary>
     /// A playout schedule for a single time window.
     ///
-    /// Files should be named `{start}_{finish}.json` using compact ISO 8601
+    /// Files should be named `{start}_{finish}.json` using compact ISO 8601 (no separators),
-    /// (no separators), e.g.
+    /// e.g. `20260413T000000.000000000-0500_20260414T002131.620000000-0500.json`, so that the
-    /// `20260413T000000.000000000-0500_20260414T002131.620000000-0500.json`,
+    /// channel can locate the correct file for the current time.
-    /// so that the channel can locate the correct file for the current time.
     /// </summary>
     public partial class Playout
     {
+        /// <summary>
+        /// Ordered list of scheduled items for this window.
+        /// </summary>
         [JsonProperty("items")]
-        public List<ItemElement> Items { get; set; }
+        public List<PlayoutItem> Items { get; set; }
 
         /// <summary>
-        /// URI identifying the schema version, e.g. "https://ersatztv.org/playout/version/0.0.1"
+        /// URI identifying the schema version, e.g. "https://ersatztv.org/playout/version/0.0.1".
         /// </summary>
         [JsonProperty("version")]
         public string Version { get; set; }
     }
 
-    public partial class ItemElement
+    /// <summary>
+    /// A single scheduled item in the playout.
+    ///
+    /// An item must supply media for its tracks. You can do this two ways, and they can be
+    /// combined:
+    ///
+    /// 1. Set `source` to use one shared source for every track. Track details (which stream,
+    /// etc.) are chosen naively, or may be refined via `tracks`.
+    /// 2. Set `tracks` to specify each track (video, audio) individually. A track may provide
+    /// its own `source`; if it doesn't, the item's top-level `source` is used.
+    ///
+    /// At least one of `source` or `tracks` must be present, and every track that is selected
+    /// must have an effective source (either its own or the item's).
+    /// </summary>
+    public partial class PlayoutItem
     {
         /// <summary>
-        /// RFC3339 formatted date/time, e.g. 2026-04-13T00:24:21.527-05:00
+        /// RFC3339 formatted finish time, e.g. 2026-04-13T00:54:21.527-05:00.
         /// </summary>
         [JsonProperty("finish")]
-        public string Finish { get; set; }
+        public DateTimeOffset Finish { get; set; }
 
+        /// <summary>
+        /// Stable identifier for this item, unique within the playout.
+        /// </summary>
         [JsonProperty("id")]
         public string Id { get; set; }
 
+        /// <summary>
+        /// The default source shared by any track that doesn't specify its own. Required unless
+        /// every selected track in `tracks` provides its own `source`.
+        /// </summary>
         [JsonProperty("source")]
-        public ItemSource Source { get; set; }
+        public Source Source { get; set; }
 
         /// <summary>
-        /// RFC3339 formatted date/time, e.g. 2026-04-13T00:24:21.527-05:00
+        /// RFC3339 formatted start time, e.g. 2026-04-13T00:24:21.527-05:00.
         /// </summary>
         [JsonProperty("start")]
-        public string Start { get; set; }
+        public DateTimeOffset Start { get; set; }
 
+        /// <summary>
+        /// Per-track selection. Omit to let the server pick the first video and first audio track of
+        /// the item's `source`.
+        /// </summary>
         [JsonProperty("tracks")]
-        public TracksClass Tracks { get; set; }
+        public PlayoutItemTracks Tracks { get; set; }
     }
 
-    public partial class ItemSource
+    /// <summary>
+    /// A file on the local filesystem reachable by the server.
+    ///
+    /// A synthetic source produced by an ffmpeg lavfi filter graph.
+    ///
+    /// A remote source fetched over HTTP(S).
+    /// </summary>
+    public partial class Source
     {
+        /// <summary>
+        /// Optional start offset into the source, in milliseconds.
+        /// </summary>
         [JsonProperty("in_point_ms")]
         public long? InPointMs { get; set; }
 
+        /// <summary>
+        /// Optional end offset into the source, in milliseconds.
+        /// </summary>
         [JsonProperty("out_point_ms")]
         public long? OutPointMs { get; set; }
 
+        /// <summary>
+        /// Absolute path to the media file.
+        /// </summary>
         [JsonProperty("path", NullValueHandling = NullValueHandling.Ignore)]
         public string Path { get; set; }
 
         [JsonProperty("source_type")]
         public SourceType SourceType { get; set; }
 
+        /// <summary>
+        /// The lavfi filter graph parameters, passed verbatim to ffmpeg's `-f lavfi -i`.
+        /// </summary>
         [JsonProperty("params", NullValueHandling = NullValueHandling.Ignore)]
         public string Params { get; set; }
 
         /// <summary>
-        /// Custom HTTP headers, e.g. ["Authorization: Bearer {{TOKEN}}"]
+        /// Custom HTTP headers, e.g. ["Authorization: Bearer {{TOKEN}}"].
         /// </summary>
         [JsonProperty("headers")]
         public List<string> Headers { get; set; }
 
         /// <summary>
-        /// Enable reconnect on failure (default: true)
+        /// Enable reconnect on failure. Default: true.
         /// </summary>
         [JsonProperty("reconnect")]
         public bool? Reconnect { get; set; }
 
         /// <summary>
-        /// Max reconnect delay in seconds
+        /// Maximum reconnect delay in seconds. Maps to ffmpeg's `reconnect_delay_max`.
         /// </summary>
         [JsonProperty("reconnect_delay_max")]
         public long? ReconnectDelayMax { get; set; }
 
         /// <summary>
-        /// Socket timeout in microseconds
+        /// Socket timeout in microseconds.
         /// </summary>
         [JsonProperty("timeout_us")]
         public long? TimeoutUs { get; set; }
 
         /// <summary>
-        /// URI template, e.g. "https://example.com/file.mkv?token={{MY_SECRET}}"
+        /// URI template, e.g. "https://example.com/file.mkv?token={{MY_SECRET}}".
         /// </summary>
         [JsonProperty("uri", NullValueHandling = NullValueHandling.Ignore)]
         public string Uri { get; set; }
 
         /// <summary>
-        /// Custom user-agent string
+        /// Custom User-Agent string.
         /// </summary>
         [JsonProperty("user_agent")]
         public string UserAgent { get; set; }
     }
 
-    public partial class TracksClass
+    /// <summary>
-    {
+    /// Per-track overrides for a playout item. Omit a field to use the server default for that
-        [JsonProperty("audio")]
+    /// track kind (first stream of that kind in the item's `source`, if any).
-        public AudioClass Audio { get; set; }
+    /// </summary>
-
+    public partial class PlayoutItemTracks
-        [JsonProperty("video")]
-        public AudioClass Video { get; set; }
-    }
-
-    public partial class AudioClass
-    {
-        [JsonProperty("source", NullValueHandling = NullValueHandling.Ignore)]
-        public AudioSource Source { get; set; }
-
-        [JsonProperty("stream_index", NullValueHandling = NullValueHandling.Ignore)]
-        public long? StreamIndex { get; set; }
-    }
-
-    public partial class AudioSource
     {
-        [JsonProperty("in_point_ms")]
-        public long? InPointMs { get; set; }
-
-        [JsonProperty("out_point_ms")]
-        public long? OutPointMs { get; set; }
-
-        [JsonProperty("path", NullValueHandling = NullValueHandling.Ignore)]
-        public string Path { get; set; }
-
-        [JsonProperty("source_type")]
-        public SourceType SourceType { get; set; }
-
-        [JsonProperty("params", NullValueHandling = NullValueHandling.Ignore)]
-        public string Params { get; set; }
-
-        /// <summary>
-        /// Custom HTTP headers, e.g. ["Authorization: Bearer {{TOKEN}}"]
-        /// </summary>
-        [JsonProperty("headers")]
-        public List<string> Headers { get; set; }
-
-        /// <summary>
-        /// Enable reconnect on failure (default: true)
-        /// </summary>
-        [JsonProperty("reconnect")]
-        public bool? Reconnect { get; set; }
-
         /// <summary>
-        /// Max reconnect delay in seconds
+        /// Audio track selection.
         /// </summary>
-        [JsonProperty("reconnect_delay_max")]
+        [JsonProperty("audio")]
-        public long? ReconnectDelayMax { get; set; }
+        public TrackSelection Audio { get; set; }
 
         /// <summary>
-        /// Socket timeout in microseconds
+        /// Video track selection.
         /// </summary>
-        [JsonProperty("timeout_us")]
+        [JsonProperty("video")]
-        public long? TimeoutUs { get; set; }
+        public TrackSelection Video { get; set; }
+    }
 
+    /// <summary>
+    /// Selects a single track (video or audio).
+    ///
+    /// - `source` absent: inherit the parent `PlayoutItem.source`.
+    /// - `source` present: use this source for this track, overriding the parent.
+    /// - `stream_index` absent: server picks the first stream of the track's kind in the
+    /// effective source.
+    /// - `stream_index` present: use this specific stream within the effective source.
+    /// </summary>
+    public partial class TrackSelection
+    {
         /// <summary>
-        /// URI template, e.g. "https://example.com/file.mkv?token={{MY_SECRET}}"
+        /// Source to pull this track from. If omitted, inherits from the parent `PlayoutItem.source`.
         /// </summary>
-        [JsonProperty("uri", NullValueHandling = NullValueHandling.Ignore)]
+        [JsonProperty("source")]
-        public string Uri { get; set; }
+        public Source Source { get; set; }
 
         /// <summary>
-        /// Custom user-agent string
+        /// Zero-based stream index within the effective source. If omitted, the server picks the
+        /// first stream of this track's kind.
         /// </summary>
-        [JsonProperty("user_agent")]
+        [JsonProperty("stream_index")]
-        public string UserAgent { get; set; }
+        public long? StreamIndex { get; set; }
     }
 
     public enum SourceType { Http, Lavfi, Local };