fix error continuity (#787)

* fix fallback filler playback

* use new transcoder logic for errors

* use realtime option for error streams

CHANGELOG.md

@@ -11,6 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix search index validation on startup; improper validation was causing a rebuild with every startup
 - Block library scanning until search index has been recreated/upgraded
 - Fix occasional erroneous log messages when HLS channel playback times out because all clients have left
+- Fix fallback filler playback
+- Fix stream continuity when error messages are displayed
 
 ### Added
 - Add `show_genre` and `show_tag` to search index for seasons and episodes

ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs

@@ -254,16 +254,16 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
         List<Subtitle> allSubtitles = playoutItemWithPath.PlayoutItem.MediaItem switch
         {
             Episode episode => Optional(episode.EpisodeMetadata).Flatten().HeadOrNone()
-                .Map(mm => mm.Subtitles)
+                .Map(mm => mm.Subtitles ?? new List<Subtitle>())
                 .IfNone(new List<Subtitle>()),
             Movie movie => Optional(movie.MovieMetadata).Flatten().HeadOrNone()
-                .Map(mm => mm.Subtitles)
+                .Map(mm => mm.Subtitles ?? new List<Subtitle>())
                 .IfNone(new List<Subtitle>()),
             MusicVideo musicVideo => Optional(musicVideo.MusicVideoMetadata).Flatten().HeadOrNone()
-                .Map(mm => mm.Subtitles)
+                .Map(mm => mm.Subtitles ?? new List<Subtitle>())
                 .IfNone(new List<Subtitle>()),
             OtherVideo otherVideo => Optional(otherVideo.OtherVideoMetadata).Flatten().HeadOrNone()
-                .Map(mm => mm.Subtitles)
+                .Map(mm => mm.Subtitles ?? new List<Subtitle>())
                 .IfNone(new List<Subtitle>()),
             _ => new List<Subtitle>()
         };

ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs

@@ -301,6 +301,7 @@ public class TranscodingTests
             oldService,
             new FFmpegPlaybackSettingsCalculator(),
             new FakeStreamSelector(),
+            new Mock<ITempFilePool>().Object,
             LoggerFactory.CreateLogger<FFmpegLibraryProcessService>());
 
         var v = new MediaVersion

ErsatzTV.Core/Domain/MediaItem/BackgroundImageMediaVersion.cs

@@ -1,5 +1,31 @@
-namespace ErsatzTV.Core.Domain;
+using ErsatzTV.Core.Interfaces.FFmpeg;
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.Core.Domain;
 
 public class BackgroundImageMediaVersion : MediaVersion
 {
+    public static BackgroundImageMediaVersion ForPath(string path, IDisplaySize resolution) =>
+        new()
+        {
+            Chapters = new List<MediaChapter>(),
+            // image has been pre-generated with correct size
+            Height = resolution.Height,
+            Width = resolution.Width,
+            SampleAspectRatio = "1:1",
+            Streams = new List<MediaStream>
+            {
+                new()
+                {
+                    MediaStreamKind = MediaStreamKind.Video,
+                    Index = 0,
+                    Codec = VideoFormat.GeneratedImage,
+                    PixelFormat = new PixelFormatUnknown().Name // the resulting pixel format is unknown
+                }
+            },
+            MediaFiles = new List<MediaFile>
+            {
+                new() { Path = path }
+            }
+        };
 }

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -18,16 +18,19 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
     private readonly IFFmpegStreamSelector _ffmpegStreamSelector;
     private readonly ILogger<FFmpegLibraryProcessService> _logger;
     private readonly FFmpegPlaybackSettingsCalculator _playbackSettingsCalculator;
+    private readonly ITempFilePool _tempFilePool;
 
     public FFmpegLibraryProcessService(
         FFmpegProcessService ffmpegProcessService,
         FFmpegPlaybackSettingsCalculator playbackSettingsCalculator,
         IFFmpegStreamSelector ffmpegStreamSelector,
+        ITempFilePool tempFilePool,
         ILogger<FFmpegLibraryProcessService> logger)
     {
         _ffmpegProcessService = ffmpegProcessService;
         _playbackSettingsCalculator = playbackSettingsCalculator;
         _ffmpegStreamSelector = ffmpegStreamSelector;
+        _tempFilePool = tempFilePool;
         _logger = logger;
     }
 
@@ -174,14 +177,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
         Option<WatermarkInputFile> watermarkInputFile = GetWatermarkInputFile(watermarkOptions, maybeFadePoints);
 
-        string videoFormat = playbackSettings.VideoFormat switch
+        string videoFormat = GetVideoFormat(playbackSettings);
-        {
-            FFmpegProfileVideoFormat.Hevc => VideoFormat.Hevc,
-            FFmpegProfileVideoFormat.H264 => VideoFormat.H264,
-            FFmpegProfileVideoFormat.Mpeg2Video => VideoFormat.Mpeg2Video,
-            FFmpegProfileVideoFormat.Copy => VideoFormat.Copy,
-            _ => throw new ArgumentOutOfRangeException($"unexpected video format {playbackSettings.VideoFormat}")
-        };
 
         HardwareAccelerationMode hwAccel = playbackSettings.HardwareAcceleration switch
         {
@@ -254,14 +250,128 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         return GetCommand(ffmpegPath, videoInputFile, audioInputFile, watermarkInputFile, None, pipeline);
     }
 
-    public Task<Command> ForError(
+    public async Task<Command> ForError(
         string ffmpegPath,
         Channel channel,
         Option<TimeSpan> duration,
         string errorMessage,
         bool hlsRealtime,
-        long ptsOffset) =>
+        long ptsOffset)
-        _ffmpegProcessService.ForError(ffmpegPath, channel, duration, errorMessage, hlsRealtime, ptsOffset);
+    {
+        FFmpegPlaybackSettings playbackSettings = _playbackSettingsCalculator.CalculateErrorSettings(
+            channel.StreamingMode,
+            channel.FFmpegProfile,
+            hlsRealtime);
+
+        IDisplaySize desiredResolution = channel.FFmpegProfile.Resolution;
+
+        var fontSize = (int)Math.Round(channel.FFmpegProfile.Resolution.Height / 20.0);
+        var margin = (int)Math.Round(channel.FFmpegProfile.Resolution.Height * 0.05);
+
+        string subtitleFile = await new SubtitleBuilder(_tempFilePool)
+            .WithResolution(desiredResolution)
+            .WithFontName("Roboto")
+            .WithFontSize(fontSize)
+            .WithAlignment(2)
+            .WithMarginV(margin)
+            .WithPrimaryColor("&HFFFFFF")
+            .WithFormattedContent(errorMessage.Replace(Environment.NewLine, "\\N"))
+            .BuildFile();
+
+        string audioFormat = playbackSettings.AudioFormat switch
+        {
+            FFmpegProfileAudioFormat.Ac3 => AudioFormat.Ac3,
+            _ => AudioFormat.Aac
+        };
+
+        var audioState = new AudioState(
+            audioFormat,
+            playbackSettings.AudioChannels,
+            playbackSettings.AudioBitrate,
+            playbackSettings.AudioBufferSize,
+            playbackSettings.AudioSampleRate,
+            Option<TimeSpan>.None,
+            playbackSettings.NormalizeLoudness);
+
+        var desiredState = new FrameState(
+            playbackSettings.RealtimeOutput,
+            false,
+            GetVideoFormat(playbackSettings),
+            new PixelFormatYuv420P(),
+            new FrameSize(desiredResolution.Width, desiredResolution.Height),
+            new FrameSize(desiredResolution.Width, desiredResolution.Height),
+            playbackSettings.FrameRate,
+            playbackSettings.VideoBitrate,
+            playbackSettings.VideoBufferSize,
+            playbackSettings.VideoTrackTimeScale,
+            playbackSettings.Deinterlace);
+
+        OutputFormatKind outputFormat = channel.StreamingMode == StreamingMode.HttpLiveStreamingSegmenter
+            ? OutputFormatKind.Hls
+            : OutputFormatKind.MpegTs;
+
+        Option<string> hlsPlaylistPath = outputFormat == OutputFormatKind.Hls
+            ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live.m3u8")
+            : Option<string>.None;
+
+        Option<string> hlsSegmentTemplate = outputFormat == OutputFormatKind.Hls
+            ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live%06d.ts")
+            : Option<string>.None;
+
+        string videoPath = Path.Combine(FileSystemLayout.ResourcesCacheFolder, "background.png");
+
+        var videoVersion = BackgroundImageMediaVersion.ForPath(videoPath, desiredResolution);
+
+        var ffmpegVideoStream = new VideoStream(
+            0,
+            VideoFormat.GeneratedImage,
+            new PixelFormatYuv420P(),
+            new FrameSize(videoVersion.Width, videoVersion.Height),
+            None,
+            true);
+
+        var videoInputFile = new VideoInputFile(videoPath, new List<VideoStream> { ffmpegVideoStream });
+
+        var ffmpegState = new FFmpegState(
+            false,
+            HardwareAccelerationMode.None,
+            None,
+            None,
+            playbackSettings.StreamSeek,
+            duration,
+            channel.StreamingMode != StreamingMode.HttpLiveStreamingDirect,
+            "ErsatzTV",
+            channel.Name,
+            None,
+            outputFormat,
+            hlsPlaylistPath,
+            hlsSegmentTemplate,
+            ptsOffset);
+
+        var ffmpegSubtitleStream = new ErsatzTV.FFmpeg.MediaStream(0, "ass", StreamKind.Video);
+
+        var audioInputFile = new NullAudioInputFile(audioState);
+
+        var subtitleInputFile = new SubtitleInputFile(
+            subtitleFile,
+            new List<ErsatzTV.FFmpeg.MediaStream> { ffmpegSubtitleStream },
+            false);
+
+        _logger.LogDebug("FFmpeg desired error state {FrameState}", desiredState);
+
+        var pipelineBuilder = new PipelineBuilder(
+            videoInputFile,
+            audioInputFile,
+            None,
+            subtitleInputFile,
+            FileSystemLayout.FFmpegReportsFolder,
+            FileSystemLayout.FontsCacheFolder,
+            _logger);
+
+        FFmpegPipeline pipeline = pipelineBuilder.Build(ffmpegState, desiredState);
+
+        return GetCommand(ffmpegPath, videoInputFile, audioInputFile, None, None, pipeline);
+    }
 
     public Command ConcatChannel(string ffmpegPath, bool saveReports, Channel channel, string scheme, string host)
     {
@@ -478,4 +588,14 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
     private static Option<string> VaapiDeviceName(HardwareAccelerationMode accelerationMode, string vaapiDevice) =>
         accelerationMode == HardwareAccelerationMode.Vaapi ? vaapiDevice : Option<string>.None;
+
+    private static string GetVideoFormat(FFmpegPlaybackSettings playbackSettings) =>
+        playbackSettings.VideoFormat switch
+        {
+            FFmpegProfileVideoFormat.Hevc => VideoFormat.Hevc,
+            FFmpegProfileVideoFormat.H264 => VideoFormat.H264,
+            FFmpegProfileVideoFormat.Mpeg2Video => VideoFormat.Mpeg2Video,
+            FFmpegProfileVideoFormat.Copy => VideoFormat.Copy,
+            _ => throw new ArgumentOutOfRangeException($"unexpected video format {playbackSettings.VideoFormat}")
+        };
 }

ErsatzTV.Core/FFmpeg/FFmpegPlaybackSettingsCalculator.cs

@@ -170,14 +170,22 @@ public class FFmpegPlaybackSettingsCalculator
         return result;
     }
 
-    public FFmpegPlaybackSettings CalculateErrorSettings(FFmpegProfile ffmpegProfile) =>
+    public FFmpegPlaybackSettings CalculateErrorSettings(
+        StreamingMode streamingMode,
+        FFmpegProfile ffmpegProfile,
+        bool hlsRealtime) =>
         new()
         {
             HardwareAcceleration = HardwareAccelerationKind.None,
-            ThreadCount = ffmpegProfile.ThreadCount,
+            ThreadCount = 1,
             FormatFlags = CommonFormatFlags,
             VideoFormat = ffmpegProfile.VideoFormat,
-            AudioFormat = ffmpegProfile.AudioFormat
+            AudioFormat = ffmpegProfile.AudioFormat,
+            RealtimeOutput = streamingMode switch
+            {
+                StreamingMode.HttpLiveStreamingSegmenter => hlsRealtime,
+                _ => true
+            }
         };
 
     private static bool NeedToScale(FFmpegProfile ffmpegProfile, MediaVersion version) =>

ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs

@@ -40,92 +40,6 @@ public class FFmpegProcessService
         _logger = logger;
     }
 
-    public async Task<Command> ForError(
-        string ffmpegPath,
-        Channel channel,
-        Option<TimeSpan> duration,
-        string errorMessage,
-        bool hlsRealtime,
-        long ptsOffset)
-    {
-        FFmpegPlaybackSettings playbackSettings =
-            _playbackSettingsCalculator.CalculateErrorSettings(channel.FFmpegProfile);
-
-        IDisplaySize desiredResolution = channel.FFmpegProfile.Resolution;
-
-        var fontSize = (int)Math.Round(channel.FFmpegProfile.Resolution.Height / 20.0);
-        var margin = (int)Math.Round(channel.FFmpegProfile.Resolution.Height * 0.05);
-
-        string subtitleFile = await new SubtitleBuilder(_tempFilePool)
-            .WithResolution(desiredResolution)
-            .WithFontName("Roboto")
-            .WithFontSize(fontSize)
-            .WithAlignment(2)
-            .WithMarginV(margin)
-            .WithPrimaryColor("&HFFFFFF")
-            .WithFormattedContent(errorMessage.Replace(Environment.NewLine, "\\N"))
-            .BuildFile();
-
-        var videoStream = new MediaStream { Index = 0 };
-        var audioStream = new MediaStream { Index = 0 };
-
-        string videoCodec = playbackSettings.VideoFormat switch
-        {
-            FFmpegProfileVideoFormat.Hevc => "libx265",
-            FFmpegProfileVideoFormat.Mpeg2Video => "mpeg2video",
-            _ => "libx264"
-        };
-
-        string audioCodec = playbackSettings.AudioFormat switch
-        {
-            FFmpegProfileAudioFormat.Ac3 => "ac3",
-            _ => "aac"
-        };
-
-        FFmpegProcessBuilder builder = new FFmpegProcessBuilder(ffmpegPath, false, _logger)
-            .WithThreads(1)
-            .WithQuiet()
-            .WithFormatFlags(playbackSettings.FormatFlags)
-            .WithRealtimeOutput(playbackSettings.RealtimeOutput)
-            .WithLoopedImage(Path.Combine(FileSystemLayout.ResourcesCacheFolder, "background.png"))
-            .WithLibavfilter()
-            .WithInput("anullsrc")
-            .WithSubtitleFile(subtitleFile)
-            .WithFilterComplex(
-                videoStream,
-                audioStream,
-                Path.Combine(FileSystemLayout.ResourcesCacheFolder, "background.png"),
-                "fake-audio-path",
-                playbackSettings.VideoFormat)
-            .WithPixfmt("yuv420p")
-            .WithPlaybackArgs(playbackSettings, videoCodec, audioCodec)
-            .WithMetadata(channel, None);
-
-        await duration.IfSomeAsync(d => builder = builder.WithDuration(d));
-
-        Process process = channel.StreamingMode switch
-        {
-            // HLS needs to segment and generate playlist
-            StreamingMode.HttpLiveStreamingSegmenter =>
-                builder.WithHls(
-                        channel.Number,
-                        None,
-                        ptsOffset,
-                        playbackSettings.VideoTrackTimeScale,
-                        playbackSettings.FrameRate)
-                    .Build(),
-            _ => builder.WithFormat("mpegts")
-                .WithPipe()
-                .Build()
-        };
-
-        return Cli.Wrap(process.StartInfo.FileName)
-            .WithArguments(process.StartInfo.ArgumentList)
-            .WithValidation(CommandResultValidation.None)
-            .WithEnvironmentVariables(process.StartInfo.Environment.ToDictionary(kvp => kvp.Key, kvp => kvp.Value))
-            .WithStandardErrorPipe(PipeTarget.ToStream(Stream.Null));
-    }
-
     public Command WrapSegmenter(string ffmpegPath, bool saveReports, Channel channel, string scheme, string host)
     {
         FFmpegPlaybackSettings playbackSettings = _playbackSettingsCalculator.ConcatSettings;
@@ -231,7 +145,10 @@ public class FFmpegProcessService
                     watermarkPath);
 
             FFmpegPlaybackSettings playbackSettings =
-                _playbackSettingsCalculator.CalculateErrorSettings(channel.FFmpegProfile);
+                _playbackSettingsCalculator.CalculateErrorSettings(
+                    StreamingMode.TransportStream,
+                    channel.FFmpegProfile,
+                    false);
 
             FFmpegPlaybackSettings scalePlaybackSettings = _playbackSettingsCalculator.CalculateSettings(
                 StreamingMode.TransportStream,

ErsatzTV.Core/FFmpeg/SongVideoGenerator.cs

@@ -2,7 +2,6 @@
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.FFmpeg;
 using ErsatzTV.Core.Interfaces.Images;
-using ErsatzTV.FFmpeg.Format;
 using ErsatzTV.FFmpeg.State;
 
 namespace ErsatzTV.Core.FFmpeg;
@@ -233,28 +232,7 @@ public class SongVideoGenerator : ISongVideoGenerator
         foreach (string si in maybeSongImage.RightToSeq())
         {
             videoPath = si;
-            videoVersion = new BackgroundImageMediaVersion
+            videoVersion = BackgroundImageMediaVersion.ForPath(si, channel.FFmpegProfile.Resolution);
-            {
-                Chapters = new List<MediaChapter>(),
-                // song image has been pre-generated with correct size
-                Height = channel.FFmpegProfile.Resolution.Height,
-                Width = channel.FFmpegProfile.Resolution.Width,
-                SampleAspectRatio = "1:1",
-                Streams = new List<MediaStream>
-                {
-                    new()
-                    {
-                        MediaStreamKind = MediaStreamKind.Video,
-                        Index = 0,
-                        Codec = VideoFormat.GeneratedImage,
-                        PixelFormat = new PixelFormatUnknown().Name // the resulting pixel format is unknown
-                    }
-                },
-                MediaFiles = new List<MediaFile>
-                {
-                    new() { Path = si }
-                }
-            };
         }
 
         return Tuple(videoPath, videoVersion);

ErsatzTV.FFmpeg/Encoder/EncoderLibx265.cs

@@ -5,7 +5,9 @@ namespace ErsatzTV.FFmpeg.Encoder;
 public class EncoderLibx265 : EncoderBase
 {
     // TODO: is tag:v needed for mpegts?
-    public override IList<string> OutputOptions => new List<string> { "-c:v", Name, "-tag:v", "hvc1" };
+    public override IList<string> OutputOptions => new List<string>
+        { "-c:v", Name, "-tag:v", "hvc1", "-x265-params", "log-level=error" };
+
     public override string Name => "libx265";
     public override StreamKind Kind => StreamKind.Video;
 

ErsatzTV.FFmpeg/Filter/ComplexFilter.cs

@@ -258,21 +258,21 @@ public class ComplexFilter : IPipelineStep
             }
         }
 
-        if (!string.IsNullOrWhiteSpace(audioFilterComplex) || !string.IsNullOrWhiteSpace(videoFilterComplex))
+        var filterComplex = string.Join(
+            ";",
+            new[]
+            {
+                audioFilterComplex,
+                videoFilterComplex,
+                watermarkFilterComplex,
+                subtitleFilterComplex,
+                watermarkOverlayFilterComplex,
+                subtitleOverlayFilterComplex
+            }.Where(
+                s => !string.IsNullOrWhiteSpace(s)));
+
+        if (!string.IsNullOrWhiteSpace(filterComplex))
         {
-            var filterComplex = string.Join(
-                ";",
-                new[]
-                {
-                    audioFilterComplex,
-                    videoFilterComplex,
-                    watermarkFilterComplex,
-                    subtitleFilterComplex,
-                    watermarkOverlayFilterComplex,
-                    subtitleOverlayFilterComplex
-                }.Where(
-                    s => !string.IsNullOrWhiteSpace(s)));
-
             result.AddRange(new[] { "-filter_complex", filterComplex });
         }
 

ErsatzTV.FFmpeg/InputFile.cs

@@ -45,6 +45,17 @@ public record AudioInputFile(string Path, IList<AudioStream> AudioStreams, Audio
     }
 }
 
+public record NullAudioInputFile : AudioInputFile
+{
+    public NullAudioInputFile(AudioState DesiredState) : base(
+        "anullsrc",
+        new List<AudioStream> { new(0, "unknown", -1) },
+        DesiredState) =>
+        InputOptions.Add(new LavfiInputOption());
+
+    public void Deconstruct(out AudioState DesiredState) => DesiredState = this.DesiredState;
+}
+
 public record VideoInputFile(string Path, IList<VideoStream> VideoStreams) : InputFile(
     Path,
     VideoStreams.Cast<MediaStream>().ToList())

ErsatzTV.FFmpeg/Option/LavfiInputOption.cs

@@ -0,0 +1,21 @@
+using ErsatzTV.FFmpeg.Environment;
+
+namespace ErsatzTV.FFmpeg.Option;
+
+public class LavfiInputOption : IInputOption
+{
+    public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
+    public IList<string> GlobalOptions => Array.Empty<string>();
+
+    public IList<string> InputOptions(InputFile inputFile) => new List<string> { "-f", "lavfi" };
+
+    public IList<string> FilterOptions => Array.Empty<string>();
+    public IList<string> OutputOptions => Array.Empty<string>();
+    public FrameState NextState(FrameState currentState) => currentState;
+
+    public bool AppliesTo(AudioInputFile audioInputFile) => true;
+
+    public bool AppliesTo(VideoInputFile videoInputFile) => false;
+
+    public bool AppliesTo(ConcatInputFile concatInputFile) => false;
+}