detect and handle nonzero hls segmenter exit code (#719)

CHANGELOG.md

@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
   - `Preferred Subtitle Language` will filter all subtitle streams based on language
   - `Subtitle Mode` will further filter subtitle streams based on attributes (forced, default)
   - If picture-based subtitles are found after filtering, they will be burned into the video stream
+- Detect non-zero ffmpeg exit code from `HLS Segmenter` and `MPEG-TS`, log error output and display error output on stream
 
 ### Changed
 - Remove legacy transcoder logic option; all channels will use the new transcoder logic

ErsatzTV.Application/Streaming/HlsSessionWorker.cs

@@ -4,7 +4,6 @@ using Bugsnag;
 using CliWrap;
 using CliWrap.Buffered;
 using ErsatzTV.Application.Channels;
-using ErsatzTV.Application.Playouts;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.FFmpeg;
@@ -212,19 +211,68 @@ public class HlsSessionWorker : IHlsSessionWorker
 
                 try
                 {
-                    await Cli.Wrap(process.StartInfo.FileName)
+                    BufferedCommandResult commandResult = await Cli.Wrap(process.StartInfo.FileName)
                         .WithArguments(process.StartInfo.ArgumentList)
                         .WithValidation(CommandResultValidation.None)
-                        .ExecuteAsync(cancellationToken);
+                        .ExecuteBufferedAsync(cancellationToken);
+
+                    if (commandResult.ExitCode == 0)
+                    {
+                        _logger.LogInformation("HLS process has completed for channel {Channel}", _channelNumber);
+                        _transcodedUntil = processModel.Until;
+                        return true;
+                    }
+                    else
+                    {
+                        // detect the non-zero exit code and transcode the ffmpeg error message instead
+
+                        string errorMessage = commandResult.StandardError;
+                        if (string.IsNullOrWhiteSpace(errorMessage))
+                        {
+                            errorMessage = $"Unknown FFMPEG error; exit code {commandResult.ExitCode}";
+                        }
+
+                        _logger.LogError(
+                            "HLS process for channel {Channel} has terminated unsuccessfully with exit code {ExitCode}: {StandardError}",
+                            _channelNumber,
+                            commandResult.ExitCode,
+                            commandResult.StandardError);
+                        
+                        Either<BaseError, PlayoutItemProcessModel> maybeOfflineProcess = await mediator.Send(
+                            new GetErrorProcess(
+                                _channelNumber,
+                                "segmenter",
+                                realtime,
+                                ptsOffset,
+                                processModel.MaybeDuration,
+                                processModel.Until,
+                                errorMessage),
+                            cancellationToken);
+
+                        foreach (PlayoutItemProcessModel errorProcessModel in maybeOfflineProcess.RightAsEnumerable())
+                        {
+                            Process errorProcess = errorProcessModel.Process;
+                            
+                            _logger.LogInformation(
+                                "ffmpeg hls error arguments {FFmpegArguments}",
+                                string.Join(" ", errorProcess.StartInfo.ArgumentList));
+
+                            commandResult = await Cli.Wrap(errorProcess.StartInfo.FileName)
+                                .WithArguments(errorProcess.StartInfo.ArgumentList)
+                                .WithValidation(CommandResultValidation.None)
+                                .ExecuteBufferedAsync(cancellationToken);
+                            
+                            return commandResult.ExitCode == 0;
+                        }
+
+                        return false;
+                    }
                 }
                 catch (TaskCanceledException)
                 {
                     _logger.LogInformation("Terminating HLS process for channel {Channel}", _channelNumber);
                     return false;
                 }
-
-                _logger.LogInformation("HLS process has completed for channel {Channel}", _channelNumber);
-                _transcodedUntil = processModel.Until;
             }
         }
         catch (Exception ex)
@@ -248,7 +296,7 @@ public class HlsSessionWorker : IHlsSessionWorker
             Interlocked.Decrement(ref _workAheadCount);
         }
 
-        return true;
+        return false;
     }
 
     private async Task TrimAndDelete(CancellationToken cancellationToken)

ErsatzTV.Application/Streaming/PlayoutItemProcessModel.cs

@@ -2,4 +2,4 @@
 
 namespace ErsatzTV.Application.Streaming;
 
-public record PlayoutItemProcessModel(Process Process, DateTimeOffset Until);
+public record PlayoutItemProcessModel(Process Process, Option<TimeSpan> MaybeDuration, DateTimeOffset Until);

ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumberHandler.cs

@@ -38,6 +38,6 @@ public class GetConcatProcessByChannelNumberHandler : FFmpegProcessHandler<GetCo
             request.Scheme,
             request.Host);
 
-        return new PlayoutItemProcessModel(process, DateTimeOffset.MaxValue);
+        return new PlayoutItemProcessModel(process, Option<TimeSpan>.None, DateTimeOffset.MaxValue);
     }
 }

ErsatzTV.Application/Streaming/Queries/GetErrorProcess.cs

@@ -0,0 +1,16 @@
+namespace ErsatzTV.Application.Streaming;
+
+public record GetErrorProcess(
+    string ChannelNumber,
+    string Mode,
+    bool HlsRealtime,
+    long PtsOffset,
+    Option<TimeSpan> MaybeDuration,
+    DateTimeOffset Until,
+    string ErrorMessage) : FFmpegProcessRequest(
+    ChannelNumber,
+    Mode,
+    DateTimeOffset.Now,
+    true,
+    HlsRealtime,
+    PtsOffset);

ErsatzTV.Application/Streaming/Queries/GetErrorProcessHandler.cs

@@ -0,0 +1,39 @@
+using System.Diagnostics;
+using ErsatzTV.Core;
+using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.Interfaces.FFmpeg;
+using ErsatzTV.Infrastructure.Data;
+using Microsoft.EntityFrameworkCore;
+
+namespace ErsatzTV.Application.Streaming;
+
+public class GetErrorProcessHandler : FFmpegProcessHandler<GetErrorProcess>
+{
+    private readonly IFFmpegProcessService _ffmpegProcessService;
+
+    public GetErrorProcessHandler(
+        IDbContextFactory<TvContext> dbContextFactory,
+        IFFmpegProcessService ffmpegProcessService)
+        : base(dbContextFactory)
+    {
+        _ffmpegProcessService = ffmpegProcessService;
+    }
+
+    protected override async Task<Either<BaseError, PlayoutItemProcessModel>> GetProcess(
+        TvContext dbContext,
+        GetErrorProcess request,
+        Channel channel,
+        string ffmpegPath,
+        CancellationToken cancellationToken)
+    {
+        Process process = await _ffmpegProcessService.ForError(
+            ffmpegPath,
+            channel,
+            request.MaybeDuration,
+            request.ErrorMessage,
+            request.HlsRealtime,
+            request.PtsOffset);
+
+        return new PlayoutItemProcessModel(process, request.MaybeDuration, request.Until);
+    }
+}

ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs

@@ -156,7 +156,10 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                 request.PtsOffset,
                 request.TargetFramerate);
 
-            var result = new PlayoutItemProcessModel(process, playoutItemWithPath.PlayoutItem.FinishOffset);
+            var result = new PlayoutItemProcessModel(
+                process,
+                playoutItemWithPath.PlayoutItem.FinishOffset - (request.StartAtZero ? playoutItemWithPath.PlayoutItem.StartOffset : now),
+                playoutItemWithPath.PlayoutItem.FinishOffset);
 
             return Right<BaseError, PlayoutItemProcessModel>(result);
         }
@@ -184,7 +187,7 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                         request.HlsRealtime,
                         request.PtsOffset);
 
-                    return new PlayoutItemProcessModel(offlineProcess, finish);
+                    return new PlayoutItemProcessModel(offlineProcess, maybeDuration, finish);
                 case PlayoutItemDoesNotExistOnDisk:
                     Process doesNotExistProcess = await _ffmpegProcessService.ForError(
                         ffmpegPath,
@@ -194,7 +197,7 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                         request.HlsRealtime,
                         request.PtsOffset);
 
-                    return new PlayoutItemProcessModel(doesNotExistProcess, finish);
+                    return new PlayoutItemProcessModel(doesNotExistProcess, maybeDuration, finish);
                 default:
                     Process errorProcess = await _ffmpegProcessService.ForError(
                         ffmpegPath,
@@ -204,7 +207,7 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler<
                         request.HlsRealtime,
                         request.PtsOffset);
 
-                    return new PlayoutItemProcessModel(errorProcess, finish);
+                    return new PlayoutItemProcessModel(errorProcess, maybeDuration, finish);
             }
         }
 

ErsatzTV.Application/Streaming/Queries/GetWrappedProcessByChannelNumberHandler.cs

@@ -38,6 +38,6 @@ public class GetWrappedProcessByChannelNumberHandler : FFmpegProcessHandler<GetW
             request.Scheme,
             request.Host);
 
-        return new PlayoutItemProcessModel(process, DateTimeOffset.MaxValue);
+        return new PlayoutItemProcessModel(process, Option<TimeSpan>.None, DateTimeOffset.MaxValue);
     }
 }

ErsatzTV/Startup.cs

@@ -296,15 +296,15 @@ public class Startup
                 endpoints.MapBlazorHub();
                 endpoints.MapFallbackToPage("/_Host");
 
-                if (env.IsDevelopment())
+                // if (env.IsDevelopment())
-                {
+                // {
-                    endpoints.MapToVueCliProxy(
+                //     endpoints.MapToVueCliProxy(
-                        "/v2/{*path}",
+                //         "/v2/{*path}",
-                        new SpaOptions { SourcePath = "client-app" },
+                //         new SpaOptions { SourcePath = "client-app" },
-                        "serve",
+                //         "serve",
-                        regex: "Compiled successfully",
+                //         regex: "Compiled successfully",
-                        forceKill: true);
+                //         forceKill: true);
-                }
+                // }
             });
     }