add troubleshooting code to hls segmenter (#685)

ErsatzTV.Application/Streaming/HlsSessionWorker.cs

@@ -281,26 +281,28 @@ public class HlsSessionWorker : IHlsSessionWorker
         }
     }
 
-    private async Task<long> GetPtsOffset(IMediator mediator, string channelNumber, CancellationToken cancellationToken)
+    private static async Task<long> GetPtsOffset(IMediator mediator, string channelNumber, CancellationToken cancellationToken)
     {
-        var directory = new DirectoryInfo(Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber));
+        await Slim.WaitAsync(cancellationToken);
-        Option<FileInfo> lastSegment =
+        try
-            Optional(directory.GetFiles("*.ts").OrderByDescending(f => f.Name).FirstOrDefault());
-
-        long result = 0;
-        foreach (FileInfo segment in lastSegment)
         {
+            long result = 0;
+
             Either<BaseError, PtsAndDuration> queryResult = await mediator.Send(
-                new GetLastPtsDuration(segment.FullName),
+                new GetLastPtsDuration(channelNumber),
                 cancellationToken);
 
-            foreach (PtsAndDuration ptsAndDuration in queryResult.RightToSeq())
+            foreach ((long pts, long duration) in queryResult.RightToSeq())
             {
-                result = ptsAndDuration.Pts + ptsAndDuration.Duration;
+                result = pts + duration;
             }
-        }
 
-        return result;
+            return result;
+        }
+        finally
+        {
+            Slim.Release();
+        }
     }
 
     private async Task<int> GetWorkAheadLimit()

ErsatzTV.Application/Streaming/Queries/GetLastPtsDuration.cs

@@ -2,4 +2,4 @@
 
 namespace ErsatzTV.Application.Streaming;
 
-public record GetLastPtsDuration(string FileName) : IRequest<Either<BaseError, PtsAndDuration>>;
+public record GetLastPtsDuration(string ChannelNumber) : IRequest<Either<BaseError, PtsAndDuration>>;

ErsatzTV.Application/Streaming/Queries/GetLastPtsDurationHandler.cs

@@ -1,18 +1,37 @@
 using System.Diagnostics;
 using System.Text;
+using Bugsnag;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.FFmpeg;
+using ErsatzTV.Core.Interfaces.FFmpeg;
+using ErsatzTV.Core.Interfaces.Metadata;
 using ErsatzTV.Core.Interfaces.Repositories;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
 
 namespace ErsatzTV.Application.Streaming;
 
 public class GetLastPtsDurationHandler : IRequestHandler<GetLastPtsDuration, Either<BaseError, PtsAndDuration>>
 {
+    private readonly IClient _client;
+    private readonly ILocalFileSystem _localFileSystem;
+    private readonly ITempFilePool _tempFilePool;
     private readonly IConfigElementRepository _configElementRepository;
+    private readonly ILogger<GetLastPtsDurationHandler> _logger;
 
-    public GetLastPtsDurationHandler(IConfigElementRepository configElementRepository)
+    public GetLastPtsDurationHandler(
+        IClient client,
+        ILocalFileSystem localFileSystem,
+        ITempFilePool tempFilePool,
+        IConfigElementRepository configElementRepository,
+        ILogger<GetLastPtsDurationHandler> logger)
     {
+        _client = client;
+        _localFileSystem = localFileSystem;
+        _tempFilePool = tempFilePool;
         _configElementRepository = configElementRepository;
+        _logger = logger;
     }
 
     public async Task<Either<BaseError, PtsAndDuration>> Handle(
@@ -21,61 +40,123 @@ public class GetLastPtsDurationHandler : IRequestHandler<GetLastPtsDuration, Eit
     {
         Validation<BaseError, RequestParameters> validation = await Validate(request);
         return await validation.Match(
-            Handle,
+            parameters => Handle(parameters, cancellationToken),
             error => Task.FromResult<Either<BaseError, PtsAndDuration>>(error.Join()));
     }
 
     private async Task<Validation<BaseError, RequestParameters>> Validate(GetLastPtsDuration request) =>
-        await ValidateFFprobePath()
+        await ValidateFFprobePath().MapT(ffprobePath => new RequestParameters(request.ChannelNumber, ffprobePath));
-            .MapT(
-                ffprobePath => new RequestParameters(
-                    request.FileName,
-                    ffprobePath));
 
-    private async Task<Either<BaseError, PtsAndDuration>> Handle(RequestParameters parameters)
+    private async Task<Either<BaseError, PtsAndDuration>> Handle(
+        RequestParameters parameters,
+        CancellationToken cancellationToken)
     {
-        var startInfo = new ProcessStartInfo
+        Option<FileInfo> maybeLastSegment = GetLastSegment(parameters.ChannelNumber);
-        {
+        foreach (FileInfo segment in maybeLastSegment)
-            FileName = parameters.FFprobePath,
-            RedirectStandardOutput = true,
-            RedirectStandardError = true,
-            UseShellExecute = false,
-            StandardOutputEncoding = Encoding.UTF8,
-            StandardErrorEncoding = Encoding.UTF8
-        };
-
-        startInfo.ArgumentList.Add("-v");
-        startInfo.ArgumentList.Add("0");
-        startInfo.ArgumentList.Add("-show_entries");
-        startInfo.ArgumentList.Add("packet=pts,duration");
-        startInfo.ArgumentList.Add("-of");
-        startInfo.ArgumentList.Add("compact=p=0:nk=1");
-        startInfo.ArgumentList.Add("-read_intervals");
-        startInfo.ArgumentList.Add("-999999");
-        startInfo.ArgumentList.Add(parameters.FileName);
-
-        var probe = new Process
         {
-            StartInfo = startInfo
+            var startInfo = new ProcessStartInfo
-        };
+            {
+                FileName = parameters.FFprobePath,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                StandardOutputEncoding = Encoding.UTF8,
+                StandardErrorEncoding = Encoding.UTF8
+            };
+
+            startInfo.ArgumentList.Add("-v");
+            startInfo.ArgumentList.Add("0");
+            startInfo.ArgumentList.Add("-show_entries");
+            startInfo.ArgumentList.Add("packet=pts,duration");
+            startInfo.ArgumentList.Add("-of");
+            startInfo.ArgumentList.Add("compact=p=0:nk=1");
+            startInfo.ArgumentList.Add("-read_intervals");
+            startInfo.ArgumentList.Add("-999999");
+            startInfo.ArgumentList.Add(segment.FullName);
+
+            var probe = new Process
+            {
+                StartInfo = startInfo
+            };
+
+            probe.Start();
+            string output = await probe.StandardOutput.ReadToEndAsync();
+            await probe.WaitForExitAsync(cancellationToken);
+            if (probe.ExitCode != 0)
+            {
+                return BaseError.New($"FFprobe at {parameters.FFprobePath} exited with code {probe.ExitCode}");
+            }
 
-        probe.Start();
+            try
-        return await probe.StandardOutput.ReadToEndAsync().MapAsync<string, Either<BaseError, PtsAndDuration>>(
+            {
-            async output =>
+                string[] lines = output.Split("\n");
+                IEnumerable<string> nonEmptyLines = lines.Filter(s => !string.IsNullOrWhiteSpace(s)).Map(l => l.Trim());
+                return PtsAndDuration.From(nonEmptyLines.Last());
+            }
+            catch (Exception ex)
             {
-                await probe.WaitForExitAsync();
+                _client.Notify(ex);
-                return probe.ExitCode == 0
+                await SaveTroubleshootingData(parameters.ChannelNumber, output);
-                    ? PtsAndDuration.From(output.Split("\n").Filter(s => !string.IsNullOrWhiteSpace(s)).Last().Trim())
+            }
-                    : BaseError.New($"FFprobe at {parameters.FFprobePath} exited with code {probe.ExitCode}");
+        }
-            });
+
+        return BaseError.New($"Failed to determine last pts duration for channel {parameters.ChannelNumber}");
     }
 
+    private static Option<FileInfo> GetLastSegment(string channelNumber)
+    {
+        var directory = new DirectoryInfo(Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber));
+        return Optional(directory.GetFiles("*.ts").OrderByDescending(f => f.Name).FirstOrDefault());
+    }
+    
     private Task<Validation<BaseError, string>> ValidateFFprobePath() =>
         _configElementRepository.GetValue<string>(ConfigElementKey.FFprobePath)
             .FilterT(File.Exists)
-            .Map(
+            .Map(ffprobePath => ffprobePath.ToValidation<BaseError>("FFprobe path does not exist on the file system"));
-                ffprobePath =>
+
-                    ffprobePath.ToValidation<BaseError>("FFprobe path does not exist on the file system"));
+    private async Task SaveTroubleshootingData(string channelNumber, string output)
+    {
+        try
+        {
+            var directory = new DirectoryInfo(Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber));
+            FileInfo[] allFiles = directory.GetFiles();
+
+            string playlistFileName = Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber, "live.m3u8");
+            string playlistContents = string.Empty;
+            if (_localFileSystem.FileExists(playlistFileName))
+            {
+                playlistContents = await File.ReadAllTextAsync(playlistFileName);
+            }
+
+            var data = new TroubleshootingData(allFiles, playlistContents, output);
+            string serialized = data.Serialize();
+
+            string file = _tempFilePool.GetNextTempFile(TempFileCategory.BadTranscodeFolder);
+            await File.WriteAllTextAsync(file, serialized);
+
+            _logger.LogWarning("Transcode folder is in bad state; troubleshooting info saved to {File}", file);
+        }
+        catch (Exception ex)
+        {
+            _client.Notify(ex);
+        }
+    }
+
+    private record RequestParameters(string ChannelNumber, string FFprobePath);
+
+    private record TroubleshootingData(IEnumerable<FileInfo> Files, string Playlist, string ProbeOutput)
+    {
+        private record FileData(string FileName, long Bytes, DateTime LastWriteTimeUtc);
+        private record InternalData(List<FileData> Files, string EncodedPlaylist, string EncodedProbeOutput);
 
-    private record RequestParameters(string FileName, string FFprobePath);
+        public string Serialize()
+        {
+            var data = new InternalData(
+                Files.Map(f => new FileData(f.FullName, f.Length, f.LastWriteTimeUtc)).ToList(),
+                Convert.ToBase64String(Encoding.UTF8.GetBytes(Playlist)),
+                Convert.ToBase64String(Encoding.UTF8.GetBytes(ProbeOutput)));
+
+            return JsonConvert.SerializeObject(data);
+        }
+    }
 }

ErsatzTV.Core/FFmpeg/TempFileCategory.cs

@@ -7,5 +7,6 @@ public enum TempFileCategory
     CoverArt = 2,
     CachedArtwork = 3,
 
+    BadTranscodeFolder = 98,
     BadPlaylist = 99
 }