fix transitions using nvidia accel (#2244)

CHANGELOG.md

@@ -50,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix building playouts with empty schedules
 - Fix schedule start time calculation when daily playout build goes beyond midnight and into a different alternate schedule
 - Fix compatibility with older NVIDIA devices (compute capability 3.0+) in unified docker image
+- Fix transitions when using NVIDIA acceleration
 
 ### Changed
 - Always tell ffmpeg to stop encoding with a specific duration

ErsatzTV.Application/Streaming/HlsSessionWorker.cs

@@ -397,7 +397,7 @@ public class HlsSessionWorker : IHlsSessionWorker
                 }
             }
 
-            double ptsOffset = await GetPtsOffset(_channelNumber, cancellationToken);
+            long ptsOffset = await GetPtsOffset(_channelNumber, cancellationToken);
             // _logger.LogInformation("PTS offset: {PtsOffset}", ptsOffset);
 
             _logger.LogDebug("HLS session state: {State}", _state);
@@ -620,12 +620,12 @@ public class HlsSessionWorker : IHlsSessionWorker
         }
     }
 
-    private async Task<double> GetPtsOffset(string channelNumber, CancellationToken cancellationToken)
+    private async Task<long> GetPtsOffset(string channelNumber, CancellationToken cancellationToken)
     {
         await _slim.WaitAsync(cancellationToken);
         try
         {
-            double result = 0;
+            long result = 0;
 
             // if we haven't yet written any segments, start at zero
             if (!_hasWrittenSegments)
@@ -644,7 +644,7 @@ public class HlsSessionWorker : IHlsSessionWorker
 
             foreach (PtsTime pts in queryResult.RightToSeq())
             {
-                result = pts.Value + 0.01;
+                result = pts.Value;
             }
 
             return result;

ErsatzTV.Application/Streaming/PtsTime.cs

@@ -2,15 +2,15 @@
 
 namespace ErsatzTV.Application.Streaming;
 
-public record PtsTime(double Value)
+public record PtsTime(long Value)
 {
     public static readonly PtsTime Zero = new(0);
 
     public static PtsTime From(string ffprobeLine)
     {
         string[] split = ffprobeLine.Split("|");
-        var ptsTime = double.Parse(split[0], CultureInfo.InvariantCulture);
+        var ptsTime = long.Parse(split[0], CultureInfo.InvariantCulture);
-        if (double.TryParse(split[1], CultureInfo.InvariantCulture, out double duration))
+        if (long.TryParse(split[1], CultureInfo.InvariantCulture, out long duration))
         {
             ptsTime += duration;
         }

ErsatzTV.Application/Streaming/Queries/FFmpegProcessRequest.cs

@@ -8,4 +8,4 @@ public record FFmpegProcessRequest(
     DateTimeOffset Now,
     bool StartAtZero,
     bool HlsRealtime,
-    double PtsOffset) : IRequest<Either<BaseError, PlayoutItemProcessModel>>;
+    long PtsOffset) : IRequest<Either<BaseError, PlayoutItemProcessModel>>;

ErsatzTV.Application/Streaming/Queries/GetErrorProcess.cs

@@ -4,7 +4,7 @@ public record GetErrorProcess(
     string ChannelNumber,
     string Mode,
     bool HlsRealtime,
-    double PtsOffset,
+    long PtsOffset,
     Option<TimeSpan> MaybeDuration,
     DateTimeOffset Until,
     string ErrorMessage) : FFmpegProcessRequest(

ErsatzTV.Application/Streaming/Queries/GetLastPtsTimeHandler.cs

@@ -54,24 +54,20 @@ public class GetLastPtsTimeHandler : IRequestHandler<GetLastPtsTime, Either<Base
         Option<FileInfo> maybeLastSegment = GetLastSegment(parameters.ChannelNumber);
         foreach (FileInfo segment in maybeLastSegment)
         {
-            PtsTime videoPts = await GetPts(parameters, segment, "v", cancellationToken).IfNoneAsync(PtsTime.Zero);
+            return await GetPts(parameters, segment, cancellationToken).IfNoneAsync(PtsTime.Zero);
-            PtsTime audioPts = await GetPts(parameters, segment, "a", cancellationToken).IfNoneAsync(PtsTime.Zero);
-            return videoPts.Value > audioPts.Value ? videoPts : audioPts;
         }
 
         return BaseError.New($"Failed to determine last pts duration for channel {parameters.ChannelNumber}");
     }
 
-    private async Task<Option<PtsTime>> GetPts(RequestParameters parameters, FileInfo segment, string audioVideo, CancellationToken cancellationToken)
+    private async Task<Option<PtsTime>> GetPts(RequestParameters parameters, FileInfo segment, CancellationToken cancellationToken)
     {
         string[] argumentList =
         {
             "-v", "0",
-            "-select_streams", $"{audioVideo}:0",
             "-show_entries",
-            "packet=pts_time,duration_time",
+            "packet=pts,duration",
             "-of", "compact=p=0:nk=1",
-            // "-read_intervals", "999999", // read_intervals causes inconsistent behavior on windows
             segment.FullName
         };
 

ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumber.cs

@@ -6,7 +6,7 @@ public record GetPlayoutItemProcessByChannelNumber(
     DateTimeOffset Now,
     bool StartAtZero,
     bool HlsRealtime,
-    double PtsOffset,
+    long PtsOffset,
     Option<int> TargetFramerate) : FFmpegProcessRequest(
     ChannelNumber,
     Mode,

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -72,7 +72,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         FillerKind fillerKind,
         TimeSpan inPoint,
         TimeSpan outPoint,
-        double ptsOffset,
+        long ptsOffset,
         Option<int> targetFramerate,
         bool disableWatermarks,
         Option<string> customReportsFolder,
@@ -471,7 +471,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         Option<TimeSpan> duration,
         string errorMessage,
         bool hlsRealtime,
-        double ptsOffset,
+        long ptsOffset,
         string vaapiDisplay,
         VaapiDriver vaapiDriver,
         string vaapiDevice,

ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs

@@ -37,7 +37,7 @@ public interface IFFmpegProcessService
         FillerKind fillerKind,
         TimeSpan inPoint,
         TimeSpan outPoint,
-        double ptsOffset,
+        long ptsOffset,
         Option<int> targetFramerate,
         bool disableWatermarks,
         Option<string> customReportsFolder,
@@ -49,7 +49,7 @@ public interface IFFmpegProcessService
         Option<TimeSpan> duration,
         string errorMessage,
         bool hlsRealtime,
-        double ptsOffset,
+        long ptsOffset,
         string vaapiDisplay,
         VaapiDriver vaapiDriver,
         string vaapiDevice,

ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs

@@ -116,7 +116,7 @@ public class PipelineBuilderBaseTests
 
         string command = PrintCommand(videoInputFile, audioInputFile, None, None, result);
         command.ShouldBe(
-            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -readrate 1.0 -i /tmp/whatever.mkv -filter_complex [0:1]aresample=async=1:first_pts=0,asetpts=N/SR/TB[a];[0:0]setpts=PTS-STARTPTS[vpf] -map [vpf] -map [a] -movflags +faststart -flags cgop -bf 0 -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -c:a aac -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -f mpegts -mpegts_flags +initial_discontinuity pipe:1");
+            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -readrate 1.0 -i /tmp/whatever.mkv -filter_complex [0:1]aresample=async=1:first_pts=0,asetpts=N/SR/TB[a];[0:0]setpts=PTS-STARTPTS[vpf] -map [vpf] -map [a] -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -bf 0 -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -c:a aac -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -f mpegts -mpegts_flags +initial_discontinuity pipe:1");
     }
 
     [Test]
@@ -213,7 +213,7 @@ public class PipelineBuilderBaseTests
 
         string command = PrintCommand(videoInputFile, audioInputFile, None, None, result);
         command.ShouldBe(
-            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -readrate 1.0 -i /tmp/whatever.mkv -filter_complex [0:1]aresample=async=1:first_pts=0,asetpts=N/SR/TB[a];[0:0]setpts=PTS-STARTPTS[vpf] -map [vpf] -map [a] -movflags +faststart -flags cgop -bf 0 -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -c:a aac -ac 6 -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -f mpegts -mpegts_flags +initial_discontinuity pipe:1");
+            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -readrate 1.0 -i /tmp/whatever.mkv -filter_complex [0:1]aresample=async=1:first_pts=0,asetpts=N/SR/TB[a];[0:0]setpts=PTS-STARTPTS[vpf] -map [vpf] -map [a] -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -bf 0 -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -c:a aac -ac 6 -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -f mpegts -mpegts_flags +initial_discontinuity pipe:1");
     }
 
     [Test]
@@ -370,7 +370,7 @@ public class PipelineBuilderBaseTests
 
         // 0.4.0 reference: "-nostdin -threads 1 -hide_banner -loglevel error -nostats -fflags +genpts+discardcorrupt+igndts -re -ss 00:14:33.6195516 -i /tmp/whatever.mkv -map 0:0 -map 0:a -c:v copy -flags cgop -sc_threshold 0 -c:a copy -movflags +faststart -metadata service_provider="ErsatzTV" -metadata service_name="ErsatzTV" -t 00:06:39.6934484 -f mpegts -mpegts_flags +initial_discontinuity pipe:1"
         command.ShouldBe(
-            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -readrate 1.0 -i /tmp/whatever.mkv -map 0:0 -map 0:1 -movflags +faststart+frag_keyframe+separate_moof+omit_tfhd_offset+empty_moov+delay_moov -flags cgop -sc_threshold 0 -c:v copy -c:a copy -f mp4 pipe:1");
+            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -readrate 1.0 -i /tmp/whatever.mkv -map 0:0 -map 0:1 -muxdelay 0 -muxpreload 0 -movflags +faststart+frag_keyframe+separate_moof+omit_tfhd_offset+empty_moov+delay_moov -flags cgop -sc_threshold 0 -c:v copy -c:a copy -f mp4 pipe:1");
     }
 
     [Test]
@@ -459,7 +459,7 @@ public class PipelineBuilderBaseTests
         string command = PrintCommand(videoInputFile, audioInputFile, None, None, result);
 
         command.ShouldBe(
-            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -readrate 1.0 -i /tmp/whatever.mkv -map 0:0 -map 0:a -movflags +faststart+frag_keyframe+separate_moof+omit_tfhd_offset+empty_moov+delay_moov -flags cgop -sc_threshold 0 -c:v copy -c:a copy -f mp4 pipe:1");
+            "-nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -readrate 1.0 -i /tmp/whatever.mkv -map 0:0 -map 0:a -muxdelay 0 -muxpreload 0 -movflags +faststart+frag_keyframe+separate_moof+omit_tfhd_offset+empty_moov+delay_moov -flags cgop -sc_threshold 0 -c:v copy -c:a copy -f mp4 pipe:1");
     }
 
     [Test]

ErsatzTV.FFmpeg/FFmpegState.cs

@@ -19,7 +19,7 @@ public record FFmpegState(
     OutputFormatKind OutputFormat,
     Option<string> HlsPlaylistPath,
     Option<string> HlsSegmentTemplate,
-    double PtsOffset,
+    long PtsOffset,
     Option<int> ThreadCount,
     Option<int> MaybeQsvExtraHardwareFrames,
     bool IsSongWithProgress,

ErsatzTV.FFmpeg/Filter/AudioPadFilter.cs

@@ -1,10 +1,8 @@
-using System.Globalization;
+namespace ErsatzTV.FFmpeg.Filter;
 
-namespace ErsatzTV.FFmpeg.Filter;
+public class AudioPadFilter : BaseFilter
-
-public class AudioPadFilter(TimeSpan audioDuration) : BaseFilter
 {
-    public override string Filter => $"apad=whole_dur={audioDuration.TotalMilliseconds.ToString(NumberFormatInfo.InvariantInfo)}ms";
+    public override string Filter => "apad";
 
     public override FrameState NextState(FrameState currentState) => currentState;
 }

ErsatzTV.FFmpeg/OutputOption/OutputTsOffsetOption.cs

@@ -2,12 +2,12 @@
 
 namespace ErsatzTV.FFmpeg.OutputOption;
 
-public class OutputTsOffsetOption(double ptsOffset) : OutputOption
+public class OutputTsOffsetOption(long ptsOffset, int videoTrackTimeScale) : OutputOption
 {
     public override string[] OutputOptions =>
     [
         "-output_ts_offset",
-        $"{ptsOffset.ToString(NumberFormatInfo.InvariantInfo)}s"
+        $"{(ptsOffset / (double)videoTrackTimeScale).ToString(NumberFormatInfo.InvariantInfo)}"
     ];
 
     // public override FrameState NextState(FrameState currentState) => currentState with { PtsOffset = _ptsOffset };

ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs

@@ -87,7 +87,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
         if (decodeCapability == FFmpegCapability.Hardware)
         {
             pipelineSteps.Add(new CudaHardwareAccelerationOption(isHdrTonemap));
-            pipelineSteps.Add(new NoAutoScaleOutputOption());
+            //pipelineSteps.Add(new NoAutoScaleOutputOption());
         }
 
         // disable hw accel if decoder/encoder isn't supported
@@ -120,6 +120,15 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
 
         Option<IDecoder> maybeDecoder = (ffmpegState.DecoderHardwareAccelerationMode, videoStream.Codec) switch
         {
+            (HardwareAccelerationMode.Nvenc, VideoFormat.Hevc) => new DecoderHevcCuvid(HardwareAccelerationMode.Nvenc),
+            (HardwareAccelerationMode.Nvenc, VideoFormat.H264) => new DecoderH264Cuvid(HardwareAccelerationMode.Nvenc),
+            (HardwareAccelerationMode.Nvenc, VideoFormat.Mpeg2Video) => new DecoderMpeg2Cuvid(
+                HardwareAccelerationMode.Nvenc, context.ShouldDeinterlace),
+            (HardwareAccelerationMode.Nvenc, VideoFormat.Vc1) => new DecoderVc1Cuvid(HardwareAccelerationMode.Nvenc),
+            (HardwareAccelerationMode.Nvenc, VideoFormat.Vp9) => new DecoderVp9Cuvid(HardwareAccelerationMode.Nvenc),
+            (HardwareAccelerationMode.Nvenc, VideoFormat.Mpeg4) =>
+                new DecoderMpeg4Cuvid(HardwareAccelerationMode.Nvenc),
+            (HardwareAccelerationMode.Nvenc, VideoFormat.Av1) => new DecoderAv1Cuvid(HardwareAccelerationMode.Nvenc),
             (HardwareAccelerationMode.Nvenc, _) => new DecoderImplicitCuda(),
             _ => GetSoftwareDecoder(videoStream)
         };

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs

@@ -193,6 +193,7 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
             new NoStatsOption(),
             new LoglevelErrorOption(),
             new StandardFormatFlags(),
+            new NoDemuxDecodeDelayOutputOption(),
             outputOption,
             new ClosedGopOutputOption()
         };
@@ -446,10 +447,9 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
             audioInputFile.FilterSteps.Add(new AudioSetPtsFilter());
         }
 
-        foreach (TimeSpan audioDuration in audioInputFile.DesiredState.AudioDuration.Filter(d => d > TimeSpan.Zero))
+        foreach (TimeSpan _ in audioInputFile.DesiredState.AudioDuration)
         {
-            audioInputFile.FilterSteps.Add(new AudioPadFilter(audioDuration));
+            audioInputFile.FilterSteps.Add(new AudioPadFilter());
-            pipelineSteps.Add(new ShortestOutputOption());
         }
     }
 
@@ -658,7 +658,10 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
 
         if (ffmpegState.PtsOffset > 0)
         {
-            pipelineSteps.Add(new OutputTsOffsetOption(ffmpegState.PtsOffset));
+            foreach (int videoTrackTimeScale in desiredState.VideoTrackTimeScale)
+            {
+                pipelineSteps.Add(new OutputTsOffsetOption(ffmpegState.PtsOffset, videoTrackTimeScale));
+            }
         }
     }