fix song playback (#644)

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -80,17 +80,26 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             videoStream.Codec,
             AvailablePixelFormats.ForPixelFormat(videoStream.PixelFormat, _logger),
             new FrameSize(videoVersion.Width, videoVersion.Height),
-            videoVersion.RFrameRate);
+            videoVersion.RFrameRate,
+            videoPath != audioPath); // still image when paths are different
 
         var inputFiles = new List<InputFile>
         {
-            new(videoVersion.MediaFiles.Head().Path, new List<ErsatzTV.FFmpeg.MediaStream> { ffmpegVideoStream })
+            new(videoPath, new List<ErsatzTV.FFmpeg.MediaStream> { ffmpegVideoStream })
         };
 
+
         foreach (MediaStream audioStream in maybeAudioStream)
         {
-            inputFiles.Head().Streams
-                .Add(new AudioStream(audioStream.Index, audioStream.Codec, audioStream.Channels));
+            var ffmpegAudioStream = new AudioStream(audioStream.Index, audioStream.Codec, audioStream.Channels);
+            if (videoPath == audioPath)
+            {
+                inputFiles.Head().Streams.Add(ffmpegAudioStream);
+            }
+            else
+            {
+                inputFiles.Add(new InputFile(audioPath, new List<ErsatzTV.FFmpeg.MediaStream> { ffmpegAudioStream }));
+            }
         }
 
         // TODO: need formats for these codecs

ErsatzTV.Core/FFmpeg/SongVideoGenerator.cs

@@ -48,7 +48,7 @@ namespace ErsatzTV.Core.FFmpeg
                 SampleAspectRatio = "1:1",
                 Streams = new List<MediaStream>
                 {
-                    new() { MediaStreamKind = MediaStreamKind.Video, Index = 0 }
+                    new() { MediaStreamKind = MediaStreamKind.Video, Index = 0, PixelFormat = "yuv420p" }
                 }
             };
             
@@ -235,7 +235,7 @@ namespace ErsatzTV.Core.FFmpeg
                     SampleAspectRatio = "1:1",
                     Streams = new List<MediaStream>
                     {
-                        new() { MediaStreamKind = MediaStreamKind.Video, Index = 0 },
+                        new() { MediaStreamKind = MediaStreamKind.Video, Index = 0, PixelFormat = "yuv420p" },
                     },
                     MediaFiles = new List<MediaFile>
                     {

ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs

@@ -23,7 +23,7 @@ public class PipelineGeneratorTests
             "/tmp/whatever.mkv",
             new List<MediaStream>
             {
-                new VideoStream(0, VideoFormat.H264, new PixelFormatYuv420P(), new FrameSize(1920, 1080), "24"),
+                new VideoStream(0, VideoFormat.H264, new PixelFormatYuv420P(), new FrameSize(1920, 1080), "24", false),
                 new AudioStream(1, AudioFormat.Aac, 2)
             });
 
@@ -79,7 +79,7 @@ public class PipelineGeneratorTests
             "/tmp/whatever.mkv",
             new List<MediaStream>
             {
-                new VideoStream(0, VideoFormat.H264, new PixelFormatYuv420P(), new FrameSize(1920, 1080), "24"),
+                new VideoStream(0, VideoFormat.H264, new PixelFormatYuv420P(), new FrameSize(1920, 1080), "24", false),
                 new AudioStream(1, AudioFormat.Aac, 2)
             });
 

ErsatzTV.FFmpeg/CommandGenerator.cs

@@ -24,7 +24,7 @@ public static class CommandGenerator
         {
             foreach (IPipelineStep step in pipelineSteps)
             {
-                arguments.AddRange(step.InputOptions);
+                arguments.AddRange(step.InputOptions(inputFile));
             }
 
             arguments.AddRange(new[] { "-i", inputFile.Path });

ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderH264Cuvid.cs

@@ -4,17 +4,14 @@ public class DecoderH264Cuvid : DecoderBase
 {
     public override string Name => "h264_cuvid";
 
-    public override IList<string> InputOptions
+    public override IList<string> InputOptions(InputFile inputFile)
     {
-        get
-        {
-            IList<string> result =  base.InputOptions;
+        IList<string> result = base.InputOptions(inputFile);
 
-            result.Add("-hwaccel_output_format");
-            result.Add("cuda");
+        result.Add("-hwaccel_output_format");
+        result.Add("cuda");
 
-            return result;
-        }
+        return result;
     }
 
     protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Hardware;

ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderHevcCuvid.cs

@@ -3,17 +3,14 @@
 public class DecoderHevcCuvid : DecoderBase
 {
     public override string Name => "hevc_cuvid";
-    public override IList<string> InputOptions
+    public override IList<string> InputOptions(InputFile inputFile)
     {
-        get
-        {
-            IList<string> result =  base.InputOptions;
-            
-            result.Add("-hwaccel_output_format");
-            result.Add("cuda");
+        IList<string> result = base.InputOptions(inputFile);
 
-            return result;
-        }
+        result.Add("-hwaccel_output_format");
+        result.Add("cuda");
+
+        return result;
     }
 
     protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Hardware;

ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderMpeg2Cuvid.cs

@@ -3,17 +3,14 @@
 public class DecoderMpeg2Cuvid : DecoderBase
 {
     public override string Name => "mpeg2_cuvid";
-    public override IList<string> InputOptions
+    public override IList<string> InputOptions(InputFile inputFile)
     {
-        get
-        {
-            IList<string> result =  base.InputOptions;
-            
-            result.Add("-hwaccel_output_format");
-            result.Add("cuda");
+        IList<string> result = base.InputOptions(inputFile);
 
-            return result;
-        }
+        result.Add("-hwaccel_output_format");
+        result.Add("cuda");
+
+        return result;
     }
 
     protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Hardware;

ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderMpeg4Cuvid.cs

@@ -4,17 +4,14 @@ public class DecoderMpeg4Cuvid : DecoderBase
 {
     public override string Name => "mpeg4_cuvid";
 
-    public override IList<string> InputOptions
+    public override IList<string> InputOptions(InputFile inputFile)
     {
-        get
-        {
-            IList<string> result =  base.InputOptions;
+        IList<string> result = base.InputOptions(inputFile);
 
-            result.Add("-hwaccel_output_format");
-            result.Add("cuda");
+        result.Add("-hwaccel_output_format");
+        result.Add("cuda");
 
-            return result;
-        }
+        return result;
     }
 
     protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Hardware;

ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderVc1Cuvid.cs

@@ -3,17 +3,14 @@
 public class DecoderVc1Cuvid : DecoderBase
 {
     public override string Name => "vc1_cuvid";
-    public override IList<string> InputOptions
+    public override IList<string> InputOptions(InputFile inputFile)
     {
-        get
-        {
-            IList<string> result =  base.InputOptions;
+        IList<string> result = base.InputOptions(inputFile);
 
-            result.Add("-hwaccel_output_format");
-            result.Add("cuda");
+        result.Add("-hwaccel_output_format");
+        result.Add("cuda");
 
-            return result;
-        }
+        return result;
     }
 
     protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Hardware;

ErsatzTV.FFmpeg/Decoder/Cuvid/DecoderVp9Cuvid.cs

@@ -3,17 +3,14 @@
 public class DecoderVp9Cuvid : DecoderBase
 {
     public override string Name => "vp9_cuvid";
-    public override IList<string> InputOptions
+    public override IList<string> InputOptions(InputFile inputFile)
     {
-        get
-        {
-            IList<string> result =  base.InputOptions;
+        IList<string> result = base.InputOptions(inputFile);
 
-            result.Add("-hwaccel_output_format");
-            result.Add("cuda");
+        result.Add("-hwaccel_output_format");
+        result.Add("cuda");
 
-            return result;
-        }
+        return result;
     }
 
     protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Hardware;

ErsatzTV.FFmpeg/Decoder/DecoderBase.cs

@@ -7,7 +7,7 @@ public abstract class DecoderBase : IDecoder
     protected abstract FrameDataLocation OutputFrameDataLocation { get; }
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public virtual IList<string> InputOptions => new List<string> { "-c:v", Name };
+    public virtual IList<string> InputOptions(InputFile inputFile) => new List<string> { "-c:v", Name };
     public IList<string> FilterOptions => Array.Empty<string>();
     public IList<string> OutputOptions => Array.Empty<string>();
     public virtual FrameState NextState(FrameState currentState) =>

ErsatzTV.FFmpeg/Decoder/DecoderImplicit.cs

@@ -4,5 +4,5 @@ public class DecoderImplicit : DecoderBase
 {
     protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Software;
     public override string Name => string.Empty;
-    public override IList<string> InputOptions => Array.Empty<string>();
+    public override IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
 }

ErsatzTV.FFmpeg/Decoder/DecoderVaapi.cs

@@ -6,7 +6,7 @@ public class DecoderVaapi : DecoderBase
 {
     protected override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Software;
     public override string Name => "implicit_vaapi";
-    public override IList<string> InputOptions => Array.Empty<string>();
+    public override IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
 
     public override FrameState NextState(FrameState currentState)
     {

ErsatzTV.FFmpeg/Encoder/EncoderBase.cs

@@ -6,7 +6,7 @@ public abstract class EncoderBase : IEncoder
 {
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public IList<string> FilterOptions => Array.Empty<string>();
     public virtual IList<string> OutputOptions => new List<string> { Kind == StreamKind.Video ? "-c:v" : "-c:a", Name };
     public abstract FrameState NextState(FrameState currentState);

ErsatzTV.FFmpeg/Environment/FFReportVariable.cs

@@ -39,7 +39,7 @@ public class FFReportVariable : IPipelineStep
     }
 
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public IList<string> FilterOptions => Array.Empty<string>();
     public IList<string> OutputOptions => Array.Empty<string>();
     public FrameState NextState(FrameState currentState) => currentState with { SaveReport = true };

ErsatzTV.FFmpeg/Environment/LibvaDriverNameVariable.cs

@@ -15,7 +15,7 @@ public class LibvaDriverNameVariable : IPipelineStep
     };
 
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public IList<string> FilterOptions => Array.Empty<string>();
     public IList<string> OutputOptions => Array.Empty<string>();
     public FrameState NextState(FrameState currentState) => currentState with { VaapiDriver = _driverName };

ErsatzTV.FFmpeg/Filter/BaseFilter.cs

@@ -6,7 +6,7 @@ public abstract class BaseFilter : IPipelineFilterStep
 {
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public virtual IList<string> GlobalOptions => Array.Empty<string>();
-    public virtual IList<string> InputOptions => Array.Empty<string>();
+    public virtual IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public virtual IList<string> FilterOptions => Array.Empty<string>();
     public virtual IList<string> OutputOptions => Array.Empty<string>();
     public abstract FrameState NextState(FrameState currentState);

ErsatzTV.FFmpeg/Filter/ComplexFilter.cs

@@ -77,7 +77,7 @@ public class ComplexFilter : IPipelineStep
 
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public IList<string> FilterOptions => Arguments();
     public IList<string> OutputOptions => Array.Empty<string>();
     public FrameState NextState(FrameState currentState) => currentState;

ErsatzTV.FFmpeg/Format/ConcatInputFormat.cs

@@ -7,7 +7,7 @@ public class ConcatInputFormat : IPipelineStep
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
 
-    public IList<string> InputOptions => new List<string>
+    public IList<string> InputOptions(InputFile inputFile) => new List<string>
     {
         "-f", "concat",
         "-safe", "0",

ErsatzTV.FFmpeg/IPipelineStep.cs

@@ -6,7 +6,7 @@ public interface IPipelineStep
 {
     IList<EnvironmentVariable> EnvironmentVariables { get; }
     IList<string> GlobalOptions { get; }
-    IList<string> InputOptions { get; }
+    IList<string> InputOptions(InputFile inputFile);
     IList<string> FilterOptions { get; }
     IList<string> OutputOptions { get; }
 

ErsatzTV.FFmpeg/InputFile.cs

@@ -12,5 +12,6 @@ public record ConcatInputFile(string Url, FrameSize Resolution) : InputFile(Url,
         string.Empty,
         Option<IPixelFormat>.None,
         Resolution,
-        Option<string>.None)
+        Option<string>.None,
+        false)
 });

ErsatzTV.FFmpeg/MediaStream.cs

@@ -15,7 +15,8 @@ public record VideoStream(
     string Codec,
     Option<IPixelFormat> PixelFormat,
     FrameSize FrameSize,
-    Option<string> FrameRate) : MediaStream(
+    Option<string> FrameRate,
+    bool StillImage) : MediaStream(
     Index,
     Codec,
     StreamKind.Video);

ErsatzTV.FFmpeg/Option/FrameRateOutputOption.cs

@@ -13,7 +13,7 @@ public class FrameRateOutputOption : IPipelineStep
 
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public IList<string> FilterOptions => Array.Empty<string>();
     public IList<string> OutputOptions => new List<string> { "-r", _frameRate.ToString(), "-vsync", "cfr" };
 

ErsatzTV.FFmpeg/Option/GlobalOption.cs

@@ -6,7 +6,7 @@ public abstract class GlobalOption : IPipelineStep
 {
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public abstract IList<string> GlobalOptions { get; }
-    public IList<string> InputOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public IList<string> FilterOptions => Array.Empty<string>();
     public IList<string> OutputOptions => Array.Empty<string>();
     public virtual FrameState NextState(FrameState currentState) => currentState;

ErsatzTV.FFmpeg/Option/InfiniteLoopInputOption.cs

@@ -13,7 +13,24 @@ public class InfiniteLoopInputOption : IPipelineStep
 
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions => new List<string> { "-stream_loop", "-1" };
+    public IList<string> InputOptions(InputFile inputFile)
+    {
+        // only loop video media
+        if (inputFile.Streams.OfType<VideoStream>().Any())
+        {
+            // loop 1 for still images
+            if (inputFile.Streams.OfType<VideoStream>().Any(s => s.StillImage))
+            {
+                return new List<string> { "-loop", "1" };
+            }
+
+            // stream_loop for looped video i.e. filler
+            return new List<string> { "-stream_loop", "-1" };
+        }
+
+        return Array.Empty<string>();
+    }
+
     public IList<string> FilterOptions => Array.Empty<string>();
 
     public IList<string> OutputOptions =>

ErsatzTV.FFmpeg/Option/OutputOption.cs

@@ -7,7 +7,7 @@ public abstract class OutputOption : IPipelineStep
     public FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Unknown;
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public IList<string> FilterOptions => Array.Empty<string>();
     public abstract IList<string> OutputOptions { get; }
 

ErsatzTV.FFmpeg/Option/RealtimeInputOption.cs

@@ -11,7 +11,7 @@ public class RealtimeInputOption : IPipelineStep
     // public IList<string> GlobalOptions => new List<string> { "-threads", "1" };
     public IList<string> GlobalOptions => Array.Empty<string>();
     
-    public IList<string> InputOptions => new List<string> { "-re" };
+    public IList<string> InputOptions(InputFile inputFile) => new List<string> { "-re" };
     public IList<string> FilterOptions => Array.Empty<string>();
     public IList<string> OutputOptions => Array.Empty<string>();
     public FrameState NextState(FrameState currentState) => currentState with { Realtime = true };

ErsatzTV.FFmpeg/Option/StreamSeekInputOption.cs

@@ -11,11 +11,15 @@ public class StreamSeekInputOption : IPipelineStep
         _start = start;
     }
 
-    public FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Unknown;
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions =>
-        _start == TimeSpan.Zero ? Array.Empty<string>() : new List<string> { "-ss", $"{_start:c}" };
+
+    // don't seek into a still image
+    public IList<string> InputOptions(InputFile inputFile) =>
+        _start == TimeSpan.Zero || inputFile.Streams.OfType<VideoStream>().Any(s => s.StillImage)
+            ? Array.Empty<string>()
+            : new List<string> { "-ss", $"{_start:c}" };
+
     public IList<string> FilterOptions => Array.Empty<string>();
     public IList<string> OutputOptions => Array.Empty<string>();
     public FrameState NextState(FrameState currentState) => currentState with { Start = _start };

ErsatzTV.FFmpeg/Option/TimeLimitOutputOption.cs

@@ -14,7 +14,7 @@ public class TimeLimitOutputOption : IPipelineStep
     public FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Unknown;
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public IList<string> FilterOptions => Array.Empty<string>();
     public IList<string> OutputOptions => new List<string> { "-t", $"{_finish:c}" };
     public FrameState NextState(FrameState currentState) => currentState with { Finish = _finish };

ErsatzTV.FFmpeg/OutputFormat/OutputFormatHls.cs

@@ -24,7 +24,7 @@ public class OutputFormatHls : IPipelineStep
 
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public IList<string> FilterOptions => Array.Empty<string>();
 
     public IList<string> OutputOptions

ErsatzTV.FFmpeg/OutputFormat/OutputFormatMpegTs.cs

@@ -6,7 +6,7 @@ public class OutputFormatMpegTs : IPipelineStep
 {
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public IList<string> FilterOptions => Array.Empty<string>();
 
     // always force an initial discontinuity

ErsatzTV.FFmpeg/PipelineBuilder.cs

@@ -49,6 +49,7 @@ public class PipelineBuilder
     {
         var allVideoStreams = _inputFiles.SelectMany(f => f.Streams)
             .Filter(s => s.Kind == StreamKind.Video)
+            .OfType<VideoStream>()
             .ToList();
 
         // -sc_threshold 0 is unsupported with mpeg2video
@@ -57,9 +58,13 @@ public class PipelineBuilder
                 ? new NoSceneDetectOutputOption(0)
                 : new NoSceneDetectOutputOption(1_000_000_000));
 
-        InputFile head = _inputFiles.First();
-        var videoStream = head.Streams.First(s => s.Kind == StreamKind.Video) as VideoStream;
-        Option<AudioStream> audioStream = head.Streams.OfType<AudioStream>().Find(s => s.Kind == StreamKind.Audio);
+        var allAudioStreams = _inputFiles.SelectMany(f => f.Streams)
+            .Filter(s => s.Kind == StreamKind.Audio)
+            .OfType<AudioStream>()
+            .ToList();
+
+        VideoStream videoStream = allVideoStreams.Head();
+        Option<AudioStream> audioStream = allAudioStreams.HeadOrNone();
         if (videoStream != null)
         {
             Option<int> initialFrameRate = Option<int>.None;
@@ -194,6 +199,11 @@ public class PipelineBuilder
                 }
             }
 
+            if (videoStream.StillImage)
+            {
+                _pipelineSteps.Add(new InfiniteLoopInputOption(currentState));
+            }
+
             // TODO: while?
             if (!IsDesiredVideoState(currentState, desiredState))
             {

ErsatzTV.FFmpeg/Protocol/PipeProtocol.cs

@@ -6,7 +6,7 @@ public class PipeProtocol : IPipelineStep
 {
     public IList<EnvironmentVariable> EnvironmentVariables => Array.Empty<EnvironmentVariable>();
     public IList<string> GlobalOptions => Array.Empty<string>();
-    public IList<string> InputOptions => Array.Empty<string>();
+    public IList<string> InputOptions(InputFile inputFile) => Array.Empty<string>();
     public IList<string> FilterOptions => Array.Empty<string>();
     public IList<string> OutputOptions => new List<string> { "pipe:1" };