fade in and fade out intermittent watermarks (#607)

* first pass at fading in/out overlay

* fix tests

* update changelog

CHANGELOG.md

@@ -6,11 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [Unreleased]
 ### Fixed
 - Normalize smart quotes in search queries as they are unsupported by the search library
+- Fix incorrect watermark time calculations caused by working ahead in `HLS Segmenter`
 
 ### Added
 - Include `Series` category tag for all episodes in XMLTV
 - Include movie, episode (show), music video (artist) genres as `category` tags in XMLTV
 
+### Changed
+- Intermittent watermarks will now fade in and out
+
 ## [0.4.0-alpha] - 2022-01-29
 ### Fixed
 - Fix m3u `mode` query param to properly override streaming mode for all channels

ErsatzTV.Core.Tests/FFmpeg/FFmpegComplexFilterBuilderTests.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.FFmpeg;
 using FluentAssertions;
@@ -183,7 +184,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg
                 ChannelWatermarkLocation.TopLeft,
                 false,
                 100,
-                "[0:0][1:v]overlay=x=134:y=54:enable='lt(mod(mod(time(0),60*60),10*60),15)'[v]",
+                "[1:v]format=yuva420p|yuva444p|yuva422p|rgba|abgr|bgra|gbrap|ya8,fade=in:st=300:d=1:alpha=1:enable='between(t,0,314)',fade=out:st=315:d=1:alpha=1:enable='between(t,301,899)',fade=in:st=900:d=1:alpha=1:enable='between(t,316,914)',fade=out:st=915:d=1:alpha=1:enable='between(t,901,1499)',fade=in:st=1500:d=1:alpha=1:enable='between(t,916,1514)',fade=out:st=1515:d=1:alpha=1:enable='between(t,1501,2099)',fade=in:st=2100:d=1:alpha=1:enable='between(t,1516,2114)',fade=out:st=2115:d=1:alpha=1:enable='between(t,2101,2699)',fade=in:st=2700:d=1:alpha=1:enable='between(t,2116,2714)',fade=out:st=2715:d=1:alpha=1:enable='between(t,2701,3300)'[wmp];[0:0][wmp]overlay=x=134:y=54,format=yuv420p[v]",
                 "0:1",
                 "[v]")]
             [TestCase(
@@ -257,23 +258,36 @@ namespace ErsatzTV.Core.Tests.FFmpeg
                 string expectedAudioLabel,
                 string expectedVideoLabel)
             {
+                var watermark = new ChannelWatermark
+                {
+                    Mode = intermittent
+                        ? ChannelWatermarkMode.Intermittent
+                        : ChannelWatermarkMode.Permanent,
+                    DurationSeconds = intermittent ? 15 : 0,
+                    FrequencyMinutes = intermittent ? 10 : 0,
+                    Location = location,
+                    Size = scaled ? ChannelWatermarkSize.Scaled : ChannelWatermarkSize.ActualSize,
+                    WidthPercent = scaled ? 20 : 0,
+                    Opacity = opacity,
+                    HorizontalMarginPercent = 7,
+                    VerticalMarginPercent = 5
+                };
+
+                Option<List<FadePoint>> maybeFadePoints = watermark.Mode == ChannelWatermarkMode.Intermittent
+                    ? Some(
+                        WatermarkCalculator.CalculateFadePoints(
+                            new DateTimeOffset(2022, 01, 31, 12, 25, 0, TimeSpan.FromHours(-5)),
+                            TimeSpan.Zero,
+                            TimeSpan.FromMinutes(55),
+                            TimeSpan.Zero,
+                            watermark.FrequencyMinutes,
+                            watermark.DurationSeconds))
+                    : None;
+                
                 FFmpegComplexFilterBuilder builder = new FFmpegComplexFilterBuilder()
                     .WithWatermark(
-                        Some(
-                            new ChannelWatermark
-                            {
-                                Mode = intermittent
-                                    ? ChannelWatermarkMode.Intermittent
-                                    : ChannelWatermarkMode.Permanent,
-                                DurationSeconds = intermittent ? 15 : 0,
-                                FrequencyMinutes = intermittent ? 10 : 0,
-                                Location = location,
-                                Size = scaled ? ChannelWatermarkSize.Scaled : ChannelWatermarkSize.ActualSize,
-                                WidthPercent = scaled ? 20 : 0,
-                                Opacity = opacity,
-                                HorizontalMarginPercent = 7,
-                                VerticalMarginPercent = 5
-                            }),
+                        Some(watermark),
+                        maybeFadePoints,
                         new Resolution { Width = 1920, Height = 1080 },
                         None)
                     .WithDeinterlace(deinterlace)

ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs

@@ -22,6 +22,7 @@ namespace ErsatzTV.Core.FFmpeg
         private IDisplaySize _resolution;
         private Option<IDisplaySize> _scaleToSize = None;
         private Option<ChannelWatermark> _watermark;
+        private Option<List<FadePoint>> _maybeFadePoints = None;
         private Option<int> _watermarkIndex;
         private string _pixelFormat;
         private string _videoEncoder;
@@ -86,10 +87,12 @@ namespace ErsatzTV.Core.FFmpeg
 
         public FFmpegComplexFilterBuilder WithWatermark(
             Option<ChannelWatermark> watermark,
+            Option<List<FadePoint>> maybeFadePoints,
             IDisplaySize resolution,
             Option<int> watermarkIndex)
         {
             _watermark = watermark;
+            _maybeFadePoints = maybeFadePoints;
             _resolution = resolution;
             _watermarkIndex = watermarkIndex;
             return this;
@@ -155,7 +158,7 @@ namespace ErsatzTV.Core.FFmpeg
 
             var audioFilterQueue = new List<string>();
             var videoFilterQueue = new List<string>();
-            string watermarkPreprocess = string.Empty;
+            var watermarkPreprocess = new List<string>();
             string watermarkOverlay = string.Empty;
             
             if (_normalizeLoudness)
@@ -281,7 +284,7 @@ namespace ErsatzTV.Core.FFmpeg
                     };
                     videoFilterQueue.Add(format);
                 }
-                
+
                 if (_boxBlur)
                 {
                     videoFilterQueue.Add("boxblur=40");
@@ -294,9 +297,11 @@ namespace ErsatzTV.Core.FFmpeg
 
                 foreach (ChannelWatermark watermark in _watermark)
                 {
-                    string enable = watermark.Mode == ChannelWatermarkMode.Intermittent
-                        ? $":enable='lt(mod(mod(time(0),60*60),{watermark.FrequencyMinutes}*60),{watermark.DurationSeconds})'"
-                        : string.Empty;
+                    if (watermark.Opacity != 100 || _maybeFadePoints.Map(fp => fp.Count).IfNone(0) > 0)
+                    {
+                        const string FORMATS = "yuva420p|yuva444p|yuva422p|rgba|abgr|bgra|gbrap|ya8";
+                        watermarkPreprocess.Add($"format={FORMATS}");
+                    }
 
                     double horizontalMargin = Math.Round(watermark.HorizontalMarginPercent / 100.0 * _resolution.Width);
                     double verticalMargin = Math.Round(watermark.VerticalMarginPercent / 100.0 * _resolution.Height);
@@ -313,26 +318,29 @@ namespace ErsatzTV.Core.FFmpeg
                         _ => $"x=W-w-{horizontalMargin}:y=H-h-{verticalMargin}"
                     };
 
+                    if (watermark.Opacity != 100)
+                    {
+                        double opacity = watermark.Opacity / 100.0;
+                        watermarkPreprocess.Add($"colorchannelmixer=aa={opacity:F2}");
+                    }
+
                     if (watermark.Size == ChannelWatermarkSize.Scaled)
                     {
                         double width = Math.Round(watermark.WidthPercent / 100.0 * _resolution.Width);
-                        watermarkPreprocess = $"scale={width}:-1";
+                        watermarkPreprocess.Add($"scale={width}:-1");
                     }
-                    
-                    if (watermark.Opacity != 100)
-                    {
-                        const string FORMATS = "yuva420p|yuva444p|yuva422p|rgba|abgr|bgra|gbrap|ya8"; 
-                        string join = string.Empty;
-                        double opacity = watermark.Opacity / 100.0;
-                        if (!string.IsNullOrWhiteSpace(watermarkPreprocess))
-                        {
-                            join = ",";
-                        }
 
-                        watermarkPreprocess = $"format={FORMATS},colorchannelmixer=aa={opacity:F2}{join}{watermarkPreprocess}";
+                    foreach (List<FadePoint> fadePoints in _maybeFadePoints)
+                    {
+                        watermarkPreprocess.AddRange(fadePoints.Map(fp => fp.ToFilter()));
                     }
 
-                    watermarkOverlay = $"overlay={position}{enable}";
+                    watermarkOverlay = $"overlay={position}";
+
+                    if (_maybeFadePoints.Map(fp => fp.Count).IfNone(0) > 0)
+                    {
+                        watermarkOverlay += ",format=yuv420p";
+                    }
                 }
             }
 
@@ -412,9 +420,10 @@ namespace ErsatzTV.Core.FFmpeg
                         watermarkLabel = $"[{audioInput+1}:{index}]";
                     }
 
-                    if (!string.IsNullOrWhiteSpace(watermarkPreprocess))
+                    if (watermarkPreprocess.Count > 0)
                     {
-                        complexFilter.Append($"{watermarkLabel}{watermarkPreprocess}[wmp];");
+                        var joined = string.Join(",", watermarkPreprocess);
+                        complexFilter.Append($"{watermarkLabel}{joined}[wmp];");
                         watermarkLabel = "[wmp]";
                     }
 

ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs

@@ -216,6 +216,7 @@ namespace ErsatzTV.Core.FFmpeg
 
         public FFmpegProcessBuilder WithWatermark(
             Option<WatermarkOptions> watermarkOptions,
+            Option<List<FadePoint>> maybeFadePoints,
             IDisplaySize resolution)
         {
             foreach (WatermarkOptions options in watermarkOptions)
@@ -228,11 +229,19 @@ namespace ErsatzTV.Core.FFmpeg
                         _arguments.Add("0");
                     }
 
+                    // when we have fade points, we need to loop the static watermark image
+                    else if (maybeFadePoints.Map(fp => fp.Count).IfNone(0) > 0)
+                    {
+                        _arguments.Add("-stream_loop");
+                        _arguments.Add("-1");
+                    }
+
                     _arguments.Add("-i");
                     _arguments.Add(path);
 
                     _complexFilterBuilder = _complexFilterBuilder.WithWatermark(
                         options.Watermark,
+                        maybeFadePoints,
                         resolution,
                         options.ImageStreamIndex);
                 }

ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Threading.Tasks;
@@ -72,6 +73,28 @@ namespace ErsatzTV.Core.FFmpeg
             Option<WatermarkOptions> watermarkOptions =
                 await GetWatermarkOptions(channel, globalWatermark, videoVersion, None, None);
 
+            Option<List<FadePoint>> maybeFadePoints = watermarkOptions
+                .Map(o => o.Watermark)
+                .Flatten()
+                .Where(wm => wm.Mode == ChannelWatermarkMode.Intermittent)
+                .Map(
+                    wm =>
+                        WatermarkCalculator.CalculateFadePoints(
+                            start,
+                            inPoint,
+                            outPoint,
+                            playbackSettings.StreamSeek,
+                            wm.FrequencyMinutes,
+                            wm.DurationSeconds));
+
+            // foreach (List<FadePoint> fadePoints in maybeFadePoints)
+            // {
+            //     foreach (FadePoint fadePoint in fadePoints)
+            //     {
+            //         _logger.LogDebug("Fade point filter: {FadePointFilter}", fadePoint.ToFilter());
+            //     }
+            // }
+
             FFmpegProcessBuilder builder = new FFmpegProcessBuilder(ffmpegPath, saveReports, _logger)
                 .WithThreads(playbackSettings.ThreadCount)
                 .WithVaapiDriver(vaapiDriver, vaapiDevice)
@@ -90,7 +113,7 @@ namespace ErsatzTV.Core.FFmpeg
                     playbackSettings.VideoDecoder,
                     videoStream.Codec,
                     videoStream.PixelFormat)
-                .WithWatermark(watermarkOptions, channel.FFmpegProfile.Resolution)
+                .WithWatermark(watermarkOptions, maybeFadePoints, channel.FFmpegProfile.Resolution)
                 .WithVideoTrackTimeScale(playbackSettings.VideoTrackTimeScale)
                 .WithAlignedAudio(videoPath == audioPath ? playbackSettings.AudioDuration : Option<TimeSpan>.None)
                 .WithNormalizeLoudness(playbackSettings.NormalizeLoudness);
@@ -345,7 +368,7 @@ namespace ErsatzTV.Core.FFmpeg
                     .WithQuiet()
                     .WithFormatFlags(playbackSettings.FormatFlags)
                     .WithSongInput(videoPath, videoStream.Codec, videoStream.PixelFormat, boxBlur)
-                    .WithWatermark(watermarkOptions, channel.FFmpegProfile.Resolution)
+                    .WithWatermark(watermarkOptions, None, channel.FFmpegProfile.Resolution)
                     .WithSubtitleFile(subtitleFile);
 
                 foreach (IDisplaySize scaledSize in scalePlaybackSettings.ScaledSize)

ErsatzTV.Core/FFmpeg/FadePoint.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Globalization;
+
+namespace ErsatzTV.Core.FFmpeg;
+
+public abstract record FadePoint(TimeSpan Time, string InOut)
+{
+    public TimeSpan EnableStart { get; set; }
+    public TimeSpan EnableFinish { get; set; }
+
+    public string ToFilter()
+    {
+        var startTime = Time.TotalSeconds.ToString(NumberFormatInfo.InvariantInfo);
+        var enableStart = EnableStart.TotalSeconds.ToString(NumberFormatInfo.InvariantInfo);
+        var enableFinish = EnableFinish.TotalSeconds.ToString(NumberFormatInfo.InvariantInfo);
+
+        return $"fade={InOut}:st={startTime}:d=1:alpha=1:enable='between(t,{enableStart},{enableFinish})'";
+    }
+}
+
+public record FadeInPoint(TimeSpan Time) : FadePoint(Time, "in");
+
+public record FadeOutPoint(TimeSpan Time) : FadePoint(Time, "out");

ErsatzTV.Core/FFmpeg/WatermarkCalculator.cs

@@ -0,0 +1,84 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using LanguageExt;
+
+namespace ErsatzTV.Core.FFmpeg;
+
+public static class WatermarkCalculator
+{
+    public static List<FadePoint> CalculateFadePoints(
+        DateTimeOffset itemStartTime,
+        TimeSpan inPoint,
+        TimeSpan outPoint,
+        Option<TimeSpan> streamSeek,
+        int frequencyMinutes,
+        int durationSeconds)
+    {
+        var result = new List<FadePoint>();
+        
+        TimeSpan duration = outPoint - inPoint;
+        DateTimeOffset itemFinishTime = itemStartTime + duration;
+
+        DateTimeOffset start = itemStartTime.AddMinutes(-16);
+
+        // find the next whole minute
+        if (start.Second > 0 || start.Millisecond > 0)
+        {
+            start = start.AddMinutes(1);
+            start = start.AddSeconds(-start.Second);
+            start = start.AddMilliseconds(-start.Millisecond);
+        }
+
+        DateTimeOffset finish = itemFinishTime;
+
+        // find the previous whole minute
+        if (finish.Second > 0 || finish.Millisecond > 0)
+        {
+            finish = finish.AddSeconds(-finish.Second);
+            finish = finish.AddMilliseconds(-finish.Millisecond);
+        }
+
+        DateTimeOffset current = start;
+        while (current <= finish)
+        {
+            current = current.AddMinutes(1);
+            if (current.Minute % frequencyMinutes == 0)
+            {
+                TimeSpan fadeInTime = inPoint + (current - itemStartTime);
+
+                result.Add(new FadeInPoint(fadeInTime));
+                result.Add(new FadeOutPoint(fadeInTime.Add(TimeSpan.FromSeconds(durationSeconds))));
+            }
+        }
+        
+        // if we're seeking, subtract the seek from each item and return that
+        foreach (TimeSpan ss in streamSeek)
+        {
+            result = result.Map(fp => fp with { Time = fp.Time - ss }).ToList();
+        }
+
+        // trim points that have already passed
+        result.RemoveAll(fp => fp.Time < TimeSpan.Zero);
+        
+        // trim points that are past the end
+        result.RemoveAll(fp => fp.Time >= outPoint);
+
+        if (result.Any())
+        {
+            for (var i = 0; i < result.Count; i++)
+            {
+                result[i].EnableStart = i == 0 ? TimeSpan.Zero : result[i - 1].Time.Add(TimeSpan.FromSeconds(1));
+            }
+
+            for (var i = 0; i < result.Count; i++)
+            {
+                result[i].EnableFinish = i == result.Count - 1
+                    ? outPoint
+                    : result[i + 1].Time.Subtract(TimeSpan.FromSeconds(1));
+            }
+        }
+
+        return result;
+    }
+}