From eed9f60273f08fbf10a90c32fc6752c3dfe85760 Mon Sep 17 00:00:00 2001 From: Jason Dove Date: Mon, 31 Jan 2022 12:31:56 -0600 Subject: [PATCH] fade in and fade out intermittent watermarks (#607) * first pass at fading in/out overlay * fix tests * update changelog --- CHANGELOG.md | 4 + .../FFmpeg/FFmpegComplexFilterBuilderTests.cs | 46 ++++++---- .../FFmpeg/FFmpegComplexFilterBuilder.cs | 49 ++++++----- ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs | 9 ++ ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs | 27 +++++- ErsatzTV.Core/FFmpeg/FadePoint.cs | 23 +++++ ErsatzTV.Core/FFmpeg/WatermarkCalculator.cs | 84 +++++++++++++++++++ 7 files changed, 204 insertions(+), 38 deletions(-) create mode 100644 ErsatzTV.Core/FFmpeg/FadePoint.cs create mode 100644 ErsatzTV.Core/FFmpeg/WatermarkCalculator.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index a954fd0de..b85b96360 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Fixed - Normalize smart quotes in search queries as they are unsupported by the search library +- Fix incorrect watermark time calculations caused by working ahead in `HLS Segmenter` ### Added - Include `Series` category tag for all episodes in XMLTV - Include movie, episode (show), music video (artist) genres as `category` tags in XMLTV +### Changed +- Intermittent watermarks will now fade in and out + ## [0.4.0-alpha] - 2022-01-29 ### Fixed - Fix m3u `mode` query param to properly override streaming mode for all channels diff --git a/ErsatzTV.Core.Tests/FFmpeg/FFmpegComplexFilterBuilderTests.cs b/ErsatzTV.Core.Tests/FFmpeg/FFmpegComplexFilterBuilderTests.cs index 0520146c5..6c757095f 100644 --- a/ErsatzTV.Core.Tests/FFmpeg/FFmpegComplexFilterBuilderTests.cs +++ b/ErsatzTV.Core.Tests/FFmpeg/FFmpegComplexFilterBuilderTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using ErsatzTV.Core.Domain; using ErsatzTV.Core.FFmpeg; using FluentAssertions; @@ -183,7 +184,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg ChannelWatermarkLocation.TopLeft, false, 100, - "[0:0][1:v]overlay=x=134:y=54:enable='lt(mod(mod(time(0),60*60),10*60),15)'[v]", + "[1:v]format=yuva420p|yuva444p|yuva422p|rgba|abgr|bgra|gbrap|ya8,fade=in:st=300:d=1:alpha=1:enable='between(t,0,314)',fade=out:st=315:d=1:alpha=1:enable='between(t,301,899)',fade=in:st=900:d=1:alpha=1:enable='between(t,316,914)',fade=out:st=915:d=1:alpha=1:enable='between(t,901,1499)',fade=in:st=1500:d=1:alpha=1:enable='between(t,916,1514)',fade=out:st=1515:d=1:alpha=1:enable='between(t,1501,2099)',fade=in:st=2100:d=1:alpha=1:enable='between(t,1516,2114)',fade=out:st=2115:d=1:alpha=1:enable='between(t,2101,2699)',fade=in:st=2700:d=1:alpha=1:enable='between(t,2116,2714)',fade=out:st=2715:d=1:alpha=1:enable='between(t,2701,3300)'[wmp];[0:0][wmp]overlay=x=134:y=54,format=yuv420p[v]", "0:1", "[v]")] [TestCase( @@ -257,23 +258,36 @@ namespace ErsatzTV.Core.Tests.FFmpeg string expectedAudioLabel, string expectedVideoLabel) { + var watermark = new ChannelWatermark + { + Mode = intermittent + ? ChannelWatermarkMode.Intermittent + : ChannelWatermarkMode.Permanent, + DurationSeconds = intermittent ? 15 : 0, + FrequencyMinutes = intermittent ? 10 : 0, + Location = location, + Size = scaled ? ChannelWatermarkSize.Scaled : ChannelWatermarkSize.ActualSize, + WidthPercent = scaled ? 20 : 0, + Opacity = opacity, + HorizontalMarginPercent = 7, + VerticalMarginPercent = 5 + }; + + Option> maybeFadePoints = watermark.Mode == ChannelWatermarkMode.Intermittent + ? Some( + WatermarkCalculator.CalculateFadePoints( + new DateTimeOffset(2022, 01, 31, 12, 25, 0, TimeSpan.FromHours(-5)), + TimeSpan.Zero, + TimeSpan.FromMinutes(55), + TimeSpan.Zero, + watermark.FrequencyMinutes, + watermark.DurationSeconds)) + : None; + FFmpegComplexFilterBuilder builder = new FFmpegComplexFilterBuilder() .WithWatermark( - Some( - new ChannelWatermark - { - Mode = intermittent - ? ChannelWatermarkMode.Intermittent - : ChannelWatermarkMode.Permanent, - DurationSeconds = intermittent ? 15 : 0, - FrequencyMinutes = intermittent ? 10 : 0, - Location = location, - Size = scaled ? ChannelWatermarkSize.Scaled : ChannelWatermarkSize.ActualSize, - WidthPercent = scaled ? 20 : 0, - Opacity = opacity, - HorizontalMarginPercent = 7, - VerticalMarginPercent = 5 - }), + Some(watermark), + maybeFadePoints, new Resolution { Width = 1920, Height = 1080 }, None) .WithDeinterlace(deinterlace) diff --git a/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs b/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs index 75b1868d9..229703389 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs @@ -22,6 +22,7 @@ namespace ErsatzTV.Core.FFmpeg private IDisplaySize _resolution; private Option _scaleToSize = None; private Option _watermark; + private Option> _maybeFadePoints = None; private Option _watermarkIndex; private string _pixelFormat; private string _videoEncoder; @@ -86,10 +87,12 @@ namespace ErsatzTV.Core.FFmpeg public FFmpegComplexFilterBuilder WithWatermark( Option watermark, + Option> maybeFadePoints, IDisplaySize resolution, Option watermarkIndex) { _watermark = watermark; + _maybeFadePoints = maybeFadePoints; _resolution = resolution; _watermarkIndex = watermarkIndex; return this; @@ -155,7 +158,7 @@ namespace ErsatzTV.Core.FFmpeg var audioFilterQueue = new List(); var videoFilterQueue = new List(); - string watermarkPreprocess = string.Empty; + var watermarkPreprocess = new List(); string watermarkOverlay = string.Empty; if (_normalizeLoudness) @@ -281,7 +284,7 @@ namespace ErsatzTV.Core.FFmpeg }; videoFilterQueue.Add(format); } - + if (_boxBlur) { videoFilterQueue.Add("boxblur=40"); @@ -294,9 +297,11 @@ namespace ErsatzTV.Core.FFmpeg foreach (ChannelWatermark watermark in _watermark) { - string enable = watermark.Mode == ChannelWatermarkMode.Intermittent - ? $":enable='lt(mod(mod(time(0),60*60),{watermark.FrequencyMinutes}*60),{watermark.DurationSeconds})'" - : string.Empty; + if (watermark.Opacity != 100 || _maybeFadePoints.Map(fp => fp.Count).IfNone(0) > 0) + { + const string FORMATS = "yuva420p|yuva444p|yuva422p|rgba|abgr|bgra|gbrap|ya8"; + watermarkPreprocess.Add($"format={FORMATS}"); + } double horizontalMargin = Math.Round(watermark.HorizontalMarginPercent / 100.0 * _resolution.Width); double verticalMargin = Math.Round(watermark.VerticalMarginPercent / 100.0 * _resolution.Height); @@ -313,26 +318,29 @@ namespace ErsatzTV.Core.FFmpeg _ => $"x=W-w-{horizontalMargin}:y=H-h-{verticalMargin}" }; + if (watermark.Opacity != 100) + { + double opacity = watermark.Opacity / 100.0; + watermarkPreprocess.Add($"colorchannelmixer=aa={opacity:F2}"); + } + if (watermark.Size == ChannelWatermarkSize.Scaled) { double width = Math.Round(watermark.WidthPercent / 100.0 * _resolution.Width); - watermarkPreprocess = $"scale={width}:-1"; + watermarkPreprocess.Add($"scale={width}:-1"); } - - if (watermark.Opacity != 100) - { - const string FORMATS = "yuva420p|yuva444p|yuva422p|rgba|abgr|bgra|gbrap|ya8"; - string join = string.Empty; - double opacity = watermark.Opacity / 100.0; - if (!string.IsNullOrWhiteSpace(watermarkPreprocess)) - { - join = ","; - } - watermarkPreprocess = $"format={FORMATS},colorchannelmixer=aa={opacity:F2}{join}{watermarkPreprocess}"; + foreach (List fadePoints in _maybeFadePoints) + { + watermarkPreprocess.AddRange(fadePoints.Map(fp => fp.ToFilter())); } - watermarkOverlay = $"overlay={position}{enable}"; + watermarkOverlay = $"overlay={position}"; + + if (_maybeFadePoints.Map(fp => fp.Count).IfNone(0) > 0) + { + watermarkOverlay += ",format=yuv420p"; + } } } @@ -412,9 +420,10 @@ namespace ErsatzTV.Core.FFmpeg watermarkLabel = $"[{audioInput+1}:{index}]"; } - if (!string.IsNullOrWhiteSpace(watermarkPreprocess)) + if (watermarkPreprocess.Count > 0) { - complexFilter.Append($"{watermarkLabel}{watermarkPreprocess}[wmp];"); + var joined = string.Join(",", watermarkPreprocess); + complexFilter.Append($"{watermarkLabel}{joined}[wmp];"); watermarkLabel = "[wmp]"; } diff --git a/ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs b/ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs index 5b43f8047..7aeddbd3d 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs @@ -216,6 +216,7 @@ namespace ErsatzTV.Core.FFmpeg public FFmpegProcessBuilder WithWatermark( Option watermarkOptions, + Option> maybeFadePoints, IDisplaySize resolution) { foreach (WatermarkOptions options in watermarkOptions) @@ -228,11 +229,19 @@ namespace ErsatzTV.Core.FFmpeg _arguments.Add("0"); } + // when we have fade points, we need to loop the static watermark image + else if (maybeFadePoints.Map(fp => fp.Count).IfNone(0) > 0) + { + _arguments.Add("-stream_loop"); + _arguments.Add("-1"); + } + _arguments.Add("-i"); _arguments.Add(path); _complexFilterBuilder = _complexFilterBuilder.WithWatermark( options.Watermark, + maybeFadePoints, resolution, options.ImageStreamIndex); } diff --git a/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs b/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs index fece5d1a8..c4348a8bc 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Threading.Tasks; @@ -72,6 +73,28 @@ namespace ErsatzTV.Core.FFmpeg Option watermarkOptions = await GetWatermarkOptions(channel, globalWatermark, videoVersion, None, None); + Option> maybeFadePoints = watermarkOptions + .Map(o => o.Watermark) + .Flatten() + .Where(wm => wm.Mode == ChannelWatermarkMode.Intermittent) + .Map( + wm => + WatermarkCalculator.CalculateFadePoints( + start, + inPoint, + outPoint, + playbackSettings.StreamSeek, + wm.FrequencyMinutes, + wm.DurationSeconds)); + + // foreach (List fadePoints in maybeFadePoints) + // { + // foreach (FadePoint fadePoint in fadePoints) + // { + // _logger.LogDebug("Fade point filter: {FadePointFilter}", fadePoint.ToFilter()); + // } + // } + FFmpegProcessBuilder builder = new FFmpegProcessBuilder(ffmpegPath, saveReports, _logger) .WithThreads(playbackSettings.ThreadCount) .WithVaapiDriver(vaapiDriver, vaapiDevice) @@ -90,7 +113,7 @@ namespace ErsatzTV.Core.FFmpeg playbackSettings.VideoDecoder, videoStream.Codec, videoStream.PixelFormat) - .WithWatermark(watermarkOptions, channel.FFmpegProfile.Resolution) + .WithWatermark(watermarkOptions, maybeFadePoints, channel.FFmpegProfile.Resolution) .WithVideoTrackTimeScale(playbackSettings.VideoTrackTimeScale) .WithAlignedAudio(videoPath == audioPath ? playbackSettings.AudioDuration : Option.None) .WithNormalizeLoudness(playbackSettings.NormalizeLoudness); @@ -345,7 +368,7 @@ namespace ErsatzTV.Core.FFmpeg .WithQuiet() .WithFormatFlags(playbackSettings.FormatFlags) .WithSongInput(videoPath, videoStream.Codec, videoStream.PixelFormat, boxBlur) - .WithWatermark(watermarkOptions, channel.FFmpegProfile.Resolution) + .WithWatermark(watermarkOptions, None, channel.FFmpegProfile.Resolution) .WithSubtitleFile(subtitleFile); foreach (IDisplaySize scaledSize in scalePlaybackSettings.ScaledSize) diff --git a/ErsatzTV.Core/FFmpeg/FadePoint.cs b/ErsatzTV.Core/FFmpeg/FadePoint.cs new file mode 100644 index 000000000..391987827 --- /dev/null +++ b/ErsatzTV.Core/FFmpeg/FadePoint.cs @@ -0,0 +1,23 @@ +using System; +using System.Globalization; + +namespace ErsatzTV.Core.FFmpeg; + +public abstract record FadePoint(TimeSpan Time, string InOut) +{ + public TimeSpan EnableStart { get; set; } + public TimeSpan EnableFinish { get; set; } + + public string ToFilter() + { + var startTime = Time.TotalSeconds.ToString(NumberFormatInfo.InvariantInfo); + var enableStart = EnableStart.TotalSeconds.ToString(NumberFormatInfo.InvariantInfo); + var enableFinish = EnableFinish.TotalSeconds.ToString(NumberFormatInfo.InvariantInfo); + + return $"fade={InOut}:st={startTime}:d=1:alpha=1:enable='between(t,{enableStart},{enableFinish})'"; + } +} + +public record FadeInPoint(TimeSpan Time) : FadePoint(Time, "in"); + +public record FadeOutPoint(TimeSpan Time) : FadePoint(Time, "out"); diff --git a/ErsatzTV.Core/FFmpeg/WatermarkCalculator.cs b/ErsatzTV.Core/FFmpeg/WatermarkCalculator.cs new file mode 100644 index 000000000..4abf02e55 --- /dev/null +++ b/ErsatzTV.Core/FFmpeg/WatermarkCalculator.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using LanguageExt; + +namespace ErsatzTV.Core.FFmpeg; + +public static class WatermarkCalculator +{ + public static List CalculateFadePoints( + DateTimeOffset itemStartTime, + TimeSpan inPoint, + TimeSpan outPoint, + Option streamSeek, + int frequencyMinutes, + int durationSeconds) + { + var result = new List(); + + TimeSpan duration = outPoint - inPoint; + DateTimeOffset itemFinishTime = itemStartTime + duration; + + DateTimeOffset start = itemStartTime.AddMinutes(-16); + + // find the next whole minute + if (start.Second > 0 || start.Millisecond > 0) + { + start = start.AddMinutes(1); + start = start.AddSeconds(-start.Second); + start = start.AddMilliseconds(-start.Millisecond); + } + + DateTimeOffset finish = itemFinishTime; + + // find the previous whole minute + if (finish.Second > 0 || finish.Millisecond > 0) + { + finish = finish.AddSeconds(-finish.Second); + finish = finish.AddMilliseconds(-finish.Millisecond); + } + + DateTimeOffset current = start; + while (current <= finish) + { + current = current.AddMinutes(1); + if (current.Minute % frequencyMinutes == 0) + { + TimeSpan fadeInTime = inPoint + (current - itemStartTime); + + result.Add(new FadeInPoint(fadeInTime)); + result.Add(new FadeOutPoint(fadeInTime.Add(TimeSpan.FromSeconds(durationSeconds)))); + } + } + + // if we're seeking, subtract the seek from each item and return that + foreach (TimeSpan ss in streamSeek) + { + result = result.Map(fp => fp with { Time = fp.Time - ss }).ToList(); + } + + // trim points that have already passed + result.RemoveAll(fp => fp.Time < TimeSpan.Zero); + + // trim points that are past the end + result.RemoveAll(fp => fp.Time >= outPoint); + + if (result.Any()) + { + for (var i = 0; i < result.Count; i++) + { + result[i].EnableStart = i == 0 ? TimeSpan.Zero : result[i - 1].Time.Add(TimeSpan.FromSeconds(1)); + } + + for (var i = 0; i < result.Count; i++) + { + result[i].EnableFinish = i == result.Count - 1 + ? outPoint + : result[i + 1].Time.Subtract(TimeSpan.FromSeconds(1)); + } + } + + return result; + } +}