From 6d32dac51bdb56c1466fc215147677a123258f27 Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Fri, 15 Aug 2025 21:59:07 -0500 Subject: [PATCH] fix graphics engine opacity (#2323) * fix skia opacity wip * fix graphics engine opacity --- .../Filter/OverlayGraphicsEngineFilter.cs | 2 +- ErsatzTV.FFmpeg/Filter/UnpremultiplyFilter.cs | 8 ++++++ .../Vaapi/OverlayGraphicsEngineVaapiFilter.cs | 2 +- .../Pipeline/NvidiaPipelineBuilder.cs | 3 +++ .../Pipeline/VaapiPipelineBuilder.cs | 3 +++ .../Streaming/Graphics/GraphicsEngine.cs | 26 +++++++++++++------ .../Graphics/Image/ImageElementBase.cs | 26 ------------------- 7 files changed, 34 insertions(+), 36 deletions(-) create mode 100644 ErsatzTV.FFmpeg/Filter/UnpremultiplyFilter.cs diff --git a/ErsatzTV.FFmpeg/Filter/OverlayGraphicsEngineFilter.cs b/ErsatzTV.FFmpeg/Filter/OverlayGraphicsEngineFilter.cs index 408baefe3..746ffbae4 100644 --- a/ErsatzTV.FFmpeg/Filter/OverlayGraphicsEngineFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/OverlayGraphicsEngineFilter.cs @@ -4,7 +4,7 @@ namespace ErsatzTV.FFmpeg.Filter; public class OverlayGraphicsEngineFilter(IPixelFormat outputPixelFormat) : BaseFilter { - public override string Filter => $"overlay=format={(outputPixelFormat.BitDepth == 10 ? '1' : '0')}"; + public override string Filter => $"overlay=format={(outputPixelFormat.BitDepth == 10 ? '1' : '0')}:alpha=premultiplied"; public override FrameState NextState(FrameState currentState) => currentState with { FrameDataLocation = FrameDataLocation.Software }; diff --git a/ErsatzTV.FFmpeg/Filter/UnpremultiplyFilter.cs b/ErsatzTV.FFmpeg/Filter/UnpremultiplyFilter.cs new file mode 100644 index 000000000..8f2102250 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/UnpremultiplyFilter.cs @@ -0,0 +1,8 @@ +namespace ErsatzTV.FFmpeg.Filter; + +public class UnpremultiplyFilter : BaseFilter +{ + public override string Filter => "unpremultiply=inplace=1"; + + public override FrameState NextState(FrameState currentState) => currentState; +} diff --git a/ErsatzTV.FFmpeg/Filter/Vaapi/OverlayGraphicsEngineVaapiFilter.cs b/ErsatzTV.FFmpeg/Filter/Vaapi/OverlayGraphicsEngineVaapiFilter.cs index cd018db0e..ba386e2ec 100644 --- a/ErsatzTV.FFmpeg/Filter/Vaapi/OverlayGraphicsEngineVaapiFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Vaapi/OverlayGraphicsEngineVaapiFilter.cs @@ -7,7 +7,7 @@ public class OverlayGraphicsEngineVaapiFilter(FrameState currentState, IPixelFor public override string Filter => currentState.FrameDataLocation is FrameDataLocation.Hardware ? "overlay_vaapi" - : $"overlay=format={(outputPixelFormat.BitDepth == 10 ? '1' : '0')}"; + : $"overlay=format={(outputPixelFormat.BitDepth == 10 ? '1' : '0')}:alpha=premultiplied"; public override FrameState NextState(FrameState currentState) => currentState; } diff --git a/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs index ba68190f8..b3d67c69d 100644 --- a/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs @@ -654,6 +654,9 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder { graphicsEngine.FilterSteps.Add(new PixelFormatFilter(new PixelFormatYuva420P())); + // overlay_cuda expects straight alpha + graphicsEngine.FilterSteps.Add(new UnpremultiplyFilter()); + graphicsEngine.FilterSteps.Add( new HardwareUploadCudaFilter(currentState with { FrameDataLocation = FrameDataLocation.Software })); diff --git a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs index 3ddf09142..a903ae42e 100644 --- a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs @@ -557,6 +557,9 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder if (currentState.FrameDataLocation is FrameDataLocation.Hardware) { + // overlay_vaapi expects straight alpha + graphicsEngine.FilterSteps.Add(new UnpremultiplyFilter()); + graphicsEngine.FilterSteps.Add(new HardwareUploadVaapiFilter(false)); } diff --git a/ErsatzTV.Infrastructure/Streaming/Graphics/GraphicsEngine.cs b/ErsatzTV.Infrastructure/Streaming/Graphics/GraphicsEngine.cs index 1c8ca4e64..cfe068e32 100644 --- a/ErsatzTV.Infrastructure/Streaming/Graphics/GraphicsEngine.cs +++ b/ErsatzTV.Infrastructure/Streaming/Graphics/GraphicsEngine.cs @@ -173,15 +173,25 @@ public class GraphicsEngine( } // draw each element - foreach (var preparedImage in preparedElementImages) + using (var paint = new SKPaint()) { - using var paint = new SKPaint(); - paint.Color = new SKColor(255, 255, 255, (byte)(preparedImage.Opacity * 255)); - canvas.DrawBitmap(preparedImage.Image, new SKPoint(preparedImage.Point.X, preparedImage.Point.Y), paint); - - if (preparedImage.Dispose) + foreach (var preparedImage in preparedElementImages) { - preparedImage.Image.Dispose(); + using (var colorFilter = SKColorFilter.CreateBlendMode( + SKColors.White.WithAlpha((byte)(preparedImage.Opacity * 255)), + SKBlendMode.Modulate)) + { + paint.ColorFilter = colorFilter; + canvas.DrawBitmap( + preparedImage.Image, + new SKPoint(preparedImage.Point.X, preparedImage.Point.Y), + paint); + } + + if (preparedImage.Dispose) + { + preparedImage.Image.Dispose(); + } } } @@ -213,4 +223,4 @@ public class GraphicsEngine( } } } -} \ No newline at end of file +} diff --git a/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElementBase.cs b/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElementBase.cs index f75570c07..e4e75fb2d 100644 --- a/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElementBase.cs +++ b/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElementBase.cs @@ -88,35 +88,13 @@ public abstract class ImageElementBase : GraphicsElement, IDisposable } } - protected static async Task GetImageStream(string image, CancellationToken cancellationToken) - { - Stream imageStream; - - bool isRemoteUri = Uri.TryCreate(image, UriKind.Absolute, out var uriResult) - && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps); - - if (isRemoteUri) - { - using var client = new HttpClient(); - imageStream = new MemoryStream(await client.GetByteArrayAsync(uriResult, cancellationToken)); - } - else - { - imageStream = new FileStream(image!, FileMode.Open, FileAccess.Read); - } - - return imageStream; - } - protected static SKBitmap ToSkiaBitmap(Image image) { - // Force a known pixel format using Image rgbaImage = image.CloneAs(); int width = rgbaImage.Width; int height = rgbaImage.Height; - // Allocate destination SKBitmap with RGBA8888 (straight alpha) var info = new SKImageInfo(width, height, SKColorType.Rgba8888, SKAlphaType.Unpremul); var skBitmap = new SKBitmap(info); if (!skBitmap.TryAllocPixels(info)) @@ -125,16 +103,12 @@ public abstract class ImageElementBase : GraphicsElement, IDisposable throw new InvalidOperationException("Failed to allocate pixels for SKBitmap."); } - // Pull pixel data from ImageSharp into a managed byte[] - // (Rgba32 is 4 bytes, so length = width * height * 4) var pixelArray = new Rgba32[width * height]; rgbaImage.CopyPixelDataTo(pixelArray); - // Convert the Rgba32[] to a byte[] without unsafe code var bytes = new byte[pixelArray.Length * 4]; MemoryMarshal.AsBytes(pixelArray.AsSpan()).CopyTo(bytes); - // Copy into Skia's pixel buffer IntPtr dstPtr = skBitmap.GetPixels(out _); Marshal.Copy(bytes, 0, dstPtr, bytes.Length);