fix graphics engine opacity (#2323)

* fix skia opacity wip

* fix graphics engine opacity

ErsatzTV.FFmpeg/Filter/OverlayGraphicsEngineFilter.cs

@@ -4,7 +4,7 @@ namespace ErsatzTV.FFmpeg.Filter;
 
 public class OverlayGraphicsEngineFilter(IPixelFormat outputPixelFormat) : BaseFilter
 {
-    public override string Filter => $"overlay=format={(outputPixelFormat.BitDepth == 10 ? '1' : '0')}";
+    public override string Filter => $"overlay=format={(outputPixelFormat.BitDepth == 10 ? '1' : '0')}:alpha=premultiplied";
 
     public override FrameState NextState(FrameState currentState) =>
         currentState with { FrameDataLocation = FrameDataLocation.Software };

ErsatzTV.FFmpeg/Filter/UnpremultiplyFilter.cs

@@ -0,0 +1,8 @@
+namespace ErsatzTV.FFmpeg.Filter;
+
+public class UnpremultiplyFilter : BaseFilter
+{
+    public override string Filter => "unpremultiply=inplace=1";
+
+    public override FrameState NextState(FrameState currentState) => currentState;
+}

ErsatzTV.FFmpeg/Filter/Vaapi/OverlayGraphicsEngineVaapiFilter.cs

@@ -7,7 +7,7 @@ public class OverlayGraphicsEngineVaapiFilter(FrameState currentState, IPixelFor
     public override string Filter =>
         currentState.FrameDataLocation is FrameDataLocation.Hardware
             ? "overlay_vaapi"
-            : $"overlay=format={(outputPixelFormat.BitDepth == 10 ? '1' : '0')}";
+            : $"overlay=format={(outputPixelFormat.BitDepth == 10 ? '1' : '0')}:alpha=premultiplied";
 
     public override FrameState NextState(FrameState currentState) => currentState;
 }

ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs

@@ -654,6 +654,9 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
         {
             graphicsEngine.FilterSteps.Add(new PixelFormatFilter(new PixelFormatYuva420P()));
 
+            // overlay_cuda expects straight alpha
+            graphicsEngine.FilterSteps.Add(new UnpremultiplyFilter());
+
             graphicsEngine.FilterSteps.Add(
                 new HardwareUploadCudaFilter(currentState with { FrameDataLocation = FrameDataLocation.Software }));
 

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -557,6 +557,9 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
 
                 if (currentState.FrameDataLocation is FrameDataLocation.Hardware)
                 {
+                    // overlay_vaapi expects straight alpha
+                    graphicsEngine.FilterSteps.Add(new UnpremultiplyFilter());
+
                     graphicsEngine.FilterSteps.Add(new HardwareUploadVaapiFilter(false));
                 }
 

ErsatzTV.Infrastructure/Streaming/Graphics/GraphicsEngine.cs

@@ -173,15 +173,25 @@ public class GraphicsEngine(
                 }
 
                 // draw each element
-                foreach (var preparedImage in preparedElementImages)
+                using (var paint = new SKPaint())
                 {
-                    using var paint = new SKPaint();
+                    foreach (var preparedImage in preparedElementImages)
-                    paint.Color = new SKColor(255, 255, 255, (byte)(preparedImage.Opacity * 255));
-                    canvas.DrawBitmap(preparedImage.Image, new SKPoint(preparedImage.Point.X, preparedImage.Point.Y), paint);
-
-                    if (preparedImage.Dispose)
                     {
-                        preparedImage.Image.Dispose();
+                        using (var colorFilter = SKColorFilter.CreateBlendMode(
+                                   SKColors.White.WithAlpha((byte)(preparedImage.Opacity * 255)),
+                                   SKBlendMode.Modulate))
+                        {
+                            paint.ColorFilter = colorFilter;
+                            canvas.DrawBitmap(
+                                preparedImage.Image,
+                                new SKPoint(preparedImage.Point.X, preparedImage.Point.Y),
+                                paint);
+                        }
+
+                        if (preparedImage.Dispose)
+                        {
+                            preparedImage.Image.Dispose();
+                        }
                     }
                 }
 
@@ -213,4 +223,4 @@ public class GraphicsEngine(
             }
         }
     }
-}
+}

ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElementBase.cs

@@ -88,35 +88,13 @@ public abstract class ImageElementBase : GraphicsElement, IDisposable
         }
     }
 
-    protected static async Task<Stream> GetImageStream(string image, CancellationToken cancellationToken)
-    {
-        Stream imageStream;
-
-        bool isRemoteUri = Uri.TryCreate(image, UriKind.Absolute, out var uriResult)
-                           && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps);
-
-        if (isRemoteUri)
-        {
-            using var client = new HttpClient();
-            imageStream = new MemoryStream(await client.GetByteArrayAsync(uriResult, cancellationToken));
-        }
-        else
-        {
-            imageStream = new FileStream(image!, FileMode.Open, FileAccess.Read);
-        }
-
-        return imageStream;
-    }
-
     protected static SKBitmap ToSkiaBitmap(Image image)
     {
-        // Force a known pixel format
         using Image<Rgba32> rgbaImage = image.CloneAs<Rgba32>();
 
         int width = rgbaImage.Width;
         int height = rgbaImage.Height;
 
-        // Allocate destination SKBitmap with RGBA8888 (straight alpha)
         var info = new SKImageInfo(width, height, SKColorType.Rgba8888, SKAlphaType.Unpremul);
         var skBitmap = new SKBitmap(info);
         if (!skBitmap.TryAllocPixels(info))
@@ -125,16 +103,12 @@ public abstract class ImageElementBase : GraphicsElement, IDisposable
             throw new InvalidOperationException("Failed to allocate pixels for SKBitmap.");
         }
 
-        // Pull pixel data from ImageSharp into a managed byte[]
-        // (Rgba32 is 4 bytes, so length = width * height * 4)
         var pixelArray = new Rgba32[width * height];
         rgbaImage.CopyPixelDataTo(pixelArray);
 
-        // Convert the Rgba32[] to a byte[] without unsafe code
         var bytes = new byte[pixelArray.Length * 4];
         MemoryMarshal.AsBytes(pixelArray.AsSpan()).CopyTo(bytes);
 
-        // Copy into Skia's pixel buffer
         IntPtr dstPtr = skBitmap.GetPixels(out _);
         Marshal.Copy(bytes, 0, dstPtr, bytes.Length);