I'm not sure if this is expected, just want to let you know.
As a workaround I can just re-order of the point and it works.
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.Drawing;
using SixLabors.ImageSharp.Drawing.Processing;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
var polygonHole = new ComplexPolygon(
[
new PointF(0, 0),
new PointF(0, 400),
new PointF(300, 400),
new PointF(300, 0),
],
/*[ By inverting the point order it works
new PointF(0, 0),
new PointF(300, 0),
new PointF(300, 400),
new PointF(0, 400),
],*/
[
new PointF(150, 100),
new PointF(50, 300),
new PointF(250, 300),
]
);
using var catImage = Image.Load("cat-original.png");
catImage.Mutate(x => x.Resize(300, 400));
using var image = new Image<Rgba32>(300, 400, Color.Transparent.ToPixel<Rgba32>());
image.Mutate(c => c.DrawImage(catImage, new Point(0, 0), new GraphicsOptions()));
image.Mutate(c => c
.SetGraphicsOptions(new GraphicsOptions
{
AlphaCompositionMode = PixelAlphaCompositionMode.DestOut,
}
)
.Fill(Color.Black, polygonHole)
);
image.SaveAsPng("output.png");