From 76dc8511395157e24b67ed38288666ed6838f9aa Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Sat, 16 Aug 2025 23:12:24 +0200 Subject: [PATCH] Reworked everything to fit the new sd.cpp API --- StableDiffusion.NET/Backends/CudaBackend.cs | 2 +- .../Enums/DiffusionModelType.cs | 8 - .../Extensions/ParameterExtension.cs | 106 ++--- StableDiffusion.NET/Helper/ImageHelper.cs | 170 +++++--- .../Models/Builder/ESRGANModelBuilder.cs | 5 +- .../DiffusionModelBuilderExtension.cs | 122 +++++- ...rExtension.cs => ModelBuilderExtension.cs} | 14 +- .../PhotomakerModelBuilderExtension.cs | 18 - .../UpscaleModelBuilderExtension.cs | 27 ++ .../Models/Builder/FluxModelBuilder.cs | 6 +- .../Interfaces/IDiffusionModelBuilder.cs | 9 +- .../Builder/Interfaces/IModelBuilder.cs | 9 + .../Interfaces/IPhotomakerModelBuilder.cs | 6 - .../Interfaces/IQuantizedModelBuilder.cs | 6 - .../Interfaces/IUpscaleModelBuilder.cs | 11 + .../Builder/StableDiffusion3_5ModelBuilder.cs | 6 +- .../Builder/StableDiffusionModelBuilder.cs | 8 +- StableDiffusion.NET/Models/DiffusionModel.cs | 375 ++-------------- .../Models/Parameter/CannyParameter.cs | 40 ++ .../Models/Parameter/ControlNetParameter.cs | 34 +- .../Parameter/DiffusionModelParameter.cs | 26 +- .../Models/Parameter/DiffusionParameter.cs | 99 ----- .../Extensions/DiffusionParameterExtension.cs | 149 ------- .../ImageGenerationParameterExtension.cs | 215 +++++++++ .../Models/Parameter/GuidanceParameter.cs | 13 + .../Parameter/ImageGenerationParameter.cs | 77 ++++ .../Interfaces/IDiffusionModelParameter.cs | 26 +- .../Parameter/Interfaces/IModelParameter.cs | 9 + .../Interfaces/IPhotomakerModelParameter.cs | 6 - .../Interfaces/IQuantizedModelParameter.cs | 8 - .../Interfaces/IUpscaleModelParameter.cs | 10 + .../Models/Parameter/PhotoMakerParameter.cs | 9 +- .../Models/Parameter/SlgParameter.cs | 27 ++ .../Models/Parameter/UpscaleModelParameter.cs | 9 +- StableDiffusion.NET/Models/UpscaleModel.cs | 21 +- .../DiffusionModelParameterMarshaller.cs | 86 ++++ .../ImageGenerationParameterMarshaller.cs | 197 +++++++++ .../Native/Marshaller/ImageMarshaller.cs | 24 + StableDiffusion.NET/Native/Native.Load.cs | 2 +- StableDiffusion.NET/Native/Native.cs | 409 ++++++++++-------- .../StableDiffusion.NET.csproj | 2 +- .../StableDiffusion.NET.csproj.DotSettings | 3 +- StableDiffusion.NET/StableDiffusionCpp.cs | 57 ++- 43 files changed, 1429 insertions(+), 1037 deletions(-) delete mode 100644 StableDiffusion.NET/Enums/DiffusionModelType.cs rename StableDiffusion.NET/Models/Builder/Extensions/{QuantizedModelBuilderExtension.cs => ModelBuilderExtension.cs} (61%) delete mode 100644 StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs create mode 100644 StableDiffusion.NET/Models/Builder/Extensions/UpscaleModelBuilderExtension.cs create mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IModelBuilder.cs delete mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs delete mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs create mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IUpscaleModelBuilder.cs create mode 100644 StableDiffusion.NET/Models/Parameter/CannyParameter.cs delete mode 100644 StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs delete mode 100644 StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs create mode 100644 StableDiffusion.NET/Models/Parameter/GuidanceParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs delete mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs delete mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/SlgParameter.cs create mode 100644 StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs create mode 100644 StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs create mode 100644 StableDiffusion.NET/Native/Marshaller/ImageMarshaller.cs diff --git a/StableDiffusion.NET/Backends/CudaBackend.cs b/StableDiffusion.NET/Backends/CudaBackend.cs index 91aa1e3..73b72f8 100644 --- a/StableDiffusion.NET/Backends/CudaBackend.cs +++ b/StableDiffusion.NET/Backends/CudaBackend.cs @@ -27,7 +27,7 @@ public partial class CudaBackend : IBackend public bool IsAvailable => (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) && (RuntimeInformation.OSArchitecture == Architecture.X64) - && CudaVersion is 11 or 12; + && CudaVersion is 12; public string PathPart => CudaVersion switch { diff --git a/StableDiffusion.NET/Enums/DiffusionModelType.cs b/StableDiffusion.NET/Enums/DiffusionModelType.cs deleted file mode 100644 index a42e137..0000000 --- a/StableDiffusion.NET/Enums/DiffusionModelType.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace StableDiffusion.NET; - -public enum DiffusionModelType -{ - None = 0, - StableDiffusion = 1, - Flux = 2 -} \ No newline at end of file diff --git a/StableDiffusion.NET/Extensions/ParameterExtension.cs b/StableDiffusion.NET/Extensions/ParameterExtension.cs index 8f33a8c..a464483 100644 --- a/StableDiffusion.NET/Extensions/ParameterExtension.cs +++ b/StableDiffusion.NET/Extensions/ParameterExtension.cs @@ -1,92 +1,100 @@ #pragma warning disable CA2208 +using JetBrains.Annotations; using System; namespace StableDiffusion.NET; +[PublicAPI] public static class ParameterExtension { public static void Validate(this UpscaleModelParameter parameter) { ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); ArgumentNullException.ThrowIfNull(parameter.ModelPath, nameof(UpscaleModelParameter.ModelPath)); - - if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(UpscaleModelParameter.Quantization)); } public static void Validate(this DiffusionModelParameter parameter) - { - ((IQuantizedModelParameter)parameter).Validate(); - ((IDiffusionModelParameter)parameter).Validate(); - ((IPhotomakerModelParameter)parameter).Validate(); - } - - public static void Validate(this IQuantizedModelParameter parameter) { ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.ThreadCount, nameof(IQuantizedModelParameter.ThreadCount)); - - if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(IQuantizedModelParameter.Quantization)); - } - - public static void Validate(this IPhotomakerModelParameter parameter) - { - ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); - ArgumentNullException.ThrowIfNull(parameter.StackedIdEmbeddingsDirectory, nameof(IPhotomakerModelParameter.StackedIdEmbeddingsDirectory)); - } - - public static void Validate(this IDiffusionModelParameter parameter) - { - ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); - ArgumentNullException.ThrowIfNull(parameter.VaePath, nameof(IDiffusionModelParameter.VaePath)); - ArgumentNullException.ThrowIfNull(parameter.TaesdPath, nameof(IDiffusionModelParameter.TaesdPath)); - ArgumentNullException.ThrowIfNull(parameter.LoraModelDirectory, nameof(IDiffusionModelParameter.LoraModelDirectory)); - ArgumentNullException.ThrowIfNull(parameter.ControlNetPath, nameof(IDiffusionModelParameter.ControlNetPath)); - ArgumentNullException.ThrowIfNull(parameter.EmbeddingsDirectory, nameof(IDiffusionModelParameter.EmbeddingsDirectory)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.ThreadCount, nameof(DiffusionModelParameter.ThreadCount)); + ArgumentNullException.ThrowIfNull(parameter.VaePath, nameof(DiffusionModelParameter.VaePath)); + ArgumentNullException.ThrowIfNull(parameter.TaesdPath, nameof(DiffusionModelParameter.TaesdPath)); + ArgumentNullException.ThrowIfNull(parameter.LoraModelDirectory, nameof(DiffusionModelParameter.LoraModelDirectory)); + ArgumentNullException.ThrowIfNull(parameter.ControlNetPath, nameof(DiffusionModelParameter.ControlNetPath)); + ArgumentNullException.ThrowIfNull(parameter.EmbeddingsDirectory, nameof(DiffusionModelParameter.EmbeddingsDirectory)); + ArgumentNullException.ThrowIfNull(parameter.StackedIdEmbeddingsDirectory, nameof(DiffusionModelParameter.StackedIdEmbeddingsDirectory)); if (!string.IsNullOrWhiteSpace(parameter.VaePath) && !string.IsNullOrWhiteSpace(parameter.TaesdPath)) throw new ArgumentException("VAE and TAESD are mutually exclusive."); - if (!Enum.IsDefined(parameter.RngType)) throw new ArgumentOutOfRangeException(nameof(IDiffusionModelParameter.RngType)); - if (!Enum.IsDefined(parameter.Schedule)) throw new ArgumentOutOfRangeException(nameof(IDiffusionModelParameter.Schedule)); + if (!Enum.IsDefined(parameter.RngType)) throw new ArgumentOutOfRangeException(nameof(DiffusionModelParameter.RngType)); + if (!Enum.IsDefined(parameter.Schedule)) throw new ArgumentOutOfRangeException(nameof(DiffusionModelParameter.Schedule)); } - public static void Validate(this DiffusionParameter parameter) + public static void Validate(this ImageGenerationParameter parameter) { ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); - ArgumentNullException.ThrowIfNull(parameter.ControlNet, nameof(DiffusionParameter.ControlNet)); - ArgumentNullException.ThrowIfNull(parameter.PhotoMaker, nameof(DiffusionParameter.PhotoMaker)); - ArgumentNullException.ThrowIfNull(parameter.NegativePrompt, nameof(DiffusionParameter.NegativePrompt)); + ArgumentNullException.ThrowIfNull(parameter.ControlNet, nameof(ImageGenerationParameter.ControlNet)); + ArgumentNullException.ThrowIfNull(parameter.PhotoMaker, nameof(ImageGenerationParameter.PhotoMaker)); + ArgumentNullException.ThrowIfNull(parameter.NegativePrompt, nameof(ImageGenerationParameter.NegativePrompt)); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Width, nameof(DiffusionParameter.Width)); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Height, nameof(DiffusionParameter.Height)); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.SampleSteps, nameof(DiffusionParameter.SampleSteps)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Width, nameof(ImageGenerationParameter.Width)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Height, nameof(ImageGenerationParameter.Height)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.SampleSteps, nameof(ImageGenerationParameter.SampleSteps)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CfgScale, nameof(DiffusionParameter.CfgScale)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.Guidance, nameof(DiffusionParameter.Guidance)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(DiffusionParameter.Strength)); - - if (!Enum.IsDefined(parameter.SampleMethod)) throw new ArgumentOutOfRangeException(nameof(DiffusionParameter.SampleMethod)); + if (!Enum.IsDefined(parameter.SampleMethod)) throw new ArgumentOutOfRangeException(nameof(ImageGenerationParameter.SampleMethod)); + parameter.Guidance.Validate(); parameter.ControlNet.Validate(); parameter.PhotoMaker.Validate(); } public static void Validate(this ControlNetParameter parameter) { - ArgumentNullException.ThrowIfNull(parameter, nameof(DiffusionParameter.ControlNet)); + ArgumentNullException.ThrowIfNull(parameter, nameof(ImageGenerationParameter.ControlNet)); ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(ControlNetParameter.Strength)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyHighThreshold, nameof(ControlNetParameter.CannyHighThreshold)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyLowThreshold, nameof(ControlNetParameter.CannyLowThreshold)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyWeak, nameof(ControlNetParameter.CannyWeak)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyStrong, nameof(ControlNetParameter.CannyStrong)); } public static void Validate(this PhotoMakerParameter parameter) { - ArgumentNullException.ThrowIfNull(parameter, nameof(DiffusionParameter.PhotoMaker)); + ArgumentNullException.ThrowIfNull(parameter, nameof(ImageGenerationParameter.PhotoMaker)); ArgumentNullException.ThrowIfNull(parameter.InputIdImageDirectory, nameof(PhotoMakerParameter.InputIdImageDirectory)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.StyleRatio, nameof(PhotoMakerParameter.StyleRatio)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.StyleStrength, nameof(PhotoMakerParameter.StyleStrength)); + } + + public static void Validate(this GuidanceParameter parameter) + { + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); + + ArgumentOutOfRangeException.ThrowIfNegative(parameter.ImgCfg); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.DistilledGuidance); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.MinCfg); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.TxtCfg); + + parameter.Slg.Validate(); + } + + public static void Validate(this SlgParameter parameter) + { + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); + ArgumentNullException.ThrowIfNull(parameter.Layers); + + ArgumentOutOfRangeException.ThrowIfNegative(parameter.Scale); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.SkipLayerStart); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.SkipLayerEnd); + } + + public static void Validate(this CannyParameter parameter) + { + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); + + ArgumentNullException.ThrowIfNull(parameter.Image); + + ArgumentOutOfRangeException.ThrowIfNegative(parameter.HighThreshold, nameof(CannyParameter.HighThreshold)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.LowThreshold, nameof(CannyParameter.LowThreshold)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.Weak, nameof(CannyParameter.Weak)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strong, nameof(CannyParameter.Strong)); } } \ No newline at end of file diff --git a/StableDiffusion.NET/Helper/ImageHelper.cs b/StableDiffusion.NET/Helper/ImageHelper.cs index f620439..de4971b 100644 --- a/StableDiffusion.NET/Helper/ImageHelper.cs +++ b/StableDiffusion.NET/Helper/ImageHelper.cs @@ -6,68 +6,140 @@ namespace StableDiffusion.NET; internal static class ImageHelper { - public static unsafe Image ToImage(Native.sd_image_t* sdImage) - { - Image image = ToImage(*sdImage); - - Marshal.FreeHGlobal((nint)sdImage); - - return image; - } - - public static unsafe Image ToImage(Native.sd_image_t sdImage) + public static unsafe Image ToImage(this Native.Types.sd_image_t sdImage) { int width = (int)sdImage.width; int height = (int)sdImage.height; int bpp = (int)sdImage.channel; - Image image = Image.Create(new ReadOnlySpan(sdImage.data, width * height * bpp), width, height, width * bpp); - - Dispose(sdImage); - - return image; - } - - public static unsafe void Dispose(Native.sd_image_t image) => Marshal.FreeHGlobal((nint)image.data); - - public static unsafe Native.sd_image_t ToSdImage(this IImage image, out nint dataPtr) - { - int sizeInBytes = image.SizeInBytes; - - dataPtr = Marshal.AllocHGlobal(sizeInBytes); - image.CopyTo(new Span((void*)dataPtr, sizeInBytes)); - - return image.ToSdImage((byte*)dataPtr); - } - - public static unsafe Native.sd_image_t ToSdImage(this IImage image, byte* pinnedReference) - => new() + switch (bpp) { - width = (uint)image.Width, - height = (uint)image.Height, - channel = (uint)image.ColorFormat.BytesPerPixel, - data = pinnedReference - }; + case 3: + return Image.Create(new ReadOnlySpan(sdImage.data, width * height * bpp), width, height, width * bpp); - public static unsafe Native.sd_image_t* ToSdImagePtr(this IImage image, out nint dataPtr) - { - int sizeInBytes = image.SizeInBytes; + case 1: + { + ColorRGB[] pixels = new ColorRGB[width * height]; + Span sdData = new(sdImage.data, pixels.Length); - dataPtr = Marshal.AllocHGlobal(sizeInBytes); - image.CopyTo(new Span((void*)dataPtr, sizeInBytes)); + for (int i = 0; i < pixels.Length; i++) + { + byte c = sdData[i]; + pixels[i] = new ColorRGB(c, c, c); + } - return image.ToSdImagePtr((byte*)dataPtr); + Image image = Image.Create(pixels, width, height); + + return image; + } + + default: + throw new ArgumentOutOfRangeException($"Image-BPP of {bpp} is not supported"); + } } - public static unsafe Native.sd_image_t* ToSdImagePtr(this IImage image, byte* pinnedReference) + public static unsafe Native.Types.sd_image_t ToSdImage(this IImage image, bool monochrome = false) { - Native.sd_image_t* nativeImage = (Native.sd_image_t*)Marshal.AllocHGlobal(sizeof(Native.sd_image_t)); + if (monochrome) + { + int sizeInBytes = image.Width * image.Height; - nativeImage->width = (uint)image.Width; - nativeImage->height = (uint)image.Height; - nativeImage->channel = (uint)image.ColorFormat.BytesPerPixel; - nativeImage->data = pinnedReference; + byte* dataPtr = (byte*)NativeMemory.Alloc((nuint)sizeInBytes); + Span data = new(dataPtr, sizeInBytes); - return nativeImage; + // DarthAffe 16.08.2025: HPPH does currently not support monochrome images, that's why we need to convert it here. We're going for the simple conversion as the source image is supposed to be monochrome anyway. + for (int y = 0; y < image.Height; y++) + for (int x = 0; x < image.Width; x++) + { + IColor color = image[x, y]; + data[(image.Width * y) + x] = (byte)Math.Round((color.R + color.G + color.B) / 3.0); + } + + return new Native.Types.sd_image_t + { + width = (uint)image.Width, + height = (uint)image.Height, + channel = 1, + data = dataPtr + }; + } + else + { + IImage img = image as IImage ?? image.ConvertTo(); + + int sizeInBytes = img.SizeInBytes; + + byte* dataPtr = (byte*)NativeMemory.Alloc((nuint)sizeInBytes); + img.CopyTo(new Span(dataPtr, sizeInBytes)); + + return new Native.Types.sd_image_t + { + width = (uint)img.Width, + height = (uint)img.Height, + channel = (uint)img.ColorFormat.BytesPerPixel, + data = dataPtr + }; + } + } + + public static unsafe Native.Types.sd_image_t* ToSdImagePtr(this IImage image, bool monochrome = false) + { + Native.Types.sd_image_t* imagePtr = (Native.Types.sd_image_t*)NativeMemory.Alloc((nuint)Marshal.SizeOf()); + imagePtr[0] = image.ToSdImage(monochrome); + + return imagePtr; + } + + public static unsafe void Free(this Native.Types.sd_image_t sdImage) + { + if (sdImage.data == null) return; + + NativeMemory.Free(sdImage.data); + } + + public static unsafe Image[] ToImageArray(Native.Types.sd_image_t* sdImage, int count) + { + if (sdImage == null) return []; + + Image[] images = new Image[count]; + + for (int i = 0; i < images.Length; i++) + images[i] = GetImage(sdImage, i); + + return images; + } + + internal static unsafe IImage[] ToImageArrayIFace(Native.Types.sd_image_t* sdImage, int count) + { + IImage[] images = new IImage[count]; + + for (int i = 0; i < images.Length; i++) + images[i] = GetImage(sdImage, i); + + return images; + } + + public static unsafe Image GetImage(Native.Types.sd_image_t* sdImage, int index) => sdImage[index].ToImage(); + + public static unsafe Native.Types.sd_image_t* ToSdImage(this IImage[] images, bool monochrome = false) + { + int count = images.Length; + + Native.Types.sd_image_t* imagePtr = (Native.Types.sd_image_t*)NativeMemory.Alloc((nuint)(count * Marshal.SizeOf())); + + for (int i = 0; i < count; i++) + imagePtr[i] = images[i].ToSdImage(monochrome); + + return imagePtr; + } + + public static unsafe void Free(Native.Types.sd_image_t* sdImage, int count) + { + if (sdImage == null) return; + + for (int i = 0; i < count; i++) + Free(sdImage[i]); + + NativeMemory.Free(sdImage); } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs b/StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs index b66452b..387f50b 100644 --- a/StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs +++ b/StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs @@ -3,12 +3,13 @@ namespace StableDiffusion.NET; [PublicAPI] -public sealed class ESRGANModelBuilder : IQuantizedModelBuilder +public sealed class ESRGANModelBuilder : IUpscaleModelBuilder { #region Properties & Fields public UpscaleModelParameter Parameter { get; } - IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; + IUpscaleModelParameter IUpscaleModelBuilder.Parameter => Parameter; + IModelParameter IModelBuilder.Parameter => Parameter; #endregion diff --git a/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs index 53ae229..62f33b2 100644 --- a/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs +++ b/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs @@ -76,10 +76,10 @@ public static class DiffusionModelBuilderExtension return builder; } - public static T KeepVaeOnCpu(this T builder, bool keepVaeOnCpu = true) + public static T KeepControlNetOnCpu(this T builder, bool keepControlNetOnCpu = true) where T : IDiffusionModelBuilder { - builder.Parameter.KeepVaeOnCPU = keepVaeOnCpu; + builder.Parameter.KeepControlNetOnCPU = keepControlNetOnCpu; return builder; } @@ -92,10 +92,34 @@ public static class DiffusionModelBuilderExtension return builder; } - public static T KeepControlNetOnCpu(this T builder, bool keepControlNetOnCpu = true) + public static T KeepVaeOnCpu(this T builder, bool keepVaeOnCpu = true) where T : IDiffusionModelBuilder { - builder.Parameter.KeepControlNetOnCPU = keepControlNetOnCpu; + builder.Parameter.KeepVaeOnCPU = keepVaeOnCpu; + + return builder; + } + + public static T WithFlashAttention(this T builder, bool flashAttention = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.FlashAttention = flashAttention; + + return builder; + } + + public static T WithDiffusionConvDirect(this T builder, bool diffusionConfDirect = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.DiffusionConvDirect = diffusionConfDirect; + + return builder; + } + + public static T WithVaeConvDirect(this T builder, bool vaeConfDirect = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.VaeConfDirect = vaeConfDirect; return builder; } @@ -120,10 +144,96 @@ public static class DiffusionModelBuilderExtension return builder; } - public static T WithFlashAttention(this T builder, bool flashAttention = true) + public static T WithQuantization(this T builder, Quantization quantization) where T : IDiffusionModelBuilder { - builder.Parameter.FlashAttention = flashAttention; + if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization)); + + builder.Parameter.Quantization = quantization; + + return builder; + } + + public static T WithPhotomaker(this T builder, string stackedIdEmbeddingsDirectory) + where T : IDiffusionModelBuilder + { + ArgumentException.ThrowIfNullOrWhiteSpace(stackedIdEmbeddingsDirectory, nameof(stackedIdEmbeddingsDirectory)); + + builder.Parameter.StackedIdEmbeddingsDirectory = stackedIdEmbeddingsDirectory; + + return builder; + } + + public static T WithModelPath(this T builder, string modelPath) + where T : IDiffusionModelBuilder + { + ArgumentNullException.ThrowIfNull(modelPath); + + builder.Parameter.ModelPath = modelPath; + + return builder; + } + + public static T WithDiffusionModelPath(this T builder, string diffusionModelPath) + where T : IDiffusionModelBuilder + { + ArgumentNullException.ThrowIfNull(diffusionModelPath); + + builder.Parameter.DiffusionModelPath = diffusionModelPath; + + return builder; + } + + public static T WithClipLPath(this T builder, string clipLPath) + where T : IDiffusionModelBuilder + { + ArgumentNullException.ThrowIfNull(clipLPath); + + builder.Parameter.ClipLPath = clipLPath; + + return builder; + } + + public static T WithT5xxlPath(this T builder, string t5xxlPath) + where T : IDiffusionModelBuilder + { + ArgumentNullException.ThrowIfNull(t5xxlPath); + + builder.Parameter.T5xxlPath = t5xxlPath; + + return builder; + } + + public static T UseChromaDitMap(this T builder, bool useChromaDitMap = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.ChromaUseDitMap = useChromaDitMap; + + return builder; + } + + public static T EnableChromaT5Map(this T builder, bool enableChromaT5Map = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.ChromaEnableT5Map = enableChromaT5Map; + + return builder; + } + + public static T WithChromaT5MaskPad(this T builder, int chromaT5MaskPad) + where T : IDiffusionModelBuilder + { + builder.Parameter.ChromaT5MaskPad = chromaT5MaskPad; + + return builder; + } + + public static T WithClipGPath(this T builder, string clipGPath) + where T : IDiffusionModelBuilder + { + ArgumentNullException.ThrowIfNull(clipGPath); + + builder.Parameter.ClipGPath = clipGPath; return builder; } diff --git a/StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/ModelBuilderExtension.cs similarity index 61% rename from StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs rename to StableDiffusion.NET/Models/Builder/Extensions/ModelBuilderExtension.cs index 671c2f1..0cf1c03 100644 --- a/StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs +++ b/StableDiffusion.NET/Models/Builder/Extensions/ModelBuilderExtension.cs @@ -4,10 +4,10 @@ using JetBrains.Annotations; namespace StableDiffusion.NET; [PublicAPI] -public static class QuantizedModelBuilderExtension +public static class ModelBuilderExtension { public static T WithoutMultithreading(this T builder) - where T : IQuantizedModelBuilder + where T : IModelBuilder { builder.Parameter.ThreadCount = 1; @@ -15,7 +15,7 @@ public static class QuantizedModelBuilderExtension } public static T WithMultithreading(this T builder, int threadCount = 0) - where T : IQuantizedModelBuilder + where T : IModelBuilder { ArgumentOutOfRangeException.ThrowIfLessThan(threadCount, 0, nameof(threadCount)); @@ -25,12 +25,4 @@ public static class QuantizedModelBuilderExtension return builder; } - - public static T WithQuantization(this T builder, Quantization quantization) - where T : IQuantizedModelBuilder - { - builder.Parameter.Quantization = quantization; - - return builder; - } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs deleted file mode 100644 index 57449f4..0000000 --- a/StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using JetBrains.Annotations; - -namespace StableDiffusion.NET; - -[PublicAPI] -public static class PhotomakerModelBuilderExtension -{ - public static T WithPhotomaker(this T builder, string stackedIdEmbeddingsDirectory) - where T : IPhotomakerModelBuilder - { - ArgumentException.ThrowIfNullOrWhiteSpace(stackedIdEmbeddingsDirectory, nameof(stackedIdEmbeddingsDirectory)); - - builder.Parameter.StackedIdEmbeddingsDirectory = stackedIdEmbeddingsDirectory; - - return builder; - } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Extensions/UpscaleModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/UpscaleModelBuilderExtension.cs new file mode 100644 index 0000000..d8beb47 --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Extensions/UpscaleModelBuilderExtension.cs @@ -0,0 +1,27 @@ +using System; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static class UpscaleModelBuilderExtension +{ + public static T WithModelPath(this T builder, string modelPath) + where T : IUpscaleModelBuilder + { + ArgumentNullException.ThrowIfNull(modelPath); + + builder.Parameter.ModelPath = modelPath; + + return builder; + } + + public static T WithConvDirect(this T builder, bool confDirect = true) + where T : IUpscaleModelBuilder + { + builder.Parameter.ConvDirect = confDirect; + + return builder; + } + +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs b/StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs index 367e23b..510bb2c 100644 --- a/StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs +++ b/StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs @@ -3,13 +3,13 @@ namespace StableDiffusion.NET; [PublicAPI] -public sealed class FluxModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder +public sealed class FluxModelBuilder : IDiffusionModelBuilder { #region Properties & Fields public DiffusionModelParameter Parameter { get; } IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter; - IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; + IModelParameter IModelBuilder.Parameter => Parameter; #endregion @@ -17,7 +17,7 @@ public sealed class FluxModelBuilder : IDiffusionModelBuilder, IQuantizedModelBu public FluxModelBuilder(string diffusionModelPath, string clipLPath, string t5xxlPath, string vaePath) { - Parameter = new DiffusionModelParameter { DiffusionModelType = DiffusionModelType.Flux, DiffusionModelPath = diffusionModelPath, ClipLPath = clipLPath, T5xxlPath = t5xxlPath, VaePath = vaePath }; + Parameter = new DiffusionModelParameter { DiffusionModelPath = diffusionModelPath, ClipLPath = clipLPath, T5xxlPath = t5xxlPath, VaePath = vaePath }; } #endregion diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs index cf4aa9f..1dc098a 100644 --- a/StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs +++ b/StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs @@ -1,6 +1,11 @@ -namespace StableDiffusion.NET; +using JetBrains.Annotations; -public interface IDiffusionModelBuilder +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IDiffusionModelBuilder : IModelBuilder { IDiffusionModelParameter Parameter { get; } + + DiffusionModel Build(); } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IModelBuilder.cs new file mode 100644 index 0000000..6119372 --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Interfaces/IModelBuilder.cs @@ -0,0 +1,9 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IModelBuilder +{ + IModelParameter Parameter { get; } +} diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs deleted file mode 100644 index 9731fa3..0000000 --- a/StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace StableDiffusion.NET; - -public interface IPhotomakerModelBuilder -{ - IPhotomakerModelParameter Parameter { get; } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs deleted file mode 100644 index 6aa95ae..0000000 --- a/StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace StableDiffusion.NET; - -public interface IQuantizedModelBuilder -{ - IQuantizedModelParameter Parameter { get; } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IUpscaleModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IUpscaleModelBuilder.cs new file mode 100644 index 0000000..f6effbb --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Interfaces/IUpscaleModelBuilder.cs @@ -0,0 +1,11 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IUpscaleModelBuilder : IModelBuilder +{ + IUpscaleModelParameter Parameter { get; } + + UpscaleModel Build(); +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/StableDiffusion3_5ModelBuilder.cs b/StableDiffusion.NET/Models/Builder/StableDiffusion3_5ModelBuilder.cs index 66ac5b1..954de7a 100644 --- a/StableDiffusion.NET/Models/Builder/StableDiffusion3_5ModelBuilder.cs +++ b/StableDiffusion.NET/Models/Builder/StableDiffusion3_5ModelBuilder.cs @@ -3,13 +3,14 @@ namespace StableDiffusion.NET; [PublicAPI] -public sealed class StableDiffusion3_5ModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder +public sealed class StableDiffusion3_5ModelBuilder : IDiffusionModelBuilder { #region Properties & Fields public DiffusionModelParameter Parameter { get; } IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter; - IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; + IModelParameter IModelBuilder.Parameter => Parameter; + #endregion @@ -19,7 +20,6 @@ public sealed class StableDiffusion3_5ModelBuilder : IDiffusionModelBuilder, IQu { Parameter = new DiffusionModelParameter { - DiffusionModelType = DiffusionModelType.StableDiffusion, ModelPath = modelPath, ClipLPath = clipLPath, ClipGPath = clipGPath, diff --git a/StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs b/StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs index d1952f2..0109d4d 100644 --- a/StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs +++ b/StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs @@ -3,14 +3,14 @@ namespace StableDiffusion.NET; [PublicAPI] -public sealed class StableDiffusionModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder, IPhotomakerModelBuilder +public sealed class StableDiffusionModelBuilder : IDiffusionModelBuilder { #region Properties & Fields public DiffusionModelParameter Parameter { get; } IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter; - IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; - IPhotomakerModelParameter IPhotomakerModelBuilder.Parameter => Parameter; + IModelParameter IModelBuilder.Parameter => Parameter; + #endregion @@ -18,7 +18,7 @@ public sealed class StableDiffusionModelBuilder : IDiffusionModelBuilder, IQuant public StableDiffusionModelBuilder(string modelPath) { - Parameter = new DiffusionModelParameter { DiffusionModelType = DiffusionModelType.StableDiffusion, ModelPath = modelPath }; + Parameter = new DiffusionModelParameter { ModelPath = modelPath }; } #endregion diff --git a/StableDiffusion.NET/Models/DiffusionModel.cs b/StableDiffusion.NET/Models/DiffusionModel.cs index 2e74e4f..7901ef1 100644 --- a/StableDiffusion.NET/Models/DiffusionModel.cs +++ b/StableDiffusion.NET/Models/DiffusionModel.cs @@ -1,8 +1,6 @@ using HPPH; using JetBrains.Annotations; using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; namespace StableDiffusion.NET; @@ -15,7 +13,7 @@ public sealed unsafe class DiffusionModel : IDisposable public DiffusionModelParameter ModelParameter { get; } - private Native.sd_ctx_t* _ctx; + private Native.Types.sd_ctx_t* _ctx; #endregion @@ -25,8 +23,6 @@ public sealed unsafe class DiffusionModel : IDisposable { ArgumentNullException.ThrowIfNull(modelParameter, nameof(modelParameter)); - modelParameter.Validate(); - this.ModelParameter = modelParameter; Initialize(); @@ -40,335 +36,53 @@ public sealed unsafe class DiffusionModel : IDisposable private void Initialize() { - _ctx = Native.new_sd_ctx(ModelParameter.ModelPath, - ModelParameter.ClipLPath, - ModelParameter.ClipGPath, - ModelParameter.T5xxlPath, - ModelParameter.DiffusionModelPath, - ModelParameter.VaePath, - ModelParameter.TaesdPath, - ModelParameter.ControlNetPath, - ModelParameter.LoraModelDirectory, - ModelParameter.EmbeddingsDirectory, - ModelParameter.StackedIdEmbeddingsDirectory, - ModelParameter.VaeDecodeOnly, - ModelParameter.VaeTiling, - false, - ModelParameter.ThreadCount, - ModelParameter.Quantization, - ModelParameter.RngType, - ModelParameter.Schedule, - ModelParameter.KeepClipOnCPU, - ModelParameter.KeepControlNetOnCPU, - ModelParameter.KeepVaeOnCPU, - ModelParameter.FlashAttention, - ModelParameter.ChromaUseDitMap, - ModelParameter.ChromaEnableT5Map, - ModelParameter.ChromaT5MaskPad); + ModelParameter.Validate(); + + _ctx = Native.new_sd_ctx(ModelParameter); if (_ctx == null) throw new NullReferenceException("Failed to initialize diffusion-model."); } - public DiffusionParameter GetDefaultParameter() => ModelParameter.DiffusionModelType switch + public Image? GenerateImage(ImageGenerationParameter parameter) { - DiffusionModelType.None => new DiffusionParameter(), - DiffusionModelType.StableDiffusion => DiffusionParameter.SDXLDefault, - DiffusionModelType.Flux => DiffusionParameter.FluxDefault, - _ => throw new ArgumentOutOfRangeException() - }; - - public IImage TextToImage(string prompt, DiffusionParameter? parameter = null) - { - parameter ??= GetDefaultParameter(); - ObjectDisposedException.ThrowIf(_disposed, this); - ArgumentNullException.ThrowIfNull(prompt); + + if (_ctx == null) throw new NullReferenceException("The model is not initialized."); parameter.Validate(); - - List ptrsToFree = []; + + Native.Types.sd_image_t* result = Native.generate_image(_ctx, parameter); + if (result == null) return null; try { - NativeParameters nativeParameters = PrefillParameters(prompt, parameter); - SetControlNetParameters(ref nativeParameters, parameter, ptrsToFree); - - Native.sd_image_t* result = Txt2Img(nativeParameters); - - return ImageHelper.ToImage(result); + return ImageHelper.GetImage(result, 0); } finally { - foreach (nint ptr in ptrsToFree) - Marshal.FreeHGlobal(ptr); + ImageHelper.Free(result, 1); } } - public IImage ImageToImage(string prompt, IImage image, DiffusionParameter? parameter = null) - { - parameter ??= GetDefaultParameter(); + // TODO DarthAffe 09.08.2025: Implement when no longer marked as broken + //public Image[] GenerateVideo() + //{ + // ObjectDisposedException.ThrowIf(_disposed, this); - ObjectDisposedException.ThrowIf(_disposed, this); - ArgumentNullException.ThrowIfNull(prompt); - ArgumentNullException.ThrowIfNull(image); + // //parameter.Validate(); - parameter.Validate(); + // int imageCount = 0; // TODO DarthAffe 09.08.2025: Set correct count - // DarthAffe 10.08.2024: Mask needs to be a 1 channel all max value image when it's not used - I really don't like this concept as it adds unnecessary allocations, but that's how it is :( - Span maskBuffer = new byte[image.Width * image.Height]; - maskBuffer.Fill(byte.MaxValue); - - return InternalImageToImage(prompt, image, maskBuffer, parameter); - } - - public IImage Inpaint(string prompt, IImage image, IImage mask, DiffusionParameter? parameter = null) - { - parameter ??= GetDefaultParameter(); - - ObjectDisposedException.ThrowIf(_disposed, this); - ArgumentNullException.ThrowIfNull(prompt); - ArgumentNullException.ThrowIfNull(image); - ArgumentNullException.ThrowIfNull(mask); - - parameter.Validate(); - - if (image.Width != mask.Width) throw new ArgumentException("The mask needs to have the same with as the image.", nameof(mask)); - if (image.Height != mask.Height) throw new ArgumentException("The mask needs to have the same height as the image.", nameof(mask)); - - // DarthAffe 10.08.2024: HPPH does currently not support monochrome images, that's why we need to convert it here. We're going for the simple conversion as the source image is supposed to be monochrome anyway. - Span maskBuffer = new byte[image.Width * image.Height]; - for (int y = 0; y < image.Height; y++) - for (int x = 0; x < image.Width; x++) - { - IColor color = mask[x, y]; - maskBuffer[(image.Width * y) + x] = (byte)Math.Round((color.R + color.G + color.B) / 3.0); - } - - return InternalImageToImage(prompt, image, maskBuffer, parameter); - } - - public IImage Edit(string prompt, IImage[] refImages, DiffusionParameter? parameter = null) - { - parameter ??= GetDefaultParameter(); - - ObjectDisposedException.ThrowIf(_disposed, this); - ArgumentNullException.ThrowIfNull(prompt); - ArgumentNullException.ThrowIfNull(refImages); - - parameter.Validate(); - - List ptrsToFree = []; - - try - { - NativeParameters nativeParameters = PrefillParameters(prompt, parameter); - SetControlNetParameters(ref nativeParameters, parameter, ptrsToFree); - - Native.sd_image_t[] nativeRefImages = new Native.sd_image_t[refImages.Length]; - - for (int i = 0; i < refImages.Length; i++) - { - IImage image = refImages[i]; - if (image is not IImage refImage) - refImage = image.ConvertTo(); - - nativeRefImages[i] = refImage.ToSdImage(out nint dataPtr); - ptrsToFree.Add(dataPtr); - } - - fixed (Native.sd_image_t* nativeRefImagesPtr = nativeRefImages) - { - nativeParameters.ref_images = nativeRefImagesPtr; - nativeParameters.ref_images_count = nativeRefImages.Length; - - Native.sd_image_t* result = Edit(nativeParameters); - - return ImageHelper.ToImage(result); - } - } - finally - { - foreach (nint ptr in ptrsToFree) - Marshal.FreeHGlobal(ptr); - } - } - - private Image InternalImageToImage(string prompt, IImage image, Span mask, DiffusionParameter parameter) - { - List ptrsToFree = []; - - try - { - NativeParameters nativeParameters = PrefillParameters(prompt, parameter); - SetControlNetParameters(ref nativeParameters, parameter, ptrsToFree); - - if (image is not IImage refImage) - refImage = image.ConvertTo(); - - nativeParameters.init_image = refImage.ToSdImage(out nint imagePtr); - ptrsToFree.Add(imagePtr); - - fixed (byte* maskPtr = mask) - { - Native.sd_image_t maskImage = new() - { - width = (uint)refImage.Width, - height = (uint)refImage.Height, - channel = 1, - data = maskPtr - }; - nativeParameters.mask_image = maskImage; - - Native.sd_image_t* result = Img2Img(nativeParameters); - - return ImageHelper.ToImage(result); - } - } - finally - { - foreach (nint ptr in ptrsToFree) - Marshal.FreeHGlobal(ptr); - } - } - - private static NativeParameters PrefillParameters(string prompt, DiffusionParameter parameter) - => new() - { - prompt = prompt, - negative_prompt = parameter.NegativePrompt, - clip_skip = parameter.ClipSkip, - cfg_scale = parameter.CfgScale, - guidance = parameter.Guidance, - eta = parameter.Eta, - width = parameter.Width, - height = parameter.Height, - sample_method = parameter.SampleMethod, - sample_steps = parameter.SampleSteps, - seed = parameter.Seed, - batch_count = 1, - control_cond = null, - control_strength = 0, - style_strength = parameter.PhotoMaker.StyleRatio, - normalize_input = parameter.PhotoMaker.NormalizeInput, - input_id_images_path = parameter.PhotoMaker.InputIdImageDirectory, - skip_layers = parameter.SkipLayers, - skip_layers_count = parameter.SkipLayers.Length, - slg_scale = parameter.SlgScale, - skip_layer_start = parameter.SkipLayerStart, - skip_layer_end = parameter.SkipLayerEnd, - strength = parameter.Strength, - }; - - private static void SetControlNetParameters(ref NativeParameters nativeParameters, DiffusionParameter parameter, List ptrsToFree) - { - if (!parameter.ControlNet.IsEnabled) return; - if (parameter.ControlNet.Image == null) return; - - if (parameter.ControlNet.Image is not IImage controlNetImage) - controlNetImage = parameter.ControlNet.Image!.ConvertTo(); - - Native.sd_image_t* nativeControlNetImage = controlNetImage.ToSdImagePtr(out nint controlNetImagePtr); - ptrsToFree.Add(controlNetImagePtr); - ptrsToFree.Add((nint)nativeControlNetImage); - - nativeParameters.control_cond = nativeControlNetImage; - nativeParameters.control_strength = parameter.ControlNet.Strength; - - if (parameter.ControlNet.CannyPreprocess) - { - nativeParameters.control_cond->data = Native.preprocess_canny(nativeParameters.control_cond->data, - parameter.Width, - parameter.Height, - parameter.ControlNet.CannyHighThreshold, - parameter.ControlNet.CannyLowThreshold, - parameter.ControlNet.CannyWeak, - parameter.ControlNet.CannyStrong, - parameter.ControlNet.CannyInverse); - ptrsToFree.Add((nint)nativeParameters.control_cond->data); - } - } - - private Native.sd_image_t* Txt2Img(NativeParameters parameter) - => Native.txt2img(_ctx, - parameter.prompt, - parameter.negative_prompt, - parameter.clip_skip, - parameter.cfg_scale, - parameter.guidance, - parameter.eta, - parameter.width, - parameter.height, - parameter.sample_method, - parameter.sample_steps, - parameter.seed, - parameter.batch_count, - parameter.control_cond, - parameter.control_strength, - parameter.style_strength, - parameter.normalize_input, - parameter.input_id_images_path, - parameter.skip_layers, - parameter.skip_layers_count, - parameter.slg_scale, - parameter.skip_layer_start, - parameter.skip_layer_end - ); - - private Native.sd_image_t* Img2Img(NativeParameters parameter) - => Native.img2img(_ctx, - parameter.init_image, - parameter.mask_image, - parameter.prompt, - parameter.negative_prompt, - parameter.clip_skip, - parameter.cfg_scale, - parameter.guidance, - parameter.width, - parameter.height, - parameter.sample_method, - parameter.sample_steps, - parameter.strength, - parameter.seed, - parameter.batch_count, - parameter.control_cond, - parameter.control_strength, - parameter.style_strength, - parameter.normalize_input, - parameter.input_id_images_path, - parameter.skip_layers, - parameter.skip_layers_count, - parameter.slg_scale, - parameter.skip_layer_start, - parameter.skip_layer_end - ); - - private Native.sd_image_t* Edit(NativeParameters parameter) - => Native.edit(_ctx, - parameter.ref_images, - parameter.ref_images_count, - parameter.prompt, - parameter.negative_prompt, - parameter.clip_skip, - parameter.cfg_scale, - parameter.guidance, - parameter.eta, - parameter.width, - parameter.height, - parameter.sample_method, - parameter.sample_steps, - parameter.strength, - parameter.seed, - parameter.batch_count, - parameter.control_cond, - parameter.control_strength, - parameter.style_strength, - parameter.normalize_input, - parameter.skip_layers, - parameter.skip_layers_count, - parameter.slg_scale, - parameter.skip_layer_start, - parameter.skip_layer_end - ); + // Native.Types.sd_image_t* result = Native.generate_video(_ctx, new Native.Types.sd_vid_gen_params_t()); // TODO DarthAffe 09.08.2025: Add Parameter + // try + // { + // return ImageHelper.ToImageArray(result, imageCount); + // } + // finally + // { + // ImageHelper.Free(result, imageCount); + // } + //} public void Dispose() { @@ -382,37 +96,4 @@ public sealed unsafe class DiffusionModel : IDisposable } #endregion - - private ref struct NativeParameters - { - internal string prompt; - internal string negative_prompt; - internal int clip_skip; - internal float cfg_scale; - internal float guidance; - internal float eta; - internal int width; - internal int height; - internal Sampler sample_method; - internal int sample_steps; - internal long seed; - internal int batch_count; - internal Native.sd_image_t* control_cond; - internal float control_strength; - internal float style_strength; - internal bool normalize_input; - internal string input_id_images_path; - internal int[] skip_layers; - internal int skip_layers_count; - internal float slg_scale; - internal float skip_layer_start; - internal float skip_layer_end; - - internal Native.sd_image_t init_image; - internal Native.sd_image_t mask_image; - - internal Native.sd_image_t* ref_images; - internal int ref_images_count; - internal float strength; - } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/CannyParameter.cs b/StableDiffusion.NET/Models/Parameter/CannyParameter.cs new file mode 100644 index 0000000..fcaa56a --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/CannyParameter.cs @@ -0,0 +1,40 @@ +using HPPH; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class CannyParameter +{ + /// + /// the image to process + /// + public IImage? Image { get; set; } = null; + + /// + /// + /// + public float HighThreshold { get; set; } = 0.08f; + + /// + /// + /// + public float LowThreshold { get; set; } = 0.08f; + + /// + /// + /// + public float Weak { get; set; } = 0.8f; + + /// + /// + /// + public float Strong { get; set; } = 1.0f; + + /// + /// + /// + public bool Inverse { get; set; } = false; + + public static CannyParameter Create() => new(); +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs b/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs index 406435c..49a906e 100644 --- a/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs @@ -1,13 +1,9 @@ using HPPH; -using JetBrains.Annotations; namespace StableDiffusion.NET; -[PublicAPI] public sealed class ControlNetParameter { - public bool IsEnabled => Image != null; - /// /// image condition, control net /// @@ -19,33 +15,5 @@ public sealed class ControlNetParameter /// public float Strength { get; set; } = 0.9f; - /// - /// apply canny preprocessor (edge detection) - /// - public bool CannyPreprocess { get; set; } = false; - - /// - /// - /// - public float CannyHighThreshold { get; set; } = 0.08f; - - /// - /// - /// - public float CannyLowThreshold { get; set; } = 0.08f; - - /// - /// - /// - public float CannyWeak { get; set; } = 0.8f; - - /// - /// - /// - public float CannyStrong { get; set; } = 1.0f; - - /// - /// - /// - public bool CannyInverse { get; set; } = false; + internal ControlNetParameter() { } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs index 6b5345c..b03c55c 100644 --- a/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs @@ -1,9 +1,10 @@ -namespace StableDiffusion.NET; +using JetBrains.Annotations; -public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantizedModelParameter, IPhotomakerModelParameter +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class DiffusionModelParameter : IDiffusionModelParameter { - public DiffusionModelType DiffusionModelType { get; set; } = DiffusionModelType.None; - /// /// path to vae /// @@ -67,6 +68,18 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantiz /// public bool FlashAttention { get; set; } = false; + /// + /// use Conv2d direct in the diffusion model + /// This might crash if it is not supported by the backend. + /// + public bool DiffusionConvDirect { get; set; } = false; + + /// + /// use Conv2d direct in the vae model (should improve the performance) + /// This might crash if it is not supported by the backend. + /// + public bool VaeConfDirect { get; set; } = false; + /// /// RNG (default: Standard) /// @@ -78,7 +91,8 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantiz public Schedule Schedule { get; set; } = Schedule.Default; /// - /// + /// quantizes on load + /// not really useful in most cases /// public Quantization Quantization { get; set; } = Quantization.Unspecified; @@ -119,4 +133,6 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantiz /// path to the clip-g text encoder /// public string ClipGPath { get; set; } = string.Empty; + + public static DiffusionModelParameter Create() => new(); } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs b/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs deleted file mode 100644 index 2c5e703..0000000 --- a/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs +++ /dev/null @@ -1,99 +0,0 @@ -using JetBrains.Annotations; - -namespace StableDiffusion.NET; - -[PublicAPI] -public sealed class DiffusionParameter -{ - #region Properties & Fields - - public static DiffusionParameter SD1Default => new() { Width = 512, Height = 512, CfgScale = 7.5f, Guidance = 1f, SampleSteps = 25, SampleMethod = Sampler.Euler_A }; - public static DiffusionParameter SDXLDefault => new() { Width = 1024, Height = 1024, CfgScale = 7f, Guidance = 1f, SampleSteps = 30, SampleMethod = Sampler.Euler_A }; - public static DiffusionParameter SD3_5Default => new() { Width = 1024, Height = 1024, CfgScale = 4.5f, Guidance = 1f, SampleSteps = 20, SampleMethod = Sampler.Euler }; - public static DiffusionParameter FluxDefault => new() { Width = 1024, Height = 1024, CfgScale = 1, Guidance = 3.5f, SampleSteps = 20, SampleMethod = Sampler.Euler }; - - /// - /// the negative prompt (default: ""); - /// - public string NegativePrompt { get; set; } = string.Empty; - - /// - /// image width, in pixel space (default: 512) - /// - public int Width { get; set; } = 512; - - /// - /// image height, in pixel space (default: 512) - /// - public int Height { get; set; } = 512; - - /// - /// sampling method (default: Euler_A) - /// - public Sampler SampleMethod { get; set; } = Sampler.Euler_A; - - /// - /// number of sample steps (default: 25) - /// - public int SampleSteps { get; set; } = 25; - - /// - /// RNG seed. use -1 for a random seed (default: -1) - /// - public long Seed { get; set; } = -1; - - /// - /// strength for noising/unnoising (default: 0.7) - /// - public float Strength { get; set; } = 0.7f; - - /// - /// ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1) - /// -1 represents unspecified, will be 1 for SD1.x, 2 for SD2.x - /// - public int ClipSkip { get; set; } = -1; - - /// - /// skip layer guidance (SLG) scale, only for DiT models: (default: 0) - /// 0 means disabled, a value of 2.5 is nice for sd3.5 medium - /// - public float SlgScale { get; set; } = 0f; - - /// - /// Layers to skip for SLG steps: (default: [7,8,9]) - /// - public int[] SkipLayers { get; set; } = [7, 8, 9]; - - /// - /// SLG enabling point: (default: 0.01) - /// - public float SkipLayerStart { get; set; } = 0.01f; - - /// - /// SLG disabling point: (default: 0.2) - /// - public float SkipLayerEnd { get; set; } = 0.2f; - - public ControlNetParameter ControlNet { get; } = new(); - - // Stable Diffusion only - /// - /// unconditional guidance scale: (default: 7.5) - /// - public float CfgScale { get; set; } = 7.5f; - - public PhotoMakerParameter PhotoMaker { get; } = new(); - - // Flux only - /// - /// - /// - public float Guidance { get; set; } = 3.5f; - - /// - /// eta in DDIM, only for DDIM and TCD (default: 0) - /// - public float Eta { get; set; } = 0f; - - #endregion -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs deleted file mode 100644 index 1372aed..0000000 --- a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs +++ /dev/null @@ -1,149 +0,0 @@ -using HPPH; - -namespace StableDiffusion.NET; - -public static class DiffusionParameterExtension -{ - public static DiffusionParameter WithSize(this DiffusionParameter parameter, int? width = null, int? height = null) - { - if (width != null) - parameter.Width = width.Value; - - if (height != null) - parameter.Height = height.Value; - - return parameter; - } - - public static DiffusionParameter WithSampler(this DiffusionParameter parameter, Sampler sampler) - { - parameter.SampleMethod = sampler; - - return parameter; - } - - public static DiffusionParameter WithSteps(this DiffusionParameter parameter, int steps) - { - parameter.SampleSteps = steps; - - return parameter; - } - - public static DiffusionParameter WithSeed(this DiffusionParameter parameter, long seed) - { - parameter.Seed = seed; - - return parameter; - } - - public static DiffusionParameter WithClipSkip(this DiffusionParameter parameter, int clipSkip) - { - parameter.ClipSkip = clipSkip; - - return parameter; - } - - public static DiffusionParameter WithCfg(this DiffusionParameter parameter, float cfg) - { - parameter.CfgScale = cfg; - - return parameter; - } - - public static DiffusionParameter WithGuidance(this DiffusionParameter parameter, float guidance) - { - parameter.Guidance = guidance; - - return parameter; - } - - public static DiffusionParameter WithNegativePrompt(this DiffusionParameter parameter, string negativePrompt) - { - parameter.NegativePrompt = negativePrompt; - - return parameter; - } - - public static DiffusionParameter WithStrength(this DiffusionParameter parameter, float strength) - { - parameter.Strength = strength; - - return parameter; - } - - public static DiffusionParameter WithSlgScale(this DiffusionParameter parameter, float slgScale) - { - parameter.SlgScale = slgScale; - - return parameter; - } - - public static DiffusionParameter WithSkipLayers(this DiffusionParameter parameter, int[] layers) - { - parameter.SkipLayers = layers; - - return parameter; - } - - public static DiffusionParameter WithSkipLayerStart(this DiffusionParameter parameter, float skipLayerStart) - { - parameter.SkipLayerStart = skipLayerStart; - - return parameter; - } - - public static DiffusionParameter WithSkipLayerEnd(this DiffusionParameter parameter, float skipLayerEnd) - { - parameter.SkipLayerEnd = skipLayerEnd; - - return parameter; - } - - public static DiffusionParameter WithControlNet(this DiffusionParameter parameter, IImage image, float? strength = null) - { - parameter.ControlNet.Image = image; - - if (strength != null) - parameter.ControlNet.Strength = strength.Value; - - return parameter; - } - - public static DiffusionParameter WithCannyPreprocessing(this DiffusionParameter parameter, - float? cannyHighThreshold = null, float? cannyLowThreshold = null, - float? cannyWeak = null, float? cannyStrong = null, - bool? cannyInverse = null) - { - parameter.ControlNet.CannyPreprocess = true; - - if (cannyHighThreshold != null) - parameter.ControlNet.CannyHighThreshold = cannyHighThreshold.Value; - - if (cannyLowThreshold != null) - parameter.ControlNet.CannyLowThreshold = cannyLowThreshold.Value; - - if (cannyWeak != null) - parameter.ControlNet.CannyWeak = cannyWeak.Value; - - if (cannyStrong != null) - parameter.ControlNet.CannyStrong = cannyStrong.Value; - - if (cannyInverse != null) - parameter.ControlNet.CannyInverse = cannyInverse.Value; - - return parameter; - } - - public static DiffusionParameter WithPhotomaker(this DiffusionParameter parameter, string inputIdImageDirectory, float? styleRatio = null, bool? normalizeInput = null) - { - parameter.PhotoMaker.InputIdImageDirectory = inputIdImageDirectory; - - if (styleRatio != null) - parameter.PhotoMaker.StyleRatio = styleRatio.Value; - - if (normalizeInput != null) - parameter.PhotoMaker.NormalizeInput = normalizeInput.Value; - - return parameter; - } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs new file mode 100644 index 0000000..8253a76 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs @@ -0,0 +1,215 @@ +using HPPH; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static class ImageGenerationParameterExtension +{ + public static ImageGenerationParameter WithPrompt(this ImageGenerationParameter parameter, string prompt) + { + parameter.Prompt = prompt; + + return parameter; + } + + public static ImageGenerationParameter WithNegativePrompt(this ImageGenerationParameter parameter, string negativePrompt) + { + parameter.NegativePrompt = negativePrompt; + + return parameter; + } + + public static ImageGenerationParameter WithClipSkip(this ImageGenerationParameter parameter, int clipSkip) + { + parameter.ClipSkip = clipSkip; + + return parameter; + } + + public static ImageGenerationParameter WithCfg(this ImageGenerationParameter parameter, float cfg) + { + parameter.WithTxtCfg(cfg); + parameter.WithImgCfg(cfg); + + return parameter; + } + + public static ImageGenerationParameter WithTxtCfg(this ImageGenerationParameter parameter, float txtCfg) + { + parameter.Guidance.TxtCfg = txtCfg; + + return parameter; + } + + public static ImageGenerationParameter WithImgCfg(this ImageGenerationParameter parameter, float imgCfg) + { + parameter.Guidance.ImgCfg = imgCfg; + + return parameter; + } + + public static ImageGenerationParameter WithMinCfg(this ImageGenerationParameter parameter, float minCfg) + { + parameter.Guidance.MinCfg = minCfg; + + return parameter; + } + + public static ImageGenerationParameter WithGuidance(this ImageGenerationParameter parameter, float guidance) + { + parameter.Guidance.DistilledGuidance = guidance; + + return parameter; + } + + public static ImageGenerationParameter WithSlgScale(this ImageGenerationParameter parameter, float slgScale) + { + parameter.Guidance.Slg.Scale = slgScale; + + return parameter; + } + + public static ImageGenerationParameter WithSkipLayers(this ImageGenerationParameter parameter, int[] layers) + { + parameter.Guidance.Slg.Layers = layers; + + return parameter; + } + + public static ImageGenerationParameter WithSkipLayerStart(this ImageGenerationParameter parameter, float skipLayerStart) + { + parameter.Guidance.Slg.SkipLayerStart = skipLayerStart; + + return parameter; + } + + public static ImageGenerationParameter WithSkipLayerEnd(this ImageGenerationParameter parameter, float skipLayerEnd) + { + parameter.Guidance.Slg.SkipLayerEnd = skipLayerEnd; + + return parameter; + } + + public static ImageGenerationParameter WithInitImage(this ImageGenerationParameter parameter, IImage image) + { + parameter.InitImage = image; + + return parameter; + } + + public static ImageGenerationParameter WithRefImages(this ImageGenerationParameter parameter, params IImage[] images) + { + parameter.RefImages = images; + + return parameter; + } + + public static ImageGenerationParameter WithMaskImage(this ImageGenerationParameter parameter, IImage image) + { + parameter.MaskImage = image; + + return parameter; + } + + public static ImageGenerationParameter WithSize(this ImageGenerationParameter parameter, int? width = null, int? height = null) + { + if (width != null) + parameter.Width = width.Value; + + if (height != null) + parameter.Height = height.Value; + + return parameter; + } + + public static ImageGenerationParameter WithSampler(this ImageGenerationParameter parameter, Sampler sampler) + { + parameter.SampleMethod = sampler; + + return parameter; + } + + public static ImageGenerationParameter WithSteps(this ImageGenerationParameter parameter, int steps) + { + parameter.SampleSteps = steps; + + return parameter; + } + + public static ImageGenerationParameter WithEta(this ImageGenerationParameter parameter, float eta) + { + parameter.Eta = eta; + + return parameter; + } + + public static ImageGenerationParameter WithStrength(this ImageGenerationParameter parameter, float strength) + { + parameter.Strength = strength; + + return parameter; + } + + public static ImageGenerationParameter WithSeed(this ImageGenerationParameter parameter, long seed) + { + parameter.Seed = seed; + + return parameter; + } + + public static ImageGenerationParameter WithControlNet(this ImageGenerationParameter parameter, IImage image, float? strength = null) + { + parameter.ControlNet.Image = image; + + if (strength != null) + parameter.ControlNet.Strength = strength.Value; + + return parameter; + } + + public static ImageGenerationParameter WithPhotomaker(this ImageGenerationParameter parameter, string inputIdImageDirectory, float? styleStrength = null, bool? normalizeInput = null) + { + parameter.PhotoMaker.InputIdImageDirectory = inputIdImageDirectory; + + if (styleStrength != null) + parameter.PhotoMaker.StyleStrength = styleStrength.Value; + + if (normalizeInput != null) + parameter.PhotoMaker.NormalizeInput = normalizeInput.Value; + + return parameter; + } + + #region Defaults + + public static ImageGenerationParameter WithSd1Defaults(this ImageGenerationParameter parameter) + => parameter.WithSize(512, 512) + .WithCfg(7.5f) + .WithGuidance(1f) + .WithSteps(25) + .WithSampler(Sampler.Euler_A); + + public static ImageGenerationParameter WithSDXLDefaults(this ImageGenerationParameter parameter) + => parameter.WithSize(1024, 1024) + .WithCfg(7f) + .WithGuidance(1f) + .WithSteps(30) + .WithSampler(Sampler.Euler_A); + + public static ImageGenerationParameter WithSD3_5Defaults(this ImageGenerationParameter parameter) + => parameter.WithSize(1024, 1024) + .WithCfg(4.5f) + .WithGuidance(1f) + .WithSteps(20) + .WithSampler(Sampler.Euler); + + public static ImageGenerationParameter WithFluxDefaults(this ImageGenerationParameter parameter) + => parameter.WithSize(1024, 1024) + .WithCfg(1f) + .WithGuidance(3.5f) + .WithSteps(20) + .WithSampler(Sampler.Euler); + + #endregion +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/GuidanceParameter.cs b/StableDiffusion.NET/Models/Parameter/GuidanceParameter.cs new file mode 100644 index 0000000..25b91e7 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/GuidanceParameter.cs @@ -0,0 +1,13 @@ +namespace StableDiffusion.NET; + +public sealed class GuidanceParameter +{ + public float TxtCfg { get; set; } = 7.0f; + public float ImgCfg { get; set; } = 7.0f; + public float MinCfg { get; set; } = 1.0f; + public float DistilledGuidance { get; set; } = 3.5f; + + public SlgParameter Slg { get; } = new(); + + internal GuidanceParameter() { } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs b/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs new file mode 100644 index 0000000..09e39dd --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs @@ -0,0 +1,77 @@ +using HPPH; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class ImageGenerationParameter +{ + #region Properties & Fields + + public string Prompt { get; set; } = string.Empty; + + /// + /// the negative prompt (default: ""); + /// + public string NegativePrompt { get; set; } = string.Empty; + + /// + /// ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1) + /// -1 represents unspecified, will be 1 for SD1.x, 2 for SD2.x + /// + public int ClipSkip { get; set; } = -1; + + public GuidanceParameter Guidance { get; } = new(); + + public IImage? InitImage { get; set; } + + public IImage[]? RefImages { get; set; } + + public IImage? MaskImage { get; set; } + + /// + /// image width, in pixel space (default: 512) + /// + public int Width { get; set; } = 512; + + /// + /// image height, in pixel space (default: 512) + /// + public int Height { get; set; } = 512; + + /// + /// sampling method (default: Euler_A) + /// + public Sampler SampleMethod { get; set; } = Sampler.Euler_A; + + /// + /// number of sample steps (default: 25) + /// + public int SampleSteps { get; set; } = 25; + + /// + /// eta in DDIM, only for DDIM and TCD (default: 0) + /// + public float Eta { get; set; } = 0f; + + /// + /// strength for noising/unnoising (default: 0.7) + /// + public float Strength { get; set; } = 0.7f; + + /// + /// RNG seed. use -1 for a random seed (default: -1) + /// + public long Seed { get; set; } = -1; + + public ControlNetParameter ControlNet { get; } = new(); + + public PhotoMakerParameter PhotoMaker { get; } = new(); + + public static ImageGenerationParameter Create() => new(); + + public static ImageGenerationParameter TextToImage(string prompt) => Create().WithPrompt(prompt); + public static ImageGenerationParameter ImageToImage(string prompt, IImage image) => Create().WithPrompt(prompt).WithInitImage(image); + + #endregion +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs index 12569ed..cadfe26 100644 --- a/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs @@ -1,9 +1,10 @@ -namespace StableDiffusion.NET; +using JetBrains.Annotations; -public interface IDiffusionModelParameter +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IDiffusionModelParameter : IModelParameter { - DiffusionModelType DiffusionModelType { get; set; } - string VaePath { get; set; } string TaesdPath { get; set; } @@ -17,7 +18,24 @@ public interface IDiffusionModelParameter bool KeepClipOnCPU { get; set; } bool KeepVaeOnCPU { get; set; } bool FlashAttention { get; set; } + bool DiffusionConvDirect { get; set; } + bool VaeConfDirect { get; set; } RngType RngType { get; set; } Schedule Schedule { get; set; } + Quantization Quantization { get; set; } + + string ModelPath { get; set; } + + string StackedIdEmbeddingsDirectory { get; set; } + + string DiffusionModelPath { get; set; } + string ClipLPath { get; set; } + string T5xxlPath { get; set; } + + bool ChromaUseDitMap { get; set; } + bool ChromaEnableT5Map { get; set; } + int ChromaT5MaskPad { get; set; } + + string ClipGPath { get; set; } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs new file mode 100644 index 0000000..ae23303 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs @@ -0,0 +1,9 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IModelParameter +{ + int ThreadCount { get; set; } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs deleted file mode 100644 index db59a09..0000000 --- a/StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace StableDiffusion.NET; - -public interface IPhotomakerModelParameter -{ - string StackedIdEmbeddingsDirectory { get; set; } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs deleted file mode 100644 index bd74727..0000000 --- a/StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace StableDiffusion.NET; - -public interface IQuantizedModelParameter -{ - int ThreadCount { get; set; } - - Quantization Quantization { get; set; } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs new file mode 100644 index 0000000..dfddf4a --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs @@ -0,0 +1,10 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IUpscaleModelParameter : IModelParameter +{ + string ModelPath { get; set; } + bool ConvDirect { get; set; } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs b/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs index 57466e1..5077158 100644 --- a/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs @@ -1,8 +1,5 @@ -using JetBrains.Annotations; +namespace StableDiffusion.NET; -namespace StableDiffusion.NET; - -[PublicAPI] public sealed class PhotoMakerParameter { /// @@ -13,10 +10,12 @@ public sealed class PhotoMakerParameter /// /// strength for keeping input identity (default: 20) /// - public float StyleRatio { get; set; } = 20f; + public float StyleStrength { get; set; } = 20f; /// /// normalize PHOTOMAKER input id images /// public bool NormalizeInput { get; set; } = false; + + internal PhotoMakerParameter() { } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/SlgParameter.cs b/StableDiffusion.NET/Models/Parameter/SlgParameter.cs new file mode 100644 index 0000000..090df64 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/SlgParameter.cs @@ -0,0 +1,27 @@ +namespace StableDiffusion.NET; + +public sealed class SlgParameter +{ + /// + /// Layers to skip for SLG steps: (default: [7,8,9]) + /// + public int[] Layers { get; set; } = [7, 8, 9]; + + /// + /// SLG enabling point: (default: 0.01) + /// + public float SkipLayerStart { get; set; } = 0.01f; + + /// + /// SLG disabling point: (default: 0.2) + /// + public float SkipLayerEnd { get; set; } = 0.2f; + + /// + /// skip layer guidance (SLG) scale, only for DiT models: (default: 0) + /// 0 means disabled, a value of 2.5 is nice for sd3.5 medium + /// + public float Scale { get; set; } = 0f; + + internal SlgParameter() { } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs index 17844de..da415e7 100644 --- a/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs @@ -3,7 +3,7 @@ namespace StableDiffusion.NET; [PublicAPI] -public sealed class UpscaleModelParameter : IQuantizedModelParameter +public sealed class UpscaleModelParameter : IUpscaleModelParameter { /// /// path to esrgan model. Upscale images after generate, just RealESRGAN_x4plus_anime_6B supported by now @@ -17,7 +17,10 @@ public sealed class UpscaleModelParameter : IQuantizedModelParameter public int ThreadCount { get; set; } = 1; /// - /// + /// use Conv2d direct in the diffusion model + /// This might crash if it is not supported by the backend. /// - public Quantization Quantization { get; set; } = Quantization.F16; + public bool ConvDirect { get; set; } = false; + + public static UpscaleModelParameter Create() => new(); } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/UpscaleModel.cs b/StableDiffusion.NET/Models/UpscaleModel.cs index 600f5b3..7d9c5b6 100644 --- a/StableDiffusion.NET/Models/UpscaleModel.cs +++ b/StableDiffusion.NET/Models/UpscaleModel.cs @@ -13,7 +13,7 @@ public sealed unsafe class UpscaleModel : IDisposable public UpscaleModelParameter ModelParameter { get; } - private Native.upscaler_ctx_t* _ctx; + private Native.Types.upscaler_ctx_t* _ctx; #endregion @@ -40,32 +40,19 @@ public sealed unsafe class UpscaleModel : IDisposable { _ctx = Native.new_upscaler_ctx(ModelParameter.ModelPath, ModelParameter.ThreadCount, - ModelParameter.Quantization); + ModelParameter.ConvDirect); if (_ctx == null) throw new NullReferenceException("Failed to initialize upscale-model."); } - public IImage Upscale(IImage image, int upscaleFactor) + public Image Upscale(IImage image, int upscaleFactor) { ObjectDisposedException.ThrowIf(_disposed, this); ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(upscaleFactor, 0, nameof(upscaleFactor)); if (_ctx == null) throw new NullReferenceException("The model is not initialized."); - if (image is not IImage sourceImage) - sourceImage = image.ConvertTo(); - - fixed (byte* imagePtr = sourceImage.AsRefImage()) - { - Native.sd_image_t result = Native.upscale(_ctx, sourceImage.ToSdImage(imagePtr), upscaleFactor); - return ImageHelper.ToImage(result); - } - } - - private IImage Upscale(Native.sd_image_t image, int upscaleFactor) - { - Native.sd_image_t result = Native.upscale(_ctx, image, upscaleFactor); - return ImageHelper.ToImage(result); + return Native.upscale(_ctx, image, (uint)upscaleFactor); } public void Dispose() diff --git a/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs new file mode 100644 index 0000000..baa2e00 --- /dev/null +++ b/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs @@ -0,0 +1,86 @@ +using System.Runtime.InteropServices.Marshalling; + +namespace StableDiffusion.NET; + +[CustomMarshaller(typeof(DiffusionModelParameter), MarshalMode.ManagedToUnmanagedIn, typeof(DiffusionModelParameterMarshaller))] +[CustomMarshaller(typeof(DiffusionModelParameter), MarshalMode.ManagedToUnmanagedRef, typeof(DiffusionModelParameterMarshaller))] +internal static unsafe class DiffusionModelParameterMarshaller +{ + public static Native.Types.sd_ctx_params_t ConvertToUnmanaged(DiffusionModelParameter managed) + => new() + { + model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ModelPath), + clip_l_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipLPath), + clip_g_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipGPath), + t5xxl_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.T5xxlPath), + diffusion_model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.DiffusionModelPath), + vae_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.VaePath), + taesd_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.TaesdPath), + control_net_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ControlNetPath), + lora_model_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.LoraModelDirectory), + embedding_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.EmbeddingsDirectory), + stacked_id_embed_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.StackedIdEmbeddingsDirectory), + vae_decode_only = (sbyte)(managed.VaeDecodeOnly ? 1 : 0), + vae_tiling = (sbyte)(managed.VaeTiling ? 1 : 0), + free_params_immediately = 0, // DarthAffe 06.08.2025: Static value + n_threads = managed.ThreadCount, + wtype = managed.Quantization, + rng_type = managed.RngType, + schedule = managed.Schedule, + keep_clip_on_cpu = (sbyte)(managed.KeepClipOnCPU ? 1 : 0), + keep_control_net_on_cpu = (sbyte)(managed.KeepControlNetOnCPU ? 1 : 0), + keep_vae_on_cpu = (sbyte)(managed.KeepVaeOnCPU ? 1 : 0), + diffusion_flash_attn = (sbyte)(managed.FlashAttention ? 1 : 0), + diffusion_conv_direct = (sbyte)(managed.DiffusionConvDirect ? 1 : 0), + vae_conv_direct = (sbyte)(managed.VaeConfDirect ? 1 : 0), + chroma_use_dit_mask = (sbyte)(managed.ChromaUseDitMap ? 1 : 0), + chroma_use_t5_mask = (sbyte)(managed.ChromaEnableT5Map ? 1 : 0), + chroma_t5_mask_pad = managed.ChromaT5MaskPad + }; + + public static DiffusionModelParameter ConvertToManaged(Native.Types.sd_ctx_params_t unmanaged) + => new() + { + ModelPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.model_path) ?? string.Empty, + ClipLPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.clip_l_path) ?? string.Empty, + ClipGPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.clip_g_path) ?? string.Empty, + T5xxlPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.t5xxl_path) ?? string.Empty, + DiffusionModelPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.diffusion_model_path) ?? string.Empty, + VaePath = AnsiStringMarshaller.ConvertToManaged(unmanaged.vae_path) ?? string.Empty, + TaesdPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.taesd_path) ?? string.Empty, + ControlNetPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.control_net_path) ?? string.Empty, + LoraModelDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.lora_model_dir) ?? string.Empty, + EmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.embedding_dir) ?? string.Empty, + StackedIdEmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.stacked_id_embed_dir) ?? string.Empty, + VaeDecodeOnly = unmanaged.vae_decode_only == 1, + VaeTiling = unmanaged.vae_tiling == 1, + ThreadCount = unmanaged.n_threads, + Quantization = unmanaged.wtype, + RngType = unmanaged.rng_type, + Schedule = unmanaged.schedule, + KeepClipOnCPU = unmanaged.keep_clip_on_cpu == 1, + KeepControlNetOnCPU = unmanaged.keep_control_net_on_cpu == 1, + KeepVaeOnCPU = unmanaged.keep_vae_on_cpu == 1, + FlashAttention = unmanaged.diffusion_flash_attn == 1, + DiffusionConvDirect = unmanaged.diffusion_conv_direct == 1, + VaeConfDirect = unmanaged.vae_conv_direct == 1, + ChromaUseDitMap = unmanaged.chroma_use_dit_mask == 1, + ChromaEnableT5Map = unmanaged.chroma_use_t5_mask == 1, + ChromaT5MaskPad = unmanaged.chroma_t5_mask_pad + }; + + public static void Free(Native.Types.sd_ctx_params_t unmanaged) + { + AnsiStringMarshaller.Free(unmanaged.model_path); + AnsiStringMarshaller.Free(unmanaged.clip_l_path); + AnsiStringMarshaller.Free(unmanaged.clip_g_path); + AnsiStringMarshaller.Free(unmanaged.t5xxl_path); + AnsiStringMarshaller.Free(unmanaged.diffusion_model_path); + AnsiStringMarshaller.Free(unmanaged.vae_path); + AnsiStringMarshaller.Free(unmanaged.taesd_path); + AnsiStringMarshaller.Free(unmanaged.control_net_path); + AnsiStringMarshaller.Free(unmanaged.lora_model_dir); + AnsiStringMarshaller.Free(unmanaged.embedding_dir); + AnsiStringMarshaller.Free(unmanaged.stacked_id_embed_dir); + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs new file mode 100644 index 0000000..08915f2 --- /dev/null +++ b/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs @@ -0,0 +1,197 @@ +// ReSharper disable MemberCanBeMadeStatic.Global + +using System; +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; + +namespace StableDiffusion.NET; + +[CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedIn, typeof(ImageGenerationParameterMarshallerIn))] +[CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedOut, typeof(ImageGenerationParameterMarshaller))] +[CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedRef, typeof(ImageGenerationParameterMarshallerRef))] +internal static class ImageGenerationParameterMarshaller +{ + public static unsafe ImageGenerationParameter ConvertToManaged(Native.Types.sd_img_gen_params_t unmanaged) + { + ImageGenerationParameter parameter = new() + { + Prompt = AnsiStringMarshaller.ConvertToManaged(unmanaged.prompt) ?? string.Empty, + NegativePrompt = AnsiStringMarshaller.ConvertToManaged(unmanaged.negative_prompt) ?? string.Empty, + ClipSkip = unmanaged.clip_skip, + Guidance = + { + TxtCfg = unmanaged.guidance.txt_cfg, + ImgCfg = unmanaged.guidance.img_cfg, + MinCfg = unmanaged.guidance.min_cfg, + DistilledGuidance = unmanaged.guidance.distilled_guidance, + Slg = + { + Layers = new int[unmanaged.guidance.slg.layer_count], + SkipLayerStart = unmanaged.guidance.slg.layer_start, + SkipLayerEnd = unmanaged.guidance.slg.layer_end, + Scale = unmanaged.guidance.slg.scale + } + }, + InitImage = unmanaged.init_image.data == null ? null : unmanaged.init_image.ToImage(), + RefImages = unmanaged.ref_images == null ? null : ImageHelper.ToImageArrayIFace(unmanaged.ref_images, unmanaged.ref_images_count), + MaskImage = unmanaged.mask_image.data == null ? null : unmanaged.mask_image.ToImage(), + Width = unmanaged.width, + Height = unmanaged.height, + SampleMethod = unmanaged.sample_method, + SampleSteps = unmanaged.sample_steps, + Eta = unmanaged.eta, + Strength = unmanaged.strength, + Seed = unmanaged.seed, + ControlNet = + { + Image = unmanaged.control_cond == null ? null : ImageHelper.GetImage(unmanaged.control_cond, 0), + Strength = unmanaged.control_strength, + }, + PhotoMaker = + { + StyleStrength = unmanaged.style_strength, + NormalizeInput = unmanaged.normalize_input == 1, + InputIdImageDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.input_id_images_path) ?? string.Empty, + } + }; + + if (unmanaged.guidance.slg.layers != null) + new Span(unmanaged.guidance.slg.layers, (int)unmanaged.guidance.slg.layer_count).CopyTo(parameter.Guidance.Slg.Layers); + + return parameter; + } + + public static unsafe void Free(Native.Types.sd_img_gen_params_t unmanaged) + { + AnsiStringMarshaller.Free(unmanaged.prompt); + AnsiStringMarshaller.Free(unmanaged.negative_prompt); + AnsiStringMarshaller.Free(unmanaged.input_id_images_path); + + unmanaged.init_image.Free(); + unmanaged.mask_image.Free(); + + if (unmanaged.ref_images != null) + ImageHelper.Free(unmanaged.ref_images, unmanaged.ref_images_count); + + if (unmanaged.control_cond != null) + ImageHelper.Free(unmanaged.control_cond, 1); + + if (unmanaged.guidance.slg.layers != null) + NativeMemory.Free(unmanaged.guidance.slg.layers); + } + + internal unsafe ref struct ImageGenerationParameterMarshallerIn + { + private Native.Types.sd_img_gen_params_t _imgGenParams; + + private Native.Types.sd_image_t _initImage; + private Native.Types.sd_image_t _maskImage; + private Native.Types.sd_image_t* _refImages; + private Native.Types.sd_image_t* _controlNetImage; + private int* _slgLayers; + + public void FromManaged(ImageGenerationParameter managed) + { + _initImage = managed.InitImage?.ToSdImage() ?? new Native.Types.sd_image_t(); + _refImages = managed.RefImages == null ? null : managed.RefImages.ToSdImage(); + _controlNetImage = managed.ControlNet.Image == null ? null : managed.ControlNet.Image.ToSdImagePtr(); + + _slgLayers = (int*)NativeMemory.Alloc((nuint)managed.Guidance.Slg.Layers.Length, (nuint)Marshal.SizeOf()); + managed.Guidance.Slg.Layers.AsSpan().CopyTo(new Span(_slgLayers, managed.Guidance.Slg.Layers.Length)); + + if (managed.MaskImage != null) + _maskImage = managed.MaskImage.ToSdImage(true); + else if (managed.InitImage != null) + { + // DarthAffe 16.08.2025: Mask needs to be a 1 channel all max value image when it's not used - I really don't like this concept as it adds unnecessary allocations, but that's how it is :( + uint maskImageByteSize = _initImage.width * _initImage.height; + _maskImage = new Native.Types.sd_image_t + { + width = _initImage.width, + height = _initImage.height, + channel = 1, + data = (byte*)NativeMemory.Alloc(maskImageByteSize) + }; + new Span(_maskImage.data, (int)maskImageByteSize).Fill(byte.MaxValue); + } + + Native.Types.sd_slg_params_t slg = new() + { + layers = _slgLayers, + layer_count = (uint)managed.Guidance.Slg.Layers.Length, + layer_start = managed.Guidance.Slg.SkipLayerStart, + layer_end = managed.Guidance.Slg.SkipLayerEnd, + scale = managed.Guidance.Slg.Scale, + }; + + Native.Types.sd_guidance_params_t guidance = new() + { + txt_cfg = managed.Guidance.TxtCfg, + img_cfg = managed.Guidance.ImgCfg, + min_cfg = managed.Guidance.MinCfg, + distilled_guidance = managed.Guidance.DistilledGuidance, + slg = slg + }; + + _imgGenParams = new Native.Types.sd_img_gen_params_t + { + prompt = AnsiStringMarshaller.ConvertToUnmanaged(managed.Prompt), + negative_prompt = AnsiStringMarshaller.ConvertToUnmanaged(managed.NegativePrompt), + clip_skip = managed.ClipSkip, + guidance = guidance, + init_image = _initImage, + ref_images = _refImages, + ref_images_count = managed.RefImages?.Length ?? 0, + mask_image = _maskImage, + width = managed.Width, + height = managed.Height, + sample_method = managed.SampleMethod, + sample_steps = managed.SampleSteps, + eta = managed.Eta, + strength = managed.Strength, + seed = managed.Seed, + batch_count = 1, + control_cond = _controlNetImage, + control_strength = managed.ControlNet.Strength, + style_strength = managed.PhotoMaker.StyleStrength, + normalize_input = (sbyte)(managed.PhotoMaker.NormalizeInput ? 1 : 0), + input_id_images_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.PhotoMaker.InputIdImageDirectory), + }; + } + + public Native.Types.sd_img_gen_params_t ToUnmanaged() => _imgGenParams; + + public void Free() + { + AnsiStringMarshaller.Free(_imgGenParams.prompt); + AnsiStringMarshaller.Free(_imgGenParams.negative_prompt); + AnsiStringMarshaller.Free(_imgGenParams.input_id_images_path); + + _initImage.Free(); + _maskImage.Free(); + + if (_refImages != null) + ImageHelper.Free(_refImages, _imgGenParams.ref_images_count); + + if (_controlNetImage != null) + ImageHelper.Free(_controlNetImage, 1); + + if (_slgLayers != null) + NativeMemory.Free(_slgLayers); + } + } + + internal ref struct ImageGenerationParameterMarshallerRef() + { + private ImageGenerationParameterMarshallerIn _inMarshaller = new(); + private ImageGenerationParameter _parameter; + + public void FromManaged(ImageGenerationParameter managed) => _inMarshaller.FromManaged(managed); + public Native.Types.sd_img_gen_params_t ToUnmanaged() => _inMarshaller.ToUnmanaged(); + + public void FromUnmanaged(Native.Types.sd_img_gen_params_t unmanaged) => _parameter = ConvertToManaged(unmanaged); + public ImageGenerationParameter ToManaged() => _parameter; + + public void Free() => _inMarshaller.Free(); + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Marshaller/ImageMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/ImageMarshaller.cs new file mode 100644 index 0000000..dd82378 --- /dev/null +++ b/StableDiffusion.NET/Native/Marshaller/ImageMarshaller.cs @@ -0,0 +1,24 @@ +using HPPH; +using System.Runtime.InteropServices.Marshalling; + +namespace StableDiffusion.NET; + +[CustomMarshaller(typeof(IImage), MarshalMode.ManagedToUnmanagedIn, typeof(ImageMarshallerIn))] +[CustomMarshaller(typeof(Image), MarshalMode.ManagedToUnmanagedOut, typeof(ImageMarshaller))] +internal static class ImageMarshaller +{ + public static Image ConvertToManaged(Native.Types.sd_image_t unmanaged) => unmanaged.ToImage(); + + public static void Free(Native.Types.sd_image_t unmanaged) => unmanaged.Free(); + + internal ref struct ImageMarshallerIn + { + private Native.Types.sd_image_t _image; + + public void FromManaged(IImage managed) => _image = managed.ToSdImage(); + + public Native.Types.sd_image_t ToUnmanaged() => _image; + + public void Free() => _image.Free(); + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Native.Load.cs b/StableDiffusion.NET/Native/Native.Load.cs index 61ecb09..6288adb 100644 --- a/StableDiffusion.NET/Native/Native.Load.cs +++ b/StableDiffusion.NET/Native/Native.Load.cs @@ -38,7 +38,7 @@ internal static partial class Native return false; } - private static nint ResolveDllImport(string libraryname, Assembly assembly, DllImportSearchPath? searchpath) + private static nint ResolveDllImport(string libraryname, Assembly _, DllImportSearchPath? __) { if (libraryname != LIB_NAME) return nint.Zero; if (_loadedLibraryHandle != nint.Zero) return _loadedLibraryHandle; diff --git a/StableDiffusion.NET/Native/Native.cs b/StableDiffusion.NET/Native/Native.cs index 757cf86..7afd2b0 100644 --- a/StableDiffusion.NET/Native/Native.cs +++ b/StableDiffusion.NET/Native/Native.cs @@ -2,8 +2,11 @@ #pragma warning disable IDE1006 // ReSharper disable InconsistentNaming // ReSharper disable ArrangeTypeMemberModifiers +// ReSharper disable UseSymbolAlias using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; +using HPPH; namespace StableDiffusion.NET; @@ -12,6 +15,17 @@ using sample_method_t = Sampler; using schedule_t = Schedule; using sd_type_t = Quantization; using sd_log_level_t = LogLevel; +using uint32_t = uint; +using uint8_t = byte; +using int64_t = long; +using size_t = nuint; +using int32_t = int; +using sd_ctx_params_t = DiffusionModelParameter; +using sd_img_gen_params_t = ImageGenerationParameter; +using sd_vid_gen_params_t = Native.Types.sd_vid_gen_params_t; +using sd_image_t = Native.Types.sd_image_t; +using sd_ctx_t = Native.Types.sd_ctx_t; +using upscaler_ctx_t = Native.Types.upscaler_ctx_t; internal unsafe partial class Native { @@ -21,6 +35,116 @@ internal unsafe partial class Native #endregion + internal static class Types + { + [StructLayout(LayoutKind.Sequential)] + internal struct sd_ctx_params_t + { + public byte* model_path; + public byte* clip_l_path; + public byte* clip_g_path; + public byte* t5xxl_path; + public byte* diffusion_model_path; + public byte* vae_path; + public byte* taesd_path; + public byte* control_net_path; + public byte* lora_model_dir; + public byte* embedding_dir; + public byte* stacked_id_embed_dir; + public sbyte vae_decode_only; + public sbyte vae_tiling; + public sbyte free_params_immediately; + public int n_threads; + public sd_type_t wtype; + public rng_type_t rng_type; + public schedule_t schedule; + public sbyte keep_clip_on_cpu; + public sbyte keep_control_net_on_cpu; + public sbyte keep_vae_on_cpu; + public sbyte diffusion_flash_attn; + public sbyte diffusion_conv_direct; + public sbyte vae_conv_direct; + public sbyte chroma_use_dit_mask; + public sbyte chroma_use_t5_mask; + public int chroma_t5_mask_pad; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct sd_image_t + { + public uint32_t width; + public uint32_t height; + public uint32_t channel; + public uint8_t* data; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct sd_slg_params_t + { + public int* layers; + public size_t layer_count; + public float layer_start; + public float layer_end; + public float scale; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct sd_guidance_params_t + { + public float txt_cfg; + public float img_cfg; + public float min_cfg; + public float distilled_guidance; + public sd_slg_params_t slg; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct sd_img_gen_params_t + { + public byte* prompt; + public byte* negative_prompt; + public int clip_skip; + public sd_guidance_params_t guidance; + public sd_image_t init_image; + public sd_image_t* ref_images; + public int ref_images_count; + public sd_image_t mask_image; + public int width; + public int height; + public sample_method_t sample_method; + public int sample_steps; + public float eta; + public float strength; + public int64_t seed; + public int batch_count; + public sd_image_t* control_cond; + public float control_strength; + public float style_strength; + public sbyte normalize_input; + public byte* input_id_images_path; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct sd_vid_gen_params_t + { + public sd_image_t init_image; + public int width; + public int height; + public sd_guidance_params_t guidance; + public sample_method_t sample_method; + public int sample_steps; + public float strength; + public int64_t seed; + public int video_frames; + public int motion_bucket_id; + public int fps; + public float augmentation_level; + } + + internal struct sd_ctx_t; + internal struct upscaler_ctx_t; + } + #region Delegates internal delegate void sd_log_cb_t(sd_log_level_t level, [MarshalAs(UnmanagedType.LPStr)] string text, void* data); @@ -28,179 +152,7 @@ internal unsafe partial class Native #endregion - #region DLL-Import - - internal struct sd_ctx_t; - internal struct upscaler_ctx_t; - - [StructLayout(LayoutKind.Sequential)] - internal struct sd_image_t - { - internal uint width; - internal uint height; - internal uint channel; - internal byte* data; - } - - [LibraryImport(LIB_NAME, EntryPoint = "get_num_physical_cores")] - internal static partial int get_num_physical_cores(); - - [LibraryImport(LIB_NAME, EntryPoint = "sd_get_system_info")] - internal static partial void* sd_get_system_info(); - - [LibraryImport(LIB_NAME, EntryPoint = "new_sd_ctx")] - internal static partial sd_ctx_t* new_sd_ctx([MarshalAs(UnmanagedType.LPStr)] string model_path, - [MarshalAs(UnmanagedType.LPStr)] string clip_l_path, - [MarshalAs(UnmanagedType.LPStr)] string clip_g_path, - [MarshalAs(UnmanagedType.LPStr)] string t5xxl_path, - [MarshalAs(UnmanagedType.LPStr)] string diffusion_model_path, - [MarshalAs(UnmanagedType.LPStr)] string vae_path, - [MarshalAs(UnmanagedType.LPStr)] string taesd_path, - [MarshalAs(UnmanagedType.LPStr)] string control_net_path_c_str, - [MarshalAs(UnmanagedType.LPStr)] string lora_model_dir, - [MarshalAs(UnmanagedType.LPStr)] string embed_dir_c_str, - [MarshalAs(UnmanagedType.LPStr)] string stacked_id_embed_dir_c_str, - [MarshalAs(UnmanagedType.I1)] bool vae_decode_only, - [MarshalAs(UnmanagedType.I1)] bool vae_tiling, - [MarshalAs(UnmanagedType.I1)] bool free_params_immediately, - int n_threads, - sd_type_t wtype, - rng_type_t rng_type, - schedule_t s, - [MarshalAs(UnmanagedType.I1)] bool keep_clip_on_cpu, - [MarshalAs(UnmanagedType.I1)] bool keep_control_net_cpu, - [MarshalAs(UnmanagedType.I1)] bool keep_vae_on_cpu, - [MarshalAs(UnmanagedType.I1)] bool diffusion_flash_attn, - [MarshalAs(UnmanagedType.I1)] bool chroma_use_dit_mask, - [MarshalAs(UnmanagedType.I1)] bool chroma_use_t5_mask, - int chroma_t5_mask_pad); - - [LibraryImport(LIB_NAME, EntryPoint = "free_sd_ctx")] - internal static partial void free_sd_ctx(sd_ctx_t* sd_ctx); - - [LibraryImport(LIB_NAME, EntryPoint = "txt2img")] - internal static partial sd_image_t* txt2img(sd_ctx_t* sd_ctx, - [MarshalAs(UnmanagedType.LPStr)] string prompt, - [MarshalAs(UnmanagedType.LPStr)] string negative_prompt, - int clip_skip, - float cfg_scale, - float guidance, - float eta, - int width, - int height, - sample_method_t sample_method, - int sample_steps, - long seed, - int batch_count, - sd_image_t* control_cond, - float control_strength, - float style_strength, - [MarshalAs(UnmanagedType.I1)] bool normalize_input, - [MarshalAs(UnmanagedType.LPStr)] string input_id_images_path, - in int[] skip_layers, - int skip_layers_count, - float slg_scale, - float skip_layer_start, - float skip_layer_end); - - [LibraryImport(LIB_NAME, EntryPoint = "img2img")] - internal static partial sd_image_t* img2img(sd_ctx_t* sd_ctx, - sd_image_t init_image, - sd_image_t mask_image, - [MarshalAs(UnmanagedType.LPStr)] string prompt, - [MarshalAs(UnmanagedType.LPStr)] string negative_prompt, - int clip_skip, - float cfg_scale, - float guidance, - int width, - int height, - sample_method_t sample_method, - int sample_steps, - float strength, - long seed, - int batch_count, - sd_image_t* control_cond, - float control_strength, - float style_strength, - [MarshalAs(UnmanagedType.I1)] bool normalize_input, - [MarshalAs(UnmanagedType.LPStr)] string input_id_images_path, - in int[] skip_layers, - int skip_layers_count, - float slg_scale, - float skip_layer_start, - float skip_layer_end); - - [LibraryImport(LIB_NAME, EntryPoint = "img2vid")] - internal static partial sd_image_t* img2vid(sd_ctx_t* sd_ctx, - sd_image_t init_image, - int width, - int height, - int video_frames, - int motion_bucket_id, - int fps, - float augmentation_level, - float min_cfg, - float cfg_scale, - sample_method_t sample_method, - int sample_steps, - float strength, - long seed); - - [LibraryImport(LIB_NAME, EntryPoint = "edit")] - internal static partial sd_image_t* edit(sd_ctx_t* sd_ctx, - sd_image_t* ref_images, - int ref_images_count, - [MarshalAs(UnmanagedType.LPStr)] string prompt, - [MarshalAs(UnmanagedType.LPStr)] string negative_prompt, - int clip_skip, - float cfg_scale, - float guidance, - float eta, - int width, - int height, - sample_method_t sample_method, - int sample_steps, - float strength, - long seed, - int batch_count, - sd_image_t* control_cond, - float control_strength, - float style_strength, - [MarshalAs(UnmanagedType.I1)] bool normalize_input, - in int[] skip_layers, - int skip_layers_count, - float slg_scale, - float skip_layer_start, - float skip_layer_end); - - [LibraryImport(LIB_NAME, EntryPoint = "new_upscaler_ctx")] - internal static partial upscaler_ctx_t* new_upscaler_ctx([MarshalAs(UnmanagedType.LPStr)] string esrgan_path, - int n_threads, - sd_type_t wtype); - - [LibraryImport(LIB_NAME, EntryPoint = "free_upscaler_ctx")] - internal static partial void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx); - - [LibraryImport(LIB_NAME, EntryPoint = "upscale")] - internal static partial sd_image_t upscale(upscaler_ctx_t* upscaler_ctx, - sd_image_t input_image, - int upscale_factor); - - [LibraryImport(LIB_NAME, EntryPoint = "convert")] - internal static partial void convert([MarshalAs(UnmanagedType.LPStr)] string input_path, - [MarshalAs(UnmanagedType.LPStr)] string vae_path, - [MarshalAs(UnmanagedType.LPStr)] string output_path, - sd_type_t output_type); - - [LibraryImport(LIB_NAME, EntryPoint = "preprocess_canny")] - internal static partial byte* preprocess_canny(byte* img, - int width, - int height, - float high_threshold, - float low_threshold, - float weak, - float strong, - [MarshalAs(UnmanagedType.I1)] bool inverse); + #region Methods [LibraryImport(LIB_NAME, EntryPoint = "sd_set_log_callback")] internal static partial void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data); @@ -208,5 +160,116 @@ internal unsafe partial class Native [LibraryImport(LIB_NAME, EntryPoint = "sd_set_progress_callback")] internal static partial void sd_set_progress_callback(sd_progress_cb_t cb, void* data); + [LibraryImport(LIB_NAME, EntryPoint = "get_num_physical_cores")] + internal static partial int32_t get_num_physical_cores(); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_get_system_info")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_get_system_info(); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "sd_type_name")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_type_name(sd_type_t type); + + [LibraryImport(LIB_NAME, EntryPoint = "str_to_sd_type")] + internal static partial sd_type_t str_to_sd_type([MarshalAs(UnmanagedType.LPStr)] string str); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_rng_type_name")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_rng_type_name(rng_type_t rng_type); + + [LibraryImport(LIB_NAME, EntryPoint = "str_to_rng_type")] + internal static partial rng_type_t str_to_rng_type([MarshalAs(UnmanagedType.LPStr)] string str); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_sample_method_name")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_sample_method_name(sample_method_t sample_method); + + [LibraryImport(LIB_NAME, EntryPoint = "str_to_sample_method")] + internal static partial sample_method_t str_to_sample_method([MarshalAs(UnmanagedType.LPStr)] string str); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_schedule_name")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_schedule_name(schedule_t schedule); + + [LibraryImport(LIB_NAME, EntryPoint = "str_to_schedule")] + internal static partial schedule_t str_to_schedule([MarshalAs(UnmanagedType.LPStr)] string str); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "sd_ctx_params_init")] + internal static partial void sd_ctx_params_init([MarshalUsing(typeof(DiffusionModelParameterMarshaller))] ref sd_ctx_params_t sd_ctx_params); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_ctx_params_to_str")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_ctx_params_to_str([MarshalUsing(typeof(DiffusionModelParameterMarshaller))] in sd_ctx_params_t sd_ctx_params); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "new_sd_ctx")] + internal static partial sd_ctx_t* new_sd_ctx([MarshalUsing(typeof(DiffusionModelParameterMarshaller))] in sd_ctx_params_t sd_ctx_params); + + [LibraryImport(LIB_NAME, EntryPoint = "free_sd_ctx")] + internal static partial void free_sd_ctx(sd_ctx_t* sd_ctx); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "sd_img_gen_params_init")] + internal static partial void sd_img_gen_params_init([MarshalUsing(typeof(ImageGenerationParameterMarshaller))] ref sd_img_gen_params_t sd_img_gen_params); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_img_gen_params_to_str")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_img_gen_params_to_str([MarshalUsing(typeof(ImageGenerationParameterMarshaller))] in sd_img_gen_params_t sd_img_gen_params); + + [LibraryImport(LIB_NAME, EntryPoint = "generate_image")] + internal static partial sd_image_t* generate_image(sd_ctx_t* sd_ctx, [MarshalUsing(typeof(ImageGenerationParameterMarshaller))] in sd_img_gen_params_t sd_img_gen_params); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "sd_vid_gen_params_init")] + internal static partial void sd_vid_gen_params_init(ref sd_vid_gen_params_t sd_vid_gen_params); + + [LibraryImport(LIB_NAME, EntryPoint = "generate_video")] + [return: MarshalUsing(typeof(ImageMarshaller))] + internal static partial sd_image_t* generate_video(sd_ctx_t* sd_ctx, in sd_vid_gen_params_t sd_vid_gen_params); // broken + + // + + [LibraryImport(LIB_NAME, EntryPoint = "new_upscaler_ctx")] + internal static partial upscaler_ctx_t* new_upscaler_ctx([MarshalAs(UnmanagedType.LPStr)] string esrgan_path, int n_threads, [MarshalAs(UnmanagedType.I1)] bool direct); + + [LibraryImport(LIB_NAME, EntryPoint = "free_upscaler_ctx")] + internal static partial void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "upscale")] + [return: MarshalUsing(typeof(ImageMarshaller))] + internal static partial Image upscale(upscaler_ctx_t* upscaler_ctx, [MarshalUsing(typeof(ImageMarshaller))] IImage input_image, uint32_t upscale_factor); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "convert")] + [return: MarshalAs(UnmanagedType.I1)] + internal static partial bool convert([MarshalAs(UnmanagedType.LPStr)] string input_path, + [MarshalAs(UnmanagedType.LPStr)] string vae_path, + [MarshalAs(UnmanagedType.LPStr)] string output_path, + sd_type_t output_type, + [MarshalAs(UnmanagedType.LPStr)] string tensor_type_rules); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "preprocess_canny")] + internal static partial uint8_t* preprocess_canny(uint8_t* img, + int width, + int height, + float high_threshold, + float low_threshold, + float weak, + float strong, + [MarshalAs(UnmanagedType.I1)] bool inverse); + #endregion } \ No newline at end of file diff --git a/StableDiffusion.NET/StableDiffusion.NET.csproj b/StableDiffusion.NET/StableDiffusion.NET.csproj index 2e33017..29afdc2 100644 --- a/StableDiffusion.NET/StableDiffusion.NET.csproj +++ b/StableDiffusion.NET/StableDiffusion.NET.csproj @@ -56,6 +56,6 @@ - + diff --git a/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings b/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings index 65e7f6a..eb80f1b 100644 --- a/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings +++ b/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings @@ -13,4 +13,5 @@ True True True - True \ No newline at end of file + True + True \ No newline at end of file diff --git a/StableDiffusion.NET/StableDiffusionCpp.cs b/StableDiffusion.NET/StableDiffusionCpp.cs index cf7e87c..71f473f 100644 --- a/StableDiffusion.NET/StableDiffusionCpp.cs +++ b/StableDiffusion.NET/StableDiffusionCpp.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.InteropServices; +using HPPH; using JetBrains.Annotations; namespace StableDiffusion.NET; @@ -10,8 +11,8 @@ public static unsafe class StableDiffusionCpp #region Properties & Fields // ReSharper disable NotAccessedField.Local - They are important, the delegate can be collected if it's not stored! - private static readonly Native.sd_log_cb_t LOG_CALLBACK; - private static readonly Native.sd_progress_cb_t PROGRESS_CALLBACK; + private static Native.sd_log_cb_t LOG_CALLBACK; + private static Native.sd_progress_cb_t PROGRESS_CALLBACK; // ReSharper restore NotAccessedField.Local #endregion @@ -23,38 +24,60 @@ public static unsafe class StableDiffusionCpp #endregion - #region Constructors + #region Methods - static StableDiffusionCpp() + public static bool LoadNativeLibrary(string libraryPath) => Native.LoadNativeLibrary(libraryPath); + + public static void InitializeEvents() { Native.sd_set_log_callback(LOG_CALLBACK = OnNativeLog, null); Native.sd_set_progress_callback(PROGRESS_CALLBACK = OnNativeProgress, null); } - #endregion - - #region Methods - - public static bool LoadNativeLibrary(string libraryPath) => Native.LoadNativeLibrary(libraryPath); - - public static void Convert(string modelPath, string vaePath, Quantization quantization, string outputPath) + public static void Convert(string modelPath, string vaePath, Quantization quantization, string outputPath, string tensorTypeRules = "") { ArgumentException.ThrowIfNullOrWhiteSpace(nameof(modelPath)); ArgumentException.ThrowIfNullOrWhiteSpace(nameof(outputPath)); ArgumentNullException.ThrowIfNull(vaePath); if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization)); - Native.convert(modelPath, vaePath, outputPath, quantization); + Native.convert(modelPath, vaePath, outputPath, quantization, tensorTypeRules); } - public static string GetSystemInfo() - { - void* s = Native.sd_get_system_info(); - return Marshal.PtrToStringUTF8((nint)s) ?? ""; - } + public static string GetSystemInfo() => Native.sd_get_system_info(); public static int GetNumPhysicalCores() => Native.get_num_physical_cores(); + public static Image PreprocessCanny(CannyParameter parameter) + { + parameter.Validate(); + + IImage controlImage = parameter.Image as IImage ?? parameter.Image!.ConvertTo(); + + byte[] controlImageData = controlImage.ToRawArray(); + fixed (byte* controlImagePtr = controlImageData) + { + byte* result = Native.preprocess_canny(controlImagePtr, + controlImage.Width, + controlImage.Height, + parameter.HighThreshold, + parameter.LowThreshold, + parameter.Weak, + parameter.Strong, + parameter.Inverse); + + try + { + return Image.Create(new ReadOnlySpan(result, controlImageData.Length), + controlImage.Width, controlImage.Height); + } + finally + { + NativeMemory.Free(result); + } + } + } + private static void OnNativeLog(LogLevel level, string text, void* data) { try