From 76dc8511395157e24b67ed38288666ed6838f9aa Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Sat, 16 Aug 2025 23:12:24 +0200 Subject: [PATCH 1/9] Reworked everything to fit the new sd.cpp API --- StableDiffusion.NET/Backends/CudaBackend.cs | 2 +- .../Enums/DiffusionModelType.cs | 8 - .../Extensions/ParameterExtension.cs | 106 ++--- StableDiffusion.NET/Helper/ImageHelper.cs | 170 +++++--- .../Models/Builder/ESRGANModelBuilder.cs | 5 +- .../DiffusionModelBuilderExtension.cs | 122 +++++- ...rExtension.cs => ModelBuilderExtension.cs} | 14 +- .../PhotomakerModelBuilderExtension.cs | 18 - .../UpscaleModelBuilderExtension.cs | 27 ++ .../Models/Builder/FluxModelBuilder.cs | 6 +- .../Interfaces/IDiffusionModelBuilder.cs | 9 +- .../Builder/Interfaces/IModelBuilder.cs | 9 + .../Interfaces/IPhotomakerModelBuilder.cs | 6 - .../Interfaces/IQuantizedModelBuilder.cs | 6 - .../Interfaces/IUpscaleModelBuilder.cs | 11 + .../Builder/StableDiffusion3_5ModelBuilder.cs | 6 +- .../Builder/StableDiffusionModelBuilder.cs | 8 +- StableDiffusion.NET/Models/DiffusionModel.cs | 375 ++-------------- .../Models/Parameter/CannyParameter.cs | 40 ++ .../Models/Parameter/ControlNetParameter.cs | 34 +- .../Parameter/DiffusionModelParameter.cs | 26 +- .../Models/Parameter/DiffusionParameter.cs | 99 ----- .../Extensions/DiffusionParameterExtension.cs | 149 ------- .../ImageGenerationParameterExtension.cs | 215 +++++++++ .../Models/Parameter/GuidanceParameter.cs | 13 + .../Parameter/ImageGenerationParameter.cs | 77 ++++ .../Interfaces/IDiffusionModelParameter.cs | 26 +- .../Parameter/Interfaces/IModelParameter.cs | 9 + .../Interfaces/IPhotomakerModelParameter.cs | 6 - .../Interfaces/IQuantizedModelParameter.cs | 8 - .../Interfaces/IUpscaleModelParameter.cs | 10 + .../Models/Parameter/PhotoMakerParameter.cs | 9 +- .../Models/Parameter/SlgParameter.cs | 27 ++ .../Models/Parameter/UpscaleModelParameter.cs | 9 +- StableDiffusion.NET/Models/UpscaleModel.cs | 21 +- .../DiffusionModelParameterMarshaller.cs | 86 ++++ .../ImageGenerationParameterMarshaller.cs | 197 +++++++++ .../Native/Marshaller/ImageMarshaller.cs | 24 + StableDiffusion.NET/Native/Native.Load.cs | 2 +- StableDiffusion.NET/Native/Native.cs | 409 ++++++++++-------- .../StableDiffusion.NET.csproj | 2 +- .../StableDiffusion.NET.csproj.DotSettings | 3 +- StableDiffusion.NET/StableDiffusionCpp.cs | 57 ++- 43 files changed, 1429 insertions(+), 1037 deletions(-) delete mode 100644 StableDiffusion.NET/Enums/DiffusionModelType.cs rename StableDiffusion.NET/Models/Builder/Extensions/{QuantizedModelBuilderExtension.cs => ModelBuilderExtension.cs} (61%) delete mode 100644 StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs create mode 100644 StableDiffusion.NET/Models/Builder/Extensions/UpscaleModelBuilderExtension.cs create mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IModelBuilder.cs delete mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs delete mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs create mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IUpscaleModelBuilder.cs create mode 100644 StableDiffusion.NET/Models/Parameter/CannyParameter.cs delete mode 100644 StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs delete mode 100644 StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs create mode 100644 StableDiffusion.NET/Models/Parameter/GuidanceParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs delete mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs delete mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/SlgParameter.cs create mode 100644 StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs create mode 100644 StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs create mode 100644 StableDiffusion.NET/Native/Marshaller/ImageMarshaller.cs diff --git a/StableDiffusion.NET/Backends/CudaBackend.cs b/StableDiffusion.NET/Backends/CudaBackend.cs index 91aa1e3..73b72f8 100644 --- a/StableDiffusion.NET/Backends/CudaBackend.cs +++ b/StableDiffusion.NET/Backends/CudaBackend.cs @@ -27,7 +27,7 @@ public partial class CudaBackend : IBackend public bool IsAvailable => (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) && (RuntimeInformation.OSArchitecture == Architecture.X64) - && CudaVersion is 11 or 12; + && CudaVersion is 12; public string PathPart => CudaVersion switch { diff --git a/StableDiffusion.NET/Enums/DiffusionModelType.cs b/StableDiffusion.NET/Enums/DiffusionModelType.cs deleted file mode 100644 index a42e137..0000000 --- a/StableDiffusion.NET/Enums/DiffusionModelType.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace StableDiffusion.NET; - -public enum DiffusionModelType -{ - None = 0, - StableDiffusion = 1, - Flux = 2 -} \ No newline at end of file diff --git a/StableDiffusion.NET/Extensions/ParameterExtension.cs b/StableDiffusion.NET/Extensions/ParameterExtension.cs index 8f33a8c..a464483 100644 --- a/StableDiffusion.NET/Extensions/ParameterExtension.cs +++ b/StableDiffusion.NET/Extensions/ParameterExtension.cs @@ -1,92 +1,100 @@ #pragma warning disable CA2208 +using JetBrains.Annotations; using System; namespace StableDiffusion.NET; +[PublicAPI] public static class ParameterExtension { public static void Validate(this UpscaleModelParameter parameter) { ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); ArgumentNullException.ThrowIfNull(parameter.ModelPath, nameof(UpscaleModelParameter.ModelPath)); - - if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(UpscaleModelParameter.Quantization)); } public static void Validate(this DiffusionModelParameter parameter) - { - ((IQuantizedModelParameter)parameter).Validate(); - ((IDiffusionModelParameter)parameter).Validate(); - ((IPhotomakerModelParameter)parameter).Validate(); - } - - public static void Validate(this IQuantizedModelParameter parameter) { ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.ThreadCount, nameof(IQuantizedModelParameter.ThreadCount)); - - if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(IQuantizedModelParameter.Quantization)); - } - - public static void Validate(this IPhotomakerModelParameter parameter) - { - ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); - ArgumentNullException.ThrowIfNull(parameter.StackedIdEmbeddingsDirectory, nameof(IPhotomakerModelParameter.StackedIdEmbeddingsDirectory)); - } - - public static void Validate(this IDiffusionModelParameter parameter) - { - ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); - ArgumentNullException.ThrowIfNull(parameter.VaePath, nameof(IDiffusionModelParameter.VaePath)); - ArgumentNullException.ThrowIfNull(parameter.TaesdPath, nameof(IDiffusionModelParameter.TaesdPath)); - ArgumentNullException.ThrowIfNull(parameter.LoraModelDirectory, nameof(IDiffusionModelParameter.LoraModelDirectory)); - ArgumentNullException.ThrowIfNull(parameter.ControlNetPath, nameof(IDiffusionModelParameter.ControlNetPath)); - ArgumentNullException.ThrowIfNull(parameter.EmbeddingsDirectory, nameof(IDiffusionModelParameter.EmbeddingsDirectory)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.ThreadCount, nameof(DiffusionModelParameter.ThreadCount)); + ArgumentNullException.ThrowIfNull(parameter.VaePath, nameof(DiffusionModelParameter.VaePath)); + ArgumentNullException.ThrowIfNull(parameter.TaesdPath, nameof(DiffusionModelParameter.TaesdPath)); + ArgumentNullException.ThrowIfNull(parameter.LoraModelDirectory, nameof(DiffusionModelParameter.LoraModelDirectory)); + ArgumentNullException.ThrowIfNull(parameter.ControlNetPath, nameof(DiffusionModelParameter.ControlNetPath)); + ArgumentNullException.ThrowIfNull(parameter.EmbeddingsDirectory, nameof(DiffusionModelParameter.EmbeddingsDirectory)); + ArgumentNullException.ThrowIfNull(parameter.StackedIdEmbeddingsDirectory, nameof(DiffusionModelParameter.StackedIdEmbeddingsDirectory)); if (!string.IsNullOrWhiteSpace(parameter.VaePath) && !string.IsNullOrWhiteSpace(parameter.TaesdPath)) throw new ArgumentException("VAE and TAESD are mutually exclusive."); - if (!Enum.IsDefined(parameter.RngType)) throw new ArgumentOutOfRangeException(nameof(IDiffusionModelParameter.RngType)); - if (!Enum.IsDefined(parameter.Schedule)) throw new ArgumentOutOfRangeException(nameof(IDiffusionModelParameter.Schedule)); + if (!Enum.IsDefined(parameter.RngType)) throw new ArgumentOutOfRangeException(nameof(DiffusionModelParameter.RngType)); + if (!Enum.IsDefined(parameter.Schedule)) throw new ArgumentOutOfRangeException(nameof(DiffusionModelParameter.Schedule)); } - public static void Validate(this DiffusionParameter parameter) + public static void Validate(this ImageGenerationParameter parameter) { ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); - ArgumentNullException.ThrowIfNull(parameter.ControlNet, nameof(DiffusionParameter.ControlNet)); - ArgumentNullException.ThrowIfNull(parameter.PhotoMaker, nameof(DiffusionParameter.PhotoMaker)); - ArgumentNullException.ThrowIfNull(parameter.NegativePrompt, nameof(DiffusionParameter.NegativePrompt)); + ArgumentNullException.ThrowIfNull(parameter.ControlNet, nameof(ImageGenerationParameter.ControlNet)); + ArgumentNullException.ThrowIfNull(parameter.PhotoMaker, nameof(ImageGenerationParameter.PhotoMaker)); + ArgumentNullException.ThrowIfNull(parameter.NegativePrompt, nameof(ImageGenerationParameter.NegativePrompt)); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Width, nameof(DiffusionParameter.Width)); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Height, nameof(DiffusionParameter.Height)); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.SampleSteps, nameof(DiffusionParameter.SampleSteps)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Width, nameof(ImageGenerationParameter.Width)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Height, nameof(ImageGenerationParameter.Height)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.SampleSteps, nameof(ImageGenerationParameter.SampleSteps)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CfgScale, nameof(DiffusionParameter.CfgScale)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.Guidance, nameof(DiffusionParameter.Guidance)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(DiffusionParameter.Strength)); - - if (!Enum.IsDefined(parameter.SampleMethod)) throw new ArgumentOutOfRangeException(nameof(DiffusionParameter.SampleMethod)); + if (!Enum.IsDefined(parameter.SampleMethod)) throw new ArgumentOutOfRangeException(nameof(ImageGenerationParameter.SampleMethod)); + parameter.Guidance.Validate(); parameter.ControlNet.Validate(); parameter.PhotoMaker.Validate(); } public static void Validate(this ControlNetParameter parameter) { - ArgumentNullException.ThrowIfNull(parameter, nameof(DiffusionParameter.ControlNet)); + ArgumentNullException.ThrowIfNull(parameter, nameof(ImageGenerationParameter.ControlNet)); ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(ControlNetParameter.Strength)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyHighThreshold, nameof(ControlNetParameter.CannyHighThreshold)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyLowThreshold, nameof(ControlNetParameter.CannyLowThreshold)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyWeak, nameof(ControlNetParameter.CannyWeak)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyStrong, nameof(ControlNetParameter.CannyStrong)); } public static void Validate(this PhotoMakerParameter parameter) { - ArgumentNullException.ThrowIfNull(parameter, nameof(DiffusionParameter.PhotoMaker)); + ArgumentNullException.ThrowIfNull(parameter, nameof(ImageGenerationParameter.PhotoMaker)); ArgumentNullException.ThrowIfNull(parameter.InputIdImageDirectory, nameof(PhotoMakerParameter.InputIdImageDirectory)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.StyleRatio, nameof(PhotoMakerParameter.StyleRatio)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.StyleStrength, nameof(PhotoMakerParameter.StyleStrength)); + } + + public static void Validate(this GuidanceParameter parameter) + { + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); + + ArgumentOutOfRangeException.ThrowIfNegative(parameter.ImgCfg); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.DistilledGuidance); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.MinCfg); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.TxtCfg); + + parameter.Slg.Validate(); + } + + public static void Validate(this SlgParameter parameter) + { + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); + ArgumentNullException.ThrowIfNull(parameter.Layers); + + ArgumentOutOfRangeException.ThrowIfNegative(parameter.Scale); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.SkipLayerStart); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.SkipLayerEnd); + } + + public static void Validate(this CannyParameter parameter) + { + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); + + ArgumentNullException.ThrowIfNull(parameter.Image); + + ArgumentOutOfRangeException.ThrowIfNegative(parameter.HighThreshold, nameof(CannyParameter.HighThreshold)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.LowThreshold, nameof(CannyParameter.LowThreshold)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.Weak, nameof(CannyParameter.Weak)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strong, nameof(CannyParameter.Strong)); } } \ No newline at end of file diff --git a/StableDiffusion.NET/Helper/ImageHelper.cs b/StableDiffusion.NET/Helper/ImageHelper.cs index f620439..de4971b 100644 --- a/StableDiffusion.NET/Helper/ImageHelper.cs +++ b/StableDiffusion.NET/Helper/ImageHelper.cs @@ -6,68 +6,140 @@ namespace StableDiffusion.NET; internal static class ImageHelper { - public static unsafe Image ToImage(Native.sd_image_t* sdImage) - { - Image image = ToImage(*sdImage); - - Marshal.FreeHGlobal((nint)sdImage); - - return image; - } - - public static unsafe Image ToImage(Native.sd_image_t sdImage) + public static unsafe Image ToImage(this Native.Types.sd_image_t sdImage) { int width = (int)sdImage.width; int height = (int)sdImage.height; int bpp = (int)sdImage.channel; - Image image = Image.Create(new ReadOnlySpan(sdImage.data, width * height * bpp), width, height, width * bpp); - - Dispose(sdImage); - - return image; - } - - public static unsafe void Dispose(Native.sd_image_t image) => Marshal.FreeHGlobal((nint)image.data); - - public static unsafe Native.sd_image_t ToSdImage(this IImage image, out nint dataPtr) - { - int sizeInBytes = image.SizeInBytes; - - dataPtr = Marshal.AllocHGlobal(sizeInBytes); - image.CopyTo(new Span((void*)dataPtr, sizeInBytes)); - - return image.ToSdImage((byte*)dataPtr); - } - - public static unsafe Native.sd_image_t ToSdImage(this IImage image, byte* pinnedReference) - => new() + switch (bpp) { - width = (uint)image.Width, - height = (uint)image.Height, - channel = (uint)image.ColorFormat.BytesPerPixel, - data = pinnedReference - }; + case 3: + return Image.Create(new ReadOnlySpan(sdImage.data, width * height * bpp), width, height, width * bpp); - public static unsafe Native.sd_image_t* ToSdImagePtr(this IImage image, out nint dataPtr) - { - int sizeInBytes = image.SizeInBytes; + case 1: + { + ColorRGB[] pixels = new ColorRGB[width * height]; + Span sdData = new(sdImage.data, pixels.Length); - dataPtr = Marshal.AllocHGlobal(sizeInBytes); - image.CopyTo(new Span((void*)dataPtr, sizeInBytes)); + for (int i = 0; i < pixels.Length; i++) + { + byte c = sdData[i]; + pixels[i] = new ColorRGB(c, c, c); + } - return image.ToSdImagePtr((byte*)dataPtr); + Image image = Image.Create(pixels, width, height); + + return image; + } + + default: + throw new ArgumentOutOfRangeException($"Image-BPP of {bpp} is not supported"); + } } - public static unsafe Native.sd_image_t* ToSdImagePtr(this IImage image, byte* pinnedReference) + public static unsafe Native.Types.sd_image_t ToSdImage(this IImage image, bool monochrome = false) { - Native.sd_image_t* nativeImage = (Native.sd_image_t*)Marshal.AllocHGlobal(sizeof(Native.sd_image_t)); + if (monochrome) + { + int sizeInBytes = image.Width * image.Height; - nativeImage->width = (uint)image.Width; - nativeImage->height = (uint)image.Height; - nativeImage->channel = (uint)image.ColorFormat.BytesPerPixel; - nativeImage->data = pinnedReference; + byte* dataPtr = (byte*)NativeMemory.Alloc((nuint)sizeInBytes); + Span data = new(dataPtr, sizeInBytes); - return nativeImage; + // DarthAffe 16.08.2025: HPPH does currently not support monochrome images, that's why we need to convert it here. We're going for the simple conversion as the source image is supposed to be monochrome anyway. + for (int y = 0; y < image.Height; y++) + for (int x = 0; x < image.Width; x++) + { + IColor color = image[x, y]; + data[(image.Width * y) + x] = (byte)Math.Round((color.R + color.G + color.B) / 3.0); + } + + return new Native.Types.sd_image_t + { + width = (uint)image.Width, + height = (uint)image.Height, + channel = 1, + data = dataPtr + }; + } + else + { + IImage img = image as IImage ?? image.ConvertTo(); + + int sizeInBytes = img.SizeInBytes; + + byte* dataPtr = (byte*)NativeMemory.Alloc((nuint)sizeInBytes); + img.CopyTo(new Span(dataPtr, sizeInBytes)); + + return new Native.Types.sd_image_t + { + width = (uint)img.Width, + height = (uint)img.Height, + channel = (uint)img.ColorFormat.BytesPerPixel, + data = dataPtr + }; + } + } + + public static unsafe Native.Types.sd_image_t* ToSdImagePtr(this IImage image, bool monochrome = false) + { + Native.Types.sd_image_t* imagePtr = (Native.Types.sd_image_t*)NativeMemory.Alloc((nuint)Marshal.SizeOf()); + imagePtr[0] = image.ToSdImage(monochrome); + + return imagePtr; + } + + public static unsafe void Free(this Native.Types.sd_image_t sdImage) + { + if (sdImage.data == null) return; + + NativeMemory.Free(sdImage.data); + } + + public static unsafe Image[] ToImageArray(Native.Types.sd_image_t* sdImage, int count) + { + if (sdImage == null) return []; + + Image[] images = new Image[count]; + + for (int i = 0; i < images.Length; i++) + images[i] = GetImage(sdImage, i); + + return images; + } + + internal static unsafe IImage[] ToImageArrayIFace(Native.Types.sd_image_t* sdImage, int count) + { + IImage[] images = new IImage[count]; + + for (int i = 0; i < images.Length; i++) + images[i] = GetImage(sdImage, i); + + return images; + } + + public static unsafe Image GetImage(Native.Types.sd_image_t* sdImage, int index) => sdImage[index].ToImage(); + + public static unsafe Native.Types.sd_image_t* ToSdImage(this IImage[] images, bool monochrome = false) + { + int count = images.Length; + + Native.Types.sd_image_t* imagePtr = (Native.Types.sd_image_t*)NativeMemory.Alloc((nuint)(count * Marshal.SizeOf())); + + for (int i = 0; i < count; i++) + imagePtr[i] = images[i].ToSdImage(monochrome); + + return imagePtr; + } + + public static unsafe void Free(Native.Types.sd_image_t* sdImage, int count) + { + if (sdImage == null) return; + + for (int i = 0; i < count; i++) + Free(sdImage[i]); + + NativeMemory.Free(sdImage); } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs b/StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs index b66452b..387f50b 100644 --- a/StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs +++ b/StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs @@ -3,12 +3,13 @@ namespace StableDiffusion.NET; [PublicAPI] -public sealed class ESRGANModelBuilder : IQuantizedModelBuilder +public sealed class ESRGANModelBuilder : IUpscaleModelBuilder { #region Properties & Fields public UpscaleModelParameter Parameter { get; } - IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; + IUpscaleModelParameter IUpscaleModelBuilder.Parameter => Parameter; + IModelParameter IModelBuilder.Parameter => Parameter; #endregion diff --git a/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs index 53ae229..62f33b2 100644 --- a/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs +++ b/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs @@ -76,10 +76,10 @@ public static class DiffusionModelBuilderExtension return builder; } - public static T KeepVaeOnCpu(this T builder, bool keepVaeOnCpu = true) + public static T KeepControlNetOnCpu(this T builder, bool keepControlNetOnCpu = true) where T : IDiffusionModelBuilder { - builder.Parameter.KeepVaeOnCPU = keepVaeOnCpu; + builder.Parameter.KeepControlNetOnCPU = keepControlNetOnCpu; return builder; } @@ -92,10 +92,34 @@ public static class DiffusionModelBuilderExtension return builder; } - public static T KeepControlNetOnCpu(this T builder, bool keepControlNetOnCpu = true) + public static T KeepVaeOnCpu(this T builder, bool keepVaeOnCpu = true) where T : IDiffusionModelBuilder { - builder.Parameter.KeepControlNetOnCPU = keepControlNetOnCpu; + builder.Parameter.KeepVaeOnCPU = keepVaeOnCpu; + + return builder; + } + + public static T WithFlashAttention(this T builder, bool flashAttention = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.FlashAttention = flashAttention; + + return builder; + } + + public static T WithDiffusionConvDirect(this T builder, bool diffusionConfDirect = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.DiffusionConvDirect = diffusionConfDirect; + + return builder; + } + + public static T WithVaeConvDirect(this T builder, bool vaeConfDirect = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.VaeConfDirect = vaeConfDirect; return builder; } @@ -120,10 +144,96 @@ public static class DiffusionModelBuilderExtension return builder; } - public static T WithFlashAttention(this T builder, bool flashAttention = true) + public static T WithQuantization(this T builder, Quantization quantization) where T : IDiffusionModelBuilder { - builder.Parameter.FlashAttention = flashAttention; + if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization)); + + builder.Parameter.Quantization = quantization; + + return builder; + } + + public static T WithPhotomaker(this T builder, string stackedIdEmbeddingsDirectory) + where T : IDiffusionModelBuilder + { + ArgumentException.ThrowIfNullOrWhiteSpace(stackedIdEmbeddingsDirectory, nameof(stackedIdEmbeddingsDirectory)); + + builder.Parameter.StackedIdEmbeddingsDirectory = stackedIdEmbeddingsDirectory; + + return builder; + } + + public static T WithModelPath(this T builder, string modelPath) + where T : IDiffusionModelBuilder + { + ArgumentNullException.ThrowIfNull(modelPath); + + builder.Parameter.ModelPath = modelPath; + + return builder; + } + + public static T WithDiffusionModelPath(this T builder, string diffusionModelPath) + where T : IDiffusionModelBuilder + { + ArgumentNullException.ThrowIfNull(diffusionModelPath); + + builder.Parameter.DiffusionModelPath = diffusionModelPath; + + return builder; + } + + public static T WithClipLPath(this T builder, string clipLPath) + where T : IDiffusionModelBuilder + { + ArgumentNullException.ThrowIfNull(clipLPath); + + builder.Parameter.ClipLPath = clipLPath; + + return builder; + } + + public static T WithT5xxlPath(this T builder, string t5xxlPath) + where T : IDiffusionModelBuilder + { + ArgumentNullException.ThrowIfNull(t5xxlPath); + + builder.Parameter.T5xxlPath = t5xxlPath; + + return builder; + } + + public static T UseChromaDitMap(this T builder, bool useChromaDitMap = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.ChromaUseDitMap = useChromaDitMap; + + return builder; + } + + public static T EnableChromaT5Map(this T builder, bool enableChromaT5Map = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.ChromaEnableT5Map = enableChromaT5Map; + + return builder; + } + + public static T WithChromaT5MaskPad(this T builder, int chromaT5MaskPad) + where T : IDiffusionModelBuilder + { + builder.Parameter.ChromaT5MaskPad = chromaT5MaskPad; + + return builder; + } + + public static T WithClipGPath(this T builder, string clipGPath) + where T : IDiffusionModelBuilder + { + ArgumentNullException.ThrowIfNull(clipGPath); + + builder.Parameter.ClipGPath = clipGPath; return builder; } diff --git a/StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/ModelBuilderExtension.cs similarity index 61% rename from StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs rename to StableDiffusion.NET/Models/Builder/Extensions/ModelBuilderExtension.cs index 671c2f1..0cf1c03 100644 --- a/StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs +++ b/StableDiffusion.NET/Models/Builder/Extensions/ModelBuilderExtension.cs @@ -4,10 +4,10 @@ using JetBrains.Annotations; namespace StableDiffusion.NET; [PublicAPI] -public static class QuantizedModelBuilderExtension +public static class ModelBuilderExtension { public static T WithoutMultithreading(this T builder) - where T : IQuantizedModelBuilder + where T : IModelBuilder { builder.Parameter.ThreadCount = 1; @@ -15,7 +15,7 @@ public static class QuantizedModelBuilderExtension } public static T WithMultithreading(this T builder, int threadCount = 0) - where T : IQuantizedModelBuilder + where T : IModelBuilder { ArgumentOutOfRangeException.ThrowIfLessThan(threadCount, 0, nameof(threadCount)); @@ -25,12 +25,4 @@ public static class QuantizedModelBuilderExtension return builder; } - - public static T WithQuantization(this T builder, Quantization quantization) - where T : IQuantizedModelBuilder - { - builder.Parameter.Quantization = quantization; - - return builder; - } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs deleted file mode 100644 index 57449f4..0000000 --- a/StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using JetBrains.Annotations; - -namespace StableDiffusion.NET; - -[PublicAPI] -public static class PhotomakerModelBuilderExtension -{ - public static T WithPhotomaker(this T builder, string stackedIdEmbeddingsDirectory) - where T : IPhotomakerModelBuilder - { - ArgumentException.ThrowIfNullOrWhiteSpace(stackedIdEmbeddingsDirectory, nameof(stackedIdEmbeddingsDirectory)); - - builder.Parameter.StackedIdEmbeddingsDirectory = stackedIdEmbeddingsDirectory; - - return builder; - } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Extensions/UpscaleModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/UpscaleModelBuilderExtension.cs new file mode 100644 index 0000000..d8beb47 --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Extensions/UpscaleModelBuilderExtension.cs @@ -0,0 +1,27 @@ +using System; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static class UpscaleModelBuilderExtension +{ + public static T WithModelPath(this T builder, string modelPath) + where T : IUpscaleModelBuilder + { + ArgumentNullException.ThrowIfNull(modelPath); + + builder.Parameter.ModelPath = modelPath; + + return builder; + } + + public static T WithConvDirect(this T builder, bool confDirect = true) + where T : IUpscaleModelBuilder + { + builder.Parameter.ConvDirect = confDirect; + + return builder; + } + +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs b/StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs index 367e23b..510bb2c 100644 --- a/StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs +++ b/StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs @@ -3,13 +3,13 @@ namespace StableDiffusion.NET; [PublicAPI] -public sealed class FluxModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder +public sealed class FluxModelBuilder : IDiffusionModelBuilder { #region Properties & Fields public DiffusionModelParameter Parameter { get; } IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter; - IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; + IModelParameter IModelBuilder.Parameter => Parameter; #endregion @@ -17,7 +17,7 @@ public sealed class FluxModelBuilder : IDiffusionModelBuilder, IQuantizedModelBu public FluxModelBuilder(string diffusionModelPath, string clipLPath, string t5xxlPath, string vaePath) { - Parameter = new DiffusionModelParameter { DiffusionModelType = DiffusionModelType.Flux, DiffusionModelPath = diffusionModelPath, ClipLPath = clipLPath, T5xxlPath = t5xxlPath, VaePath = vaePath }; + Parameter = new DiffusionModelParameter { DiffusionModelPath = diffusionModelPath, ClipLPath = clipLPath, T5xxlPath = t5xxlPath, VaePath = vaePath }; } #endregion diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs index cf4aa9f..1dc098a 100644 --- a/StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs +++ b/StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs @@ -1,6 +1,11 @@ -namespace StableDiffusion.NET; +using JetBrains.Annotations; -public interface IDiffusionModelBuilder +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IDiffusionModelBuilder : IModelBuilder { IDiffusionModelParameter Parameter { get; } + + DiffusionModel Build(); } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IModelBuilder.cs new file mode 100644 index 0000000..6119372 --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Interfaces/IModelBuilder.cs @@ -0,0 +1,9 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IModelBuilder +{ + IModelParameter Parameter { get; } +} diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs deleted file mode 100644 index 9731fa3..0000000 --- a/StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace StableDiffusion.NET; - -public interface IPhotomakerModelBuilder -{ - IPhotomakerModelParameter Parameter { get; } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs deleted file mode 100644 index 6aa95ae..0000000 --- a/StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace StableDiffusion.NET; - -public interface IQuantizedModelBuilder -{ - IQuantizedModelParameter Parameter { get; } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IUpscaleModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IUpscaleModelBuilder.cs new file mode 100644 index 0000000..f6effbb --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Interfaces/IUpscaleModelBuilder.cs @@ -0,0 +1,11 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IUpscaleModelBuilder : IModelBuilder +{ + IUpscaleModelParameter Parameter { get; } + + UpscaleModel Build(); +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/StableDiffusion3_5ModelBuilder.cs b/StableDiffusion.NET/Models/Builder/StableDiffusion3_5ModelBuilder.cs index 66ac5b1..954de7a 100644 --- a/StableDiffusion.NET/Models/Builder/StableDiffusion3_5ModelBuilder.cs +++ b/StableDiffusion.NET/Models/Builder/StableDiffusion3_5ModelBuilder.cs @@ -3,13 +3,14 @@ namespace StableDiffusion.NET; [PublicAPI] -public sealed class StableDiffusion3_5ModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder +public sealed class StableDiffusion3_5ModelBuilder : IDiffusionModelBuilder { #region Properties & Fields public DiffusionModelParameter Parameter { get; } IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter; - IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; + IModelParameter IModelBuilder.Parameter => Parameter; + #endregion @@ -19,7 +20,6 @@ public sealed class StableDiffusion3_5ModelBuilder : IDiffusionModelBuilder, IQu { Parameter = new DiffusionModelParameter { - DiffusionModelType = DiffusionModelType.StableDiffusion, ModelPath = modelPath, ClipLPath = clipLPath, ClipGPath = clipGPath, diff --git a/StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs b/StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs index d1952f2..0109d4d 100644 --- a/StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs +++ b/StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs @@ -3,14 +3,14 @@ namespace StableDiffusion.NET; [PublicAPI] -public sealed class StableDiffusionModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder, IPhotomakerModelBuilder +public sealed class StableDiffusionModelBuilder : IDiffusionModelBuilder { #region Properties & Fields public DiffusionModelParameter Parameter { get; } IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter; - IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; - IPhotomakerModelParameter IPhotomakerModelBuilder.Parameter => Parameter; + IModelParameter IModelBuilder.Parameter => Parameter; + #endregion @@ -18,7 +18,7 @@ public sealed class StableDiffusionModelBuilder : IDiffusionModelBuilder, IQuant public StableDiffusionModelBuilder(string modelPath) { - Parameter = new DiffusionModelParameter { DiffusionModelType = DiffusionModelType.StableDiffusion, ModelPath = modelPath }; + Parameter = new DiffusionModelParameter { ModelPath = modelPath }; } #endregion diff --git a/StableDiffusion.NET/Models/DiffusionModel.cs b/StableDiffusion.NET/Models/DiffusionModel.cs index 2e74e4f..7901ef1 100644 --- a/StableDiffusion.NET/Models/DiffusionModel.cs +++ b/StableDiffusion.NET/Models/DiffusionModel.cs @@ -1,8 +1,6 @@ using HPPH; using JetBrains.Annotations; using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; namespace StableDiffusion.NET; @@ -15,7 +13,7 @@ public sealed unsafe class DiffusionModel : IDisposable public DiffusionModelParameter ModelParameter { get; } - private Native.sd_ctx_t* _ctx; + private Native.Types.sd_ctx_t* _ctx; #endregion @@ -25,8 +23,6 @@ public sealed unsafe class DiffusionModel : IDisposable { ArgumentNullException.ThrowIfNull(modelParameter, nameof(modelParameter)); - modelParameter.Validate(); - this.ModelParameter = modelParameter; Initialize(); @@ -40,335 +36,53 @@ public sealed unsafe class DiffusionModel : IDisposable private void Initialize() { - _ctx = Native.new_sd_ctx(ModelParameter.ModelPath, - ModelParameter.ClipLPath, - ModelParameter.ClipGPath, - ModelParameter.T5xxlPath, - ModelParameter.DiffusionModelPath, - ModelParameter.VaePath, - ModelParameter.TaesdPath, - ModelParameter.ControlNetPath, - ModelParameter.LoraModelDirectory, - ModelParameter.EmbeddingsDirectory, - ModelParameter.StackedIdEmbeddingsDirectory, - ModelParameter.VaeDecodeOnly, - ModelParameter.VaeTiling, - false, - ModelParameter.ThreadCount, - ModelParameter.Quantization, - ModelParameter.RngType, - ModelParameter.Schedule, - ModelParameter.KeepClipOnCPU, - ModelParameter.KeepControlNetOnCPU, - ModelParameter.KeepVaeOnCPU, - ModelParameter.FlashAttention, - ModelParameter.ChromaUseDitMap, - ModelParameter.ChromaEnableT5Map, - ModelParameter.ChromaT5MaskPad); + ModelParameter.Validate(); + + _ctx = Native.new_sd_ctx(ModelParameter); if (_ctx == null) throw new NullReferenceException("Failed to initialize diffusion-model."); } - public DiffusionParameter GetDefaultParameter() => ModelParameter.DiffusionModelType switch + public Image? GenerateImage(ImageGenerationParameter parameter) { - DiffusionModelType.None => new DiffusionParameter(), - DiffusionModelType.StableDiffusion => DiffusionParameter.SDXLDefault, - DiffusionModelType.Flux => DiffusionParameter.FluxDefault, - _ => throw new ArgumentOutOfRangeException() - }; - - public IImage TextToImage(string prompt, DiffusionParameter? parameter = null) - { - parameter ??= GetDefaultParameter(); - ObjectDisposedException.ThrowIf(_disposed, this); - ArgumentNullException.ThrowIfNull(prompt); + + if (_ctx == null) throw new NullReferenceException("The model is not initialized."); parameter.Validate(); - - List ptrsToFree = []; + + Native.Types.sd_image_t* result = Native.generate_image(_ctx, parameter); + if (result == null) return null; try { - NativeParameters nativeParameters = PrefillParameters(prompt, parameter); - SetControlNetParameters(ref nativeParameters, parameter, ptrsToFree); - - Native.sd_image_t* result = Txt2Img(nativeParameters); - - return ImageHelper.ToImage(result); + return ImageHelper.GetImage(result, 0); } finally { - foreach (nint ptr in ptrsToFree) - Marshal.FreeHGlobal(ptr); + ImageHelper.Free(result, 1); } } - public IImage ImageToImage(string prompt, IImage image, DiffusionParameter? parameter = null) - { - parameter ??= GetDefaultParameter(); + // TODO DarthAffe 09.08.2025: Implement when no longer marked as broken + //public Image[] GenerateVideo() + //{ + // ObjectDisposedException.ThrowIf(_disposed, this); - ObjectDisposedException.ThrowIf(_disposed, this); - ArgumentNullException.ThrowIfNull(prompt); - ArgumentNullException.ThrowIfNull(image); + // //parameter.Validate(); - parameter.Validate(); + // int imageCount = 0; // TODO DarthAffe 09.08.2025: Set correct count - // DarthAffe 10.08.2024: Mask needs to be a 1 channel all max value image when it's not used - I really don't like this concept as it adds unnecessary allocations, but that's how it is :( - Span maskBuffer = new byte[image.Width * image.Height]; - maskBuffer.Fill(byte.MaxValue); - - return InternalImageToImage(prompt, image, maskBuffer, parameter); - } - - public IImage Inpaint(string prompt, IImage image, IImage mask, DiffusionParameter? parameter = null) - { - parameter ??= GetDefaultParameter(); - - ObjectDisposedException.ThrowIf(_disposed, this); - ArgumentNullException.ThrowIfNull(prompt); - ArgumentNullException.ThrowIfNull(image); - ArgumentNullException.ThrowIfNull(mask); - - parameter.Validate(); - - if (image.Width != mask.Width) throw new ArgumentException("The mask needs to have the same with as the image.", nameof(mask)); - if (image.Height != mask.Height) throw new ArgumentException("The mask needs to have the same height as the image.", nameof(mask)); - - // DarthAffe 10.08.2024: HPPH does currently not support monochrome images, that's why we need to convert it here. We're going for the simple conversion as the source image is supposed to be monochrome anyway. - Span maskBuffer = new byte[image.Width * image.Height]; - for (int y = 0; y < image.Height; y++) - for (int x = 0; x < image.Width; x++) - { - IColor color = mask[x, y]; - maskBuffer[(image.Width * y) + x] = (byte)Math.Round((color.R + color.G + color.B) / 3.0); - } - - return InternalImageToImage(prompt, image, maskBuffer, parameter); - } - - public IImage Edit(string prompt, IImage[] refImages, DiffusionParameter? parameter = null) - { - parameter ??= GetDefaultParameter(); - - ObjectDisposedException.ThrowIf(_disposed, this); - ArgumentNullException.ThrowIfNull(prompt); - ArgumentNullException.ThrowIfNull(refImages); - - parameter.Validate(); - - List ptrsToFree = []; - - try - { - NativeParameters nativeParameters = PrefillParameters(prompt, parameter); - SetControlNetParameters(ref nativeParameters, parameter, ptrsToFree); - - Native.sd_image_t[] nativeRefImages = new Native.sd_image_t[refImages.Length]; - - for (int i = 0; i < refImages.Length; i++) - { - IImage image = refImages[i]; - if (image is not IImage refImage) - refImage = image.ConvertTo(); - - nativeRefImages[i] = refImage.ToSdImage(out nint dataPtr); - ptrsToFree.Add(dataPtr); - } - - fixed (Native.sd_image_t* nativeRefImagesPtr = nativeRefImages) - { - nativeParameters.ref_images = nativeRefImagesPtr; - nativeParameters.ref_images_count = nativeRefImages.Length; - - Native.sd_image_t* result = Edit(nativeParameters); - - return ImageHelper.ToImage(result); - } - } - finally - { - foreach (nint ptr in ptrsToFree) - Marshal.FreeHGlobal(ptr); - } - } - - private Image InternalImageToImage(string prompt, IImage image, Span mask, DiffusionParameter parameter) - { - List ptrsToFree = []; - - try - { - NativeParameters nativeParameters = PrefillParameters(prompt, parameter); - SetControlNetParameters(ref nativeParameters, parameter, ptrsToFree); - - if (image is not IImage refImage) - refImage = image.ConvertTo(); - - nativeParameters.init_image = refImage.ToSdImage(out nint imagePtr); - ptrsToFree.Add(imagePtr); - - fixed (byte* maskPtr = mask) - { - Native.sd_image_t maskImage = new() - { - width = (uint)refImage.Width, - height = (uint)refImage.Height, - channel = 1, - data = maskPtr - }; - nativeParameters.mask_image = maskImage; - - Native.sd_image_t* result = Img2Img(nativeParameters); - - return ImageHelper.ToImage(result); - } - } - finally - { - foreach (nint ptr in ptrsToFree) - Marshal.FreeHGlobal(ptr); - } - } - - private static NativeParameters PrefillParameters(string prompt, DiffusionParameter parameter) - => new() - { - prompt = prompt, - negative_prompt = parameter.NegativePrompt, - clip_skip = parameter.ClipSkip, - cfg_scale = parameter.CfgScale, - guidance = parameter.Guidance, - eta = parameter.Eta, - width = parameter.Width, - height = parameter.Height, - sample_method = parameter.SampleMethod, - sample_steps = parameter.SampleSteps, - seed = parameter.Seed, - batch_count = 1, - control_cond = null, - control_strength = 0, - style_strength = parameter.PhotoMaker.StyleRatio, - normalize_input = parameter.PhotoMaker.NormalizeInput, - input_id_images_path = parameter.PhotoMaker.InputIdImageDirectory, - skip_layers = parameter.SkipLayers, - skip_layers_count = parameter.SkipLayers.Length, - slg_scale = parameter.SlgScale, - skip_layer_start = parameter.SkipLayerStart, - skip_layer_end = parameter.SkipLayerEnd, - strength = parameter.Strength, - }; - - private static void SetControlNetParameters(ref NativeParameters nativeParameters, DiffusionParameter parameter, List ptrsToFree) - { - if (!parameter.ControlNet.IsEnabled) return; - if (parameter.ControlNet.Image == null) return; - - if (parameter.ControlNet.Image is not IImage controlNetImage) - controlNetImage = parameter.ControlNet.Image!.ConvertTo(); - - Native.sd_image_t* nativeControlNetImage = controlNetImage.ToSdImagePtr(out nint controlNetImagePtr); - ptrsToFree.Add(controlNetImagePtr); - ptrsToFree.Add((nint)nativeControlNetImage); - - nativeParameters.control_cond = nativeControlNetImage; - nativeParameters.control_strength = parameter.ControlNet.Strength; - - if (parameter.ControlNet.CannyPreprocess) - { - nativeParameters.control_cond->data = Native.preprocess_canny(nativeParameters.control_cond->data, - parameter.Width, - parameter.Height, - parameter.ControlNet.CannyHighThreshold, - parameter.ControlNet.CannyLowThreshold, - parameter.ControlNet.CannyWeak, - parameter.ControlNet.CannyStrong, - parameter.ControlNet.CannyInverse); - ptrsToFree.Add((nint)nativeParameters.control_cond->data); - } - } - - private Native.sd_image_t* Txt2Img(NativeParameters parameter) - => Native.txt2img(_ctx, - parameter.prompt, - parameter.negative_prompt, - parameter.clip_skip, - parameter.cfg_scale, - parameter.guidance, - parameter.eta, - parameter.width, - parameter.height, - parameter.sample_method, - parameter.sample_steps, - parameter.seed, - parameter.batch_count, - parameter.control_cond, - parameter.control_strength, - parameter.style_strength, - parameter.normalize_input, - parameter.input_id_images_path, - parameter.skip_layers, - parameter.skip_layers_count, - parameter.slg_scale, - parameter.skip_layer_start, - parameter.skip_layer_end - ); - - private Native.sd_image_t* Img2Img(NativeParameters parameter) - => Native.img2img(_ctx, - parameter.init_image, - parameter.mask_image, - parameter.prompt, - parameter.negative_prompt, - parameter.clip_skip, - parameter.cfg_scale, - parameter.guidance, - parameter.width, - parameter.height, - parameter.sample_method, - parameter.sample_steps, - parameter.strength, - parameter.seed, - parameter.batch_count, - parameter.control_cond, - parameter.control_strength, - parameter.style_strength, - parameter.normalize_input, - parameter.input_id_images_path, - parameter.skip_layers, - parameter.skip_layers_count, - parameter.slg_scale, - parameter.skip_layer_start, - parameter.skip_layer_end - ); - - private Native.sd_image_t* Edit(NativeParameters parameter) - => Native.edit(_ctx, - parameter.ref_images, - parameter.ref_images_count, - parameter.prompt, - parameter.negative_prompt, - parameter.clip_skip, - parameter.cfg_scale, - parameter.guidance, - parameter.eta, - parameter.width, - parameter.height, - parameter.sample_method, - parameter.sample_steps, - parameter.strength, - parameter.seed, - parameter.batch_count, - parameter.control_cond, - parameter.control_strength, - parameter.style_strength, - parameter.normalize_input, - parameter.skip_layers, - parameter.skip_layers_count, - parameter.slg_scale, - parameter.skip_layer_start, - parameter.skip_layer_end - ); + // Native.Types.sd_image_t* result = Native.generate_video(_ctx, new Native.Types.sd_vid_gen_params_t()); // TODO DarthAffe 09.08.2025: Add Parameter + // try + // { + // return ImageHelper.ToImageArray(result, imageCount); + // } + // finally + // { + // ImageHelper.Free(result, imageCount); + // } + //} public void Dispose() { @@ -382,37 +96,4 @@ public sealed unsafe class DiffusionModel : IDisposable } #endregion - - private ref struct NativeParameters - { - internal string prompt; - internal string negative_prompt; - internal int clip_skip; - internal float cfg_scale; - internal float guidance; - internal float eta; - internal int width; - internal int height; - internal Sampler sample_method; - internal int sample_steps; - internal long seed; - internal int batch_count; - internal Native.sd_image_t* control_cond; - internal float control_strength; - internal float style_strength; - internal bool normalize_input; - internal string input_id_images_path; - internal int[] skip_layers; - internal int skip_layers_count; - internal float slg_scale; - internal float skip_layer_start; - internal float skip_layer_end; - - internal Native.sd_image_t init_image; - internal Native.sd_image_t mask_image; - - internal Native.sd_image_t* ref_images; - internal int ref_images_count; - internal float strength; - } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/CannyParameter.cs b/StableDiffusion.NET/Models/Parameter/CannyParameter.cs new file mode 100644 index 0000000..fcaa56a --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/CannyParameter.cs @@ -0,0 +1,40 @@ +using HPPH; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class CannyParameter +{ + /// + /// the image to process + /// + public IImage? Image { get; set; } = null; + + /// + /// + /// + public float HighThreshold { get; set; } = 0.08f; + + /// + /// + /// + public float LowThreshold { get; set; } = 0.08f; + + /// + /// + /// + public float Weak { get; set; } = 0.8f; + + /// + /// + /// + public float Strong { get; set; } = 1.0f; + + /// + /// + /// + public bool Inverse { get; set; } = false; + + public static CannyParameter Create() => new(); +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs b/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs index 406435c..49a906e 100644 --- a/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs @@ -1,13 +1,9 @@ using HPPH; -using JetBrains.Annotations; namespace StableDiffusion.NET; -[PublicAPI] public sealed class ControlNetParameter { - public bool IsEnabled => Image != null; - /// /// image condition, control net /// @@ -19,33 +15,5 @@ public sealed class ControlNetParameter /// public float Strength { get; set; } = 0.9f; - /// - /// apply canny preprocessor (edge detection) - /// - public bool CannyPreprocess { get; set; } = false; - - /// - /// - /// - public float CannyHighThreshold { get; set; } = 0.08f; - - /// - /// - /// - public float CannyLowThreshold { get; set; } = 0.08f; - - /// - /// - /// - public float CannyWeak { get; set; } = 0.8f; - - /// - /// - /// - public float CannyStrong { get; set; } = 1.0f; - - /// - /// - /// - public bool CannyInverse { get; set; } = false; + internal ControlNetParameter() { } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs index 6b5345c..b03c55c 100644 --- a/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs @@ -1,9 +1,10 @@ -namespace StableDiffusion.NET; +using JetBrains.Annotations; -public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantizedModelParameter, IPhotomakerModelParameter +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class DiffusionModelParameter : IDiffusionModelParameter { - public DiffusionModelType DiffusionModelType { get; set; } = DiffusionModelType.None; - /// /// path to vae /// @@ -67,6 +68,18 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantiz /// public bool FlashAttention { get; set; } = false; + /// + /// use Conv2d direct in the diffusion model + /// This might crash if it is not supported by the backend. + /// + public bool DiffusionConvDirect { get; set; } = false; + + /// + /// use Conv2d direct in the vae model (should improve the performance) + /// This might crash if it is not supported by the backend. + /// + public bool VaeConfDirect { get; set; } = false; + /// /// RNG (default: Standard) /// @@ -78,7 +91,8 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantiz public Schedule Schedule { get; set; } = Schedule.Default; /// - /// + /// quantizes on load + /// not really useful in most cases /// public Quantization Quantization { get; set; } = Quantization.Unspecified; @@ -119,4 +133,6 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantiz /// path to the clip-g text encoder /// public string ClipGPath { get; set; } = string.Empty; + + public static DiffusionModelParameter Create() => new(); } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs b/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs deleted file mode 100644 index 2c5e703..0000000 --- a/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs +++ /dev/null @@ -1,99 +0,0 @@ -using JetBrains.Annotations; - -namespace StableDiffusion.NET; - -[PublicAPI] -public sealed class DiffusionParameter -{ - #region Properties & Fields - - public static DiffusionParameter SD1Default => new() { Width = 512, Height = 512, CfgScale = 7.5f, Guidance = 1f, SampleSteps = 25, SampleMethod = Sampler.Euler_A }; - public static DiffusionParameter SDXLDefault => new() { Width = 1024, Height = 1024, CfgScale = 7f, Guidance = 1f, SampleSteps = 30, SampleMethod = Sampler.Euler_A }; - public static DiffusionParameter SD3_5Default => new() { Width = 1024, Height = 1024, CfgScale = 4.5f, Guidance = 1f, SampleSteps = 20, SampleMethod = Sampler.Euler }; - public static DiffusionParameter FluxDefault => new() { Width = 1024, Height = 1024, CfgScale = 1, Guidance = 3.5f, SampleSteps = 20, SampleMethod = Sampler.Euler }; - - /// - /// the negative prompt (default: ""); - /// - public string NegativePrompt { get; set; } = string.Empty; - - /// - /// image width, in pixel space (default: 512) - /// - public int Width { get; set; } = 512; - - /// - /// image height, in pixel space (default: 512) - /// - public int Height { get; set; } = 512; - - /// - /// sampling method (default: Euler_A) - /// - public Sampler SampleMethod { get; set; } = Sampler.Euler_A; - - /// - /// number of sample steps (default: 25) - /// - public int SampleSteps { get; set; } = 25; - - /// - /// RNG seed. use -1 for a random seed (default: -1) - /// - public long Seed { get; set; } = -1; - - /// - /// strength for noising/unnoising (default: 0.7) - /// - public float Strength { get; set; } = 0.7f; - - /// - /// ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1) - /// -1 represents unspecified, will be 1 for SD1.x, 2 for SD2.x - /// - public int ClipSkip { get; set; } = -1; - - /// - /// skip layer guidance (SLG) scale, only for DiT models: (default: 0) - /// 0 means disabled, a value of 2.5 is nice for sd3.5 medium - /// - public float SlgScale { get; set; } = 0f; - - /// - /// Layers to skip for SLG steps: (default: [7,8,9]) - /// - public int[] SkipLayers { get; set; } = [7, 8, 9]; - - /// - /// SLG enabling point: (default: 0.01) - /// - public float SkipLayerStart { get; set; } = 0.01f; - - /// - /// SLG disabling point: (default: 0.2) - /// - public float SkipLayerEnd { get; set; } = 0.2f; - - public ControlNetParameter ControlNet { get; } = new(); - - // Stable Diffusion only - /// - /// unconditional guidance scale: (default: 7.5) - /// - public float CfgScale { get; set; } = 7.5f; - - public PhotoMakerParameter PhotoMaker { get; } = new(); - - // Flux only - /// - /// - /// - public float Guidance { get; set; } = 3.5f; - - /// - /// eta in DDIM, only for DDIM and TCD (default: 0) - /// - public float Eta { get; set; } = 0f; - - #endregion -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs deleted file mode 100644 index 1372aed..0000000 --- a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs +++ /dev/null @@ -1,149 +0,0 @@ -using HPPH; - -namespace StableDiffusion.NET; - -public static class DiffusionParameterExtension -{ - public static DiffusionParameter WithSize(this DiffusionParameter parameter, int? width = null, int? height = null) - { - if (width != null) - parameter.Width = width.Value; - - if (height != null) - parameter.Height = height.Value; - - return parameter; - } - - public static DiffusionParameter WithSampler(this DiffusionParameter parameter, Sampler sampler) - { - parameter.SampleMethod = sampler; - - return parameter; - } - - public static DiffusionParameter WithSteps(this DiffusionParameter parameter, int steps) - { - parameter.SampleSteps = steps; - - return parameter; - } - - public static DiffusionParameter WithSeed(this DiffusionParameter parameter, long seed) - { - parameter.Seed = seed; - - return parameter; - } - - public static DiffusionParameter WithClipSkip(this DiffusionParameter parameter, int clipSkip) - { - parameter.ClipSkip = clipSkip; - - return parameter; - } - - public static DiffusionParameter WithCfg(this DiffusionParameter parameter, float cfg) - { - parameter.CfgScale = cfg; - - return parameter; - } - - public static DiffusionParameter WithGuidance(this DiffusionParameter parameter, float guidance) - { - parameter.Guidance = guidance; - - return parameter; - } - - public static DiffusionParameter WithNegativePrompt(this DiffusionParameter parameter, string negativePrompt) - { - parameter.NegativePrompt = negativePrompt; - - return parameter; - } - - public static DiffusionParameter WithStrength(this DiffusionParameter parameter, float strength) - { - parameter.Strength = strength; - - return parameter; - } - - public static DiffusionParameter WithSlgScale(this DiffusionParameter parameter, float slgScale) - { - parameter.SlgScale = slgScale; - - return parameter; - } - - public static DiffusionParameter WithSkipLayers(this DiffusionParameter parameter, int[] layers) - { - parameter.SkipLayers = layers; - - return parameter; - } - - public static DiffusionParameter WithSkipLayerStart(this DiffusionParameter parameter, float skipLayerStart) - { - parameter.SkipLayerStart = skipLayerStart; - - return parameter; - } - - public static DiffusionParameter WithSkipLayerEnd(this DiffusionParameter parameter, float skipLayerEnd) - { - parameter.SkipLayerEnd = skipLayerEnd; - - return parameter; - } - - public static DiffusionParameter WithControlNet(this DiffusionParameter parameter, IImage image, float? strength = null) - { - parameter.ControlNet.Image = image; - - if (strength != null) - parameter.ControlNet.Strength = strength.Value; - - return parameter; - } - - public static DiffusionParameter WithCannyPreprocessing(this DiffusionParameter parameter, - float? cannyHighThreshold = null, float? cannyLowThreshold = null, - float? cannyWeak = null, float? cannyStrong = null, - bool? cannyInverse = null) - { - parameter.ControlNet.CannyPreprocess = true; - - if (cannyHighThreshold != null) - parameter.ControlNet.CannyHighThreshold = cannyHighThreshold.Value; - - if (cannyLowThreshold != null) - parameter.ControlNet.CannyLowThreshold = cannyLowThreshold.Value; - - if (cannyWeak != null) - parameter.ControlNet.CannyWeak = cannyWeak.Value; - - if (cannyStrong != null) - parameter.ControlNet.CannyStrong = cannyStrong.Value; - - if (cannyInverse != null) - parameter.ControlNet.CannyInverse = cannyInverse.Value; - - return parameter; - } - - public static DiffusionParameter WithPhotomaker(this DiffusionParameter parameter, string inputIdImageDirectory, float? styleRatio = null, bool? normalizeInput = null) - { - parameter.PhotoMaker.InputIdImageDirectory = inputIdImageDirectory; - - if (styleRatio != null) - parameter.PhotoMaker.StyleRatio = styleRatio.Value; - - if (normalizeInput != null) - parameter.PhotoMaker.NormalizeInput = normalizeInput.Value; - - return parameter; - } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs new file mode 100644 index 0000000..8253a76 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs @@ -0,0 +1,215 @@ +using HPPH; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static class ImageGenerationParameterExtension +{ + public static ImageGenerationParameter WithPrompt(this ImageGenerationParameter parameter, string prompt) + { + parameter.Prompt = prompt; + + return parameter; + } + + public static ImageGenerationParameter WithNegativePrompt(this ImageGenerationParameter parameter, string negativePrompt) + { + parameter.NegativePrompt = negativePrompt; + + return parameter; + } + + public static ImageGenerationParameter WithClipSkip(this ImageGenerationParameter parameter, int clipSkip) + { + parameter.ClipSkip = clipSkip; + + return parameter; + } + + public static ImageGenerationParameter WithCfg(this ImageGenerationParameter parameter, float cfg) + { + parameter.WithTxtCfg(cfg); + parameter.WithImgCfg(cfg); + + return parameter; + } + + public static ImageGenerationParameter WithTxtCfg(this ImageGenerationParameter parameter, float txtCfg) + { + parameter.Guidance.TxtCfg = txtCfg; + + return parameter; + } + + public static ImageGenerationParameter WithImgCfg(this ImageGenerationParameter parameter, float imgCfg) + { + parameter.Guidance.ImgCfg = imgCfg; + + return parameter; + } + + public static ImageGenerationParameter WithMinCfg(this ImageGenerationParameter parameter, float minCfg) + { + parameter.Guidance.MinCfg = minCfg; + + return parameter; + } + + public static ImageGenerationParameter WithGuidance(this ImageGenerationParameter parameter, float guidance) + { + parameter.Guidance.DistilledGuidance = guidance; + + return parameter; + } + + public static ImageGenerationParameter WithSlgScale(this ImageGenerationParameter parameter, float slgScale) + { + parameter.Guidance.Slg.Scale = slgScale; + + return parameter; + } + + public static ImageGenerationParameter WithSkipLayers(this ImageGenerationParameter parameter, int[] layers) + { + parameter.Guidance.Slg.Layers = layers; + + return parameter; + } + + public static ImageGenerationParameter WithSkipLayerStart(this ImageGenerationParameter parameter, float skipLayerStart) + { + parameter.Guidance.Slg.SkipLayerStart = skipLayerStart; + + return parameter; + } + + public static ImageGenerationParameter WithSkipLayerEnd(this ImageGenerationParameter parameter, float skipLayerEnd) + { + parameter.Guidance.Slg.SkipLayerEnd = skipLayerEnd; + + return parameter; + } + + public static ImageGenerationParameter WithInitImage(this ImageGenerationParameter parameter, IImage image) + { + parameter.InitImage = image; + + return parameter; + } + + public static ImageGenerationParameter WithRefImages(this ImageGenerationParameter parameter, params IImage[] images) + { + parameter.RefImages = images; + + return parameter; + } + + public static ImageGenerationParameter WithMaskImage(this ImageGenerationParameter parameter, IImage image) + { + parameter.MaskImage = image; + + return parameter; + } + + public static ImageGenerationParameter WithSize(this ImageGenerationParameter parameter, int? width = null, int? height = null) + { + if (width != null) + parameter.Width = width.Value; + + if (height != null) + parameter.Height = height.Value; + + return parameter; + } + + public static ImageGenerationParameter WithSampler(this ImageGenerationParameter parameter, Sampler sampler) + { + parameter.SampleMethod = sampler; + + return parameter; + } + + public static ImageGenerationParameter WithSteps(this ImageGenerationParameter parameter, int steps) + { + parameter.SampleSteps = steps; + + return parameter; + } + + public static ImageGenerationParameter WithEta(this ImageGenerationParameter parameter, float eta) + { + parameter.Eta = eta; + + return parameter; + } + + public static ImageGenerationParameter WithStrength(this ImageGenerationParameter parameter, float strength) + { + parameter.Strength = strength; + + return parameter; + } + + public static ImageGenerationParameter WithSeed(this ImageGenerationParameter parameter, long seed) + { + parameter.Seed = seed; + + return parameter; + } + + public static ImageGenerationParameter WithControlNet(this ImageGenerationParameter parameter, IImage image, float? strength = null) + { + parameter.ControlNet.Image = image; + + if (strength != null) + parameter.ControlNet.Strength = strength.Value; + + return parameter; + } + + public static ImageGenerationParameter WithPhotomaker(this ImageGenerationParameter parameter, string inputIdImageDirectory, float? styleStrength = null, bool? normalizeInput = null) + { + parameter.PhotoMaker.InputIdImageDirectory = inputIdImageDirectory; + + if (styleStrength != null) + parameter.PhotoMaker.StyleStrength = styleStrength.Value; + + if (normalizeInput != null) + parameter.PhotoMaker.NormalizeInput = normalizeInput.Value; + + return parameter; + } + + #region Defaults + + public static ImageGenerationParameter WithSd1Defaults(this ImageGenerationParameter parameter) + => parameter.WithSize(512, 512) + .WithCfg(7.5f) + .WithGuidance(1f) + .WithSteps(25) + .WithSampler(Sampler.Euler_A); + + public static ImageGenerationParameter WithSDXLDefaults(this ImageGenerationParameter parameter) + => parameter.WithSize(1024, 1024) + .WithCfg(7f) + .WithGuidance(1f) + .WithSteps(30) + .WithSampler(Sampler.Euler_A); + + public static ImageGenerationParameter WithSD3_5Defaults(this ImageGenerationParameter parameter) + => parameter.WithSize(1024, 1024) + .WithCfg(4.5f) + .WithGuidance(1f) + .WithSteps(20) + .WithSampler(Sampler.Euler); + + public static ImageGenerationParameter WithFluxDefaults(this ImageGenerationParameter parameter) + => parameter.WithSize(1024, 1024) + .WithCfg(1f) + .WithGuidance(3.5f) + .WithSteps(20) + .WithSampler(Sampler.Euler); + + #endregion +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/GuidanceParameter.cs b/StableDiffusion.NET/Models/Parameter/GuidanceParameter.cs new file mode 100644 index 0000000..25b91e7 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/GuidanceParameter.cs @@ -0,0 +1,13 @@ +namespace StableDiffusion.NET; + +public sealed class GuidanceParameter +{ + public float TxtCfg { get; set; } = 7.0f; + public float ImgCfg { get; set; } = 7.0f; + public float MinCfg { get; set; } = 1.0f; + public float DistilledGuidance { get; set; } = 3.5f; + + public SlgParameter Slg { get; } = new(); + + internal GuidanceParameter() { } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs b/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs new file mode 100644 index 0000000..09e39dd --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs @@ -0,0 +1,77 @@ +using HPPH; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class ImageGenerationParameter +{ + #region Properties & Fields + + public string Prompt { get; set; } = string.Empty; + + /// + /// the negative prompt (default: ""); + /// + public string NegativePrompt { get; set; } = string.Empty; + + /// + /// ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1) + /// -1 represents unspecified, will be 1 for SD1.x, 2 for SD2.x + /// + public int ClipSkip { get; set; } = -1; + + public GuidanceParameter Guidance { get; } = new(); + + public IImage? InitImage { get; set; } + + public IImage[]? RefImages { get; set; } + + public IImage? MaskImage { get; set; } + + /// + /// image width, in pixel space (default: 512) + /// + public int Width { get; set; } = 512; + + /// + /// image height, in pixel space (default: 512) + /// + public int Height { get; set; } = 512; + + /// + /// sampling method (default: Euler_A) + /// + public Sampler SampleMethod { get; set; } = Sampler.Euler_A; + + /// + /// number of sample steps (default: 25) + /// + public int SampleSteps { get; set; } = 25; + + /// + /// eta in DDIM, only for DDIM and TCD (default: 0) + /// + public float Eta { get; set; } = 0f; + + /// + /// strength for noising/unnoising (default: 0.7) + /// + public float Strength { get; set; } = 0.7f; + + /// + /// RNG seed. use -1 for a random seed (default: -1) + /// + public long Seed { get; set; } = -1; + + public ControlNetParameter ControlNet { get; } = new(); + + public PhotoMakerParameter PhotoMaker { get; } = new(); + + public static ImageGenerationParameter Create() => new(); + + public static ImageGenerationParameter TextToImage(string prompt) => Create().WithPrompt(prompt); + public static ImageGenerationParameter ImageToImage(string prompt, IImage image) => Create().WithPrompt(prompt).WithInitImage(image); + + #endregion +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs index 12569ed..cadfe26 100644 --- a/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs @@ -1,9 +1,10 @@ -namespace StableDiffusion.NET; +using JetBrains.Annotations; -public interface IDiffusionModelParameter +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IDiffusionModelParameter : IModelParameter { - DiffusionModelType DiffusionModelType { get; set; } - string VaePath { get; set; } string TaesdPath { get; set; } @@ -17,7 +18,24 @@ public interface IDiffusionModelParameter bool KeepClipOnCPU { get; set; } bool KeepVaeOnCPU { get; set; } bool FlashAttention { get; set; } + bool DiffusionConvDirect { get; set; } + bool VaeConfDirect { get; set; } RngType RngType { get; set; } Schedule Schedule { get; set; } + Quantization Quantization { get; set; } + + string ModelPath { get; set; } + + string StackedIdEmbeddingsDirectory { get; set; } + + string DiffusionModelPath { get; set; } + string ClipLPath { get; set; } + string T5xxlPath { get; set; } + + bool ChromaUseDitMap { get; set; } + bool ChromaEnableT5Map { get; set; } + int ChromaT5MaskPad { get; set; } + + string ClipGPath { get; set; } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs new file mode 100644 index 0000000..ae23303 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs @@ -0,0 +1,9 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IModelParameter +{ + int ThreadCount { get; set; } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs deleted file mode 100644 index db59a09..0000000 --- a/StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace StableDiffusion.NET; - -public interface IPhotomakerModelParameter -{ - string StackedIdEmbeddingsDirectory { get; set; } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs deleted file mode 100644 index bd74727..0000000 --- a/StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace StableDiffusion.NET; - -public interface IQuantizedModelParameter -{ - int ThreadCount { get; set; } - - Quantization Quantization { get; set; } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs new file mode 100644 index 0000000..dfddf4a --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs @@ -0,0 +1,10 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public interface IUpscaleModelParameter : IModelParameter +{ + string ModelPath { get; set; } + bool ConvDirect { get; set; } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs b/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs index 57466e1..5077158 100644 --- a/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs @@ -1,8 +1,5 @@ -using JetBrains.Annotations; +namespace StableDiffusion.NET; -namespace StableDiffusion.NET; - -[PublicAPI] public sealed class PhotoMakerParameter { /// @@ -13,10 +10,12 @@ public sealed class PhotoMakerParameter /// /// strength for keeping input identity (default: 20) /// - public float StyleRatio { get; set; } = 20f; + public float StyleStrength { get; set; } = 20f; /// /// normalize PHOTOMAKER input id images /// public bool NormalizeInput { get; set; } = false; + + internal PhotoMakerParameter() { } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/SlgParameter.cs b/StableDiffusion.NET/Models/Parameter/SlgParameter.cs new file mode 100644 index 0000000..090df64 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/SlgParameter.cs @@ -0,0 +1,27 @@ +namespace StableDiffusion.NET; + +public sealed class SlgParameter +{ + /// + /// Layers to skip for SLG steps: (default: [7,8,9]) + /// + public int[] Layers { get; set; } = [7, 8, 9]; + + /// + /// SLG enabling point: (default: 0.01) + /// + public float SkipLayerStart { get; set; } = 0.01f; + + /// + /// SLG disabling point: (default: 0.2) + /// + public float SkipLayerEnd { get; set; } = 0.2f; + + /// + /// skip layer guidance (SLG) scale, only for DiT models: (default: 0) + /// 0 means disabled, a value of 2.5 is nice for sd3.5 medium + /// + public float Scale { get; set; } = 0f; + + internal SlgParameter() { } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs index 17844de..da415e7 100644 --- a/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs @@ -3,7 +3,7 @@ namespace StableDiffusion.NET; [PublicAPI] -public sealed class UpscaleModelParameter : IQuantizedModelParameter +public sealed class UpscaleModelParameter : IUpscaleModelParameter { /// /// path to esrgan model. Upscale images after generate, just RealESRGAN_x4plus_anime_6B supported by now @@ -17,7 +17,10 @@ public sealed class UpscaleModelParameter : IQuantizedModelParameter public int ThreadCount { get; set; } = 1; /// - /// + /// use Conv2d direct in the diffusion model + /// This might crash if it is not supported by the backend. /// - public Quantization Quantization { get; set; } = Quantization.F16; + public bool ConvDirect { get; set; } = false; + + public static UpscaleModelParameter Create() => new(); } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/UpscaleModel.cs b/StableDiffusion.NET/Models/UpscaleModel.cs index 600f5b3..7d9c5b6 100644 --- a/StableDiffusion.NET/Models/UpscaleModel.cs +++ b/StableDiffusion.NET/Models/UpscaleModel.cs @@ -13,7 +13,7 @@ public sealed unsafe class UpscaleModel : IDisposable public UpscaleModelParameter ModelParameter { get; } - private Native.upscaler_ctx_t* _ctx; + private Native.Types.upscaler_ctx_t* _ctx; #endregion @@ -40,32 +40,19 @@ public sealed unsafe class UpscaleModel : IDisposable { _ctx = Native.new_upscaler_ctx(ModelParameter.ModelPath, ModelParameter.ThreadCount, - ModelParameter.Quantization); + ModelParameter.ConvDirect); if (_ctx == null) throw new NullReferenceException("Failed to initialize upscale-model."); } - public IImage Upscale(IImage image, int upscaleFactor) + public Image Upscale(IImage image, int upscaleFactor) { ObjectDisposedException.ThrowIf(_disposed, this); ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(upscaleFactor, 0, nameof(upscaleFactor)); if (_ctx == null) throw new NullReferenceException("The model is not initialized."); - if (image is not IImage sourceImage) - sourceImage = image.ConvertTo(); - - fixed (byte* imagePtr = sourceImage.AsRefImage()) - { - Native.sd_image_t result = Native.upscale(_ctx, sourceImage.ToSdImage(imagePtr), upscaleFactor); - return ImageHelper.ToImage(result); - } - } - - private IImage Upscale(Native.sd_image_t image, int upscaleFactor) - { - Native.sd_image_t result = Native.upscale(_ctx, image, upscaleFactor); - return ImageHelper.ToImage(result); + return Native.upscale(_ctx, image, (uint)upscaleFactor); } public void Dispose() diff --git a/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs new file mode 100644 index 0000000..baa2e00 --- /dev/null +++ b/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs @@ -0,0 +1,86 @@ +using System.Runtime.InteropServices.Marshalling; + +namespace StableDiffusion.NET; + +[CustomMarshaller(typeof(DiffusionModelParameter), MarshalMode.ManagedToUnmanagedIn, typeof(DiffusionModelParameterMarshaller))] +[CustomMarshaller(typeof(DiffusionModelParameter), MarshalMode.ManagedToUnmanagedRef, typeof(DiffusionModelParameterMarshaller))] +internal static unsafe class DiffusionModelParameterMarshaller +{ + public static Native.Types.sd_ctx_params_t ConvertToUnmanaged(DiffusionModelParameter managed) + => new() + { + model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ModelPath), + clip_l_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipLPath), + clip_g_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipGPath), + t5xxl_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.T5xxlPath), + diffusion_model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.DiffusionModelPath), + vae_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.VaePath), + taesd_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.TaesdPath), + control_net_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ControlNetPath), + lora_model_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.LoraModelDirectory), + embedding_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.EmbeddingsDirectory), + stacked_id_embed_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.StackedIdEmbeddingsDirectory), + vae_decode_only = (sbyte)(managed.VaeDecodeOnly ? 1 : 0), + vae_tiling = (sbyte)(managed.VaeTiling ? 1 : 0), + free_params_immediately = 0, // DarthAffe 06.08.2025: Static value + n_threads = managed.ThreadCount, + wtype = managed.Quantization, + rng_type = managed.RngType, + schedule = managed.Schedule, + keep_clip_on_cpu = (sbyte)(managed.KeepClipOnCPU ? 1 : 0), + keep_control_net_on_cpu = (sbyte)(managed.KeepControlNetOnCPU ? 1 : 0), + keep_vae_on_cpu = (sbyte)(managed.KeepVaeOnCPU ? 1 : 0), + diffusion_flash_attn = (sbyte)(managed.FlashAttention ? 1 : 0), + diffusion_conv_direct = (sbyte)(managed.DiffusionConvDirect ? 1 : 0), + vae_conv_direct = (sbyte)(managed.VaeConfDirect ? 1 : 0), + chroma_use_dit_mask = (sbyte)(managed.ChromaUseDitMap ? 1 : 0), + chroma_use_t5_mask = (sbyte)(managed.ChromaEnableT5Map ? 1 : 0), + chroma_t5_mask_pad = managed.ChromaT5MaskPad + }; + + public static DiffusionModelParameter ConvertToManaged(Native.Types.sd_ctx_params_t unmanaged) + => new() + { + ModelPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.model_path) ?? string.Empty, + ClipLPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.clip_l_path) ?? string.Empty, + ClipGPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.clip_g_path) ?? string.Empty, + T5xxlPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.t5xxl_path) ?? string.Empty, + DiffusionModelPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.diffusion_model_path) ?? string.Empty, + VaePath = AnsiStringMarshaller.ConvertToManaged(unmanaged.vae_path) ?? string.Empty, + TaesdPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.taesd_path) ?? string.Empty, + ControlNetPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.control_net_path) ?? string.Empty, + LoraModelDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.lora_model_dir) ?? string.Empty, + EmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.embedding_dir) ?? string.Empty, + StackedIdEmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.stacked_id_embed_dir) ?? string.Empty, + VaeDecodeOnly = unmanaged.vae_decode_only == 1, + VaeTiling = unmanaged.vae_tiling == 1, + ThreadCount = unmanaged.n_threads, + Quantization = unmanaged.wtype, + RngType = unmanaged.rng_type, + Schedule = unmanaged.schedule, + KeepClipOnCPU = unmanaged.keep_clip_on_cpu == 1, + KeepControlNetOnCPU = unmanaged.keep_control_net_on_cpu == 1, + KeepVaeOnCPU = unmanaged.keep_vae_on_cpu == 1, + FlashAttention = unmanaged.diffusion_flash_attn == 1, + DiffusionConvDirect = unmanaged.diffusion_conv_direct == 1, + VaeConfDirect = unmanaged.vae_conv_direct == 1, + ChromaUseDitMap = unmanaged.chroma_use_dit_mask == 1, + ChromaEnableT5Map = unmanaged.chroma_use_t5_mask == 1, + ChromaT5MaskPad = unmanaged.chroma_t5_mask_pad + }; + + public static void Free(Native.Types.sd_ctx_params_t unmanaged) + { + AnsiStringMarshaller.Free(unmanaged.model_path); + AnsiStringMarshaller.Free(unmanaged.clip_l_path); + AnsiStringMarshaller.Free(unmanaged.clip_g_path); + AnsiStringMarshaller.Free(unmanaged.t5xxl_path); + AnsiStringMarshaller.Free(unmanaged.diffusion_model_path); + AnsiStringMarshaller.Free(unmanaged.vae_path); + AnsiStringMarshaller.Free(unmanaged.taesd_path); + AnsiStringMarshaller.Free(unmanaged.control_net_path); + AnsiStringMarshaller.Free(unmanaged.lora_model_dir); + AnsiStringMarshaller.Free(unmanaged.embedding_dir); + AnsiStringMarshaller.Free(unmanaged.stacked_id_embed_dir); + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs new file mode 100644 index 0000000..08915f2 --- /dev/null +++ b/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs @@ -0,0 +1,197 @@ +// ReSharper disable MemberCanBeMadeStatic.Global + +using System; +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; + +namespace StableDiffusion.NET; + +[CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedIn, typeof(ImageGenerationParameterMarshallerIn))] +[CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedOut, typeof(ImageGenerationParameterMarshaller))] +[CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedRef, typeof(ImageGenerationParameterMarshallerRef))] +internal static class ImageGenerationParameterMarshaller +{ + public static unsafe ImageGenerationParameter ConvertToManaged(Native.Types.sd_img_gen_params_t unmanaged) + { + ImageGenerationParameter parameter = new() + { + Prompt = AnsiStringMarshaller.ConvertToManaged(unmanaged.prompt) ?? string.Empty, + NegativePrompt = AnsiStringMarshaller.ConvertToManaged(unmanaged.negative_prompt) ?? string.Empty, + ClipSkip = unmanaged.clip_skip, + Guidance = + { + TxtCfg = unmanaged.guidance.txt_cfg, + ImgCfg = unmanaged.guidance.img_cfg, + MinCfg = unmanaged.guidance.min_cfg, + DistilledGuidance = unmanaged.guidance.distilled_guidance, + Slg = + { + Layers = new int[unmanaged.guidance.slg.layer_count], + SkipLayerStart = unmanaged.guidance.slg.layer_start, + SkipLayerEnd = unmanaged.guidance.slg.layer_end, + Scale = unmanaged.guidance.slg.scale + } + }, + InitImage = unmanaged.init_image.data == null ? null : unmanaged.init_image.ToImage(), + RefImages = unmanaged.ref_images == null ? null : ImageHelper.ToImageArrayIFace(unmanaged.ref_images, unmanaged.ref_images_count), + MaskImage = unmanaged.mask_image.data == null ? null : unmanaged.mask_image.ToImage(), + Width = unmanaged.width, + Height = unmanaged.height, + SampleMethod = unmanaged.sample_method, + SampleSteps = unmanaged.sample_steps, + Eta = unmanaged.eta, + Strength = unmanaged.strength, + Seed = unmanaged.seed, + ControlNet = + { + Image = unmanaged.control_cond == null ? null : ImageHelper.GetImage(unmanaged.control_cond, 0), + Strength = unmanaged.control_strength, + }, + PhotoMaker = + { + StyleStrength = unmanaged.style_strength, + NormalizeInput = unmanaged.normalize_input == 1, + InputIdImageDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.input_id_images_path) ?? string.Empty, + } + }; + + if (unmanaged.guidance.slg.layers != null) + new Span(unmanaged.guidance.slg.layers, (int)unmanaged.guidance.slg.layer_count).CopyTo(parameter.Guidance.Slg.Layers); + + return parameter; + } + + public static unsafe void Free(Native.Types.sd_img_gen_params_t unmanaged) + { + AnsiStringMarshaller.Free(unmanaged.prompt); + AnsiStringMarshaller.Free(unmanaged.negative_prompt); + AnsiStringMarshaller.Free(unmanaged.input_id_images_path); + + unmanaged.init_image.Free(); + unmanaged.mask_image.Free(); + + if (unmanaged.ref_images != null) + ImageHelper.Free(unmanaged.ref_images, unmanaged.ref_images_count); + + if (unmanaged.control_cond != null) + ImageHelper.Free(unmanaged.control_cond, 1); + + if (unmanaged.guidance.slg.layers != null) + NativeMemory.Free(unmanaged.guidance.slg.layers); + } + + internal unsafe ref struct ImageGenerationParameterMarshallerIn + { + private Native.Types.sd_img_gen_params_t _imgGenParams; + + private Native.Types.sd_image_t _initImage; + private Native.Types.sd_image_t _maskImage; + private Native.Types.sd_image_t* _refImages; + private Native.Types.sd_image_t* _controlNetImage; + private int* _slgLayers; + + public void FromManaged(ImageGenerationParameter managed) + { + _initImage = managed.InitImage?.ToSdImage() ?? new Native.Types.sd_image_t(); + _refImages = managed.RefImages == null ? null : managed.RefImages.ToSdImage(); + _controlNetImage = managed.ControlNet.Image == null ? null : managed.ControlNet.Image.ToSdImagePtr(); + + _slgLayers = (int*)NativeMemory.Alloc((nuint)managed.Guidance.Slg.Layers.Length, (nuint)Marshal.SizeOf()); + managed.Guidance.Slg.Layers.AsSpan().CopyTo(new Span(_slgLayers, managed.Guidance.Slg.Layers.Length)); + + if (managed.MaskImage != null) + _maskImage = managed.MaskImage.ToSdImage(true); + else if (managed.InitImage != null) + { + // DarthAffe 16.08.2025: Mask needs to be a 1 channel all max value image when it's not used - I really don't like this concept as it adds unnecessary allocations, but that's how it is :( + uint maskImageByteSize = _initImage.width * _initImage.height; + _maskImage = new Native.Types.sd_image_t + { + width = _initImage.width, + height = _initImage.height, + channel = 1, + data = (byte*)NativeMemory.Alloc(maskImageByteSize) + }; + new Span(_maskImage.data, (int)maskImageByteSize).Fill(byte.MaxValue); + } + + Native.Types.sd_slg_params_t slg = new() + { + layers = _slgLayers, + layer_count = (uint)managed.Guidance.Slg.Layers.Length, + layer_start = managed.Guidance.Slg.SkipLayerStart, + layer_end = managed.Guidance.Slg.SkipLayerEnd, + scale = managed.Guidance.Slg.Scale, + }; + + Native.Types.sd_guidance_params_t guidance = new() + { + txt_cfg = managed.Guidance.TxtCfg, + img_cfg = managed.Guidance.ImgCfg, + min_cfg = managed.Guidance.MinCfg, + distilled_guidance = managed.Guidance.DistilledGuidance, + slg = slg + }; + + _imgGenParams = new Native.Types.sd_img_gen_params_t + { + prompt = AnsiStringMarshaller.ConvertToUnmanaged(managed.Prompt), + negative_prompt = AnsiStringMarshaller.ConvertToUnmanaged(managed.NegativePrompt), + clip_skip = managed.ClipSkip, + guidance = guidance, + init_image = _initImage, + ref_images = _refImages, + ref_images_count = managed.RefImages?.Length ?? 0, + mask_image = _maskImage, + width = managed.Width, + height = managed.Height, + sample_method = managed.SampleMethod, + sample_steps = managed.SampleSteps, + eta = managed.Eta, + strength = managed.Strength, + seed = managed.Seed, + batch_count = 1, + control_cond = _controlNetImage, + control_strength = managed.ControlNet.Strength, + style_strength = managed.PhotoMaker.StyleStrength, + normalize_input = (sbyte)(managed.PhotoMaker.NormalizeInput ? 1 : 0), + input_id_images_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.PhotoMaker.InputIdImageDirectory), + }; + } + + public Native.Types.sd_img_gen_params_t ToUnmanaged() => _imgGenParams; + + public void Free() + { + AnsiStringMarshaller.Free(_imgGenParams.prompt); + AnsiStringMarshaller.Free(_imgGenParams.negative_prompt); + AnsiStringMarshaller.Free(_imgGenParams.input_id_images_path); + + _initImage.Free(); + _maskImage.Free(); + + if (_refImages != null) + ImageHelper.Free(_refImages, _imgGenParams.ref_images_count); + + if (_controlNetImage != null) + ImageHelper.Free(_controlNetImage, 1); + + if (_slgLayers != null) + NativeMemory.Free(_slgLayers); + } + } + + internal ref struct ImageGenerationParameterMarshallerRef() + { + private ImageGenerationParameterMarshallerIn _inMarshaller = new(); + private ImageGenerationParameter _parameter; + + public void FromManaged(ImageGenerationParameter managed) => _inMarshaller.FromManaged(managed); + public Native.Types.sd_img_gen_params_t ToUnmanaged() => _inMarshaller.ToUnmanaged(); + + public void FromUnmanaged(Native.Types.sd_img_gen_params_t unmanaged) => _parameter = ConvertToManaged(unmanaged); + public ImageGenerationParameter ToManaged() => _parameter; + + public void Free() => _inMarshaller.Free(); + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Marshaller/ImageMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/ImageMarshaller.cs new file mode 100644 index 0000000..dd82378 --- /dev/null +++ b/StableDiffusion.NET/Native/Marshaller/ImageMarshaller.cs @@ -0,0 +1,24 @@ +using HPPH; +using System.Runtime.InteropServices.Marshalling; + +namespace StableDiffusion.NET; + +[CustomMarshaller(typeof(IImage), MarshalMode.ManagedToUnmanagedIn, typeof(ImageMarshallerIn))] +[CustomMarshaller(typeof(Image), MarshalMode.ManagedToUnmanagedOut, typeof(ImageMarshaller))] +internal static class ImageMarshaller +{ + public static Image ConvertToManaged(Native.Types.sd_image_t unmanaged) => unmanaged.ToImage(); + + public static void Free(Native.Types.sd_image_t unmanaged) => unmanaged.Free(); + + internal ref struct ImageMarshallerIn + { + private Native.Types.sd_image_t _image; + + public void FromManaged(IImage managed) => _image = managed.ToSdImage(); + + public Native.Types.sd_image_t ToUnmanaged() => _image; + + public void Free() => _image.Free(); + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Native.Load.cs b/StableDiffusion.NET/Native/Native.Load.cs index 61ecb09..6288adb 100644 --- a/StableDiffusion.NET/Native/Native.Load.cs +++ b/StableDiffusion.NET/Native/Native.Load.cs @@ -38,7 +38,7 @@ internal static partial class Native return false; } - private static nint ResolveDllImport(string libraryname, Assembly assembly, DllImportSearchPath? searchpath) + private static nint ResolveDllImport(string libraryname, Assembly _, DllImportSearchPath? __) { if (libraryname != LIB_NAME) return nint.Zero; if (_loadedLibraryHandle != nint.Zero) return _loadedLibraryHandle; diff --git a/StableDiffusion.NET/Native/Native.cs b/StableDiffusion.NET/Native/Native.cs index 757cf86..7afd2b0 100644 --- a/StableDiffusion.NET/Native/Native.cs +++ b/StableDiffusion.NET/Native/Native.cs @@ -2,8 +2,11 @@ #pragma warning disable IDE1006 // ReSharper disable InconsistentNaming // ReSharper disable ArrangeTypeMemberModifiers +// ReSharper disable UseSymbolAlias using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; +using HPPH; namespace StableDiffusion.NET; @@ -12,6 +15,17 @@ using sample_method_t = Sampler; using schedule_t = Schedule; using sd_type_t = Quantization; using sd_log_level_t = LogLevel; +using uint32_t = uint; +using uint8_t = byte; +using int64_t = long; +using size_t = nuint; +using int32_t = int; +using sd_ctx_params_t = DiffusionModelParameter; +using sd_img_gen_params_t = ImageGenerationParameter; +using sd_vid_gen_params_t = Native.Types.sd_vid_gen_params_t; +using sd_image_t = Native.Types.sd_image_t; +using sd_ctx_t = Native.Types.sd_ctx_t; +using upscaler_ctx_t = Native.Types.upscaler_ctx_t; internal unsafe partial class Native { @@ -21,6 +35,116 @@ internal unsafe partial class Native #endregion + internal static class Types + { + [StructLayout(LayoutKind.Sequential)] + internal struct sd_ctx_params_t + { + public byte* model_path; + public byte* clip_l_path; + public byte* clip_g_path; + public byte* t5xxl_path; + public byte* diffusion_model_path; + public byte* vae_path; + public byte* taesd_path; + public byte* control_net_path; + public byte* lora_model_dir; + public byte* embedding_dir; + public byte* stacked_id_embed_dir; + public sbyte vae_decode_only; + public sbyte vae_tiling; + public sbyte free_params_immediately; + public int n_threads; + public sd_type_t wtype; + public rng_type_t rng_type; + public schedule_t schedule; + public sbyte keep_clip_on_cpu; + public sbyte keep_control_net_on_cpu; + public sbyte keep_vae_on_cpu; + public sbyte diffusion_flash_attn; + public sbyte diffusion_conv_direct; + public sbyte vae_conv_direct; + public sbyte chroma_use_dit_mask; + public sbyte chroma_use_t5_mask; + public int chroma_t5_mask_pad; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct sd_image_t + { + public uint32_t width; + public uint32_t height; + public uint32_t channel; + public uint8_t* data; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct sd_slg_params_t + { + public int* layers; + public size_t layer_count; + public float layer_start; + public float layer_end; + public float scale; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct sd_guidance_params_t + { + public float txt_cfg; + public float img_cfg; + public float min_cfg; + public float distilled_guidance; + public sd_slg_params_t slg; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct sd_img_gen_params_t + { + public byte* prompt; + public byte* negative_prompt; + public int clip_skip; + public sd_guidance_params_t guidance; + public sd_image_t init_image; + public sd_image_t* ref_images; + public int ref_images_count; + public sd_image_t mask_image; + public int width; + public int height; + public sample_method_t sample_method; + public int sample_steps; + public float eta; + public float strength; + public int64_t seed; + public int batch_count; + public sd_image_t* control_cond; + public float control_strength; + public float style_strength; + public sbyte normalize_input; + public byte* input_id_images_path; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct sd_vid_gen_params_t + { + public sd_image_t init_image; + public int width; + public int height; + public sd_guidance_params_t guidance; + public sample_method_t sample_method; + public int sample_steps; + public float strength; + public int64_t seed; + public int video_frames; + public int motion_bucket_id; + public int fps; + public float augmentation_level; + } + + internal struct sd_ctx_t; + internal struct upscaler_ctx_t; + } + #region Delegates internal delegate void sd_log_cb_t(sd_log_level_t level, [MarshalAs(UnmanagedType.LPStr)] string text, void* data); @@ -28,179 +152,7 @@ internal unsafe partial class Native #endregion - #region DLL-Import - - internal struct sd_ctx_t; - internal struct upscaler_ctx_t; - - [StructLayout(LayoutKind.Sequential)] - internal struct sd_image_t - { - internal uint width; - internal uint height; - internal uint channel; - internal byte* data; - } - - [LibraryImport(LIB_NAME, EntryPoint = "get_num_physical_cores")] - internal static partial int get_num_physical_cores(); - - [LibraryImport(LIB_NAME, EntryPoint = "sd_get_system_info")] - internal static partial void* sd_get_system_info(); - - [LibraryImport(LIB_NAME, EntryPoint = "new_sd_ctx")] - internal static partial sd_ctx_t* new_sd_ctx([MarshalAs(UnmanagedType.LPStr)] string model_path, - [MarshalAs(UnmanagedType.LPStr)] string clip_l_path, - [MarshalAs(UnmanagedType.LPStr)] string clip_g_path, - [MarshalAs(UnmanagedType.LPStr)] string t5xxl_path, - [MarshalAs(UnmanagedType.LPStr)] string diffusion_model_path, - [MarshalAs(UnmanagedType.LPStr)] string vae_path, - [MarshalAs(UnmanagedType.LPStr)] string taesd_path, - [MarshalAs(UnmanagedType.LPStr)] string control_net_path_c_str, - [MarshalAs(UnmanagedType.LPStr)] string lora_model_dir, - [MarshalAs(UnmanagedType.LPStr)] string embed_dir_c_str, - [MarshalAs(UnmanagedType.LPStr)] string stacked_id_embed_dir_c_str, - [MarshalAs(UnmanagedType.I1)] bool vae_decode_only, - [MarshalAs(UnmanagedType.I1)] bool vae_tiling, - [MarshalAs(UnmanagedType.I1)] bool free_params_immediately, - int n_threads, - sd_type_t wtype, - rng_type_t rng_type, - schedule_t s, - [MarshalAs(UnmanagedType.I1)] bool keep_clip_on_cpu, - [MarshalAs(UnmanagedType.I1)] bool keep_control_net_cpu, - [MarshalAs(UnmanagedType.I1)] bool keep_vae_on_cpu, - [MarshalAs(UnmanagedType.I1)] bool diffusion_flash_attn, - [MarshalAs(UnmanagedType.I1)] bool chroma_use_dit_mask, - [MarshalAs(UnmanagedType.I1)] bool chroma_use_t5_mask, - int chroma_t5_mask_pad); - - [LibraryImport(LIB_NAME, EntryPoint = "free_sd_ctx")] - internal static partial void free_sd_ctx(sd_ctx_t* sd_ctx); - - [LibraryImport(LIB_NAME, EntryPoint = "txt2img")] - internal static partial sd_image_t* txt2img(sd_ctx_t* sd_ctx, - [MarshalAs(UnmanagedType.LPStr)] string prompt, - [MarshalAs(UnmanagedType.LPStr)] string negative_prompt, - int clip_skip, - float cfg_scale, - float guidance, - float eta, - int width, - int height, - sample_method_t sample_method, - int sample_steps, - long seed, - int batch_count, - sd_image_t* control_cond, - float control_strength, - float style_strength, - [MarshalAs(UnmanagedType.I1)] bool normalize_input, - [MarshalAs(UnmanagedType.LPStr)] string input_id_images_path, - in int[] skip_layers, - int skip_layers_count, - float slg_scale, - float skip_layer_start, - float skip_layer_end); - - [LibraryImport(LIB_NAME, EntryPoint = "img2img")] - internal static partial sd_image_t* img2img(sd_ctx_t* sd_ctx, - sd_image_t init_image, - sd_image_t mask_image, - [MarshalAs(UnmanagedType.LPStr)] string prompt, - [MarshalAs(UnmanagedType.LPStr)] string negative_prompt, - int clip_skip, - float cfg_scale, - float guidance, - int width, - int height, - sample_method_t sample_method, - int sample_steps, - float strength, - long seed, - int batch_count, - sd_image_t* control_cond, - float control_strength, - float style_strength, - [MarshalAs(UnmanagedType.I1)] bool normalize_input, - [MarshalAs(UnmanagedType.LPStr)] string input_id_images_path, - in int[] skip_layers, - int skip_layers_count, - float slg_scale, - float skip_layer_start, - float skip_layer_end); - - [LibraryImport(LIB_NAME, EntryPoint = "img2vid")] - internal static partial sd_image_t* img2vid(sd_ctx_t* sd_ctx, - sd_image_t init_image, - int width, - int height, - int video_frames, - int motion_bucket_id, - int fps, - float augmentation_level, - float min_cfg, - float cfg_scale, - sample_method_t sample_method, - int sample_steps, - float strength, - long seed); - - [LibraryImport(LIB_NAME, EntryPoint = "edit")] - internal static partial sd_image_t* edit(sd_ctx_t* sd_ctx, - sd_image_t* ref_images, - int ref_images_count, - [MarshalAs(UnmanagedType.LPStr)] string prompt, - [MarshalAs(UnmanagedType.LPStr)] string negative_prompt, - int clip_skip, - float cfg_scale, - float guidance, - float eta, - int width, - int height, - sample_method_t sample_method, - int sample_steps, - float strength, - long seed, - int batch_count, - sd_image_t* control_cond, - float control_strength, - float style_strength, - [MarshalAs(UnmanagedType.I1)] bool normalize_input, - in int[] skip_layers, - int skip_layers_count, - float slg_scale, - float skip_layer_start, - float skip_layer_end); - - [LibraryImport(LIB_NAME, EntryPoint = "new_upscaler_ctx")] - internal static partial upscaler_ctx_t* new_upscaler_ctx([MarshalAs(UnmanagedType.LPStr)] string esrgan_path, - int n_threads, - sd_type_t wtype); - - [LibraryImport(LIB_NAME, EntryPoint = "free_upscaler_ctx")] - internal static partial void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx); - - [LibraryImport(LIB_NAME, EntryPoint = "upscale")] - internal static partial sd_image_t upscale(upscaler_ctx_t* upscaler_ctx, - sd_image_t input_image, - int upscale_factor); - - [LibraryImport(LIB_NAME, EntryPoint = "convert")] - internal static partial void convert([MarshalAs(UnmanagedType.LPStr)] string input_path, - [MarshalAs(UnmanagedType.LPStr)] string vae_path, - [MarshalAs(UnmanagedType.LPStr)] string output_path, - sd_type_t output_type); - - [LibraryImport(LIB_NAME, EntryPoint = "preprocess_canny")] - internal static partial byte* preprocess_canny(byte* img, - int width, - int height, - float high_threshold, - float low_threshold, - float weak, - float strong, - [MarshalAs(UnmanagedType.I1)] bool inverse); + #region Methods [LibraryImport(LIB_NAME, EntryPoint = "sd_set_log_callback")] internal static partial void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data); @@ -208,5 +160,116 @@ internal unsafe partial class Native [LibraryImport(LIB_NAME, EntryPoint = "sd_set_progress_callback")] internal static partial void sd_set_progress_callback(sd_progress_cb_t cb, void* data); + [LibraryImport(LIB_NAME, EntryPoint = "get_num_physical_cores")] + internal static partial int32_t get_num_physical_cores(); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_get_system_info")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_get_system_info(); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "sd_type_name")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_type_name(sd_type_t type); + + [LibraryImport(LIB_NAME, EntryPoint = "str_to_sd_type")] + internal static partial sd_type_t str_to_sd_type([MarshalAs(UnmanagedType.LPStr)] string str); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_rng_type_name")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_rng_type_name(rng_type_t rng_type); + + [LibraryImport(LIB_NAME, EntryPoint = "str_to_rng_type")] + internal static partial rng_type_t str_to_rng_type([MarshalAs(UnmanagedType.LPStr)] string str); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_sample_method_name")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_sample_method_name(sample_method_t sample_method); + + [LibraryImport(LIB_NAME, EntryPoint = "str_to_sample_method")] + internal static partial sample_method_t str_to_sample_method([MarshalAs(UnmanagedType.LPStr)] string str); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_schedule_name")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_schedule_name(schedule_t schedule); + + [LibraryImport(LIB_NAME, EntryPoint = "str_to_schedule")] + internal static partial schedule_t str_to_schedule([MarshalAs(UnmanagedType.LPStr)] string str); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "sd_ctx_params_init")] + internal static partial void sd_ctx_params_init([MarshalUsing(typeof(DiffusionModelParameterMarshaller))] ref sd_ctx_params_t sd_ctx_params); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_ctx_params_to_str")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_ctx_params_to_str([MarshalUsing(typeof(DiffusionModelParameterMarshaller))] in sd_ctx_params_t sd_ctx_params); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "new_sd_ctx")] + internal static partial sd_ctx_t* new_sd_ctx([MarshalUsing(typeof(DiffusionModelParameterMarshaller))] in sd_ctx_params_t sd_ctx_params); + + [LibraryImport(LIB_NAME, EntryPoint = "free_sd_ctx")] + internal static partial void free_sd_ctx(sd_ctx_t* sd_ctx); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "sd_img_gen_params_init")] + internal static partial void sd_img_gen_params_init([MarshalUsing(typeof(ImageGenerationParameterMarshaller))] ref sd_img_gen_params_t sd_img_gen_params); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_img_gen_params_to_str")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_img_gen_params_to_str([MarshalUsing(typeof(ImageGenerationParameterMarshaller))] in sd_img_gen_params_t sd_img_gen_params); + + [LibraryImport(LIB_NAME, EntryPoint = "generate_image")] + internal static partial sd_image_t* generate_image(sd_ctx_t* sd_ctx, [MarshalUsing(typeof(ImageGenerationParameterMarshaller))] in sd_img_gen_params_t sd_img_gen_params); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "sd_vid_gen_params_init")] + internal static partial void sd_vid_gen_params_init(ref sd_vid_gen_params_t sd_vid_gen_params); + + [LibraryImport(LIB_NAME, EntryPoint = "generate_video")] + [return: MarshalUsing(typeof(ImageMarshaller))] + internal static partial sd_image_t* generate_video(sd_ctx_t* sd_ctx, in sd_vid_gen_params_t sd_vid_gen_params); // broken + + // + + [LibraryImport(LIB_NAME, EntryPoint = "new_upscaler_ctx")] + internal static partial upscaler_ctx_t* new_upscaler_ctx([MarshalAs(UnmanagedType.LPStr)] string esrgan_path, int n_threads, [MarshalAs(UnmanagedType.I1)] bool direct); + + [LibraryImport(LIB_NAME, EntryPoint = "free_upscaler_ctx")] + internal static partial void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "upscale")] + [return: MarshalUsing(typeof(ImageMarshaller))] + internal static partial Image upscale(upscaler_ctx_t* upscaler_ctx, [MarshalUsing(typeof(ImageMarshaller))] IImage input_image, uint32_t upscale_factor); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "convert")] + [return: MarshalAs(UnmanagedType.I1)] + internal static partial bool convert([MarshalAs(UnmanagedType.LPStr)] string input_path, + [MarshalAs(UnmanagedType.LPStr)] string vae_path, + [MarshalAs(UnmanagedType.LPStr)] string output_path, + sd_type_t output_type, + [MarshalAs(UnmanagedType.LPStr)] string tensor_type_rules); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "preprocess_canny")] + internal static partial uint8_t* preprocess_canny(uint8_t* img, + int width, + int height, + float high_threshold, + float low_threshold, + float weak, + float strong, + [MarshalAs(UnmanagedType.I1)] bool inverse); + #endregion } \ No newline at end of file diff --git a/StableDiffusion.NET/StableDiffusion.NET.csproj b/StableDiffusion.NET/StableDiffusion.NET.csproj index 2e33017..29afdc2 100644 --- a/StableDiffusion.NET/StableDiffusion.NET.csproj +++ b/StableDiffusion.NET/StableDiffusion.NET.csproj @@ -56,6 +56,6 @@ - + diff --git a/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings b/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings index 65e7f6a..eb80f1b 100644 --- a/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings +++ b/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings @@ -13,4 +13,5 @@ True True True - True \ No newline at end of file + True + True \ No newline at end of file diff --git a/StableDiffusion.NET/StableDiffusionCpp.cs b/StableDiffusion.NET/StableDiffusionCpp.cs index cf7e87c..71f473f 100644 --- a/StableDiffusion.NET/StableDiffusionCpp.cs +++ b/StableDiffusion.NET/StableDiffusionCpp.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.InteropServices; +using HPPH; using JetBrains.Annotations; namespace StableDiffusion.NET; @@ -10,8 +11,8 @@ public static unsafe class StableDiffusionCpp #region Properties & Fields // ReSharper disable NotAccessedField.Local - They are important, the delegate can be collected if it's not stored! - private static readonly Native.sd_log_cb_t LOG_CALLBACK; - private static readonly Native.sd_progress_cb_t PROGRESS_CALLBACK; + private static Native.sd_log_cb_t LOG_CALLBACK; + private static Native.sd_progress_cb_t PROGRESS_CALLBACK; // ReSharper restore NotAccessedField.Local #endregion @@ -23,38 +24,60 @@ public static unsafe class StableDiffusionCpp #endregion - #region Constructors + #region Methods - static StableDiffusionCpp() + public static bool LoadNativeLibrary(string libraryPath) => Native.LoadNativeLibrary(libraryPath); + + public static void InitializeEvents() { Native.sd_set_log_callback(LOG_CALLBACK = OnNativeLog, null); Native.sd_set_progress_callback(PROGRESS_CALLBACK = OnNativeProgress, null); } - #endregion - - #region Methods - - public static bool LoadNativeLibrary(string libraryPath) => Native.LoadNativeLibrary(libraryPath); - - public static void Convert(string modelPath, string vaePath, Quantization quantization, string outputPath) + public static void Convert(string modelPath, string vaePath, Quantization quantization, string outputPath, string tensorTypeRules = "") { ArgumentException.ThrowIfNullOrWhiteSpace(nameof(modelPath)); ArgumentException.ThrowIfNullOrWhiteSpace(nameof(outputPath)); ArgumentNullException.ThrowIfNull(vaePath); if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization)); - Native.convert(modelPath, vaePath, outputPath, quantization); + Native.convert(modelPath, vaePath, outputPath, quantization, tensorTypeRules); } - public static string GetSystemInfo() - { - void* s = Native.sd_get_system_info(); - return Marshal.PtrToStringUTF8((nint)s) ?? ""; - } + public static string GetSystemInfo() => Native.sd_get_system_info(); public static int GetNumPhysicalCores() => Native.get_num_physical_cores(); + public static Image PreprocessCanny(CannyParameter parameter) + { + parameter.Validate(); + + IImage controlImage = parameter.Image as IImage ?? parameter.Image!.ConvertTo(); + + byte[] controlImageData = controlImage.ToRawArray(); + fixed (byte* controlImagePtr = controlImageData) + { + byte* result = Native.preprocess_canny(controlImagePtr, + controlImage.Width, + controlImage.Height, + parameter.HighThreshold, + parameter.LowThreshold, + parameter.Weak, + parameter.Strong, + parameter.Inverse); + + try + { + return Image.Create(new ReadOnlySpan(result, controlImageData.Length), + controlImage.Width, controlImage.Height); + } + finally + { + NativeMemory.Free(result); + } + } + } + private static void OnNativeLog(LogLevel level, string text, void* data) { try From 1ec8d67357e0faf82cab4b56014393a72ff27251 Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Sun, 17 Aug 2025 00:06:05 +0200 Subject: [PATCH 2/9] Updated readme --- README.md | 59 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 48c484d..e96dbe9 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Based on https://github.com/leejet/stable-diffusion.cpp ## Usage ### Setup Install the [StableDiffusion.NET](https://www.nuget.org/packages/StableDiffusion.NET)-Nuget and at least one of the [Backend-Packages](https://www.nuget.org/packages?q=StableDiffusion.NET.Backend). +StableDiffusion.NET is using semantic versioning. Backend-packages are compatible as long as the version does only differ in the last digit. If GPU-support is available it will prefer this over CPU. If you want to add your own native-libraries or need more control over which backend to load, check the static `Backends` class. @@ -17,37 +18,43 @@ If you want to add your own native-libraries or need more control over which bac stable diffusion: ```csharp -using DiffusionModel model = ModelBuilder.StableDiffusion(@"") - .WithMultithreading() - .Build(); -``` +// Enable the Log- and Progress-events +StableDiffusionCpp.InitializeEvents(); -flux: -```csharp -using DiffusionModel model = ModelBuilder.Flux(@"", - @"", - @"", - @"") - .WithMultithreading() - .Build(); -``` +// Register the Log and Progress-events to capture stable-diffusion.cpp output +StableDiffusionCpp.Log += (_, args) => Console.WriteLine($"LOG [{args.Level}]: {args.Text}"); +StableDiffusionCpp.Progress += (_, args) => Console.WriteLine($"PROGRESS {args.Step} / {args.Steps} ({(args.Progress * 100):N2} %) {args.IterationsPerSecond:N2} it/s ({args.Time})"); -#### 2. create image +Image? treeWithTiger; +// Load a StableDiffusion model in a using block to unload it again after the two images are created +using (DiffusionModel sd = ModelBuilder.StableDiffusion(@"") + .WithMultithreading() + .WithFlashAttention() + .Build()) +{ + // Create a image from a prompt + Image? tree = sd.GenerateImage(ImageGenerationParameter.TextToImage("A beautiful tree standing on a small hill").WithSDXLDefaults()); + // (optional) Save the image (requires the HPPH System.Dawing or SkiaSharp extension) + File.WriteAllBytes("image1.png", tree.ToPng()); -with default parameters: -```csharp -IImage image = model.TextToImage(""); -``` + // Use the previously created image for an image-to-image creation + treeWithTiger = sd.GenerateImage(ImageGenerationParameter.ImageToImage("A cute tiger in front of a tree on a small hill", tree).WithSDXLDefaults()); + File.WriteAllBytes("image2.png", treeWithTiger.ToPng()); +} -with custom parameters: -```csharp -IImage image = model.TextToImage("", model.GetDefaultParameter().WithSeed(1234).WithSize(1344, 768)); -``` +// Load a flux kontext model +using DiffusionModel flux = ModelBuilder.Flux(@"", + @"", + @"", + @"") + .WithMultithreading() + .WithFlashAttention() + .Build(); -#### 3. (optional) save the image (requires System.Dawing or SkiaSharp extension) -```csharp -File.WriteAllBytes("output.png", image.ToPng()); +// Perform an edit on the previosly created image +Image? tigerOnMoon = flux.GenerateImage(ImageGenerationParameter.TextToImage("Remove the hill with the grass and place the tree with the tiger on the moon").WithFluxDefaults().WithRefImages(treeWithTiger)); +File.WriteAllBytes("image3.png", tigerOnMoon.ToPng()); ``` To process the resulting image further you can write your own extensions or install one of the [HPPH](https://github.com/DarthAffe/HPPH)-extension sets: From 07a65ead6d64d25f37adfc76d2b9903bec218587 Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Sat, 25 Oct 2025 17:19:11 +0200 Subject: [PATCH 3/9] Finished update to latest sd.cpp --- Examples/ImageCreationUI/ActionCommand.cs | 42 -- Examples/ImageCreationUI/App.xaml | 4 - Examples/ImageCreationUI/App.xaml.cs | 5 - .../Converter/ImageToImageSourceConverter.cs | 34 -- .../ImageCreationUI/ImageCreationUI.csproj | 23 - .../ImageCreationUI.csproj.DotSettings | 2 - Examples/ImageCreationUI/MainWindow.xaml | 165 ------- Examples/ImageCreationUI/MainWindow.xaml.cs | 11 - .../ImageCreationUI/MainWindowViewModel.cs | 422 ------------------ Header/stable-diffusion.h | 322 +++++++++++++ StableDiffusion.NET.sln | 12 +- StableDiffusion.NET/Enums/Prediction.cs | 11 + StableDiffusion.NET/Enums/Quantization.cs | 3 +- StableDiffusion.NET/Enums/Sampler.cs | 5 +- StableDiffusion.NET/Enums/Schedule.cs | 7 +- .../Extensions/ParameterExtension.cs | 37 +- StableDiffusion.NET/Helper/ImageHelper.cs | 2 + .../Models/Builder/ESRGANModelBuilder.cs | 30 -- .../DiffusionModelBuilderExtension.cs | 240 ---------- .../Extensions/ModelBuilderExtension.cs | 28 -- .../UpscaleModelBuilderExtension.cs | 27 -- .../Models/Builder/FluxModelBuilder.cs | 30 -- .../Interfaces/IDiffusionModelBuilder.cs | 11 - .../Builder/Interfaces/IModelBuilder.cs | 9 - .../Interfaces/IUpscaleModelBuilder.cs | 11 - .../Models/Builder/ModelBuilder.cs | 12 - .../Builder/StableDiffusion3_5ModelBuilder.cs | 37 -- .../Builder/StableDiffusionModelBuilder.cs | 31 -- StableDiffusion.NET/Models/DiffusionModel.cs | 31 +- .../Parameter/DiffusionModelParameter.cs | 60 +-- .../DiffusionModelBuilderExtension.cs | 280 ++++++++++++ .../ImageGenerationParameterExtension.cs | 231 +++++++--- .../UpscaleModelBuilderExtension.cs | 49 ++ .../VideoGenerationParameterExtension.cs | 318 +++++++++++++ .../Parameter/ImageGenerationParameter.cs | 27 +- .../Interfaces/IDiffusionModelParameter.cs | 41 -- .../Parameter/Interfaces/IModelParameter.cs | 9 - .../Interfaces/IUpscaleModelParameter.cs | 10 - .../Models/Parameter/PhotoMakerParameter.cs | 16 +- .../Models/Parameter/SampleParameter.cs | 30 ++ .../Models/Parameter/TilingParameter.cs | 14 + .../Models/Parameter/UpscaleModelParameter.cs | 4 +- .../Parameter/VideoGenerationParameter.cs | 46 ++ StableDiffusion.NET/Models/UpscaleModel.cs | 7 +- .../DiffusionModelParameterMarshaller.cs | 36 +- .../ImageGenerationParameterMarshaller.cs | 116 +++-- .../Marshaller/SampleParameterMarshaller.cs | 113 +++++ .../VideoGenerationParameterMarshaller.cs | 124 +++++ StableDiffusion.NET/Native/Native.cs | 148 ++++-- .../StableDiffusion.NET.csproj.DotSettings | 2 + StableDiffusion.NET/StableDiffusionCpp.cs | 40 +- 51 files changed, 1780 insertions(+), 1545 deletions(-) delete mode 100644 Examples/ImageCreationUI/ActionCommand.cs delete mode 100644 Examples/ImageCreationUI/App.xaml delete mode 100644 Examples/ImageCreationUI/App.xaml.cs delete mode 100644 Examples/ImageCreationUI/Converter/ImageToImageSourceConverter.cs delete mode 100644 Examples/ImageCreationUI/ImageCreationUI.csproj delete mode 100644 Examples/ImageCreationUI/ImageCreationUI.csproj.DotSettings delete mode 100644 Examples/ImageCreationUI/MainWindow.xaml delete mode 100644 Examples/ImageCreationUI/MainWindow.xaml.cs delete mode 100644 Examples/ImageCreationUI/MainWindowViewModel.cs create mode 100644 Header/stable-diffusion.h create mode 100644 StableDiffusion.NET/Enums/Prediction.cs delete mode 100644 StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs delete mode 100644 StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs delete mode 100644 StableDiffusion.NET/Models/Builder/Extensions/ModelBuilderExtension.cs delete mode 100644 StableDiffusion.NET/Models/Builder/Extensions/UpscaleModelBuilderExtension.cs delete mode 100644 StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs delete mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs delete mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IModelBuilder.cs delete mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IUpscaleModelBuilder.cs delete mode 100644 StableDiffusion.NET/Models/Builder/ModelBuilder.cs delete mode 100644 StableDiffusion.NET/Models/Builder/StableDiffusion3_5ModelBuilder.cs delete mode 100644 StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Extensions/UpscaleModelBuilderExtension.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Extensions/VideoGenerationParameterExtension.cs delete mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs delete mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs delete mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/SampleParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/TilingParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/VideoGenerationParameter.cs create mode 100644 StableDiffusion.NET/Native/Marshaller/SampleParameterMarshaller.cs create mode 100644 StableDiffusion.NET/Native/Marshaller/VideoGenerationParameterMarshaller.cs diff --git a/Examples/ImageCreationUI/ActionCommand.cs b/Examples/ImageCreationUI/ActionCommand.cs deleted file mode 100644 index 2b87fdb..0000000 --- a/Examples/ImageCreationUI/ActionCommand.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System.Windows.Input; - -namespace ImageCreationUI; - -public class ActionCommand(Action command, Func? canExecute = null) : ICommand -{ - #region Events - - public event EventHandler? CanExecuteChanged; - - #endregion - - #region Methods - - public bool CanExecute(object? parameter) => canExecute?.Invoke() ?? true; - - public void Execute(object? parameter) => command.Invoke(); - - public void RaiseCanExecuteChanged() => CanExecuteChanged?.Invoke(this, EventArgs.Empty); - - #endregion -} - -public class ActionCommand(Action command, Func? canExecute = null) : ICommand - where T : class -{ - #region Events - - public event EventHandler? CanExecuteChanged; - - #endregion - - #region Methods - - public bool CanExecute(object? parameter) => canExecute?.Invoke((T)parameter!) ?? true; - - public void Execute(object? parameter) => command.Invoke((T)parameter!); - - public void RaiseCanExecuteChanged() => CanExecuteChanged?.Invoke(this, EventArgs.Empty); - - #endregion -} \ No newline at end of file diff --git a/Examples/ImageCreationUI/App.xaml b/Examples/ImageCreationUI/App.xaml deleted file mode 100644 index 84ead37..0000000 --- a/Examples/ImageCreationUI/App.xaml +++ /dev/null @@ -1,4 +0,0 @@ - diff --git a/Examples/ImageCreationUI/App.xaml.cs b/Examples/ImageCreationUI/App.xaml.cs deleted file mode 100644 index 4095cd8..0000000 --- a/Examples/ImageCreationUI/App.xaml.cs +++ /dev/null @@ -1,5 +0,0 @@ -using System.Windows; - -namespace ImageCreationUI; - -public partial class App : Application; diff --git a/Examples/ImageCreationUI/Converter/ImageToImageSourceConverter.cs b/Examples/ImageCreationUI/Converter/ImageToImageSourceConverter.cs deleted file mode 100644 index c3d8cb6..0000000 --- a/Examples/ImageCreationUI/Converter/ImageToImageSourceConverter.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System.Drawing; -using System.Globalization; -using System.IO; -using System.Windows.Data; -using System.Windows.Media; -using System.Windows.Media.Imaging; -using HPPH; -using HPPH.System.Drawing; - -namespace ImageCreationUI.Converter; - -[ValueConversion(typeof(IImage), typeof(ImageSource))] -public class ImageToImageSourceConverter : IValueConverter -{ - public object? Convert(object? value, Type targetType, object? parameter, CultureInfo culture) - { - using Bitmap? bitmap = (value as IImage)?.ToBitmap(); - if (bitmap == null) return null; - - using MemoryStream ms = new(); - bitmap.Save(ms, System.Drawing.Imaging.ImageFormat.Bmp); - ms.Position = 0; - - BitmapImage bitmapImage = new(); - bitmapImage.BeginInit(); - bitmapImage.StreamSource = ms; - bitmapImage.CacheOption = BitmapCacheOption.OnLoad; - bitmapImage.EndInit(); - - return bitmapImage; - } - - public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) => throw new NotSupportedException(); -} \ No newline at end of file diff --git a/Examples/ImageCreationUI/ImageCreationUI.csproj b/Examples/ImageCreationUI/ImageCreationUI.csproj deleted file mode 100644 index d1afed9..0000000 --- a/Examples/ImageCreationUI/ImageCreationUI.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - - WinExe - net9.0-windows - enable - enable - true - true - - - - - - - - - - - - - - diff --git a/Examples/ImageCreationUI/ImageCreationUI.csproj.DotSettings b/Examples/ImageCreationUI/ImageCreationUI.csproj.DotSettings deleted file mode 100644 index 17962b1..0000000 --- a/Examples/ImageCreationUI/ImageCreationUI.csproj.DotSettings +++ /dev/null @@ -1,2 +0,0 @@ - - True \ No newline at end of file diff --git a/Examples/ImageCreationUI/MainWindow.xaml b/Examples/ImageCreationUI/MainWindow.xaml deleted file mode 100644 index a549ca1..0000000 --- a/Examples/ImageCreationUI/MainWindow.xaml +++ /dev/null @@ -1,165 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - public bool VaeConfDirect { get; set; } = false; + public bool VaeConvDirect { get; set; } = false; /// /// RNG (default: Standard) /// public RngType RngType { get; set; } = RngType.Standard; - /// - /// Denoiser sigma schedule (default: Default) - /// - public Schedule Schedule { get; set; } = Schedule.Default; + public Prediction Prediction { get; set; } = Prediction.Default; /// /// quantizes on load @@ -96,18 +95,20 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter /// public Quantization Quantization { get; set; } = Quantization.Unspecified; - // SD <= 3 only - /// - /// path to full model - /// - public string ModelPath { get; set; } = string.Empty; + public float FlowShift { get; set; } = 0; + + public bool ForceSdxlVaeConvScale { get; set; } = false; /// /// path to PHOTOMAKER stacked id embeddings /// public string StackedIdEmbeddingsDirectory { get; set; } = string.Empty; + + /// + /// path to full model + /// + public string ModelPath { get; set; } = string.Empty; - // Flux & SD3.5 only /// /// path to the standalone diffusion model /// @@ -118,21 +119,26 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter /// public string ClipLPath { get; set; } = string.Empty; - /// - /// path to the the t5xxl text encoder - /// - public string T5xxlPath { get; set; } = string.Empty; - - // Flux Chroma specific - public bool ChromaUseDitMap { get; set; } = true; - public bool ChromaEnableT5Map { get; set; } = false; - public int ChromaT5MaskPad { get; set; } = 1; - - // SD3.5 only /// /// path to the clip-g text encoder /// public string ClipGPath { get; set; } = string.Empty; + /// + /// path to the the t5xxl text encoder + /// + public string T5xxlPath { get; set; } = string.Empty; + + public string Qwen2VLPath { get; set; } = string.Empty; + + public string Qwen2VLVisionPath { get; set; } = string.Empty; + + public string ClipVisionPath { get; set; } = string.Empty; + public string HighNoiseDiffusionModelPath { get; set; } = string.Empty; + + public bool ChromaUseDitMap { get; set; } = true; + public bool ChromaEnableT5Map { get; set; } = false; + public int ChromaT5MaskPad { get; set; } = 1; + public static DiffusionModelParameter Create() => new(); } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs new file mode 100644 index 0000000..3a33c60 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs @@ -0,0 +1,280 @@ +using System; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static class DiffusionModelBuilderExtension +{ + public static DiffusionModelParameter WithVae(this DiffusionModelParameter parameter, string vaePath) + { + ArgumentNullException.ThrowIfNull(vaePath); + + if (!string.IsNullOrWhiteSpace(parameter.TaesdPath)) throw new ArgumentException("TAESD is already enabled. VAE and TAESD are mutually exclusive.", nameof(vaePath)); + + parameter.VaePath = vaePath; + + return parameter; + } + + public static DiffusionModelParameter WithTaesd(this DiffusionModelParameter parameter, string taesdPath) + { + ArgumentNullException.ThrowIfNull(taesdPath); + + if (!string.IsNullOrWhiteSpace(parameter.VaePath)) throw new ArgumentException("VAE is already enabled. TAESD and VAE are mutually exclusive.", nameof(taesdPath)); + + parameter.TaesdPath = taesdPath; + + return parameter; + } + + public static DiffusionModelParameter WithLoraSupport(this DiffusionModelParameter parameter, string loraModelDirectory) + { + ArgumentNullException.ThrowIfNull(loraModelDirectory); + + parameter.LoraModelDirectory = loraModelDirectory; + + return parameter; + } + + public static DiffusionModelParameter WithEmbeddingSupport(this DiffusionModelParameter parameter, string embeddingsDirectory) + { + ArgumentNullException.ThrowIfNull(embeddingsDirectory); + + parameter.EmbeddingsDirectory = embeddingsDirectory; + + return parameter; + } + + public static DiffusionModelParameter WithControlNet(this DiffusionModelParameter parameter, string controlNetPath) + { + ArgumentNullException.ThrowIfNull(controlNetPath); + + parameter.ControlNetPath = controlNetPath; + + return parameter; + } + + public static DiffusionModelParameter WithoutMultithreading(this DiffusionModelParameter parameter) + { + parameter.ThreadCount = 1; + + return parameter; + } + + public static DiffusionModelParameter WithMultithreading(this DiffusionModelParameter parameter, int threadCount = 0) + { + ArgumentOutOfRangeException.ThrowIfLessThan(threadCount, 0, nameof(threadCount)); + + if (threadCount == 0) threadCount = Environment.ProcessorCount; + + parameter.ThreadCount = threadCount; + + return parameter; + } + + public static DiffusionModelParameter WithVaeDecodeOnly(this DiffusionModelParameter parameter, bool vaeDecodeOnly = true) + { + parameter.VaeDecodeOnly = vaeDecodeOnly; + + return parameter; + } + + public static DiffusionModelParameter WithVaeTiling(this DiffusionModelParameter parameter, bool vaeTiling = true) + { + parameter.VaeTiling = vaeTiling; + + return parameter; + } + + public static DiffusionModelParameter WithOffloadedParamsToCPU(this DiffusionModelParameter parameter, bool offloadParamsToCPU = true) + { + parameter.OffloadParamsToCPU = offloadParamsToCPU; + + return parameter; + } + + public static DiffusionModelParameter WithClipNetOnCpu(this DiffusionModelParameter parameter, bool keepClipNetOnCpu = true) + { + parameter.KeepClipOnCPU = keepClipNetOnCpu; + + return parameter; + } + + public static DiffusionModelParameter WithControlNetOnCpu(this DiffusionModelParameter parameter, bool keepControlNetOnCpu = true) + { + parameter.KeepControlNetOnCPU = keepControlNetOnCpu; + + return parameter; + } + + public static DiffusionModelParameter WithVaeOnCpu(this DiffusionModelParameter parameter, bool keepVaeOnCpu = true) + { + parameter.KeepVaeOnCPU = keepVaeOnCpu; + + return parameter; + } + + public static DiffusionModelParameter WithFlashAttention(this DiffusionModelParameter parameter, bool flashAttention = true) + { + parameter.FlashAttention = flashAttention; + + return parameter; + } + + public static DiffusionModelParameter WithDiffusionConvDirect(this DiffusionModelParameter parameter, bool diffusionConvDirect = true) + { + parameter.DiffusionConvDirect = diffusionConvDirect; + + return parameter; + } + + public static DiffusionModelParameter WithVaeConvDirect(this DiffusionModelParameter parameter, bool vaeConvDirect = true) + { + parameter.VaeConvDirect = vaeConvDirect; + + return parameter; + } + + public static DiffusionModelParameter WithRngType(this DiffusionModelParameter parameter, RngType rngType) + { + if (!Enum.IsDefined(rngType)) throw new ArgumentOutOfRangeException(nameof(rngType)); + + parameter.RngType = rngType; + + return parameter; + } + + public static DiffusionModelParameter WithPrediction(this DiffusionModelParameter parameter, Prediction prediction) + { + parameter.Prediction = prediction; + + return parameter; + } + + public static DiffusionModelParameter WithQuantization(this DiffusionModelParameter parameter, Quantization quantization) + { + if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization)); + + parameter.Quantization = quantization; + + return parameter; + } + + public static DiffusionModelParameter WithFlowShift(this DiffusionModelParameter parameter, float flowShift) + { + parameter.FlowShift = flowShift; + + return parameter; + } + + public static DiffusionModelParameter WithForcedSdxlVaeConvScale(this DiffusionModelParameter parameter, bool forcedScale = true) + { + parameter.ForceSdxlVaeConvScale = forcedScale; + + return parameter; + } + + public static DiffusionModelParameter WithPhotomaker(this DiffusionModelParameter parameter, string stackedIdEmbeddingsDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(stackedIdEmbeddingsDirectory, nameof(stackedIdEmbeddingsDirectory)); + + parameter.StackedIdEmbeddingsDirectory = stackedIdEmbeddingsDirectory; + + return parameter; + } + + public static DiffusionModelParameter WithModelPath(this DiffusionModelParameter parameter, string modelPath) + { + ArgumentNullException.ThrowIfNull(modelPath); + + parameter.ModelPath = modelPath; + + return parameter; + } + + public static DiffusionModelParameter WithDiffusionModelPath(this DiffusionModelParameter parameter, string diffusionModelPath) + { + ArgumentNullException.ThrowIfNull(diffusionModelPath); + + parameter.DiffusionModelPath = diffusionModelPath; + + return parameter; + } + + public static DiffusionModelParameter WithClipLPath(this DiffusionModelParameter parameter, string clipLPath) + { + ArgumentNullException.ThrowIfNull(clipLPath); + + parameter.ClipLPath = clipLPath; + + return parameter; + } + + public static DiffusionModelParameter WithClipGPath(this DiffusionModelParameter parameter, string clipGPath) + { + ArgumentNullException.ThrowIfNull(clipGPath); + + parameter.ClipGPath = clipGPath; + + return parameter; + } + + public static DiffusionModelParameter WithT5xxlPath(this DiffusionModelParameter parameter, string t5xxlPath) + { + ArgumentNullException.ThrowIfNull(t5xxlPath); + + parameter.T5xxlPath = t5xxlPath; + + return parameter; + } + + public static DiffusionModelParameter WithQwen2VLPath(this DiffusionModelParameter parameter, string qwen2VLPath) + { + parameter.Qwen2VLPath = qwen2VLPath; + + return parameter; + } + + public static DiffusionModelParameter WithQwen2VLVisionPath(this DiffusionModelParameter parameter, string qwen2VLVisionPath) + { + parameter.Qwen2VLVisionPath = qwen2VLVisionPath; + + return parameter; + } + + public static DiffusionModelParameter WithClipVisionPath(this DiffusionModelParameter parameter, string clipVisionPath) + { + parameter.ClipVisionPath = clipVisionPath; + + return parameter; + } + + public static DiffusionModelParameter WithHighNoiseDiffusionModelPath(this DiffusionModelParameter parameter, string highNoiseDiffusionModelPath) + { + parameter.HighNoiseDiffusionModelPath = highNoiseDiffusionModelPath; + + return parameter; + } + + public static DiffusionModelParameter UseChromaDitMap(this DiffusionModelParameter parameter, bool useChromaDitMap = true) + { + parameter.ChromaUseDitMap = useChromaDitMap; + + return parameter; + } + + public static DiffusionModelParameter EnableChromaT5Map(this DiffusionModelParameter parameter, bool enableChromaT5Map = true) + { + parameter.ChromaEnableT5Map = enableChromaT5Map; + + return parameter; + } + + public static DiffusionModelParameter WithChromaT5MaskPad(this DiffusionModelParameter parameter, int chromaT5MaskPad) + { + parameter.ChromaT5MaskPad = chromaT5MaskPad; + + return parameter; + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs index 8253a76..9459e81 100644 --- a/StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs +++ b/StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs @@ -27,70 +27,6 @@ public static class ImageGenerationParameterExtension return parameter; } - public static ImageGenerationParameter WithCfg(this ImageGenerationParameter parameter, float cfg) - { - parameter.WithTxtCfg(cfg); - parameter.WithImgCfg(cfg); - - return parameter; - } - - public static ImageGenerationParameter WithTxtCfg(this ImageGenerationParameter parameter, float txtCfg) - { - parameter.Guidance.TxtCfg = txtCfg; - - return parameter; - } - - public static ImageGenerationParameter WithImgCfg(this ImageGenerationParameter parameter, float imgCfg) - { - parameter.Guidance.ImgCfg = imgCfg; - - return parameter; - } - - public static ImageGenerationParameter WithMinCfg(this ImageGenerationParameter parameter, float minCfg) - { - parameter.Guidance.MinCfg = minCfg; - - return parameter; - } - - public static ImageGenerationParameter WithGuidance(this ImageGenerationParameter parameter, float guidance) - { - parameter.Guidance.DistilledGuidance = guidance; - - return parameter; - } - - public static ImageGenerationParameter WithSlgScale(this ImageGenerationParameter parameter, float slgScale) - { - parameter.Guidance.Slg.Scale = slgScale; - - return parameter; - } - - public static ImageGenerationParameter WithSkipLayers(this ImageGenerationParameter parameter, int[] layers) - { - parameter.Guidance.Slg.Layers = layers; - - return parameter; - } - - public static ImageGenerationParameter WithSkipLayerStart(this ImageGenerationParameter parameter, float skipLayerStart) - { - parameter.Guidance.Slg.SkipLayerStart = skipLayerStart; - - return parameter; - } - - public static ImageGenerationParameter WithSkipLayerEnd(this ImageGenerationParameter parameter, float skipLayerEnd) - { - parameter.Guidance.Slg.SkipLayerEnd = skipLayerEnd; - - return parameter; - } - public static ImageGenerationParameter WithInitImage(this ImageGenerationParameter parameter, IImage image) { parameter.InitImage = image; @@ -112,6 +48,20 @@ public static class ImageGenerationParameterExtension return parameter; } + public static ImageGenerationParameter WithRefIndexIncrease(this ImageGenerationParameter parameter, bool refIndexIncrease = true) + { + parameter.IncreaseRefIndex = refIndexIncrease; + + return parameter; + } + + public static ImageGenerationParameter WithRefImageAutoResize(this ImageGenerationParameter parameter, bool refImageAutoResize = true) + { + parameter.AutoResizeRefImage = refImageAutoResize; + + return parameter; + } + public static ImageGenerationParameter WithSize(this ImageGenerationParameter parameter, int? width = null, int? height = null) { if (width != null) @@ -123,27 +73,117 @@ public static class ImageGenerationParameterExtension return parameter; } + #region SampleParameter + + #region Guidance + + public static ImageGenerationParameter WithCfg(this ImageGenerationParameter parameter, float cfg) + { + parameter.WithTxtCfg(cfg); + parameter.WithImgCfg(cfg); + + return parameter; + } + + public static ImageGenerationParameter WithTxtCfg(this ImageGenerationParameter parameter, float txtCfg) + { + parameter.SampleParameter.Guidance.TxtCfg = txtCfg; + + return parameter; + } + + public static ImageGenerationParameter WithImgCfg(this ImageGenerationParameter parameter, float imgCfg) + { + parameter.SampleParameter.Guidance.ImgCfg = imgCfg; + + return parameter; + } + + public static ImageGenerationParameter WithMinCfg(this ImageGenerationParameter parameter, float minCfg) + { + parameter.SampleParameter.Guidance.MinCfg = minCfg; + + return parameter; + } + + public static ImageGenerationParameter WithGuidance(this ImageGenerationParameter parameter, float guidance) + { + parameter.SampleParameter.Guidance.DistilledGuidance = guidance; + + return parameter; + } + + #region Slg + + public static ImageGenerationParameter WithSkipLayers(this ImageGenerationParameter parameter, int[] layers) + { + parameter.SampleParameter.Guidance.Slg.Layers = layers; + + return parameter; + } + + public static ImageGenerationParameter WithSkipLayerStart(this ImageGenerationParameter parameter, float skipLayerStart) + { + parameter.SampleParameter.Guidance.Slg.SkipLayerStart = skipLayerStart; + + return parameter; + } + + public static ImageGenerationParameter WithSkipLayerEnd(this ImageGenerationParameter parameter, float skipLayerEnd) + { + parameter.SampleParameter.Guidance.Slg.SkipLayerEnd = skipLayerEnd; + + return parameter; + } + + public static ImageGenerationParameter WithSlgScale(this ImageGenerationParameter parameter, float slgScale) + { + parameter.SampleParameter.Guidance.Slg.Scale = slgScale; + + return parameter; + } + + #endregion + + #endregion + + public static ImageGenerationParameter WithScheduler(this ImageGenerationParameter parameter, Scheduler scheduler) + { + parameter.SampleParameter.Scheduler = scheduler; + + return parameter; + } + public static ImageGenerationParameter WithSampler(this ImageGenerationParameter parameter, Sampler sampler) { - parameter.SampleMethod = sampler; + parameter.SampleParameter.SampleMethod = sampler; return parameter; } public static ImageGenerationParameter WithSteps(this ImageGenerationParameter parameter, int steps) { - parameter.SampleSteps = steps; + parameter.SampleParameter.SampleSteps = steps; return parameter; } public static ImageGenerationParameter WithEta(this ImageGenerationParameter parameter, float eta) { - parameter.Eta = eta; + parameter.SampleParameter.Eta = eta; return parameter; } + public static ImageGenerationParameter WithShiftedTimestep(this ImageGenerationParameter parameter, int shiftedTimestep) + { + parameter.SampleParameter.ShiftedTimestep = shiftedTimestep; + + return parameter; + } + + #endregion + public static ImageGenerationParameter WithStrength(this ImageGenerationParameter parameter, float strength) { parameter.Strength = strength; @@ -170,17 +210,64 @@ public static class ImageGenerationParameterExtension public static ImageGenerationParameter WithPhotomaker(this ImageGenerationParameter parameter, string inputIdImageDirectory, float? styleStrength = null, bool? normalizeInput = null) { - parameter.PhotoMaker.InputIdImageDirectory = inputIdImageDirectory; + //todo + //parameter.PhotoMaker.InputIdImageDirectory = inputIdImageDirectory; - if (styleStrength != null) - parameter.PhotoMaker.StyleStrength = styleStrength.Value; + //if (styleStrength != null) + // parameter.PhotoMaker.StyleStrength = styleStrength.Value; - if (normalizeInput != null) - parameter.PhotoMaker.NormalizeInput = normalizeInput.Value; + //if (normalizeInput != null) + // parameter.PhotoMaker.NormalizeInput = normalizeInput.Value; return parameter; } + #region VaeTiling + + public static ImageGenerationParameter WithVaeTiling(this ImageGenerationParameter parameter, bool tiling = true) + { + parameter.VaeTiling.IsEnabled = tiling; + + return parameter; + } + + public static ImageGenerationParameter WithVaeTileSizeX(this ImageGenerationParameter parameter, int tileSizeX) + { + parameter.VaeTiling.TileSizeX = tileSizeX; + + return parameter; + } + + public static ImageGenerationParameter WithVaeTileSizeY(this ImageGenerationParameter parameter, int tileSizeY) + { + parameter.VaeTiling.TileSizeY = tileSizeY; + + return parameter; + } + + public static ImageGenerationParameter WithVaeTargetOverlap(this ImageGenerationParameter parameter, float targetOverlap) + { + parameter.VaeTiling.TargetOverlap = targetOverlap; + + return parameter; + } + + public static ImageGenerationParameter WithVaeRelSizeX(this ImageGenerationParameter parameter, float relSizeX) + { + parameter.VaeTiling.RelSizeX = relSizeX; + + return parameter; + } + + public static ImageGenerationParameter WithVaeRelSizeY(this ImageGenerationParameter parameter, float relSizeY) + { + parameter.VaeTiling.RelSizeY = relSizeY; + + return parameter; + } + + #endregion + #region Defaults public static ImageGenerationParameter WithSd1Defaults(this ImageGenerationParameter parameter) diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/UpscaleModelBuilderExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/UpscaleModelBuilderExtension.cs new file mode 100644 index 0000000..cfcd04d --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Extensions/UpscaleModelBuilderExtension.cs @@ -0,0 +1,49 @@ +using System; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static class UpscaleModelBuilderExtension +{ + public static UpscaleModelParameter WithModelPath(this UpscaleModelParameter parameter, string modelPath) + { + ArgumentNullException.ThrowIfNull(modelPath); + + parameter.ModelPath = modelPath; + + return parameter; + } + + public static UpscaleModelParameter WithOffloadedParamsToCPU(this UpscaleModelParameter parameter, bool offloadParamsToCPU = true) + { + parameter.OffloadParamsToCPU = offloadParamsToCPU; + + return parameter; + } + + public static UpscaleModelParameter WithoutMultithreading(this UpscaleModelParameter parameter) + { + parameter.ThreadCount = 1; + + return parameter; + } + + public static UpscaleModelParameter WithMultithreading(this UpscaleModelParameter parameter, int threadCount = 0) + { + ArgumentOutOfRangeException.ThrowIfLessThan(threadCount, 0, nameof(threadCount)); + + if (threadCount == 0) threadCount = Environment.ProcessorCount; + + parameter.ThreadCount = threadCount; + + return parameter; + } + + public static UpscaleModelParameter WithConvDirect(this UpscaleModelParameter parameter, bool convDirect = true) + { + parameter.ConvDirect = convDirect; + + return parameter; + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/VideoGenerationParameterExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/VideoGenerationParameterExtension.cs new file mode 100644 index 0000000..12ecd9b --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Extensions/VideoGenerationParameterExtension.cs @@ -0,0 +1,318 @@ +using HPPH; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static class VideoGenerationParameterExtension +{ + public static VideoGenerationParameter WithPrompt(this VideoGenerationParameter parameter, string prompt) + { + parameter.Prompt = prompt; + + return parameter; + } + + public static VideoGenerationParameter WithNegativePrompt(this VideoGenerationParameter parameter, string negativePrompt) + { + parameter.NegativePrompt = negativePrompt; + + return parameter; + } + + public static VideoGenerationParameter WithClipSkip(this VideoGenerationParameter parameter, int clipSkip) + { + parameter.ClipSkip = clipSkip; + + return parameter; + } + + public static VideoGenerationParameter WithSize(this VideoGenerationParameter parameter, int? width = null, int? height = null) + { + if (width != null) + parameter.Width = width.Value; + + if (height != null) + parameter.Height = height.Value; + + return parameter; + } + + public static VideoGenerationParameter WithInitImage(this VideoGenerationParameter parameter, IImage image) + { + parameter.InitImage = image; + + return parameter; + } + + public static VideoGenerationParameter WithEndImage(this VideoGenerationParameter parameter, IImage image) + { + parameter.EndImage = image; + + return parameter; + } + + public static VideoGenerationParameter WithControlFrames(this VideoGenerationParameter parameter, params IImage[] images) + { + parameter.ControlFrames = images; + + return parameter; + } + + #region SampleParameter + + #region Guidance + + public static VideoGenerationParameter WithCfg(this VideoGenerationParameter parameter, float cfg) + { + parameter.WithTxtCfg(cfg); + parameter.WithImgCfg(cfg); + + return parameter; + } + + public static VideoGenerationParameter WithTxtCfg(this VideoGenerationParameter parameter, float txtCfg) + { + parameter.SampleParameter.Guidance.TxtCfg = txtCfg; + + return parameter; + } + + public static VideoGenerationParameter WithImgCfg(this VideoGenerationParameter parameter, float imgCfg) + { + parameter.SampleParameter.Guidance.ImgCfg = imgCfg; + + return parameter; + } + + public static VideoGenerationParameter WithMinCfg(this VideoGenerationParameter parameter, float minCfg) + { + parameter.SampleParameter.Guidance.MinCfg = minCfg; + + return parameter; + } + + public static VideoGenerationParameter WithGuidance(this VideoGenerationParameter parameter, float guidance) + { + parameter.SampleParameter.Guidance.DistilledGuidance = guidance; + + return parameter; + } + + #region Slg + + public static VideoGenerationParameter WithSkipLayers(this VideoGenerationParameter parameter, int[] layers) + { + parameter.SampleParameter.Guidance.Slg.Layers = layers; + + return parameter; + } + + public static VideoGenerationParameter WithSkipLayerStart(this VideoGenerationParameter parameter, float skipLayerStart) + { + parameter.SampleParameter.Guidance.Slg.SkipLayerStart = skipLayerStart; + + return parameter; + } + + public static VideoGenerationParameter WithSkipLayerEnd(this VideoGenerationParameter parameter, float skipLayerEnd) + { + parameter.SampleParameter.Guidance.Slg.SkipLayerEnd = skipLayerEnd; + + return parameter; + } + + public static VideoGenerationParameter WithSlgScale(this VideoGenerationParameter parameter, float slgScale) + { + parameter.SampleParameter.Guidance.Slg.Scale = slgScale; + + return parameter; + } + + #endregion + + #endregion + + public static VideoGenerationParameter WithScheduler(this VideoGenerationParameter parameter, Scheduler scheduler) + { + parameter.SampleParameter.Scheduler = scheduler; + + return parameter; + } + + public static VideoGenerationParameter WithSampler(this VideoGenerationParameter parameter, Sampler sampler) + { + parameter.SampleParameter.SampleMethod = sampler; + + return parameter; + } + + public static VideoGenerationParameter WithSteps(this VideoGenerationParameter parameter, int steps) + { + parameter.SampleParameter.SampleSteps = steps; + + return parameter; + } + + public static VideoGenerationParameter WithEta(this VideoGenerationParameter parameter, float eta) + { + parameter.SampleParameter.Eta = eta; + + return parameter; + } + + public static VideoGenerationParameter WithShiftedTimestep(this VideoGenerationParameter parameter, int shiftedTimestep) + { + parameter.SampleParameter.ShiftedTimestep = shiftedTimestep; + + return parameter; + } + + #endregion + + #region HighNoiseSampleParameter + + #region Guidance + + public static VideoGenerationParameter WithHighNoiseCfg(this VideoGenerationParameter parameter, float cfg) + { + parameter.WithTxtCfg(cfg); + parameter.WithImgCfg(cfg); + + return parameter; + } + + public static VideoGenerationParameter WithHighNoiseTxtCfg(this VideoGenerationParameter parameter, float txtCfg) + { + parameter.HighNoiseSampleParameter.Guidance.TxtCfg = txtCfg; + + return parameter; + } + + public static VideoGenerationParameter WithHighNoiseImgCfg(this VideoGenerationParameter parameter, float imgCfg) + { + parameter.HighNoiseSampleParameter.Guidance.ImgCfg = imgCfg; + + return parameter; + } + + public static VideoGenerationParameter WithHighNoiseMinCfg(this VideoGenerationParameter parameter, float minCfg) + { + parameter.HighNoiseSampleParameter.Guidance.MinCfg = minCfg; + + return parameter; + } + + public static VideoGenerationParameter WithHighNoiseGuidance(this VideoGenerationParameter parameter, float guidance) + { + parameter.HighNoiseSampleParameter.Guidance.DistilledGuidance = guidance; + + return parameter; + } + + #region Slg + + public static VideoGenerationParameter WithHighNoiseSkipLayers(this VideoGenerationParameter parameter, int[] layers) + { + parameter.HighNoiseSampleParameter.Guidance.Slg.Layers = layers; + + return parameter; + } + + public static VideoGenerationParameter WithHighNoiseSkipLayerStart(this VideoGenerationParameter parameter, float skipLayerStart) + { + parameter.HighNoiseSampleParameter.Guidance.Slg.SkipLayerStart = skipLayerStart; + + return parameter; + } + + public static VideoGenerationParameter WithHighNoiseSkipLayerEnd(this VideoGenerationParameter parameter, float skipLayerEnd) + { + parameter.HighNoiseSampleParameter.Guidance.Slg.SkipLayerEnd = skipLayerEnd; + + return parameter; + } + + public static VideoGenerationParameter WithHighNoiseSlgScale(this VideoGenerationParameter parameter, float slgScale) + { + parameter.HighNoiseSampleParameter.Guidance.Slg.Scale = slgScale; + + return parameter; + } + + #endregion + + #endregion + + public static VideoGenerationParameter WithHighNoiseScheduler(this VideoGenerationParameter parameter, Scheduler scheduler) + { + parameter.HighNoiseSampleParameter.Scheduler = scheduler; + + return parameter; + } + + public static VideoGenerationParameter WithHighNoiseSampler(this VideoGenerationParameter parameter, Sampler sampler) + { + parameter.HighNoiseSampleParameter.SampleMethod = sampler; + + return parameter; + } + + public static VideoGenerationParameter WithHighNoiseSteps(this VideoGenerationParameter parameter, int steps) + { + parameter.HighNoiseSampleParameter.SampleSteps = steps; + + return parameter; + } + + public static VideoGenerationParameter WithHighNoiseEta(this VideoGenerationParameter parameter, float eta) + { + parameter.HighNoiseSampleParameter.Eta = eta; + + return parameter; + } + + public static VideoGenerationParameter WithHighNoiseShiftedTimestep(this VideoGenerationParameter parameter, int shiftedTimestep) + { + parameter.HighNoiseSampleParameter.ShiftedTimestep = shiftedTimestep; + + return parameter; + } + + #endregion + + public static VideoGenerationParameter WithMoeBoundry(this VideoGenerationParameter parameter, float moeBoundry) + { + parameter.MoeBoundry = moeBoundry; + + return parameter; + } + + public static VideoGenerationParameter WithStrength(this VideoGenerationParameter parameter, float strength) + { + parameter.Strength = strength; + + return parameter; + } + + public static VideoGenerationParameter WithSeed(this VideoGenerationParameter parameter, long seed) + { + parameter.Seed = seed; + + return parameter; + } + + public static VideoGenerationParameter WithFrameCount(this VideoGenerationParameter parameter, int frameCount) + { + parameter.FrameCount = frameCount; + + return parameter; + } + + public static VideoGenerationParameter WithVaceStrength(this VideoGenerationParameter parameter, float vaceStrength) + { + parameter.VaceStrength = vaceStrength; + + return parameter; + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs b/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs index 09e39dd..c65d8df 100644 --- a/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs @@ -21,12 +21,14 @@ public sealed class ImageGenerationParameter /// public int ClipSkip { get; set; } = -1; - public GuidanceParameter Guidance { get; } = new(); - public IImage? InitImage { get; set; } public IImage[]? RefImages { get; set; } + public bool IncreaseRefIndex { get; set; } = false; + + public bool AutoResizeRefImage { get; set; } = false; + public IImage? MaskImage { get; set; } /// @@ -39,20 +41,7 @@ public sealed class ImageGenerationParameter /// public int Height { get; set; } = 512; - /// - /// sampling method (default: Euler_A) - /// - public Sampler SampleMethod { get; set; } = Sampler.Euler_A; - - /// - /// number of sample steps (default: 25) - /// - public int SampleSteps { get; set; } = 25; - - /// - /// eta in DDIM, only for DDIM and TCD (default: 0) - /// - public float Eta { get; set; } = 0f; + public SampleParameter SampleParameter { get; internal init; } = new(); /// /// strength for noising/unnoising (default: 0.7) @@ -68,10 +57,12 @@ public sealed class ImageGenerationParameter public PhotoMakerParameter PhotoMaker { get; } = new(); + public TilingParameter VaeTiling { get; } = new(); + + #endregion + public static ImageGenerationParameter Create() => new(); public static ImageGenerationParameter TextToImage(string prompt) => Create().WithPrompt(prompt); public static ImageGenerationParameter ImageToImage(string prompt, IImage image) => Create().WithPrompt(prompt).WithInitImage(image); - - #endregion } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs deleted file mode 100644 index cadfe26..0000000 --- a/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs +++ /dev/null @@ -1,41 +0,0 @@ -using JetBrains.Annotations; - -namespace StableDiffusion.NET; - -[PublicAPI] -public interface IDiffusionModelParameter : IModelParameter -{ - string VaePath { get; set; } - string TaesdPath { get; set; } - - string LoraModelDirectory { get; set; } - string EmbeddingsDirectory { get; set; } - string ControlNetPath { get; set; } - - bool VaeDecodeOnly { get; set; } - bool VaeTiling { get; set; } - bool KeepControlNetOnCPU { get; set; } - bool KeepClipOnCPU { get; set; } - bool KeepVaeOnCPU { get; set; } - bool FlashAttention { get; set; } - bool DiffusionConvDirect { get; set; } - bool VaeConfDirect { get; set; } - - RngType RngType { get; set; } - Schedule Schedule { get; set; } - Quantization Quantization { get; set; } - - string ModelPath { get; set; } - - string StackedIdEmbeddingsDirectory { get; set; } - - string DiffusionModelPath { get; set; } - string ClipLPath { get; set; } - string T5xxlPath { get; set; } - - bool ChromaUseDitMap { get; set; } - bool ChromaEnableT5Map { get; set; } - int ChromaT5MaskPad { get; set; } - - string ClipGPath { get; set; } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs deleted file mode 100644 index ae23303..0000000 --- a/StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs +++ /dev/null @@ -1,9 +0,0 @@ -using JetBrains.Annotations; - -namespace StableDiffusion.NET; - -[PublicAPI] -public interface IModelParameter -{ - int ThreadCount { get; set; } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs deleted file mode 100644 index dfddf4a..0000000 --- a/StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs +++ /dev/null @@ -1,10 +0,0 @@ -using JetBrains.Annotations; - -namespace StableDiffusion.NET; - -[PublicAPI] -public interface IUpscaleModelParameter : IModelParameter -{ - string ModelPath { get; set; } - bool ConvDirect { get; set; } -} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs b/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs index 5077158..6380c92 100644 --- a/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs @@ -1,21 +1,17 @@ -namespace StableDiffusion.NET; +using HPPH; + +namespace StableDiffusion.NET; public sealed class PhotoMakerParameter { - /// - /// path to PHOTOMAKER input id images dir - /// - public string InputIdImageDirectory { get; set; } = string.Empty; + public IImage[]? IdImages { get; set; } + + public string IdEmbedPath { get; set; } = string.Empty; /// /// strength for keeping input identity (default: 20) /// public float StyleStrength { get; set; } = 20f; - /// - /// normalize PHOTOMAKER input id images - /// - public bool NormalizeInput { get; set; } = false; - internal PhotoMakerParameter() { } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/SampleParameter.cs b/StableDiffusion.NET/Models/Parameter/SampleParameter.cs new file mode 100644 index 0000000..39e988c --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/SampleParameter.cs @@ -0,0 +1,30 @@ +namespace StableDiffusion.NET; + +public sealed class SampleParameter +{ + public GuidanceParameter Guidance { get; } = new(); + + /// + /// Denoiser sigma schedule (default: Default) + /// + public Scheduler Scheduler { get; set; } = Scheduler.Default; + + /// + /// sampling method (default: Euler_A) + /// + public Sampler SampleMethod { get; set; } = Sampler.Euler_A; + + /// + /// number of sample steps (default: 25) + /// + public int SampleSteps { get; set; } = 25; + + /// + /// eta in DDIM, only for DDIM and TCD (default: 0) + /// + public float Eta { get; set; } = 0f; + + public int ShiftedTimestep { get; set; } = 0; + + internal SampleParameter() { } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/TilingParameter.cs b/StableDiffusion.NET/Models/Parameter/TilingParameter.cs new file mode 100644 index 0000000..6f9934e --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/TilingParameter.cs @@ -0,0 +1,14 @@ +namespace StableDiffusion.NET; + +public sealed class TilingParameter +{ + public bool IsEnabled { get; set; } + + public int TileSizeX { get; set; } + public int TileSizeY { get; set; } + public float TargetOverlap { get; set; } + public float RelSizeX { get; set; } + public float RelSizeY { get; set; } + + internal TilingParameter() { } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs index da415e7..b5ae8ff 100644 --- a/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs @@ -3,13 +3,15 @@ namespace StableDiffusion.NET; [PublicAPI] -public sealed class UpscaleModelParameter : IUpscaleModelParameter +public sealed class UpscaleModelParameter { /// /// path to esrgan model. Upscale images after generate, just RealESRGAN_x4plus_anime_6B supported by now /// public string ModelPath { get; set; } = string.Empty; + public bool OffloadParamsToCPU { get; set; } = false; + /// /// number of threads to use during computation (default: -1) /// If threads = -1, then threads will be set to the number of CPU physical cores diff --git a/StableDiffusion.NET/Models/Parameter/VideoGenerationParameter.cs b/StableDiffusion.NET/Models/Parameter/VideoGenerationParameter.cs new file mode 100644 index 0000000..22f6224 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/VideoGenerationParameter.cs @@ -0,0 +1,46 @@ +using HPPH; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class VideoGenerationParameter +{ + #region Properties & Fields + + public string Prompt { get; set; } = string.Empty; + + public string NegativePrompt { get; set; } = string.Empty; + + public int ClipSkip { get; set; } = -1; + + public IImage? InitImage { get; set; } + + public IImage? EndImage { get; set; } + + public IImage[]? ControlFrames { get; set; } + + public int Width { get; set; } + + public int Height { get; set; } + + public SampleParameter SampleParameter { get; internal init; } = new(); + + public SampleParameter HighNoiseSampleParameter { get; internal init; } = new(); + + public float MoeBoundry { get; set; } + + public float Strength { get; set; } + + public long Seed { get; set; } + + public int FrameCount { get; set; } + + public float VaceStrength { get; set; } + + #endregion + + public static VideoGenerationParameter Create() => new(); + + public static VideoGenerationParameter TextToVideo(string prompt) => Create().WithPrompt(prompt); +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/UpscaleModel.cs b/StableDiffusion.NET/Models/UpscaleModel.cs index 7d9c5b6..9450422 100644 --- a/StableDiffusion.NET/Models/UpscaleModel.cs +++ b/StableDiffusion.NET/Models/UpscaleModel.cs @@ -16,7 +16,7 @@ public sealed unsafe class UpscaleModel : IDisposable private Native.Types.upscaler_ctx_t* _ctx; #endregion - + #region Constructors public UpscaleModel(UpscaleModelParameter modelParameter) @@ -39,8 +39,9 @@ public sealed unsafe class UpscaleModel : IDisposable private void Initialize() { _ctx = Native.new_upscaler_ctx(ModelParameter.ModelPath, - ModelParameter.ThreadCount, - ModelParameter.ConvDirect); + ModelParameter.OffloadParamsToCPU, + ModelParameter.ConvDirect, + ModelParameter.ThreadCount); if (_ctx == null) throw new NullReferenceException("Failed to initialize upscale-model."); } diff --git a/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs index baa2e00..8f14ec8 100644 --- a/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs +++ b/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs @@ -12,30 +12,36 @@ internal static unsafe class DiffusionModelParameterMarshaller model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ModelPath), clip_l_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipLPath), clip_g_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipGPath), + clip_vision_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipVisionPath), t5xxl_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.T5xxlPath), + qwen2vl_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.Qwen2VLPath), + qwen2vl_vision_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.Qwen2VLVisionPath), diffusion_model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.DiffusionModelPath), + high_noise_diffusion_model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.HighNoiseDiffusionModelPath), vae_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.VaePath), taesd_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.TaesdPath), control_net_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ControlNetPath), lora_model_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.LoraModelDirectory), embedding_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.EmbeddingsDirectory), - stacked_id_embed_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.StackedIdEmbeddingsDirectory), + photo_maker_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.StackedIdEmbeddingsDirectory), vae_decode_only = (sbyte)(managed.VaeDecodeOnly ? 1 : 0), - vae_tiling = (sbyte)(managed.VaeTiling ? 1 : 0), free_params_immediately = 0, // DarthAffe 06.08.2025: Static value n_threads = managed.ThreadCount, wtype = managed.Quantization, rng_type = managed.RngType, - schedule = managed.Schedule, + prediction = managed.Prediction, + offload_params_to_cpu = (sbyte)(managed.OffloadParamsToCPU ? 1 : 0), keep_clip_on_cpu = (sbyte)(managed.KeepClipOnCPU ? 1 : 0), keep_control_net_on_cpu = (sbyte)(managed.KeepControlNetOnCPU ? 1 : 0), keep_vae_on_cpu = (sbyte)(managed.KeepVaeOnCPU ? 1 : 0), diffusion_flash_attn = (sbyte)(managed.FlashAttention ? 1 : 0), diffusion_conv_direct = (sbyte)(managed.DiffusionConvDirect ? 1 : 0), - vae_conv_direct = (sbyte)(managed.VaeConfDirect ? 1 : 0), + vae_conv_direct = (sbyte)(managed.VaeConvDirect ? 1 : 0), + force_sdxl_vae_conv_scale = (sbyte)(managed.ForceSdxlVaeConvScale ? 1 : 0), chroma_use_dit_mask = (sbyte)(managed.ChromaUseDitMap ? 1 : 0), chroma_use_t5_mask = (sbyte)(managed.ChromaEnableT5Map ? 1 : 0), - chroma_t5_mask_pad = managed.ChromaT5MaskPad + chroma_t5_mask_pad = managed.ChromaT5MaskPad, + flow_shift = managed.FlowShift }; public static DiffusionModelParameter ConvertToManaged(Native.Types.sd_ctx_params_t unmanaged) @@ -44,29 +50,35 @@ internal static unsafe class DiffusionModelParameterMarshaller ModelPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.model_path) ?? string.Empty, ClipLPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.clip_l_path) ?? string.Empty, ClipGPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.clip_g_path) ?? string.Empty, + ClipVisionPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.clip_vision_path) ?? string.Empty, T5xxlPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.t5xxl_path) ?? string.Empty, + Qwen2VLPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.qwen2vl_path) ?? string.Empty, + Qwen2VLVisionPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.qwen2vl_vision_path) ?? string.Empty, DiffusionModelPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.diffusion_model_path) ?? string.Empty, + HighNoiseDiffusionModelPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.high_noise_diffusion_model_path) ?? string.Empty, VaePath = AnsiStringMarshaller.ConvertToManaged(unmanaged.vae_path) ?? string.Empty, TaesdPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.taesd_path) ?? string.Empty, ControlNetPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.control_net_path) ?? string.Empty, LoraModelDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.lora_model_dir) ?? string.Empty, EmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.embedding_dir) ?? string.Empty, - StackedIdEmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.stacked_id_embed_dir) ?? string.Empty, + StackedIdEmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.photo_maker_path) ?? string.Empty, VaeDecodeOnly = unmanaged.vae_decode_only == 1, - VaeTiling = unmanaged.vae_tiling == 1, ThreadCount = unmanaged.n_threads, Quantization = unmanaged.wtype, RngType = unmanaged.rng_type, - Schedule = unmanaged.schedule, + Prediction = unmanaged.prediction, + OffloadParamsToCPU = unmanaged.offload_params_to_cpu == 1, KeepClipOnCPU = unmanaged.keep_clip_on_cpu == 1, KeepControlNetOnCPU = unmanaged.keep_control_net_on_cpu == 1, KeepVaeOnCPU = unmanaged.keep_vae_on_cpu == 1, FlashAttention = unmanaged.diffusion_flash_attn == 1, DiffusionConvDirect = unmanaged.diffusion_conv_direct == 1, - VaeConfDirect = unmanaged.vae_conv_direct == 1, + VaeConvDirect = unmanaged.vae_conv_direct == 1, + ForceSdxlVaeConvScale = unmanaged.force_sdxl_vae_conv_scale == 1, ChromaUseDitMap = unmanaged.chroma_use_dit_mask == 1, ChromaEnableT5Map = unmanaged.chroma_use_t5_mask == 1, - ChromaT5MaskPad = unmanaged.chroma_t5_mask_pad + ChromaT5MaskPad = unmanaged.chroma_t5_mask_pad, + FlowShift = unmanaged.flow_shift }; public static void Free(Native.Types.sd_ctx_params_t unmanaged) @@ -75,12 +87,14 @@ internal static unsafe class DiffusionModelParameterMarshaller AnsiStringMarshaller.Free(unmanaged.clip_l_path); AnsiStringMarshaller.Free(unmanaged.clip_g_path); AnsiStringMarshaller.Free(unmanaged.t5xxl_path); + AnsiStringMarshaller.Free(unmanaged.qwen2vl_path); + AnsiStringMarshaller.Free(unmanaged.qwen2vl_vision_path); AnsiStringMarshaller.Free(unmanaged.diffusion_model_path); AnsiStringMarshaller.Free(unmanaged.vae_path); AnsiStringMarshaller.Free(unmanaged.taesd_path); AnsiStringMarshaller.Free(unmanaged.control_net_path); AnsiStringMarshaller.Free(unmanaged.lora_model_dir); AnsiStringMarshaller.Free(unmanaged.embedding_dir); - AnsiStringMarshaller.Free(unmanaged.stacked_id_embed_dir); + AnsiStringMarshaller.Free(unmanaged.photo_maker_path); } } \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs index 08915f2..2dd1ee9 100644 --- a/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs +++ b/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs @@ -18,46 +18,37 @@ internal static class ImageGenerationParameterMarshaller Prompt = AnsiStringMarshaller.ConvertToManaged(unmanaged.prompt) ?? string.Empty, NegativePrompt = AnsiStringMarshaller.ConvertToManaged(unmanaged.negative_prompt) ?? string.Empty, ClipSkip = unmanaged.clip_skip, - Guidance = - { - TxtCfg = unmanaged.guidance.txt_cfg, - ImgCfg = unmanaged.guidance.img_cfg, - MinCfg = unmanaged.guidance.min_cfg, - DistilledGuidance = unmanaged.guidance.distilled_guidance, - Slg = - { - Layers = new int[unmanaged.guidance.slg.layer_count], - SkipLayerStart = unmanaged.guidance.slg.layer_start, - SkipLayerEnd = unmanaged.guidance.slg.layer_end, - Scale = unmanaged.guidance.slg.scale - } - }, + SampleParameter = SampleParameterMarshaller.ConvertToManaged(unmanaged.sample_params), InitImage = unmanaged.init_image.data == null ? null : unmanaged.init_image.ToImage(), RefImages = unmanaged.ref_images == null ? null : ImageHelper.ToImageArrayIFace(unmanaged.ref_images, unmanaged.ref_images_count), + AutoResizeRefImage = unmanaged.auto_resize_ref_image == 1, MaskImage = unmanaged.mask_image.data == null ? null : unmanaged.mask_image.ToImage(), Width = unmanaged.width, Height = unmanaged.height, - SampleMethod = unmanaged.sample_method, - SampleSteps = unmanaged.sample_steps, - Eta = unmanaged.eta, Strength = unmanaged.strength, Seed = unmanaged.seed, ControlNet = { - Image = unmanaged.control_cond == null ? null : ImageHelper.GetImage(unmanaged.control_cond, 0), + Image = unmanaged.control_image.ToImage(), Strength = unmanaged.control_strength, }, PhotoMaker = { - StyleStrength = unmanaged.style_strength, - NormalizeInput = unmanaged.normalize_input == 1, - InputIdImageDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.input_id_images_path) ?? string.Empty, + IdImages = unmanaged.pm_params.id_images == null ? null : ImageHelper.ToImageArrayIFace(unmanaged.pm_params.id_images, unmanaged.pm_params.id_images_count), + IdEmbedPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.pm_params.id_embed_path) ?? string.Empty, + StyleStrength = unmanaged.pm_params.style_strength, + }, + VaeTiling = + { + IsEnabled = unmanaged.vae_tiling_params.enabled == 1, + TileSizeX = unmanaged.vae_tiling_params.tile_size_x, + TileSizeY = unmanaged.vae_tiling_params.tile_size_y, + TargetOverlap = unmanaged.vae_tiling_params.target_overlap, + RelSizeX = unmanaged.vae_tiling_params.rel_size_x, + RelSizeY = unmanaged.vae_tiling_params.rel_size_y } }; - if (unmanaged.guidance.slg.layers != null) - new Span(unmanaged.guidance.slg.layers, (int)unmanaged.guidance.slg.layer_count).CopyTo(parameter.Guidance.Slg.Layers); - return parameter; } @@ -65,39 +56,41 @@ internal static class ImageGenerationParameterMarshaller { AnsiStringMarshaller.Free(unmanaged.prompt); AnsiStringMarshaller.Free(unmanaged.negative_prompt); - AnsiStringMarshaller.Free(unmanaged.input_id_images_path); unmanaged.init_image.Free(); unmanaged.mask_image.Free(); + unmanaged.control_image.Free(); if (unmanaged.ref_images != null) ImageHelper.Free(unmanaged.ref_images, unmanaged.ref_images_count); - if (unmanaged.control_cond != null) - ImageHelper.Free(unmanaged.control_cond, 1); + if (unmanaged.pm_params.id_images != null) + ImageHelper.Free(unmanaged.pm_params.id_images, unmanaged.pm_params.id_images_count); - if (unmanaged.guidance.slg.layers != null) - NativeMemory.Free(unmanaged.guidance.slg.layers); + SampleParameterMarshaller.Free(unmanaged.sample_params); } internal unsafe ref struct ImageGenerationParameterMarshallerIn { + private SampleParameterMarshaller.SampleParameterMarshallerIn _sampleParameterMarshaller = new(); private Native.Types.sd_img_gen_params_t _imgGenParams; private Native.Types.sd_image_t _initImage; private Native.Types.sd_image_t _maskImage; + private Native.Types.sd_image_t _controlNetImage; private Native.Types.sd_image_t* _refImages; - private Native.Types.sd_image_t* _controlNetImage; - private int* _slgLayers; + private Native.Types.sd_image_t* _pmIdImages; + + public ImageGenerationParameterMarshallerIn() { } public void FromManaged(ImageGenerationParameter managed) { - _initImage = managed.InitImage?.ToSdImage() ?? new Native.Types.sd_image_t(); - _refImages = managed.RefImages == null ? null : managed.RefImages.ToSdImage(); - _controlNetImage = managed.ControlNet.Image == null ? null : managed.ControlNet.Image.ToSdImagePtr(); + _sampleParameterMarshaller.FromManaged(managed.SampleParameter); - _slgLayers = (int*)NativeMemory.Alloc((nuint)managed.Guidance.Slg.Layers.Length, (nuint)Marshal.SizeOf()); - managed.Guidance.Slg.Layers.AsSpan().CopyTo(new Span(_slgLayers, managed.Guidance.Slg.Layers.Length)); + _initImage = managed.InitImage?.ToSdImage() ?? new Native.Types.sd_image_t(); + _controlNetImage = managed.ControlNet.Image?.ToSdImage() ?? new Native.Types.sd_image_t(); + _refImages = managed.RefImages == null ? null : managed.RefImages.ToSdImage(); + _pmIdImages = managed.PhotoMaker.IdImages == null ? null : managed.PhotoMaker.IdImages.ToSdImage(); if (managed.MaskImage != null) _maskImage = managed.MaskImage.ToSdImage(true); @@ -115,22 +108,22 @@ internal static class ImageGenerationParameterMarshaller new Span(_maskImage.data, (int)maskImageByteSize).Fill(byte.MaxValue); } - Native.Types.sd_slg_params_t slg = new() + Native.Types.sd_pm_params_t photoMakerParams = new() { - layers = _slgLayers, - layer_count = (uint)managed.Guidance.Slg.Layers.Length, - layer_start = managed.Guidance.Slg.SkipLayerStart, - layer_end = managed.Guidance.Slg.SkipLayerEnd, - scale = managed.Guidance.Slg.Scale, + id_images = _pmIdImages, + id_images_count = managed.PhotoMaker.IdImages?.Length ?? 0, + id_embed_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.PhotoMaker.IdEmbedPath), + style_strength = managed.PhotoMaker.StyleStrength }; - Native.Types.sd_guidance_params_t guidance = new() + Native.Types.sd_tiling_params_t tilingParams = new() { - txt_cfg = managed.Guidance.TxtCfg, - img_cfg = managed.Guidance.ImgCfg, - min_cfg = managed.Guidance.MinCfg, - distilled_guidance = managed.Guidance.DistilledGuidance, - slg = slg + enabled = (sbyte)(managed.VaeTiling.IsEnabled ? 1 : 0), + tile_size_x = managed.VaeTiling.TileSizeX, + tile_size_y = managed.VaeTiling.TileSizeY, + target_overlap = managed.VaeTiling.TargetOverlap, + rel_size_x = managed.VaeTiling.RelSizeX, + rel_size_y = managed.VaeTiling.RelSizeY }; _imgGenParams = new Native.Types.sd_img_gen_params_t @@ -138,24 +131,21 @@ internal static class ImageGenerationParameterMarshaller prompt = AnsiStringMarshaller.ConvertToUnmanaged(managed.Prompt), negative_prompt = AnsiStringMarshaller.ConvertToUnmanaged(managed.NegativePrompt), clip_skip = managed.ClipSkip, - guidance = guidance, + sample_params = _sampleParameterMarshaller.ToUnmanaged(), init_image = _initImage, ref_images = _refImages, ref_images_count = managed.RefImages?.Length ?? 0, + auto_resize_ref_image = (sbyte)(managed.AutoResizeRefImage ? 1 : 0), mask_image = _maskImage, width = managed.Width, height = managed.Height, - sample_method = managed.SampleMethod, - sample_steps = managed.SampleSteps, - eta = managed.Eta, strength = managed.Strength, seed = managed.Seed, batch_count = 1, - control_cond = _controlNetImage, + control_image = _controlNetImage, control_strength = managed.ControlNet.Strength, - style_strength = managed.PhotoMaker.StyleStrength, - normalize_input = (sbyte)(managed.PhotoMaker.NormalizeInput ? 1 : 0), - input_id_images_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.PhotoMaker.InputIdImageDirectory), + pm_params = photoMakerParams, + vae_tiling_params = tilingParams }; } @@ -165,32 +155,32 @@ internal static class ImageGenerationParameterMarshaller { AnsiStringMarshaller.Free(_imgGenParams.prompt); AnsiStringMarshaller.Free(_imgGenParams.negative_prompt); - AnsiStringMarshaller.Free(_imgGenParams.input_id_images_path); + AnsiStringMarshaller.Free(_imgGenParams.pm_params.id_embed_path); _initImage.Free(); _maskImage.Free(); + _controlNetImage.Free(); if (_refImages != null) ImageHelper.Free(_refImages, _imgGenParams.ref_images_count); - if (_controlNetImage != null) - ImageHelper.Free(_controlNetImage, 1); + if (_pmIdImages != null) + ImageHelper.Free(_pmIdImages, _imgGenParams.pm_params.id_images_count); - if (_slgLayers != null) - NativeMemory.Free(_slgLayers); + _sampleParameterMarshaller.Free(); } } internal ref struct ImageGenerationParameterMarshallerRef() { private ImageGenerationParameterMarshallerIn _inMarshaller = new(); - private ImageGenerationParameter _parameter; + private ImageGenerationParameter? _parameter; public void FromManaged(ImageGenerationParameter managed) => _inMarshaller.FromManaged(managed); public Native.Types.sd_img_gen_params_t ToUnmanaged() => _inMarshaller.ToUnmanaged(); public void FromUnmanaged(Native.Types.sd_img_gen_params_t unmanaged) => _parameter = ConvertToManaged(unmanaged); - public ImageGenerationParameter ToManaged() => _parameter; + public ImageGenerationParameter ToManaged() => _parameter!; public void Free() => _inMarshaller.Free(); } diff --git a/StableDiffusion.NET/Native/Marshaller/SampleParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/SampleParameterMarshaller.cs new file mode 100644 index 0000000..79c79ac --- /dev/null +++ b/StableDiffusion.NET/Native/Marshaller/SampleParameterMarshaller.cs @@ -0,0 +1,113 @@ +// ReSharper disable MemberCanBeMadeStatic.Global + +using System; +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; + +namespace StableDiffusion.NET; + +[CustomMarshaller(typeof(SampleParameter), MarshalMode.ManagedToUnmanagedIn, typeof(SampleParameterMarshallerIn))] +[CustomMarshaller(typeof(SampleParameter), MarshalMode.ManagedToUnmanagedOut, typeof(SampleParameterMarshaller))] +[CustomMarshaller(typeof(SampleParameter), MarshalMode.ManagedToUnmanagedRef, typeof(SampleParameterMarshallerRef))] +internal static class SampleParameterMarshaller +{ + public static unsafe SampleParameter ConvertToManaged(Native.Types.sd_sample_params_t unmanaged) + { + SampleParameter parameter = new() + { + Guidance = + { + TxtCfg = unmanaged.guidance.txt_cfg, + ImgCfg = unmanaged.guidance.img_cfg, + MinCfg = unmanaged.guidance.min_cfg, + DistilledGuidance = unmanaged.guidance.distilled_guidance, + Slg = + { + Layers = new int[unmanaged.guidance.slg.layer_count], + SkipLayerStart = unmanaged.guidance.slg.layer_start, + SkipLayerEnd = unmanaged.guidance.slg.layer_end, + Scale = unmanaged.guidance.slg.scale + } + }, + Scheduler = unmanaged.scheduler, + SampleMethod = unmanaged.sample_method, + SampleSteps = unmanaged.sample_steps, + Eta = unmanaged.eta, + ShiftedTimestep = unmanaged.shifted_timestep + }; + + if (unmanaged.guidance.slg.layers != null) + new Span(unmanaged.guidance.slg.layers, (int)unmanaged.guidance.slg.layer_count).CopyTo(parameter.Guidance.Slg.Layers); + + return parameter; + } + + public static unsafe void Free(Native.Types.sd_sample_params_t unmanaged) + { + if (unmanaged.guidance.slg.layers != null) + NativeMemory.Free(unmanaged.guidance.slg.layers); + } + + internal unsafe ref struct SampleParameterMarshallerIn + { + private Native.Types.sd_sample_params_t _sampleParams; + + private int* _slgLayers; + + public void FromManaged(SampleParameter managed) + { + _slgLayers = (int*)NativeMemory.Alloc((nuint)managed.Guidance.Slg.Layers.Length, (nuint)Marshal.SizeOf()); + managed.Guidance.Slg.Layers.AsSpan().CopyTo(new Span(_slgLayers, managed.Guidance.Slg.Layers.Length)); + + Native.Types.sd_slg_params_t slg = new() + { + layers = _slgLayers, + layer_count = (uint)managed.Guidance.Slg.Layers.Length, + layer_start = managed.Guidance.Slg.SkipLayerStart, + layer_end = managed.Guidance.Slg.SkipLayerEnd, + scale = managed.Guidance.Slg.Scale, + }; + + Native.Types.sd_guidance_params_t guidance = new() + { + txt_cfg = managed.Guidance.TxtCfg, + img_cfg = managed.Guidance.ImgCfg, + min_cfg = managed.Guidance.MinCfg, + distilled_guidance = managed.Guidance.DistilledGuidance, + slg = slg + }; + + _sampleParams = new Native.Types.sd_sample_params_t + { + guidance = guidance, + scheduler = managed.Scheduler, + sample_method = managed.SampleMethod, + sample_steps = managed.SampleSteps, + eta = managed.Eta, + shifted_timestep = managed.ShiftedTimestep + }; + } + + public Native.Types.sd_sample_params_t ToUnmanaged() => _sampleParams; + + public void Free() + { + if (_slgLayers != null) + NativeMemory.Free(_slgLayers); + } + } + + internal ref struct SampleParameterMarshallerRef() + { + private SampleParameterMarshallerIn _inMarshaller = new(); + private SampleParameter? _parameter; + + public void FromManaged(SampleParameter managed) => _inMarshaller.FromManaged(managed); + public Native.Types.sd_sample_params_t ToUnmanaged() => _inMarshaller.ToUnmanaged(); + + public void FromUnmanaged(Native.Types.sd_sample_params_t unmanaged) => _parameter = ConvertToManaged(unmanaged); + public SampleParameter ToManaged() => _parameter ?? throw new NullReferenceException($"{nameof(FromUnmanaged)} needs to be called before ToManaged."); + + public void Free() => _inMarshaller.Free(); + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Marshaller/VideoGenerationParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/VideoGenerationParameterMarshaller.cs new file mode 100644 index 0000000..9af8e8f --- /dev/null +++ b/StableDiffusion.NET/Native/Marshaller/VideoGenerationParameterMarshaller.cs @@ -0,0 +1,124 @@ +// ReSharper disable MemberCanBeMadeStatic.Global + +using System.Runtime.InteropServices.Marshalling; + +namespace StableDiffusion.NET; + +[CustomMarshaller(typeof(VideoGenerationParameter), MarshalMode.ManagedToUnmanagedIn, typeof(VideoGenerationParameterMarshallerIn))] +[CustomMarshaller(typeof(VideoGenerationParameter), MarshalMode.ManagedToUnmanagedOut, typeof(VideoGenerationParameterMarshaller))] +[CustomMarshaller(typeof(VideoGenerationParameter), MarshalMode.ManagedToUnmanagedRef, typeof(VideoGenerationParameterMarshallerRef))] +internal static class VideoGenerationParameterMarshaller +{ + public static unsafe VideoGenerationParameter ConvertToManaged(Native.Types.sd_vid_gen_params_t unmanaged) + { + VideoGenerationParameter parameter = new() + { + Prompt = AnsiStringMarshaller.ConvertToManaged(unmanaged.prompt) ?? string.Empty, + NegativePrompt = AnsiStringMarshaller.ConvertToManaged(unmanaged.negative_prompt) ?? string.Empty, + ClipSkip = unmanaged.clip_skip, + InitImage = unmanaged.init_image.data == null ? null : unmanaged.init_image.ToImage(), + EndImage = unmanaged.end_image.data == null ? null : unmanaged.end_image.ToImage(), + ControlFrames = unmanaged.control_frames == null ? null : ImageHelper.ToImageArrayIFace(unmanaged.control_frames, unmanaged.control_frames_size), + Width = unmanaged.width, + Height = unmanaged.height, + SampleParameter = SampleParameterMarshaller.ConvertToManaged(unmanaged.sample_params), + HighNoiseSampleParameter = SampleParameterMarshaller.ConvertToManaged(unmanaged.high_noise_sample_params), + MoeBoundry = unmanaged.moe_boundary, + Strength = unmanaged.strength, + Seed = unmanaged.seed, + FrameCount = unmanaged.video_frames, + VaceStrength = unmanaged.vace_strength + }; + + return parameter; + } + + public static unsafe void Free(Native.Types.sd_vid_gen_params_t unmanaged) + { + AnsiStringMarshaller.Free(unmanaged.prompt); + AnsiStringMarshaller.Free(unmanaged.negative_prompt); + + unmanaged.init_image.Free(); + unmanaged.end_image.Free(); + + if (unmanaged.control_frames != null) + ImageHelper.Free(unmanaged.control_frames, unmanaged.control_frames_size); + + SampleParameterMarshaller.Free(unmanaged.sample_params); + SampleParameterMarshaller.Free(unmanaged.high_noise_sample_params); + } + + internal unsafe ref struct VideoGenerationParameterMarshallerIn + { + private SampleParameterMarshaller.SampleParameterMarshallerIn _sampleParameterMarshaller = new(); + private SampleParameterMarshaller.SampleParameterMarshallerIn _highNoiseSampleParameterMarshaller = new(); + private Native.Types.sd_vid_gen_params_t _vidGenParams; + + private Native.Types.sd_image_t _initImage; + private Native.Types.sd_image_t _endImage; + private Native.Types.sd_image_t* _controlFrames; + + public VideoGenerationParameterMarshallerIn() { } + + public void FromManaged(VideoGenerationParameter managed) + { + _sampleParameterMarshaller.FromManaged(managed.SampleParameter); + _highNoiseSampleParameterMarshaller.FromManaged(managed.HighNoiseSampleParameter); + + _initImage = managed.InitImage?.ToSdImage() ?? new Native.Types.sd_image_t(); + _endImage = managed.EndImage?.ToSdImage() ?? new Native.Types.sd_image_t(); + _controlFrames = managed.ControlFrames == null ? null : managed.ControlFrames.ToSdImage(); + + _vidGenParams = new Native.Types.sd_vid_gen_params_t + { + prompt = AnsiStringMarshaller.ConvertToUnmanaged(managed.Prompt), + negative_prompt = AnsiStringMarshaller.ConvertToUnmanaged(managed.NegativePrompt), + clip_skip = managed.ClipSkip, + init_image = _initImage, + end_image = _endImage, + control_frames = _controlFrames, + control_frames_size = managed.ControlFrames?.Length ?? 0, + width = managed.Width, + height = managed.Height, + sample_params = _sampleParameterMarshaller.ToUnmanaged(), + high_noise_sample_params = _highNoiseSampleParameterMarshaller.ToUnmanaged(), + moe_boundary = managed.MoeBoundry, + strength = managed.Strength, + seed = managed.Seed, + video_frames = managed.FrameCount, + vace_strength = managed.VaceStrength, + }; + } + + public Native.Types.sd_vid_gen_params_t ToUnmanaged() => _vidGenParams; + + public void Free() + { + AnsiStringMarshaller.Free(_vidGenParams.prompt); + AnsiStringMarshaller.Free(_vidGenParams.negative_prompt); + + _initImage.Free(); + _endImage.Free(); + + if (_controlFrames != null) + ImageHelper.Free(_controlFrames, _vidGenParams.control_frames_size); + + _sampleParameterMarshaller.Free(); + _highNoiseSampleParameterMarshaller.Free(); + } + } + + internal ref struct VideoGenerationParameterMarshallerRef() + { + private VideoGenerationParameterMarshallerIn _inMarshaller = new(); + private VideoGenerationParameter? _parameter; + + public void FromManaged(VideoGenerationParameter managed) => _inMarshaller.FromManaged(managed); + public Native.Types.sd_vid_gen_params_t ToUnmanaged() => _inMarshaller.ToUnmanaged(); + + public void FromUnmanaged(Native.Types.sd_vid_gen_params_t unmanaged) => _parameter = ConvertToManaged(unmanaged); + public VideoGenerationParameter ToManaged() => _parameter!; + + public void Free() => _inMarshaller.Free(); + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Native.cs b/StableDiffusion.NET/Native/Native.cs index 7afd2b0..b007d65 100644 --- a/StableDiffusion.NET/Native/Native.cs +++ b/StableDiffusion.NET/Native/Native.cs @@ -1,30 +1,31 @@ -#pragma warning disable CS0169 // Field is never used -#pragma warning disable IDE1006 +#pragma warning disable IDE1006 // ReSharper disable InconsistentNaming // ReSharper disable ArrangeTypeMemberModifiers // ReSharper disable UseSymbolAlias +using HPPH; using System.Runtime.InteropServices; using System.Runtime.InteropServices.Marshalling; -using HPPH; namespace StableDiffusion.NET; +using int32_t = int; +using int64_t = long; using rng_type_t = RngType; using sample_method_t = Sampler; -using schedule_t = Schedule; -using sd_type_t = Quantization; +using scheduler_t = Scheduler; +using prediction_t = Prediction; +using sd_ctx_params_t = DiffusionModelParameter; +using sd_ctx_t = Native.Types.sd_ctx_t; +using sd_image_t = Native.Types.sd_image_t; +using sd_sample_params_t = SampleParameter; +using sd_img_gen_params_t = ImageGenerationParameter; using sd_log_level_t = LogLevel; +using sd_type_t = Quantization; +using sd_vid_gen_params_t = VideoGenerationParameter; +using size_t = nuint; using uint32_t = uint; using uint8_t = byte; -using int64_t = long; -using size_t = nuint; -using int32_t = int; -using sd_ctx_params_t = DiffusionModelParameter; -using sd_img_gen_params_t = ImageGenerationParameter; -using sd_vid_gen_params_t = Native.Types.sd_vid_gen_params_t; -using sd_image_t = Native.Types.sd_image_t; -using sd_ctx_t = Native.Types.sd_ctx_t; using upscaler_ctx_t = Native.Types.upscaler_ctx_t; internal unsafe partial class Native @@ -37,36 +38,53 @@ internal unsafe partial class Native internal static class Types { + [StructLayout(LayoutKind.Sequential)] + public struct sd_tiling_params_t + { + public sbyte enabled; + public int tile_size_x; + public int tile_size_y; + public float target_overlap; + public float rel_size_x; + public float rel_size_y; + } + [StructLayout(LayoutKind.Sequential)] internal struct sd_ctx_params_t { public byte* model_path; public byte* clip_l_path; public byte* clip_g_path; + public byte* clip_vision_path; public byte* t5xxl_path; + public byte* qwen2vl_path; + public byte* qwen2vl_vision_path; public byte* diffusion_model_path; + public byte* high_noise_diffusion_model_path; public byte* vae_path; public byte* taesd_path; public byte* control_net_path; public byte* lora_model_dir; public byte* embedding_dir; - public byte* stacked_id_embed_dir; + public byte* photo_maker_path; public sbyte vae_decode_only; - public sbyte vae_tiling; public sbyte free_params_immediately; public int n_threads; public sd_type_t wtype; public rng_type_t rng_type; - public schedule_t schedule; + public prediction_t prediction; + public sbyte offload_params_to_cpu; public sbyte keep_clip_on_cpu; public sbyte keep_control_net_on_cpu; public sbyte keep_vae_on_cpu; public sbyte diffusion_flash_attn; public sbyte diffusion_conv_direct; public sbyte vae_conv_direct; + public sbyte force_sdxl_vae_conv_scale; public sbyte chroma_use_dit_mask; public sbyte chroma_use_t5_mask; public int chroma_t5_mask_pad; + public float flow_shift; } [StructLayout(LayoutKind.Sequential)] @@ -98,47 +116,68 @@ internal unsafe partial class Native public sd_slg_params_t slg; } + [StructLayout(LayoutKind.Sequential)] + internal struct sd_sample_params_t + { + public sd_guidance_params_t guidance; + public scheduler_t scheduler; + public sample_method_t sample_method; + public int sample_steps; + public float eta; + public int shifted_timestep; + } + + internal struct sd_pm_params_t + { + public sd_image_t* id_images; + public int id_images_count; + public byte* id_embed_path; + public float style_strength; + } + [StructLayout(LayoutKind.Sequential)] internal struct sd_img_gen_params_t { public byte* prompt; public byte* negative_prompt; public int clip_skip; - public sd_guidance_params_t guidance; public sd_image_t init_image; public sd_image_t* ref_images; public int ref_images_count; + public sbyte auto_resize_ref_image; + public sbyte increase_ref_index; public sd_image_t mask_image; public int width; public int height; - public sample_method_t sample_method; - public int sample_steps; - public float eta; + public sd_sample_params_t sample_params; public float strength; public int64_t seed; public int batch_count; - public sd_image_t* control_cond; + public sd_image_t control_image; public float control_strength; - public float style_strength; - public sbyte normalize_input; - public byte* input_id_images_path; + public sd_pm_params_t pm_params; + public sd_tiling_params_t vae_tiling_params; } [StructLayout(LayoutKind.Sequential)] internal struct sd_vid_gen_params_t { + public byte* prompt; + public byte* negative_prompt; + public int clip_skip; public sd_image_t init_image; + public sd_image_t end_image; + public sd_image_t* control_frames; + public int control_frames_size; public int width; public int height; - public sd_guidance_params_t guidance; - public sample_method_t sample_method; - public int sample_steps; + public sd_sample_params_t sample_params; + public sd_sample_params_t high_noise_sample_params; + public float moe_boundary; public float strength; public int64_t seed; public int video_frames; - public int motion_bucket_id; - public int fps; - public float augmentation_level; + public float vace_strength; } internal struct sd_ctx_t; @@ -192,10 +231,17 @@ internal unsafe partial class Native [LibraryImport(LIB_NAME, EntryPoint = "sd_schedule_name")] [return: MarshalAs(UnmanagedType.LPStr)] - internal static partial string sd_schedule_name(schedule_t schedule); + internal static partial string sd_schedule_name(scheduler_t schedule); [LibraryImport(LIB_NAME, EntryPoint = "str_to_schedule")] - internal static partial schedule_t str_to_schedule([MarshalAs(UnmanagedType.LPStr)] string str); + internal static partial scheduler_t str_to_schedule([MarshalAs(UnmanagedType.LPStr)] string str); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_prediction_name")] + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_prediction_name(prediction_t prediction); + + [LibraryImport(LIB_NAME, EntryPoint = "str_to_prediction")] + internal static partial prediction_t str_to_prediction([MarshalAs(UnmanagedType.LPStr)] string str); // @@ -214,6 +260,16 @@ internal unsafe partial class Native [LibraryImport(LIB_NAME, EntryPoint = "free_sd_ctx")] internal static partial void free_sd_ctx(sd_ctx_t* sd_ctx); + [LibraryImport(LIB_NAME, EntryPoint = "sd_get_default_sample_method")] + internal static partial sample_method_t sd_get_default_sample_method(sd_ctx_t* sd_ctx); + + // + + [LibraryImport(LIB_NAME, EntryPoint = "sd_sample_params_init")] + internal static partial void sd_sample_params_init([MarshalUsing(typeof(SampleParameterMarshaller))] ref sd_sample_params_t sample_params); + [LibraryImport(LIB_NAME, EntryPoint = "sd_sample_params_to_str")] + internal static partial char* sd_sample_params_to_str([MarshalUsing(typeof(SampleParameterMarshaller))] in sd_sample_params_t sample_params); + // [LibraryImport(LIB_NAME, EntryPoint = "sd_img_gen_params_init")] @@ -229,16 +285,18 @@ internal unsafe partial class Native // [LibraryImport(LIB_NAME, EntryPoint = "sd_vid_gen_params_init")] - internal static partial void sd_vid_gen_params_init(ref sd_vid_gen_params_t sd_vid_gen_params); + internal static partial void sd_vid_gen_params_init([MarshalUsing(typeof(VideoGenerationParameterMarshaller))] ref sd_vid_gen_params_t sd_vid_gen_params); [LibraryImport(LIB_NAME, EntryPoint = "generate_video")] - [return: MarshalUsing(typeof(ImageMarshaller))] - internal static partial sd_image_t* generate_video(sd_ctx_t* sd_ctx, in sd_vid_gen_params_t sd_vid_gen_params); // broken + internal static partial sd_image_t* generate_video(sd_ctx_t* sd_ctx, [MarshalUsing(typeof(VideoGenerationParameterMarshaller))] in sd_vid_gen_params_t sd_vid_gen_params, out int num_frames_out); // [LibraryImport(LIB_NAME, EntryPoint = "new_upscaler_ctx")] - internal static partial upscaler_ctx_t* new_upscaler_ctx([MarshalAs(UnmanagedType.LPStr)] string esrgan_path, int n_threads, [MarshalAs(UnmanagedType.I1)] bool direct); + internal static partial upscaler_ctx_t* new_upscaler_ctx([MarshalAs(UnmanagedType.LPStr)] string esrgan_path, + [MarshalAs(UnmanagedType.I1)] bool offload_params_to_cpu, + [MarshalAs(UnmanagedType.I1)] bool direct, + int n_threads); [LibraryImport(LIB_NAME, EntryPoint = "free_upscaler_ctx")] internal static partial void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx); @@ -249,6 +307,9 @@ internal unsafe partial class Native [return: MarshalUsing(typeof(ImageMarshaller))] internal static partial Image upscale(upscaler_ctx_t* upscaler_ctx, [MarshalUsing(typeof(ImageMarshaller))] IImage input_image, uint32_t upscale_factor); + [LibraryImport(LIB_NAME, EntryPoint = "get_upscale_factor")] + internal static partial int get_upscale_factor(upscaler_ctx_t* upscaler_ctx); + // [LibraryImport(LIB_NAME, EntryPoint = "convert")] @@ -262,14 +323,13 @@ internal unsafe partial class Native // [LibraryImport(LIB_NAME, EntryPoint = "preprocess_canny")] - internal static partial uint8_t* preprocess_canny(uint8_t* img, - int width, - int height, - float high_threshold, - float low_threshold, - float weak, - float strong, - [MarshalAs(UnmanagedType.I1)] bool inverse); + [return: MarshalAs(UnmanagedType.I1)] + internal static partial bool preprocess_canny(sd_image_t image, + float high_threshold, + float low_threshold, + float weak, + float strong, + [MarshalAs(UnmanagedType.I1)] bool inverse); #endregion } \ No newline at end of file diff --git a/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings b/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings index eb80f1b..0982152 100644 --- a/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings +++ b/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings @@ -10,6 +10,8 @@ True True True + True + True True True True diff --git a/StableDiffusion.NET/StableDiffusionCpp.cs b/StableDiffusion.NET/StableDiffusionCpp.cs index 71f473f..13676c7 100644 --- a/StableDiffusion.NET/StableDiffusionCpp.cs +++ b/StableDiffusion.NET/StableDiffusionCpp.cs @@ -11,8 +11,8 @@ public static unsafe class StableDiffusionCpp #region Properties & Fields // ReSharper disable NotAccessedField.Local - They are important, the delegate can be collected if it's not stored! - private static Native.sd_log_cb_t LOG_CALLBACK; - private static Native.sd_progress_cb_t PROGRESS_CALLBACK; + private static Native.sd_log_cb_t? _logCallback; + private static Native.sd_progress_cb_t? _progressCallback; // ReSharper restore NotAccessedField.Local #endregion @@ -30,8 +30,8 @@ public static unsafe class StableDiffusionCpp public static void InitializeEvents() { - Native.sd_set_log_callback(LOG_CALLBACK = OnNativeLog, null); - Native.sd_set_progress_callback(PROGRESS_CALLBACK = OnNativeProgress, null); + Native.sd_set_log_callback(_logCallback = OnNativeLog, null); + Native.sd_set_progress_callback(_progressCallback = OnNativeProgress, null); } public static void Convert(string modelPath, string vaePath, Quantization quantization, string outputPath, string tensorTypeRules = "") @@ -54,27 +54,21 @@ public static unsafe class StableDiffusionCpp IImage controlImage = parameter.Image as IImage ?? parameter.Image!.ConvertTo(); - byte[] controlImageData = controlImage.ToRawArray(); - fixed (byte* controlImagePtr = controlImageData) + Native.Types.sd_image_t sdImage = controlImage.ToSdImage(); + try { - byte* result = Native.preprocess_canny(controlImagePtr, - controlImage.Width, - controlImage.Height, - parameter.HighThreshold, - parameter.LowThreshold, - parameter.Weak, - parameter.Strong, - parameter.Inverse); + bool result = Native.preprocess_canny(sdImage, + parameter.HighThreshold, + parameter.LowThreshold, + parameter.Weak, + parameter.Strong, + parameter.Inverse); - try - { - return Image.Create(new ReadOnlySpan(result, controlImageData.Length), - controlImage.Width, controlImage.Height); - } - finally - { - NativeMemory.Free(result); - } + return sdImage.ToImage(); + } + finally + { + sdImage.Free(); } } From e25d8ad5c90eef142083659a4c29b9ac6e012a69 Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Mon, 27 Oct 2025 19:34:14 +0100 Subject: [PATCH 4/9] Removed unused method --- StableDiffusion.NET/Helper/ImageHelper.cs | 10 +--------- StableDiffusion.NET/StableDiffusionCpp.cs | 1 - 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/StableDiffusion.NET/Helper/ImageHelper.cs b/StableDiffusion.NET/Helper/ImageHelper.cs index 9b36f97..d61d27c 100644 --- a/StableDiffusion.NET/Helper/ImageHelper.cs +++ b/StableDiffusion.NET/Helper/ImageHelper.cs @@ -82,14 +82,6 @@ internal static class ImageHelper } } - public static unsafe Native.Types.sd_image_t* ToSdImagePtr(this IImage image, bool monochrome = false) - { - Native.Types.sd_image_t* imagePtr = (Native.Types.sd_image_t*)NativeMemory.Alloc((nuint)Marshal.SizeOf()); - imagePtr[0] = image.ToSdImage(monochrome); - - return imagePtr; - } - public static unsafe void Free(this Native.Types.sd_image_t sdImage) { if (sdImage.data == null) return; @@ -127,7 +119,7 @@ internal static class ImageHelper { int count = images.Length; - Native.Types.sd_image_t* imagePtr = (Native.Types.sd_image_t*)NativeMemory.Alloc((nuint)(count * Marshal.SizeOf())); + Native.Types.sd_image_t* imagePtr = (Native.Types.sd_image_t*)NativeMemory.Alloc((nuint)count, (nuint)Marshal.SizeOf()); for (int i = 0; i < count; i++) imagePtr[i] = images[i].ToSdImage(monochrome); diff --git a/StableDiffusion.NET/StableDiffusionCpp.cs b/StableDiffusion.NET/StableDiffusionCpp.cs index 13676c7..eb3d650 100644 --- a/StableDiffusion.NET/StableDiffusionCpp.cs +++ b/StableDiffusion.NET/StableDiffusionCpp.cs @@ -1,5 +1,4 @@ using System; -using System.Runtime.InteropServices; using HPPH; using JetBrains.Annotations; From 305d0a6f3a55925e9f23d2e25285932037731de0 Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Mon, 27 Oct 2025 19:43:07 +0100 Subject: [PATCH 5/9] Updated backends-build --- .github/workflows/backends.yml | 103 +----------------- .../StableDiffusion.NET.Backend.Cuda.nuspec | 4 +- ...eDiffusion.NET.Backend.Cuda11.Linux.nuspec | 28 ----- ...iffusion.NET.Backend.Cuda11.Windows.nuspec | 28 ----- 4 files changed, 3 insertions(+), 160 deletions(-) delete mode 100644 Backends/StableDiffusion.NET.Backend.Cuda11.Linux.nuspec delete mode 100644 Backends/StableDiffusion.NET.Backend.Cuda11.Windows.nuspec diff --git a/.github/workflows/backends.yml b/.github/workflows/backends.yml index 795b02f..e265828 100644 --- a/.github/workflows/backends.yml +++ b/.github/workflows/backends.yml @@ -111,65 +111,6 @@ jobs: name: windows-cuda12 path: .\build\bin\Release\stable-diffusion.dll - windows-cuda11: - runs-on: windows-2019 - - steps: - - name: Checkout - id: checkout - uses: actions/checkout@v4.2.2 - with: - repository: 'leejet/stable-diffusion.cpp' - ref: '${{ github.event.inputs.commit }}' - submodules: recursive - - - name: Install Cuda Toolkit 11.8 - id: cuda-11 - run: | - mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" - choco install unzip -y - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-11.8.89-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-11.8.89-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-11.8.89-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-11.8.1.74-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-11.8.86-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-11.8.86-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-11.8.86-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-11.8.87-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.8.89-archive.zip" - unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_cudart-windows-x86_64-11.8.89-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_nvcc-windows-x86_64-11.8.89-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_nvrtc-windows-x86_64-11.8.89-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\libcublas-windows-x86_64-11.8.1.74-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_nvtx-windows-x86_64-11.8.86-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_profiler_api-windows-x86_64-11.8.86-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\visual_studio_integration-windows-x86_64-11.8.86-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_nvprof-windows-x86_64-11.8.87-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_cccl-windows-x86_64-11.8.89-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\visual_studio_integration\MSBuildExtensions\*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\BuildCustomizations" /E /I /H /Y - echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\libnvvp" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - echo "CUDA_PATH_V11_8=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - - - name: Build - id: cmake_build - shell: cmd - run: | - mkdir build - cd build - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - cmake .. -DSD_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="61;75;86;89" -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler" - cmake --build . --config Release - - - name: Upload artifact - id: upload_artifact - uses: actions/upload-artifact@v4.6.1 - with: - name: windows-cuda11 - path: .\build\bin\Release\stable-diffusion.dll - windows-sycl: runs-on: windows-latest @@ -370,42 +311,7 @@ jobs: with: name: linux-cuda12 path: ./build/bin/libstable-diffusion.so - - linux-cuda11: - runs-on: ubuntu-latest - container: nvidia/cuda:11.8.0-devel-ubuntu22.04 - - steps: - - name: Install dependencies - env: - DEBIAN_FRONTEND: noninteractive - run: | - apt update - apt install -y cmake build-essential ninja-build libgomp1 git - - - name: Checkout - id: checkout - uses: actions/checkout@v4.2.2 - with: - repository: 'leejet/stable-diffusion.cpp' - ref: '${{ github.event.inputs.commit }}' - submodules: recursive - - - name: Build - id: cmake_build - run: | - mkdir build - cd build - cmake .. -DSD_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="61;75;86;89" -DSD_BUILD_SHARED_LIBS=ON - cmake --build . --config Release - - - name: Upload artifact - id: upload_artifact - uses: actions/upload-artifact@v4.6.1 - with: - name: linux-cuda11 - path: ./build/bin/libstable-diffusion.so - + linux-sycl: runs-on: ubuntu-latest @@ -526,13 +432,11 @@ jobs: needs: - windows-cpu - windows-cuda12 - - windows-cuda11 # - windows-sycl - windows-hip - windows-vulkan - linux-cpu - linux-cuda12 - - linux-cuda11 - linux-sycl - linux-hip - osx-cpu @@ -559,13 +463,10 @@ jobs: id: pack run: | nuget pack ./Backends/StableDiffusion.NET.Backend.Cpu.nuspec -version ${{ github.event.inputs.version }} - nuget pack ./Backends/StableDiffusion.NET.Backend.Cuda11.Windows.nuspec -version ${{ github.event.inputs.version }} nuget pack ./Backends/StableDiffusion.NET.Backend.Cuda12.Windows.nuspec -version ${{ github.event.inputs.version }} - nuget pack ./Backends/StableDiffusion.NET.Backend.Cuda11.Linux.nuspec -version ${{ github.event.inputs.version }} nuget pack ./Backends/StableDiffusion.NET.Backend.Cuda12.Linux.nuspec -version ${{ github.event.inputs.version }} nuget pack ./Backends/StableDiffusion.NET.Backend.Cuda.nuspec -version ${{ github.event.inputs.version }} nuget pack ./Backends/StableDiffusion.NET.Backend.Rocm.nuspec -version ${{ github.event.inputs.version }} - nuget pack ./Backends/StableDiffusion.NET.Backend.Sycl.nuspec -version ${{ github.event.inputs.version }} nuget pack ./Backends/StableDiffusion.NET.Backend.Vulkan.nuspec -version ${{ github.event.inputs.version }} - name: Upload artifacts @@ -577,4 +478,4 @@ jobs: - name: Nuget Push id: nuget_push - run: dotnet nuget push **\*.nupkg --skip-duplicate --api-key ${{ secrets.NUGET_TOKEN }} --source https://api.nuget.org/v3/index.json + run: dotnet nuget push **\*.nupkg --skip-duplicate --api-key ${{ secrets.NUGET_TOKEN }} --source https://api.nuget.org/v3/index.json \ No newline at end of file diff --git a/Backends/StableDiffusion.NET.Backend.Cuda.nuspec b/Backends/StableDiffusion.NET.Backend.Cuda.nuspec index 08c3c63..a467d88 100644 --- a/Backends/StableDiffusion.NET.Backend.Cuda.nuspec +++ b/Backends/StableDiffusion.NET.Backend.Cuda.nuspec @@ -9,15 +9,13 @@ MIT sd_net_cuda.png https://github.com/DarthAffe/StableDiffusion.NET - CUDA-Backend (11 and 12) for StableDiffusion.NET. + CUDA-Backend (12) for StableDiffusion.NET. Copyright © Darth Affe 2024 readme.md - - diff --git a/Backends/StableDiffusion.NET.Backend.Cuda11.Linux.nuspec b/Backends/StableDiffusion.NET.Backend.Cuda11.Linux.nuspec deleted file mode 100644 index af81315..0000000 --- a/Backends/StableDiffusion.NET.Backend.Cuda11.Linux.nuspec +++ /dev/null @@ -1,28 +0,0 @@ - - - - StableDiffusion.NET.Backend.Cuda11.Linux - $version$ - StableDiffusion.NET.Backend.Cuda11.Linux - Darth Affe & stable-diffusion.cpp Authors - false - MIT - sd_net_cuda.png - https://github.com/DarthAffe/StableDiffusion.NET - CUDA 11 Linux Backend for StableDiffusion.NET. - - Copyright © Darth Affe 2024 - readme.md - - - - - - - - - - - - - diff --git a/Backends/StableDiffusion.NET.Backend.Cuda11.Windows.nuspec b/Backends/StableDiffusion.NET.Backend.Cuda11.Windows.nuspec deleted file mode 100644 index 1421635..0000000 --- a/Backends/StableDiffusion.NET.Backend.Cuda11.Windows.nuspec +++ /dev/null @@ -1,28 +0,0 @@ - - - - StableDiffusion.NET.Backend.Cuda11.Windows - $version$ - StableDiffusion.NET.Backend.Cuda11.Windows - Darth Affe & stable-diffusion.cpp Authors - false - MIT - sd_net_cuda.png - https://github.com/DarthAffe/StableDiffusion.NET - CUDA 11 Windows Backend for StableDiffusion.NET. - - Copyright © Darth Affe 2024 - readme.md - - - - - - - - - - - - - From e6297d485f1c0fc18fa3a6b17e71a141c1985bc7 Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Mon, 27 Oct 2025 19:46:46 +0100 Subject: [PATCH 6/9] Mapped FreeParamsImmediately-Parameter --- .../Models/Parameter/DiffusionModelParameter.cs | 4 +++- .../Native/Marshaller/DiffusionModelParameterMarshaller.cs | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs index 89391cd..64d375c 100644 --- a/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs @@ -41,6 +41,8 @@ public sealed class DiffusionModelParameter /// public bool VaeDecodeOnly { get; set; } = false; + public bool FreeParamsImmediately { get; set; } = false; + /// /// process vae in tiles to reduce memory usage /// @@ -103,7 +105,7 @@ public sealed class DiffusionModelParameter /// path to PHOTOMAKER stacked id embeddings /// public string StackedIdEmbeddingsDirectory { get; set; } = string.Empty; - + /// /// path to full model /// diff --git a/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs index 8f14ec8..b975543 100644 --- a/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs +++ b/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs @@ -25,7 +25,7 @@ internal static unsafe class DiffusionModelParameterMarshaller embedding_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.EmbeddingsDirectory), photo_maker_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.StackedIdEmbeddingsDirectory), vae_decode_only = (sbyte)(managed.VaeDecodeOnly ? 1 : 0), - free_params_immediately = 0, // DarthAffe 06.08.2025: Static value + free_params_immediately = (sbyte)(managed.FreeParamsImmediately ? 1 : 0), n_threads = managed.ThreadCount, wtype = managed.Quantization, rng_type = managed.RngType, @@ -63,6 +63,7 @@ internal static unsafe class DiffusionModelParameterMarshaller EmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.embedding_dir) ?? string.Empty, StackedIdEmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.photo_maker_path) ?? string.Empty, VaeDecodeOnly = unmanaged.vae_decode_only == 1, + FreeParamsImmediately = unmanaged.free_params_immediately == 1, ThreadCount = unmanaged.n_threads, Quantization = unmanaged.wtype, RngType = unmanaged.rng_type, From 4348323abc47b8f28200e634291e75d031a8b34f Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Mon, 27 Oct 2025 19:56:39 +0100 Subject: [PATCH 7/9] Added Params-Extension for FreeParamsImmediately --- .../Parameter/Extensions/DiffusionModelBuilderExtension.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs index 3a33c60..6f9ec7f 100644 --- a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs +++ b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs @@ -80,6 +80,13 @@ public static class DiffusionModelBuilderExtension return parameter; } + public static DiffusionModelParameter WithImmediatelyFreedParams(this DiffusionModelParameter parameter, bool immediatelyFreedParams = true) + { + parameter.FreeParamsImmediately = immediatelyFreedParams; + + return parameter; + } + public static DiffusionModelParameter WithVaeTiling(this DiffusionModelParameter parameter, bool vaeTiling = true) { parameter.VaeTiling = vaeTiling; From 68064e282dba34cfd4f5f1300d06ecf8cf277744 Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Mon, 27 Oct 2025 19:56:48 +0100 Subject: [PATCH 8/9] Updated readme --- README.md | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index e96dbe9..0d88a85 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,11 @@ StableDiffusionCpp.Progress += (_, args) => Console.WriteLine($"PROGRESS {args.S Image? treeWithTiger; // Load a StableDiffusion model in a using block to unload it again after the two images are created -using (DiffusionModel sd = ModelBuilder.StableDiffusion(@"") - .WithMultithreading() - .WithFlashAttention() - .Build()) +using (DiffusionModel sd = new(DiffusionModelParameter.Create() + .WithModelPath(@"N:\StableDiffusion\stable-diffusion-webui\models\Stable-diffusion\animagineXLV3_v30.safetensors") + // .WithVae(@"") + .WithMultithreading() + .WithFlashAttention())) { // Create a image from a prompt Image? tree = sd.GenerateImage(ImageGenerationParameter.TextToImage("A beautiful tree standing on a small hill").WithSDXLDefaults()); @@ -43,17 +43,24 @@ using (DiffusionModel sd = ModelBuilder.StableDiffusion(@"", - @"", - @"", - @"") - .WithMultithreading() - .WithFlashAttention() - .Build(); +// Load the qwen image edit model +using DiffusionModel qwenContext = new(DiffusionModelParameter.Create() + .WithDiffusionModelPath(@"") + .WithQwen2VLPath(@"") + .WithQwen2VLVisionPath(@"") + .WithVae(@"") + .WithMultithreading() + .WithFlashAttention() + .WithFlowShift(3) + .WithOffloadedParamsToCPU() + .WithImmediatelyFreedParams()); -// Perform an edit on the previosly created image -Image? tigerOnMoon = flux.GenerateImage(ImageGenerationParameter.TextToImage("Remove the hill with the grass and place the tree with the tiger on the moon").WithFluxDefaults().WithRefImages(treeWithTiger)); +// Perform an edit on the previously created image +Image? tigerOnMoon = qwenContext.GenerateImage(ImageGenerationParameter.TextToImage("Remove the background and place the tree and the tiger on the moon.") + .WithSize(1024, 1024) + .WithCfg(2.5f) + .WithSampler(Sampler.Euler) + .WithRefImages(treeWithTiger)); File.WriteAllBytes("image3.png", tigerOnMoon.ToPng()); ``` From e564a6453cf154fa151bede3b05b6f1f017664ab Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Tue, 28 Oct 2025 09:16:01 +0100 Subject: [PATCH 9/9] Updated readme, removed cuda 11 from build --- .github/workflows/backends.yml | 59 ---------------------------------- README.md | 2 +- 2 files changed, 1 insertion(+), 60 deletions(-) diff --git a/.github/workflows/backends.yml b/.github/workflows/backends.yml index f21e9c2..e265828 100644 --- a/.github/workflows/backends.yml +++ b/.github/workflows/backends.yml @@ -111,65 +111,6 @@ jobs: name: windows-cuda12 path: .\build\bin\Release\stable-diffusion.dll - windows-cuda11: - runs-on: windows-2019 - - steps: - - name: Checkout - id: checkout - uses: actions/checkout@v4.2.2 - with: - repository: 'leejet/stable-diffusion.cpp' - ref: '${{ github.event.inputs.commit }}' - submodules: recursive - - - name: Install Cuda Toolkit 11.8 - id: cuda-11 - run: | - mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" - choco install unzip -y - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-11.8.89-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-11.8.89-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-11.8.89-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-11.8.1.74-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-11.8.86-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-11.8.86-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-11.8.86-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-11.8.87-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.8.89-archive.zip" - unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_cudart-windows-x86_64-11.8.89-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_nvcc-windows-x86_64-11.8.89-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_nvrtc-windows-x86_64-11.8.89-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\libcublas-windows-x86_64-11.8.1.74-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_nvtx-windows-x86_64-11.8.86-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_profiler_api-windows-x86_64-11.8.86-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\visual_studio_integration-windows-x86_64-11.8.86-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_nvprof-windows-x86_64-11.8.87-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\cuda_cccl-windows-x86_64-11.8.89-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\visual_studio_integration\MSBuildExtensions\*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\BuildCustomizations" /E /I /H /Y - echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\libnvvp" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - echo "CUDA_PATH_V11_8=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - - - name: Build - id: cmake_build - shell: cmd - run: | - mkdir build - cd build - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - cmake .. -DSD_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="61;75;86;89" -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler" - cmake --build . --config Release - - - name: Upload artifact - id: upload_artifact - uses: actions/upload-artifact@v4.6.1 - with: - name: windows-cuda11 - path: .\build\bin\Release\stable-diffusion.dll - windows-sycl: runs-on: windows-latest diff --git a/README.md b/README.md index 0d88a85..4dfea82 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ StableDiffusionCpp.Progress += (_, args) => Console.WriteLine($"PROGRESS {args.S Image? treeWithTiger; // Load a StableDiffusion model in a using block to unload it again after the two images are created using (DiffusionModel sd = new(DiffusionModelParameter.Create() - .WithModelPath(@"N:\StableDiffusion\stable-diffusion-webui\models\Stable-diffusion\animagineXLV3_v30.safetensors") + .WithModelPath(@"") // .WithVae(@"") .WithMultithreading() .WithFlashAttention()))