DarthAffe/StableDiffusion.NET
1
0
Fork 0
mirror of https://github.com/DarthAffe/StableDiffusion.NET.git synced 2025-12-12 13:28:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reworked everything to fit the new sd.cpp API

Browse Source
This commit is contained in:
DarthAffe 2025-08-16 23:12:24 +02:00
parent 89cf3108df
commit 76dc851139
43 changed files with 1429 additions and 1037 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
StableDiffusion.NET/Backends/CudaBackend.cs
View File

@ -27,7 +27,7 @@ public partial class CudaBackend : IBackend
public bool IsAvailable => (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) public bool IsAvailable => (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
|| RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
&& (RuntimeInformation.OSArchitecture == Architecture.X64) && (RuntimeInformation.OSArchitecture == Architecture.X64)
&& CudaVersion is 11 or 12; && CudaVersion is 12;
public string PathPart => CudaVersion switch public string PathPart => CudaVersion switch
{ {

8
StableDiffusion.NET/Enums/DiffusionModelType.cs
View File

@ -1,8 +0,0 @@
namespace StableDiffusion.NET;
public enum DiffusionModelType
{
None = 0,
StableDiffusion = 1,
Flux = 2
}

106
StableDiffusion.NET/Extensions/ParameterExtension.cs
View File

@ -1,92 +1,100 @@
#pragma warning disable CA2208 #pragma warning disable CA2208
using JetBrains.Annotations;
using System; using System;
namespace StableDiffusion.NET; namespace StableDiffusion.NET;
[PublicAPI]
public static class ParameterExtension public static class ParameterExtension
{ {
public static void Validate(this UpscaleModelParameter parameter) public static void Validate(this UpscaleModelParameter parameter)
{ {
ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
ArgumentNullException.ThrowIfNull(parameter.ModelPath, nameof(UpscaleModelParameter.ModelPath)); ArgumentNullException.ThrowIfNull(parameter.ModelPath, nameof(UpscaleModelParameter.ModelPath));
if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(UpscaleModelParameter.Quantization));
} }
public static void Validate(this DiffusionModelParameter parameter) public static void Validate(this DiffusionModelParameter parameter)
{
((IQuantizedModelParameter)parameter).Validate();
((IDiffusionModelParameter)parameter).Validate();
((IPhotomakerModelParameter)parameter).Validate();
}
public static void Validate(this IQuantizedModelParameter parameter)
{ {
ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.ThreadCount, nameof(IQuantizedModelParameter.ThreadCount)); ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.ThreadCount, nameof(DiffusionModelParameter.ThreadCount));
ArgumentNullException.ThrowIfNull(parameter.VaePath, nameof(DiffusionModelParameter.VaePath));
if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(IQuantizedModelParameter.Quantization)); ArgumentNullException.ThrowIfNull(parameter.TaesdPath, nameof(DiffusionModelParameter.TaesdPath));
} ArgumentNullException.ThrowIfNull(parameter.LoraModelDirectory, nameof(DiffusionModelParameter.LoraModelDirectory));
ArgumentNullException.ThrowIfNull(parameter.ControlNetPath, nameof(DiffusionModelParameter.ControlNetPath));
public static void Validate(this IPhotomakerModelParameter parameter) ArgumentNullException.ThrowIfNull(parameter.EmbeddingsDirectory, nameof(DiffusionModelParameter.EmbeddingsDirectory));
{ ArgumentNullException.ThrowIfNull(parameter.StackedIdEmbeddingsDirectory, nameof(DiffusionModelParameter.StackedIdEmbeddingsDirectory));
ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
ArgumentNullException.ThrowIfNull(parameter.StackedIdEmbeddingsDirectory, nameof(IPhotomakerModelParameter.StackedIdEmbeddingsDirectory));
}
public static void Validate(this IDiffusionModelParameter parameter)
{
ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
ArgumentNullException.ThrowIfNull(parameter.VaePath, nameof(IDiffusionModelParameter.VaePath));
ArgumentNullException.ThrowIfNull(parameter.TaesdPath, nameof(IDiffusionModelParameter.TaesdPath));
ArgumentNullException.ThrowIfNull(parameter.LoraModelDirectory, nameof(IDiffusionModelParameter.LoraModelDirectory));
ArgumentNullException.ThrowIfNull(parameter.ControlNetPath, nameof(IDiffusionModelParameter.ControlNetPath));
ArgumentNullException.ThrowIfNull(parameter.EmbeddingsDirectory, nameof(IDiffusionModelParameter.EmbeddingsDirectory));
if (!string.IsNullOrWhiteSpace(parameter.VaePath) && !string.IsNullOrWhiteSpace(parameter.TaesdPath)) throw new ArgumentException("VAE and TAESD are mutually exclusive."); if (!string.IsNullOrWhiteSpace(parameter.VaePath) && !string.IsNullOrWhiteSpace(parameter.TaesdPath)) throw new ArgumentException("VAE and TAESD are mutually exclusive.");
if (!Enum.IsDefined(parameter.RngType)) throw new ArgumentOutOfRangeException(nameof(IDiffusionModelParameter.RngType)); if (!Enum.IsDefined(parameter.RngType)) throw new ArgumentOutOfRangeException(nameof(DiffusionModelParameter.RngType));
if (!Enum.IsDefined(parameter.Schedule)) throw new ArgumentOutOfRangeException(nameof(IDiffusionModelParameter.Schedule)); if (!Enum.IsDefined(parameter.Schedule)) throw new ArgumentOutOfRangeException(nameof(DiffusionModelParameter.Schedule));
} }
public static void Validate(this DiffusionParameter parameter) public static void Validate(this ImageGenerationParameter parameter)
{ {
ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
ArgumentNullException.ThrowIfNull(parameter.ControlNet, nameof(DiffusionParameter.ControlNet)); ArgumentNullException.ThrowIfNull(parameter.ControlNet, nameof(ImageGenerationParameter.ControlNet));
ArgumentNullException.ThrowIfNull(parameter.PhotoMaker, nameof(DiffusionParameter.PhotoMaker)); ArgumentNullException.ThrowIfNull(parameter.PhotoMaker, nameof(ImageGenerationParameter.PhotoMaker));
ArgumentNullException.ThrowIfNull(parameter.NegativePrompt, nameof(DiffusionParameter.NegativePrompt)); ArgumentNullException.ThrowIfNull(parameter.NegativePrompt, nameof(ImageGenerationParameter.NegativePrompt));
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Width, nameof(DiffusionParameter.Width)); ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Width, nameof(ImageGenerationParameter.Width));
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Height, nameof(DiffusionParameter.Height)); ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Height, nameof(ImageGenerationParameter.Height));
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.SampleSteps, nameof(DiffusionParameter.SampleSteps)); ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.SampleSteps, nameof(ImageGenerationParameter.SampleSteps));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.CfgScale, nameof(DiffusionParameter.CfgScale)); if (!Enum.IsDefined(parameter.SampleMethod)) throw new ArgumentOutOfRangeException(nameof(ImageGenerationParameter.SampleMethod));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.Guidance, nameof(DiffusionParameter.Guidance));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(DiffusionParameter.Strength));
if (!Enum.IsDefined(parameter.SampleMethod)) throw new ArgumentOutOfRangeException(nameof(DiffusionParameter.SampleMethod));
parameter.Guidance.Validate();
parameter.ControlNet.Validate(); parameter.ControlNet.Validate();
parameter.PhotoMaker.Validate(); parameter.PhotoMaker.Validate();
} }
public static void Validate(this ControlNetParameter parameter) public static void Validate(this ControlNetParameter parameter)
{ {
ArgumentNullException.ThrowIfNull(parameter, nameof(DiffusionParameter.ControlNet)); ArgumentNullException.ThrowIfNull(parameter, nameof(ImageGenerationParameter.ControlNet));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(ControlNetParameter.Strength)); ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(ControlNetParameter.Strength));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyHighThreshold, nameof(ControlNetParameter.CannyHighThreshold));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyLowThreshold, nameof(ControlNetParameter.CannyLowThreshold));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyWeak, nameof(ControlNetParameter.CannyWeak));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyStrong, nameof(ControlNetParameter.CannyStrong));
} }
public static void Validate(this PhotoMakerParameter parameter) public static void Validate(this PhotoMakerParameter parameter)
{ {
ArgumentNullException.ThrowIfNull(parameter, nameof(DiffusionParameter.PhotoMaker)); ArgumentNullException.ThrowIfNull(parameter, nameof(ImageGenerationParameter.PhotoMaker));
ArgumentNullException.ThrowIfNull(parameter.InputIdImageDirectory, nameof(PhotoMakerParameter.InputIdImageDirectory)); ArgumentNullException.ThrowIfNull(parameter.InputIdImageDirectory, nameof(PhotoMakerParameter.InputIdImageDirectory));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.StyleRatio, nameof(PhotoMakerParameter.StyleRatio)); ArgumentOutOfRangeException.ThrowIfNegative(parameter.StyleStrength, nameof(PhotoMakerParameter.StyleStrength));
}
public static void Validate(this GuidanceParameter parameter)
{
ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.ImgCfg);
ArgumentOutOfRangeException.ThrowIfNegative(parameter.DistilledGuidance);
ArgumentOutOfRangeException.ThrowIfNegative(parameter.MinCfg);
ArgumentOutOfRangeException.ThrowIfNegative(parameter.TxtCfg);
parameter.Slg.Validate();
}
public static void Validate(this SlgParameter parameter)
{
ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
ArgumentNullException.ThrowIfNull(parameter.Layers);
ArgumentOutOfRangeException.ThrowIfNegative(parameter.Scale);
ArgumentOutOfRangeException.ThrowIfNegative(parameter.SkipLayerStart);
ArgumentOutOfRangeException.ThrowIfNegative(parameter.SkipLayerEnd);
}
public static void Validate(this CannyParameter parameter)
{
ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
ArgumentNullException.ThrowIfNull(parameter.Image);
ArgumentOutOfRangeException.ThrowIfNegative(parameter.HighThreshold, nameof(CannyParameter.HighThreshold));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.LowThreshold, nameof(CannyParameter.LowThreshold));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.Weak, nameof(CannyParameter.Weak));
ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strong, nameof(CannyParameter.Strong));
} }
} }

170
StableDiffusion.NET/Helper/ImageHelper.cs
View File

@ -6,68 +6,140 @@ namespace StableDiffusion.NET;
internal static class ImageHelper internal static class ImageHelper
{ {
public static unsafe Image<ColorRGB> ToImage(Native.sd_image_t* sdImage) public static unsafe Image<ColorRGB> ToImage(this Native.Types.sd_image_t sdImage)
{
Image<ColorRGB> image = ToImage(*sdImage);
Marshal.FreeHGlobal((nint)sdImage);
return image;
}
public static unsafe Image<ColorRGB> ToImage(Native.sd_image_t sdImage)
{ {
int width = (int)sdImage.width; int width = (int)sdImage.width;
int height = (int)sdImage.height; int height = (int)sdImage.height;
int bpp = (int)sdImage.channel; int bpp = (int)sdImage.channel;
Image<ColorRGB> image = Image<ColorRGB>.Create(new ReadOnlySpan<byte>(sdImage.data, width * height * bpp), width, height, width * bpp); switch (bpp)
Dispose(sdImage);
return image;
}
public static unsafe void Dispose(Native.sd_image_t image) => Marshal.FreeHGlobal((nint)image.data);
public static unsafe Native.sd_image_t ToSdImage(this IImage image, out nint dataPtr)
{
int sizeInBytes = image.SizeInBytes;
dataPtr = Marshal.AllocHGlobal(sizeInBytes);
image.CopyTo(new Span<byte>((void*)dataPtr, sizeInBytes));
return image.ToSdImage((byte*)dataPtr);
}
public static unsafe Native.sd_image_t ToSdImage(this IImage image, byte* pinnedReference)
=> new()
{ {
width = (uint)image.Width, case 3:
height = (uint)image.Height, return Image<ColorRGB>.Create(new ReadOnlySpan<byte>(sdImage.data, width * height * bpp), width, height, width * bpp);
channel = (uint)image.ColorFormat.BytesPerPixel,
data = pinnedReference
};
public static unsafe Native.sd_image_t* ToSdImagePtr(this IImage image, out nint dataPtr) case 1:
{ {
int sizeInBytes = image.SizeInBytes; ColorRGB[] pixels = new ColorRGB[width * height];
Span<byte> sdData = new(sdImage.data, pixels.Length);
dataPtr = Marshal.AllocHGlobal(sizeInBytes); for (int i = 0; i < pixels.Length; i++)
image.CopyTo(new Span<byte>((void*)dataPtr, sizeInBytes)); {
byte c = sdData[i];
pixels[i] = new ColorRGB(c, c, c);
}
return image.ToSdImagePtr((byte*)dataPtr); Image<ColorRGB> image = Image<ColorRGB>.Create(pixels, width, height);
return image;
}
default:
throw new ArgumentOutOfRangeException($"Image-BPP of {bpp} is not supported");
}
} }
public static unsafe Native.sd_image_t* ToSdImagePtr(this IImage image, byte* pinnedReference) public static unsafe Native.Types.sd_image_t ToSdImage(this IImage image, bool monochrome = false)
{ {
Native.sd_image_t* nativeImage = (Native.sd_image_t*)Marshal.AllocHGlobal(sizeof(Native.sd_image_t)); if (monochrome)
{
int sizeInBytes = image.Width * image.Height;
nativeImage->width = (uint)image.Width; byte* dataPtr = (byte*)NativeMemory.Alloc((nuint)sizeInBytes);
nativeImage->height = (uint)image.Height; Span<byte> data = new(dataPtr, sizeInBytes);
nativeImage->channel = (uint)image.ColorFormat.BytesPerPixel;
nativeImage->data = pinnedReference;
return nativeImage; // DarthAffe 16.08.2025: HPPH does currently not support monochrome images, that's why we need to convert it here. We're going for the simple conversion as the source image is supposed to be monochrome anyway.
for (int y = 0; y < image.Height; y++)
for (int x = 0; x < image.Width; x++)
{
IColor color = image[x, y];
data[(image.Width * y) + x] = (byte)Math.Round((color.R + color.G + color.B) / 3.0);
}
return new Native.Types.sd_image_t
{
width = (uint)image.Width,
height = (uint)image.Height,
channel = 1,
data = dataPtr
};
}
else
{
IImage<ColorRGB> img = image as IImage<ColorRGB> ?? image.ConvertTo<ColorRGB>();
int sizeInBytes = img.SizeInBytes;
byte* dataPtr = (byte*)NativeMemory.Alloc((nuint)sizeInBytes);
img.CopyTo(new Span<byte>(dataPtr, sizeInBytes));
return new Native.Types.sd_image_t
{
width = (uint)img.Width,
height = (uint)img.Height,
channel = (uint)img.ColorFormat.BytesPerPixel,
data = dataPtr
};
}
}
public static unsafe Native.Types.sd_image_t* ToSdImagePtr(this IImage image, bool monochrome = false)
{
Native.Types.sd_image_t* imagePtr = (Native.Types.sd_image_t*)NativeMemory.Alloc((nuint)Marshal.SizeOf<Native.Types.sd_image_t>());
imagePtr[0] = image.ToSdImage(monochrome);
return imagePtr;
}
public static unsafe void Free(this Native.Types.sd_image_t sdImage)
{
if (sdImage.data == null) return;
NativeMemory.Free(sdImage.data);
}
public static unsafe Image<ColorRGB>[] ToImageArray(Native.Types.sd_image_t* sdImage, int count)
{
if (sdImage == null) return [];
Image<ColorRGB>[] images = new Image<ColorRGB>[count];
for (int i = 0; i < images.Length; i++)
images[i] = GetImage(sdImage, i);
return images;
}
internal static unsafe IImage[] ToImageArrayIFace(Native.Types.sd_image_t* sdImage, int count)
{
IImage[] images = new IImage[count];
for (int i = 0; i < images.Length; i++)
images[i] = GetImage(sdImage, i);
return images;
}
public static unsafe Image<ColorRGB> GetImage(Native.Types.sd_image_t* sdImage, int index) => sdImage[index].ToImage();
public static unsafe Native.Types.sd_image_t* ToSdImage(this IImage[] images, bool monochrome = false)
{
int count = images.Length;
Native.Types.sd_image_t* imagePtr = (Native.Types.sd_image_t*)NativeMemory.Alloc((nuint)(count * Marshal.SizeOf<Native.Types.sd_image_t>()));
for (int i = 0; i < count; i++)
imagePtr[i] = images[i].ToSdImage(monochrome);
return imagePtr;
}
public static unsafe void Free(Native.Types.sd_image_t* sdImage, int count)
{
if (sdImage == null) return;
for (int i = 0; i < count; i++)
Free(sdImage[i]);
NativeMemory.Free(sdImage);
} }
} }

5
StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs
View File

@ -3,12 +3,13 @@
namespace StableDiffusion.NET; namespace StableDiffusion.NET;
[PublicAPI] [PublicAPI]
public sealed class ESRGANModelBuilder : IQuantizedModelBuilder public sealed class ESRGANModelBuilder : IUpscaleModelBuilder
{ {
#region Properties & Fields #region Properties & Fields
public UpscaleModelParameter Parameter { get; } public UpscaleModelParameter Parameter { get; }
IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; IUpscaleModelParameter IUpscaleModelBuilder.Parameter => Parameter;
IModelParameter IModelBuilder.Parameter => Parameter;
#endregion #endregion

122
StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs
View File

@ -76,10 +76,10 @@ public static class DiffusionModelBuilderExtension
return builder; return builder;
} }
public static T KeepVaeOnCpu<T>(this T builder, bool keepVaeOnCpu = true) public static T KeepControlNetOnCpu<T>(this T builder, bool keepControlNetOnCpu = true)
where T : IDiffusionModelBuilder where T : IDiffusionModelBuilder
{ {
builder.Parameter.KeepVaeOnCPU = keepVaeOnCpu; builder.Parameter.KeepControlNetOnCPU = keepControlNetOnCpu;
return builder; return builder;
} }
@ -92,10 +92,34 @@ public static class DiffusionModelBuilderExtension
return builder; return builder;
} }
public static T KeepControlNetOnCpu<T>(this T builder, bool keepControlNetOnCpu = true) public static T KeepVaeOnCpu<T>(this T builder, bool keepVaeOnCpu = true)
where T : IDiffusionModelBuilder where T : IDiffusionModelBuilder
{ {
builder.Parameter.KeepControlNetOnCPU = keepControlNetOnCpu; builder.Parameter.KeepVaeOnCPU = keepVaeOnCpu;
return builder;
}
public static T WithFlashAttention<T>(this T builder, bool flashAttention = true)
where T : IDiffusionModelBuilder
{
builder.Parameter.FlashAttention = flashAttention;
return builder;
}
public static T WithDiffusionConvDirect<T>(this T builder, bool diffusionConfDirect = true)
where T : IDiffusionModelBuilder
{
builder.Parameter.DiffusionConvDirect = diffusionConfDirect;
return builder;
}
public static T WithVaeConvDirect<T>(this T builder, bool vaeConfDirect = true)
where T : IDiffusionModelBuilder
{
builder.Parameter.VaeConfDirect = vaeConfDirect;
return builder; return builder;
} }
@ -120,10 +144,96 @@ public static class DiffusionModelBuilderExtension
return builder; return builder;
} }
public static T WithFlashAttention<T>(this T builder, bool flashAttention = true) public static T WithQuantization<T>(this T builder, Quantization quantization)
where T : IDiffusionModelBuilder where T : IDiffusionModelBuilder
{ {
builder.Parameter.FlashAttention = flashAttention; if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization));
builder.Parameter.Quantization = quantization;
return builder;
}
public static T WithPhotomaker<T>(this T builder, string stackedIdEmbeddingsDirectory)
where T : IDiffusionModelBuilder
{
ArgumentException.ThrowIfNullOrWhiteSpace(stackedIdEmbeddingsDirectory, nameof(stackedIdEmbeddingsDirectory));
builder.Parameter.StackedIdEmbeddingsDirectory = stackedIdEmbeddingsDirectory;
return builder;
}
public static T WithModelPath<T>(this T builder, string modelPath)
where T : IDiffusionModelBuilder
{
ArgumentNullException.ThrowIfNull(modelPath);
builder.Parameter.ModelPath = modelPath;
return builder;
}
public static T WithDiffusionModelPath<T>(this T builder, string diffusionModelPath)
where T : IDiffusionModelBuilder
{
ArgumentNullException.ThrowIfNull(diffusionModelPath);
builder.Parameter.DiffusionModelPath = diffusionModelPath;
return builder;
}
public static T WithClipLPath<T>(this T builder, string clipLPath)
where T : IDiffusionModelBuilder
{
ArgumentNullException.ThrowIfNull(clipLPath);
builder.Parameter.ClipLPath = clipLPath;
return builder;
}
public static T WithT5xxlPath<T>(this T builder, string t5xxlPath)
where T : IDiffusionModelBuilder
{
ArgumentNullException.ThrowIfNull(t5xxlPath);
builder.Parameter.T5xxlPath = t5xxlPath;
return builder;
}
public static T UseChromaDitMap<T>(this T builder, bool useChromaDitMap = true)
where T : IDiffusionModelBuilder
{
builder.Parameter.ChromaUseDitMap = useChromaDitMap;
return builder;
}
public static T EnableChromaT5Map<T>(this T builder, bool enableChromaT5Map = true)
where T : IDiffusionModelBuilder
{
builder.Parameter.ChromaEnableT5Map = enableChromaT5Map;
return builder;
}
public static T WithChromaT5MaskPad<T>(this T builder, int chromaT5MaskPad)
where T : IDiffusionModelBuilder
{
builder.Parameter.ChromaT5MaskPad = chromaT5MaskPad;
return builder;
}
public static T WithClipGPath<T>(this T builder, string clipGPath)
where T : IDiffusionModelBuilder
{
ArgumentNullException.ThrowIfNull(clipGPath);
builder.Parameter.ClipGPath = clipGPath;
return builder; return builder;
} }

14
StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs → StableDiffusion.NET/Models/Builder/Extensions/ModelBuilderExtension.cs
View File

@ -4,10 +4,10 @@ using JetBrains.Annotations;
namespace StableDiffusion.NET; namespace StableDiffusion.NET;
[PublicAPI] [PublicAPI]
public static class QuantizedModelBuilderExtension public static class ModelBuilderExtension
{ {
public static T WithoutMultithreading<T>(this T builder) public static T WithoutMultithreading<T>(this T builder)
where T : IQuantizedModelBuilder where T : IModelBuilder
{ {
builder.Parameter.ThreadCount = 1; builder.Parameter.ThreadCount = 1;
@ -15,7 +15,7 @@ public static class QuantizedModelBuilderExtension
} }
public static T WithMultithreading<T>(this T builder, int threadCount = 0) public static T WithMultithreading<T>(this T builder, int threadCount = 0)
where T : IQuantizedModelBuilder where T : IModelBuilder
{ {
ArgumentOutOfRangeException.ThrowIfLessThan(threadCount, 0, nameof(threadCount)); ArgumentOutOfRangeException.ThrowIfLessThan(threadCount, 0, nameof(threadCount));
@ -25,12 +25,4 @@ public static class QuantizedModelBuilderExtension
return builder; return builder;
} }
public static T WithQuantization<T>(this T builder, Quantization quantization)
where T : IQuantizedModelBuilder
{
builder.Parameter.Quantization = quantization;
return builder;
}
} }

18
StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs
View File

@ -1,18 +0,0 @@
using System;
using JetBrains.Annotations;
namespace StableDiffusion.NET;
[PublicAPI]
public static class PhotomakerModelBuilderExtension
{
public static T WithPhotomaker<T>(this T builder, string stackedIdEmbeddingsDirectory)
where T : IPhotomakerModelBuilder
{
ArgumentException.ThrowIfNullOrWhiteSpace(stackedIdEmbeddingsDirectory, nameof(stackedIdEmbeddingsDirectory));
builder.Parameter.StackedIdEmbeddingsDirectory = stackedIdEmbeddingsDirectory;
return builder;
}
}

27
StableDiffusion.NET/Models/Builder/Extensions/UpscaleModelBuilderExtension.cs Normal file
View File

@ -0,0 +1,27 @@
using System;
using JetBrains.Annotations;
namespace StableDiffusion.NET;
[PublicAPI]
public static class UpscaleModelBuilderExtension
{
public static T WithModelPath<T>(this T builder, string modelPath)
where T : IUpscaleModelBuilder
{
ArgumentNullException.ThrowIfNull(modelPath);
builder.Parameter.ModelPath = modelPath;
return builder;
}
public static T WithConvDirect<T>(this T builder, bool confDirect = true)
where T : IUpscaleModelBuilder
{
builder.Parameter.ConvDirect = confDirect;
return builder;
}
}

6
StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs
View File

@ -3,13 +3,13 @@
namespace StableDiffusion.NET; namespace StableDiffusion.NET;
[PublicAPI] [PublicAPI]
public sealed class FluxModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder public sealed class FluxModelBuilder : IDiffusionModelBuilder
{ {
#region Properties & Fields #region Properties & Fields
public DiffusionModelParameter Parameter { get; } public DiffusionModelParameter Parameter { get; }
IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter; IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter;
IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; IModelParameter IModelBuilder.Parameter => Parameter;
#endregion #endregion
@ -17,7 +17,7 @@ public sealed class FluxModelBuilder : IDiffusionModelBuilder, IQuantizedModelBu
public FluxModelBuilder(string diffusionModelPath, string clipLPath, string t5xxlPath, string vaePath) public FluxModelBuilder(string diffusionModelPath, string clipLPath, string t5xxlPath, string vaePath)
{ {
Parameter = new DiffusionModelParameter { DiffusionModelType = DiffusionModelType.Flux, DiffusionModelPath = diffusionModelPath, ClipLPath = clipLPath, T5xxlPath = t5xxlPath, VaePath = vaePath }; Parameter = new DiffusionModelParameter { DiffusionModelPath = diffusionModelPath, ClipLPath = clipLPath, T5xxlPath = t5xxlPath, VaePath = vaePath };
} }
#endregion #endregion

9
StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs
View File

@ -1,6 +1,11 @@
namespace StableDiffusion.NET; using JetBrains.Annotations;
public interface IDiffusionModelBuilder namespace StableDiffusion.NET;
[PublicAPI]
public interface IDiffusionModelBuilder : IModelBuilder
{ {
IDiffusionModelParameter Parameter { get; } IDiffusionModelParameter Parameter { get; }
DiffusionModel Build();
} }

9
StableDiffusion.NET/Models/Builder/Interfaces/IModelBuilder.cs Normal file
View File

@ -0,0 +1,9 @@
using JetBrains.Annotations;
namespace StableDiffusion.NET;
[PublicAPI]
public interface IModelBuilder
{
IModelParameter Parameter { get; }
}

6
StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs
View File

@ -1,6 +0,0 @@
namespace StableDiffusion.NET;
public interface IPhotomakerModelBuilder
{
IPhotomakerModelParameter Parameter { get; }
}

6
StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs
View File

@ -1,6 +0,0 @@
namespace StableDiffusion.NET;
public interface IQuantizedModelBuilder
{
IQuantizedModelParameter Parameter { get; }
}

11
StableDiffusion.NET/Models/Builder/Interfaces/IUpscaleModelBuilder.cs Normal file
View File

@ -0,0 +1,11 @@
using JetBrains.Annotations;
namespace StableDiffusion.NET;
[PublicAPI]
public interface IUpscaleModelBuilder : IModelBuilder
{
IUpscaleModelParameter Parameter { get; }
UpscaleModel Build();
}

6
StableDiffusion.NET/Models/Builder/StableDiffusion3_5ModelBuilder.cs
View File

@ -3,13 +3,14 @@
namespace StableDiffusion.NET; namespace StableDiffusion.NET;
[PublicAPI] [PublicAPI]
public sealed class StableDiffusion3_5ModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder public sealed class StableDiffusion3_5ModelBuilder : IDiffusionModelBuilder
{ {
#region Properties & Fields #region Properties & Fields
public DiffusionModelParameter Parameter { get; } public DiffusionModelParameter Parameter { get; }
IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter; IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter;
IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; IModelParameter IModelBuilder.Parameter => Parameter;
#endregion #endregion
@ -19,7 +20,6 @@ public sealed class StableDiffusion3_5ModelBuilder : IDiffusionModelBuilder, IQu
{ {
Parameter = new DiffusionModelParameter Parameter = new DiffusionModelParameter
{ {
DiffusionModelType = DiffusionModelType.StableDiffusion,
ModelPath = modelPath, ModelPath = modelPath,
ClipLPath = clipLPath, ClipLPath = clipLPath,
ClipGPath = clipGPath, ClipGPath = clipGPath,

8
StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs
View File

@ -3,14 +3,14 @@
namespace StableDiffusion.NET; namespace StableDiffusion.NET;
[PublicAPI] [PublicAPI]
public sealed class StableDiffusionModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder, IPhotomakerModelBuilder public sealed class StableDiffusionModelBuilder : IDiffusionModelBuilder
{ {
#region Properties & Fields #region Properties & Fields
public DiffusionModelParameter Parameter { get; } public DiffusionModelParameter Parameter { get; }
IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter; IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter;
IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; IModelParameter IModelBuilder.Parameter => Parameter;
IPhotomakerModelParameter IPhotomakerModelBuilder.Parameter => Parameter;
#endregion #endregion
@ -18,7 +18,7 @@ public sealed class StableDiffusionModelBuilder : IDiffusionModelBuilder, IQuant
public StableDiffusionModelBuilder(string modelPath) public StableDiffusionModelBuilder(string modelPath)
{ {
Parameter = new DiffusionModelParameter { DiffusionModelType = DiffusionModelType.StableDiffusion, ModelPath = modelPath }; Parameter = new DiffusionModelParameter { ModelPath = modelPath };
} }
#endregion #endregion

375
StableDiffusion.NET/Models/DiffusionModel.cs
View File

@ -1,8 +1,6 @@
using HPPH; using HPPH;
using JetBrains.Annotations; using JetBrains.Annotations;
using System; using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
namespace StableDiffusion.NET; namespace StableDiffusion.NET;
@ -15,7 +13,7 @@ public sealed unsafe class DiffusionModel : IDisposable
public DiffusionModelParameter ModelParameter { get; } public DiffusionModelParameter ModelParameter { get; }
private Native.sd_ctx_t* _ctx; private Native.Types.sd_ctx_t* _ctx;
#endregion #endregion
@ -25,8 +23,6 @@ public sealed unsafe class DiffusionModel : IDisposable
{ {
ArgumentNullException.ThrowIfNull(modelParameter, nameof(modelParameter)); ArgumentNullException.ThrowIfNull(modelParameter, nameof(modelParameter));
modelParameter.Validate();
this.ModelParameter = modelParameter; this.ModelParameter = modelParameter;
Initialize(); Initialize();
@ -40,335 +36,53 @@ public sealed unsafe class DiffusionModel : IDisposable
private void Initialize() private void Initialize()
{ {
_ctx = Native.new_sd_ctx(ModelParameter.ModelPath, ModelParameter.Validate();
ModelParameter.ClipLPath,
ModelParameter.ClipGPath, _ctx = Native.new_sd_ctx(ModelParameter);
ModelParameter.T5xxlPath,
ModelParameter.DiffusionModelPath,
ModelParameter.VaePath,
ModelParameter.TaesdPath,
ModelParameter.ControlNetPath,
ModelParameter.LoraModelDirectory,
ModelParameter.EmbeddingsDirectory,
ModelParameter.StackedIdEmbeddingsDirectory,
ModelParameter.VaeDecodeOnly,
ModelParameter.VaeTiling,
false,
ModelParameter.ThreadCount,
ModelParameter.Quantization,
ModelParameter.RngType,
ModelParameter.Schedule,
ModelParameter.KeepClipOnCPU,
ModelParameter.KeepControlNetOnCPU,
ModelParameter.KeepVaeOnCPU,
ModelParameter.FlashAttention,
ModelParameter.ChromaUseDitMap,
ModelParameter.ChromaEnableT5Map,
ModelParameter.ChromaT5MaskPad);
if (_ctx == null) throw new NullReferenceException("Failed to initialize diffusion-model."); if (_ctx == null) throw new NullReferenceException("Failed to initialize diffusion-model.");
} }
public DiffusionParameter GetDefaultParameter() => ModelParameter.DiffusionModelType switch public Image<ColorRGB>? GenerateImage(ImageGenerationParameter parameter)
{ {
DiffusionModelType.None => new DiffusionParameter(),
DiffusionModelType.StableDiffusion => DiffusionParameter.SDXLDefault,
DiffusionModelType.Flux => DiffusionParameter.FluxDefault,
_ => throw new ArgumentOutOfRangeException()
};
public IImage<ColorRGB> TextToImage(string prompt, DiffusionParameter? parameter = null)
{
parameter ??= GetDefaultParameter();
ObjectDisposedException.ThrowIf(_disposed, this); ObjectDisposedException.ThrowIf(_disposed, this);
ArgumentNullException.ThrowIfNull(prompt);
if (_ctx == null) throw new NullReferenceException("The model is not initialized.");
parameter.Validate(); parameter.Validate();
List<nint> ptrsToFree = []; Native.Types.sd_image_t* result = Native.generate_image(_ctx, parameter);
if (result == null) return null;
try try
{ {
NativeParameters nativeParameters = PrefillParameters(prompt, parameter); return ImageHelper.GetImage(result, 0);
SetControlNetParameters(ref nativeParameters, parameter, ptrsToFree);
Native.sd_image_t* result = Txt2Img(nativeParameters);
return ImageHelper.ToImage(result);
} }
finally finally
{ {
foreach (nint ptr in ptrsToFree) ImageHelper.Free(result, 1);
Marshal.FreeHGlobal(ptr);
} }
} }
public IImage<ColorRGB> ImageToImage(string prompt, IImage image, DiffusionParameter? parameter = null) // TODO DarthAffe 09.08.2025: Implement when no longer marked as broken
{ //public Image<ColorRGB>[] GenerateVideo()
parameter ??= GetDefaultParameter(); //{
// ObjectDisposedException.ThrowIf(_disposed, this);
ObjectDisposedException.ThrowIf(_disposed, this); // //parameter.Validate();
ArgumentNullException.ThrowIfNull(prompt);
ArgumentNullException.ThrowIfNull(image);
parameter.Validate(); // int imageCount = 0; // TODO DarthAffe 09.08.2025: Set correct count
// DarthAffe 10.08.2024: Mask needs to be a 1 channel all max value image when it's not used - I really don't like this concept as it adds unnecessary allocations, but that's how it is :( // Native.Types.sd_image_t* result = Native.generate_video(_ctx, new Native.Types.sd_vid_gen_params_t()); // TODO DarthAffe 09.08.2025: Add Parameter
Span<byte> maskBuffer = new byte[image.Width * image.Height]; // try
maskBuffer.Fill(byte.MaxValue); // {
// return ImageHelper.ToImageArray(result, imageCount);
return InternalImageToImage(prompt, image, maskBuffer, parameter); // }
} // finally
// {
public IImage<ColorRGB> Inpaint(string prompt, IImage image, IImage mask, DiffusionParameter? parameter = null) // ImageHelper.Free(result, imageCount);
{ // }
parameter ??= GetDefaultParameter(); //}
ObjectDisposedException.ThrowIf(_disposed, this);
ArgumentNullException.ThrowIfNull(prompt);
ArgumentNullException.ThrowIfNull(image);
ArgumentNullException.ThrowIfNull(mask);
parameter.Validate();
if (image.Width != mask.Width) throw new ArgumentException("The mask needs to have the same with as the image.", nameof(mask));
if (image.Height != mask.Height) throw new ArgumentException("The mask needs to have the same height as the image.", nameof(mask));
// DarthAffe 10.08.2024: HPPH does currently not support monochrome images, that's why we need to convert it here. We're going for the simple conversion as the source image is supposed to be monochrome anyway.
Span<byte> maskBuffer = new byte[image.Width * image.Height];
for (int y = 0; y < image.Height; y++)
for (int x = 0; x < image.Width; x++)
{
IColor color = mask[x, y];
maskBuffer[(image.Width * y) + x] = (byte)Math.Round((color.R + color.G + color.B) / 3.0);
}
return InternalImageToImage(prompt, image, maskBuffer, parameter);
}
public IImage<ColorRGB> Edit(string prompt, IImage[] refImages, DiffusionParameter? parameter = null)
{
parameter ??= GetDefaultParameter();
ObjectDisposedException.ThrowIf(_disposed, this);
ArgumentNullException.ThrowIfNull(prompt);
ArgumentNullException.ThrowIfNull(refImages);
parameter.Validate();
List<nint> ptrsToFree = [];
try
{
NativeParameters nativeParameters = PrefillParameters(prompt, parameter);
SetControlNetParameters(ref nativeParameters, parameter, ptrsToFree);
Native.sd_image_t[] nativeRefImages = new Native.sd_image_t[refImages.Length];
for (int i = 0; i < refImages.Length; i++)
{
IImage image = refImages[i];
if (image is not IImage<ColorRGB> refImage)
refImage = image.ConvertTo<ColorRGB>();
nativeRefImages[i] = refImage.ToSdImage(out nint dataPtr);
ptrsToFree.Add(dataPtr);
}
fixed (Native.sd_image_t* nativeRefImagesPtr = nativeRefImages)
{
nativeParameters.ref_images = nativeRefImagesPtr;
nativeParameters.ref_images_count = nativeRefImages.Length;
Native.sd_image_t* result = Edit(nativeParameters);
return ImageHelper.ToImage(result);
}
}
finally
{
foreach (nint ptr in ptrsToFree)
Marshal.FreeHGlobal(ptr);
}
}
private Image<ColorRGB> InternalImageToImage(string prompt, IImage image, Span<byte> mask, DiffusionParameter parameter)
{
List<nint> ptrsToFree = [];
try
{
NativeParameters nativeParameters = PrefillParameters(prompt, parameter);
SetControlNetParameters(ref nativeParameters, parameter, ptrsToFree);
if (image is not IImage<ColorRGB> refImage)
refImage = image.ConvertTo<ColorRGB>();
nativeParameters.init_image = refImage.ToSdImage(out nint imagePtr);
ptrsToFree.Add(imagePtr);
fixed (byte* maskPtr = mask)
{
Native.sd_image_t maskImage = new()
{
width = (uint)refImage.Width,
height = (uint)refImage.Height,
channel = 1,
data = maskPtr
};
nativeParameters.mask_image = maskImage;
Native.sd_image_t* result = Img2Img(nativeParameters);
return ImageHelper.ToImage(result);
}
}
finally
{
foreach (nint ptr in ptrsToFree)
Marshal.FreeHGlobal(ptr);
}
}
private static NativeParameters PrefillParameters(string prompt, DiffusionParameter parameter)
=> new()
{
prompt = prompt,
negative_prompt = parameter.NegativePrompt,
clip_skip = parameter.ClipSkip,
cfg_scale = parameter.CfgScale,
guidance = parameter.Guidance,
eta = parameter.Eta,
width = parameter.Width,
height = parameter.Height,
sample_method = parameter.SampleMethod,
sample_steps = parameter.SampleSteps,
seed = parameter.Seed,
batch_count = 1,
control_cond = null,
control_strength = 0,
style_strength = parameter.PhotoMaker.StyleRatio,
normalize_input = parameter.PhotoMaker.NormalizeInput,
input_id_images_path = parameter.PhotoMaker.InputIdImageDirectory,
skip_layers = parameter.SkipLayers,
skip_layers_count = parameter.SkipLayers.Length,
slg_scale = parameter.SlgScale,
skip_layer_start = parameter.SkipLayerStart,
skip_layer_end = parameter.SkipLayerEnd,
strength = parameter.Strength,
};
private static void SetControlNetParameters(ref NativeParameters nativeParameters, DiffusionParameter parameter, List<nint> ptrsToFree)
{
if (!parameter.ControlNet.IsEnabled) return;
if (parameter.ControlNet.Image == null) return;
if (parameter.ControlNet.Image is not IImage<ColorRGB> controlNetImage)
controlNetImage = parameter.ControlNet.Image!.ConvertTo<ColorRGB>();
Native.sd_image_t* nativeControlNetImage = controlNetImage.ToSdImagePtr(out nint controlNetImagePtr);
ptrsToFree.Add(controlNetImagePtr);
ptrsToFree.Add((nint)nativeControlNetImage);
nativeParameters.control_cond = nativeControlNetImage;
nativeParameters.control_strength = parameter.ControlNet.Strength;
if (parameter.ControlNet.CannyPreprocess)
{
nativeParameters.control_cond->data = Native.preprocess_canny(nativeParameters.control_cond->data,
parameter.Width,
parameter.Height,
parameter.ControlNet.CannyHighThreshold,
parameter.ControlNet.CannyLowThreshold,
parameter.ControlNet.CannyWeak,
parameter.ControlNet.CannyStrong,
parameter.ControlNet.CannyInverse);
ptrsToFree.Add((nint)nativeParameters.control_cond->data);
}
}
private Native.sd_image_t* Txt2Img(NativeParameters parameter)
=> Native.txt2img(_ctx,
parameter.prompt,
parameter.negative_prompt,
parameter.clip_skip,
parameter.cfg_scale,
parameter.guidance,
parameter.eta,
parameter.width,
parameter.height,
parameter.sample_method,
parameter.sample_steps,
parameter.seed,
parameter.batch_count,
parameter.control_cond,
parameter.control_strength,
parameter.style_strength,
parameter.normalize_input,
parameter.input_id_images_path,
parameter.skip_layers,
parameter.skip_layers_count,
parameter.slg_scale,
parameter.skip_layer_start,
parameter.skip_layer_end
);
private Native.sd_image_t* Img2Img(NativeParameters parameter)
=> Native.img2img(_ctx,
parameter.init_image,
parameter.mask_image,
parameter.prompt,
parameter.negative_prompt,
parameter.clip_skip,
parameter.cfg_scale,
parameter.guidance,
parameter.width,
parameter.height,
parameter.sample_method,
parameter.sample_steps,
parameter.strength,
parameter.seed,
parameter.batch_count,
parameter.control_cond,
parameter.control_strength,
parameter.style_strength,
parameter.normalize_input,
parameter.input_id_images_path,
parameter.skip_layers,
parameter.skip_layers_count,
parameter.slg_scale,
parameter.skip_layer_start,
parameter.skip_layer_end
);
private Native.sd_image_t* Edit(NativeParameters parameter)
=> Native.edit(_ctx,
parameter.ref_images,
parameter.ref_images_count,
parameter.prompt,
parameter.negative_prompt,
parameter.clip_skip,
parameter.cfg_scale,
parameter.guidance,
parameter.eta,
parameter.width,
parameter.height,
parameter.sample_method,
parameter.sample_steps,
parameter.strength,
parameter.seed,
parameter.batch_count,
parameter.control_cond,
parameter.control_strength,
parameter.style_strength,
parameter.normalize_input,
parameter.skip_layers,
parameter.skip_layers_count,
parameter.slg_scale,
parameter.skip_layer_start,
parameter.skip_layer_end
);
public void Dispose() public void Dispose()
{ {
@ -382,37 +96,4 @@ public sealed unsafe class DiffusionModel : IDisposable
} }
#endregion #endregion
private ref struct NativeParameters
{
internal string prompt;
internal string negative_prompt;
internal int clip_skip;
internal float cfg_scale;
internal float guidance;
internal float eta;
internal int width;
internal int height;
internal Sampler sample_method;
internal int sample_steps;
internal long seed;
internal int batch_count;
internal Native.sd_image_t* control_cond;
internal float control_strength;
internal float style_strength;
internal bool normalize_input;
internal string input_id_images_path;
internal int[] skip_layers;
internal int skip_layers_count;
internal float slg_scale;
internal float skip_layer_start;
internal float skip_layer_end;
internal Native.sd_image_t init_image;
internal Native.sd_image_t mask_image;
internal Native.sd_image_t* ref_images;
internal int ref_images_count;
internal float strength;
}
} }

40
StableDiffusion.NET/Models/Parameter/CannyParameter.cs Normal file
View File

@ -0,0 +1,40 @@
using HPPH;
using JetBrains.Annotations;
namespace StableDiffusion.NET;
[PublicAPI]
public sealed class CannyParameter
{
/// <summary>
/// the image to process
/// </summary>
public IImage? Image { get; set; } = null;
/// <summary>
///
/// </summary>
public float HighThreshold { get; set; } = 0.08f;
/// <summary>
///
/// </summary>
public float LowThreshold { get; set; } = 0.08f;
/// <summary>
///
/// </summary>
public float Weak { get; set; } = 0.8f;
/// <summary>
///
/// </summary>
public float Strong { get; set; } = 1.0f;
/// <summary>
///
/// </summary>
public bool Inverse { get; set; } = false;
public static CannyParameter Create() => new();
}

34
StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs
View File

@ -1,13 +1,9 @@
using HPPH; using HPPH;
using JetBrains.Annotations;
namespace StableDiffusion.NET; namespace StableDiffusion.NET;
[PublicAPI]
public sealed class ControlNetParameter public sealed class ControlNetParameter
{ {
public bool IsEnabled => Image != null;
/// <summary> /// <summary>
/// image condition, control net /// image condition, control net
/// </summary> /// </summary>
@ -19,33 +15,5 @@ public sealed class ControlNetParameter
/// </summary> /// </summary>
public float Strength { get; set; } = 0.9f; public float Strength { get; set; } = 0.9f;
/// <summary> internal ControlNetParameter() { }
/// apply canny preprocessor (edge detection)
/// </summary>
public bool CannyPreprocess { get; set; } = false;
/// <summary>
///
/// </summary>
public float CannyHighThreshold { get; set; } = 0.08f;
/// <summary>
///
/// </summary>
public float CannyLowThreshold { get; set; } = 0.08f;
/// <summary>
///
/// </summary>
public float CannyWeak { get; set; } = 0.8f;
/// <summary>
///
/// </summary>
public float CannyStrong { get; set; } = 1.0f;
/// <summary>
///
/// </summary>
public bool CannyInverse { get; set; } = false;
} }

26
StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs
View File

@ -1,9 +1,10 @@
namespace StableDiffusion.NET; using JetBrains.Annotations;
public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantizedModelParameter, IPhotomakerModelParameter namespace StableDiffusion.NET;
[PublicAPI]
public sealed class DiffusionModelParameter : IDiffusionModelParameter
{ {
public DiffusionModelType DiffusionModelType { get; set; } = DiffusionModelType.None;
/// <summary> /// <summary>
/// path to vae /// path to vae
/// </summary> /// </summary>
@ -67,6 +68,18 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantiz
/// </summary> /// </summary>
public bool FlashAttention { get; set; } = false; public bool FlashAttention { get; set; } = false;
/// <summary>
/// use Conv2d direct in the diffusion model
/// This might crash if it is not supported by the backend.
/// </summary>
public bool DiffusionConvDirect { get; set; } = false;
/// <summary>
/// use Conv2d direct in the vae model (should improve the performance)
/// This might crash if it is not supported by the backend.
/// </summary>
public bool VaeConfDirect { get; set; } = false;
/// <summary> /// <summary>
/// RNG (default: Standard) /// RNG (default: Standard)
/// </summary> /// </summary>
@ -78,7 +91,8 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantiz
public Schedule Schedule { get; set; } = Schedule.Default; public Schedule Schedule { get; set; } = Schedule.Default;
/// <summary> /// <summary>
/// /// quantizes on load
/// not really useful in most cases
/// </summary> /// </summary>
public Quantization Quantization { get; set; } = Quantization.Unspecified; public Quantization Quantization { get; set; } = Quantization.Unspecified;
@ -119,4 +133,6 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantiz
/// path to the clip-g text encoder /// path to the clip-g text encoder
/// </summary> /// </summary>
public string ClipGPath { get; set; } = string.Empty; public string ClipGPath { get; set; } = string.Empty;
public static DiffusionModelParameter Create() => new();
} }

99
StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs
View File

@ -1,99 +0,0 @@
using JetBrains.Annotations;
namespace StableDiffusion.NET;
[PublicAPI]
public sealed class DiffusionParameter
{
#region Properties & Fields
public static DiffusionParameter SD1Default => new() { Width = 512, Height = 512, CfgScale = 7.5f, Guidance = 1f, SampleSteps = 25, SampleMethod = Sampler.Euler_A };
public static DiffusionParameter SDXLDefault => new() { Width = 1024, Height = 1024, CfgScale = 7f, Guidance = 1f, SampleSteps = 30, SampleMethod = Sampler.Euler_A };
public static DiffusionParameter SD3_5Default => new() { Width = 1024, Height = 1024, CfgScale = 4.5f, Guidance = 1f, SampleSteps = 20, SampleMethod = Sampler.Euler };
public static DiffusionParameter FluxDefault => new() { Width = 1024, Height = 1024, CfgScale = 1, Guidance = 3.5f, SampleSteps = 20, SampleMethod = Sampler.Euler };
/// <summary>
/// the negative prompt (default: "");
/// </summary>
public string NegativePrompt { get; set; } = string.Empty;
/// <summary>
/// image width, in pixel space (default: 512)
/// </summary>
public int Width { get; set; } = 512;
/// <summary>
/// image height, in pixel space (default: 512)
/// </summary>
public int Height { get; set; } = 512;
/// <summary>
/// sampling method (default: Euler_A)
/// </summary>
public Sampler SampleMethod { get; set; } = Sampler.Euler_A;
/// <summary>
/// number of sample steps (default: 25)
/// </summary>
public int SampleSteps { get; set; } = 25;
/// <summary>
/// RNG seed. use -1 for a random seed (default: -1)
/// </summary>
public long Seed { get; set; } = -1;
/// <summary>
/// strength for noising/unnoising (default: 0.7)
/// </summary>
public float Strength { get; set; } = 0.7f;
/// <summary>
/// ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1)
/// -1 represents unspecified, will be 1 for SD1.x, 2 for SD2.x
/// </summary>
public int ClipSkip { get; set; } = -1;
/// <summary>
/// skip layer guidance (SLG) scale, only for DiT models: (default: 0)
/// 0 means disabled, a value of 2.5 is nice for sd3.5 medium
/// </summary>
public float SlgScale { get; set; } = 0f;
/// <summary>
/// Layers to skip for SLG steps: (default: [7,8,9])
/// </summary>
public int[] SkipLayers { get; set; } = [7, 8, 9];
/// <summary>
/// SLG enabling point: (default: 0.01)
/// </summary>
public float SkipLayerStart { get; set; } = 0.01f;
/// <summary>
/// SLG disabling point: (default: 0.2)
/// </summary>
public float SkipLayerEnd { get; set; } = 0.2f;
public ControlNetParameter ControlNet { get; } = new();
// Stable Diffusion only
/// <summary>
/// unconditional guidance scale: (default: 7.5)
/// </summary>
public float CfgScale { get; set; } = 7.5f;
public PhotoMakerParameter PhotoMaker { get; } = new();
// Flux only
/// <summary>
///
/// </summary>
public float Guidance { get; set; } = 3.5f;
/// <summary>
/// eta in DDIM, only for DDIM and TCD (default: 0)
/// </summary>
public float Eta { get; set; } = 0f;
#endregion
}

149
StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs
View File

@ -1,149 +0,0 @@
using HPPH;
namespace StableDiffusion.NET;
public static class DiffusionParameterExtension
{
public static DiffusionParameter WithSize(this DiffusionParameter parameter, int? width = null, int? height = null)
{
if (width != null)
parameter.Width = width.Value;
if (height != null)
parameter.Height = height.Value;
return parameter;
}
public static DiffusionParameter WithSampler(this DiffusionParameter parameter, Sampler sampler)
{
parameter.SampleMethod = sampler;
return parameter;
}
public static DiffusionParameter WithSteps(this DiffusionParameter parameter, int steps)
{
parameter.SampleSteps = steps;
return parameter;
}
public static DiffusionParameter WithSeed(this DiffusionParameter parameter, long seed)
{
parameter.Seed = seed;
return parameter;
}
public static DiffusionParameter WithClipSkip(this DiffusionParameter parameter, int clipSkip)
{
parameter.ClipSkip = clipSkip;
return parameter;
}
public static DiffusionParameter WithCfg(this DiffusionParameter parameter, float cfg)
{
parameter.CfgScale = cfg;
return parameter;
}
public static DiffusionParameter WithGuidance(this DiffusionParameter parameter, float guidance)
{
parameter.Guidance = guidance;
return parameter;
}
public static DiffusionParameter WithNegativePrompt(this DiffusionParameter parameter, string negativePrompt)
{
parameter.NegativePrompt = negativePrompt;
return parameter;
}
public static DiffusionParameter WithStrength(this DiffusionParameter parameter, float strength)
{
parameter.Strength = strength;
return parameter;
}
public static DiffusionParameter WithSlgScale(this DiffusionParameter parameter, float slgScale)
{
parameter.SlgScale = slgScale;
return parameter;
}
public static DiffusionParameter WithSkipLayers(this DiffusionParameter parameter, int[] layers)
{
parameter.SkipLayers = layers;
return parameter;
}
public static DiffusionParameter WithSkipLayerStart(this DiffusionParameter parameter, float skipLayerStart)
{
parameter.SkipLayerStart = skipLayerStart;
return parameter;
}
public static DiffusionParameter WithSkipLayerEnd(this DiffusionParameter parameter, float skipLayerEnd)
{
parameter.SkipLayerEnd = skipLayerEnd;
return parameter;
}
public static DiffusionParameter WithControlNet(this DiffusionParameter parameter, IImage image, float? strength = null)
{
parameter.ControlNet.Image = image;
if (strength != null)
parameter.ControlNet.Strength = strength.Value;
return parameter;
}
public static DiffusionParameter WithCannyPreprocessing(this DiffusionParameter parameter,
float? cannyHighThreshold = null, float? cannyLowThreshold = null,
float? cannyWeak = null, float? cannyStrong = null,
bool? cannyInverse = null)
{
parameter.ControlNet.CannyPreprocess = true;
if (cannyHighThreshold != null)
parameter.ControlNet.CannyHighThreshold = cannyHighThreshold.Value;
if (cannyLowThreshold != null)
parameter.ControlNet.CannyLowThreshold = cannyLowThreshold.Value;
if (cannyWeak != null)
parameter.ControlNet.CannyWeak = cannyWeak.Value;
if (cannyStrong != null)
parameter.ControlNet.CannyStrong = cannyStrong.Value;
if (cannyInverse != null)
parameter.ControlNet.CannyInverse = cannyInverse.Value;
return parameter;
}
public static DiffusionParameter WithPhotomaker(this DiffusionParameter parameter, string inputIdImageDirectory, float? styleRatio = null, bool? normalizeInput = null)
{
parameter.PhotoMaker.InputIdImageDirectory = inputIdImageDirectory;
if (styleRatio != null)
parameter.PhotoMaker.StyleRatio = styleRatio.Value;
if (normalizeInput != null)
parameter.PhotoMaker.NormalizeInput = normalizeInput.Value;
return parameter;
}
}

215
StableDiffusion.NET/Models/Parameter/Extensions/ImageGenerationParameterExtension.cs Normal file
View File

@ -0,0 +1,215 @@
using HPPH;
using JetBrains.Annotations;
namespace StableDiffusion.NET;
[PublicAPI]
public static class ImageGenerationParameterExtension
{
public static ImageGenerationParameter WithPrompt(this ImageGenerationParameter parameter, string prompt)
{
parameter.Prompt = prompt;
return parameter;
}
public static ImageGenerationParameter WithNegativePrompt(this ImageGenerationParameter parameter, string negativePrompt)
{
parameter.NegativePrompt = negativePrompt;
return parameter;
}
public static ImageGenerationParameter WithClipSkip(this ImageGenerationParameter parameter, int clipSkip)
{
parameter.ClipSkip = clipSkip;
return parameter;
}
public static ImageGenerationParameter WithCfg(this ImageGenerationParameter parameter, float cfg)
{
parameter.WithTxtCfg(cfg);
parameter.WithImgCfg(cfg);
return parameter;
}
public static ImageGenerationParameter WithTxtCfg(this ImageGenerationParameter parameter, float txtCfg)
{
parameter.Guidance.TxtCfg = txtCfg;
return parameter;
}
public static ImageGenerationParameter WithImgCfg(this ImageGenerationParameter parameter, float imgCfg)
{
parameter.Guidance.ImgCfg = imgCfg;
return parameter;
}
public static ImageGenerationParameter WithMinCfg(this ImageGenerationParameter parameter, float minCfg)
{
parameter.Guidance.MinCfg = minCfg;
return parameter;
}
public static ImageGenerationParameter WithGuidance(this ImageGenerationParameter parameter, float guidance)
{
parameter.Guidance.DistilledGuidance = guidance;
return parameter;
}
public static ImageGenerationParameter WithSlgScale(this ImageGenerationParameter parameter, float slgScale)
{
parameter.Guidance.Slg.Scale = slgScale;
return parameter;
}
public static ImageGenerationParameter WithSkipLayers(this ImageGenerationParameter parameter, int[] layers)
{
parameter.Guidance.Slg.Layers = layers;
return parameter;
}
public static ImageGenerationParameter WithSkipLayerStart(this ImageGenerationParameter parameter, float skipLayerStart)
{
parameter.Guidance.Slg.SkipLayerStart = skipLayerStart;
return parameter;
}
public static ImageGenerationParameter WithSkipLayerEnd(this ImageGenerationParameter parameter, float skipLayerEnd)
{
parameter.Guidance.Slg.SkipLayerEnd = skipLayerEnd;
return parameter;
}
public static ImageGenerationParameter WithInitImage(this ImageGenerationParameter parameter, IImage image)
{
parameter.InitImage = image;
return parameter;
}
public static ImageGenerationParameter WithRefImages(this ImageGenerationParameter parameter, params IImage[] images)
{
parameter.RefImages = images;
return parameter;
}
public static ImageGenerationParameter WithMaskImage(this ImageGenerationParameter parameter, IImage image)
{
parameter.MaskImage = image;
return parameter;
}
public static ImageGenerationParameter WithSize(this ImageGenerationParameter parameter, int? width = null, int? height = null)
{
if (width != null)
parameter.Width = width.Value;
if (height != null)
parameter.Height = height.Value;
return parameter;
}
public static ImageGenerationParameter WithSampler(this ImageGenerationParameter parameter, Sampler sampler)
{
parameter.SampleMethod = sampler;
return parameter;
}
public static ImageGenerationParameter WithSteps(this ImageGenerationParameter parameter, int steps)
{
parameter.SampleSteps = steps;
return parameter;
}
public static ImageGenerationParameter WithEta(this ImageGenerationParameter parameter, float eta)
{
parameter.Eta = eta;
return parameter;
}
public static ImageGenerationParameter WithStrength(this ImageGenerationParameter parameter, float strength)
{
parameter.Strength = strength;
return parameter;
}
public static ImageGenerationParameter WithSeed(this ImageGenerationParameter parameter, long seed)
{
parameter.Seed = seed;
return parameter;
}
public static ImageGenerationParameter WithControlNet(this ImageGenerationParameter parameter, IImage image, float? strength = null)
{
parameter.ControlNet.Image = image;
if (strength != null)
parameter.ControlNet.Strength = strength.Value;
return parameter;
}
public static ImageGenerationParameter WithPhotomaker(this ImageGenerationParameter parameter, string inputIdImageDirectory, float? styleStrength = null, bool? normalizeInput = null)
{
parameter.PhotoMaker.InputIdImageDirectory = inputIdImageDirectory;
if (styleStrength != null)
parameter.PhotoMaker.StyleStrength = styleStrength.Value;
if (normalizeInput != null)
parameter.PhotoMaker.NormalizeInput = normalizeInput.Value;
return parameter;
}
#region Defaults
public static ImageGenerationParameter WithSd1Defaults(this ImageGenerationParameter parameter)
=> parameter.WithSize(512, 512)
.WithCfg(7.5f)
.WithGuidance(1f)
.WithSteps(25)
.WithSampler(Sampler.Euler_A);
public static ImageGenerationParameter WithSDXLDefaults(this ImageGenerationParameter parameter)
=> parameter.WithSize(1024, 1024)
.WithCfg(7f)
.WithGuidance(1f)
.WithSteps(30)
.WithSampler(Sampler.Euler_A);
public static ImageGenerationParameter WithSD3_5Defaults(this ImageGenerationParameter parameter)
=> parameter.WithSize(1024, 1024)
.WithCfg(4.5f)
.WithGuidance(1f)
.WithSteps(20)
.WithSampler(Sampler.Euler);
public static ImageGenerationParameter WithFluxDefaults(this ImageGenerationParameter parameter)
=> parameter.WithSize(1024, 1024)
.WithCfg(1f)
.WithGuidance(3.5f)
.WithSteps(20)
.WithSampler(Sampler.Euler);
#endregion
}

13
StableDiffusion.NET/Models/Parameter/GuidanceParameter.cs Normal file
View File

@ -0,0 +1,13 @@
namespace StableDiffusion.NET;
public sealed class GuidanceParameter
{
public float TxtCfg { get; set; } = 7.0f;
public float ImgCfg { get; set; } = 7.0f;
public float MinCfg { get; set; } = 1.0f;
public float DistilledGuidance { get; set; } = 3.5f;
public SlgParameter Slg { get; } = new();
internal GuidanceParameter() { }
}

77
StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs Normal file
View File

@ -0,0 +1,77 @@
using HPPH;
using JetBrains.Annotations;
namespace StableDiffusion.NET;
[PublicAPI]
public sealed class ImageGenerationParameter
{
#region Properties & Fields
public string Prompt { get; set; } = string.Empty;
/// <summary>
/// the negative prompt (default: "");
/// </summary>
public string NegativePrompt { get; set; } = string.Empty;
/// <summary>
/// ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1)
/// -1 represents unspecified, will be 1 for SD1.x, 2 for SD2.x
/// </summary>
public int ClipSkip { get; set; } = -1;
public GuidanceParameter Guidance { get; } = new();
public IImage? InitImage { get; set; }
public IImage[]? RefImages { get; set; }
public IImage? MaskImage { get; set; }
/// <summary>
/// image width, in pixel space (default: 512)
/// </summary>
public int Width { get; set; } = 512;
/// <summary>
/// image height, in pixel space (default: 512)
/// </summary>
public int Height { get; set; } = 512;
/// <summary>
/// sampling method (default: Euler_A)
/// </summary>
public Sampler SampleMethod { get; set; } = Sampler.Euler_A;
/// <summary>
/// number of sample steps (default: 25)
/// </summary>
public int SampleSteps { get; set; } = 25;
/// <summary>
/// eta in DDIM, only for DDIM and TCD (default: 0)
/// </summary>
public float Eta { get; set; } = 0f;
/// <summary>
/// strength for noising/unnoising (default: 0.7)
/// </summary>
public float Strength { get; set; } = 0.7f;
/// <summary>
/// RNG seed. use -1 for a random seed (default: -1)
/// </summary>
public long Seed { get; set; } = -1;
public ControlNetParameter ControlNet { get; } = new();
public PhotoMakerParameter PhotoMaker { get; } = new();
public static ImageGenerationParameter Create() => new();
public static ImageGenerationParameter TextToImage(string prompt) => Create().WithPrompt(prompt);
public static ImageGenerationParameter ImageToImage(string prompt, IImage image) => Create().WithPrompt(prompt).WithInitImage(image);
#endregion
}

26
StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs
View File

@ -1,9 +1,10 @@
namespace StableDiffusion.NET; using JetBrains.Annotations;
public interface IDiffusionModelParameter namespace StableDiffusion.NET;
[PublicAPI]
public interface IDiffusionModelParameter : IModelParameter
{ {
DiffusionModelType DiffusionModelType { get; set; }
string VaePath { get; set; } string VaePath { get; set; }
string TaesdPath { get; set; } string TaesdPath { get; set; }
@ -17,7 +18,24 @@ public interface IDiffusionModelParameter
bool KeepClipOnCPU { get; set; } bool KeepClipOnCPU { get; set; }
bool KeepVaeOnCPU { get; set; } bool KeepVaeOnCPU { get; set; }
bool FlashAttention { get; set; } bool FlashAttention { get; set; }
bool DiffusionConvDirect { get; set; }
bool VaeConfDirect { get; set; }
RngType RngType { get; set; } RngType RngType { get; set; }
Schedule Schedule { get; set; } Schedule Schedule { get; set; }
Quantization Quantization { get; set; }
string ModelPath { get; set; }
string StackedIdEmbeddingsDirectory { get; set; }
string DiffusionModelPath { get; set; }
string ClipLPath { get; set; }
string T5xxlPath { get; set; }
bool ChromaUseDitMap { get; set; }
bool ChromaEnableT5Map { get; set; }
int ChromaT5MaskPad { get; set; }
string ClipGPath { get; set; }
} }

9
StableDiffusion.NET/Models/Parameter/Interfaces/IModelParameter.cs Normal file
View File

@ -0,0 +1,9 @@
using JetBrains.Annotations;
namespace StableDiffusion.NET;
[PublicAPI]
public interface IModelParameter
{
int ThreadCount { get; set; }
}

6
StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs
View File

@ -1,6 +0,0 @@
namespace StableDiffusion.NET;
public interface IPhotomakerModelParameter
{
string StackedIdEmbeddingsDirectory { get; set; }
}

8
StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs
View File

@ -1,8 +0,0 @@
namespace StableDiffusion.NET;
public interface IQuantizedModelParameter
{
int ThreadCount { get; set; }
Quantization Quantization { get; set; }
}

10
StableDiffusion.NET/Models/Parameter/Interfaces/IUpscaleModelParameter.cs Normal file
View File

@ -0,0 +1,10 @@
using JetBrains.Annotations;
namespace StableDiffusion.NET;
[PublicAPI]
public interface IUpscaleModelParameter : IModelParameter
{
string ModelPath { get; set; }
bool ConvDirect { get; set; }
}

9
StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs
View File

@ -1,8 +1,5 @@
using JetBrains.Annotations; namespace StableDiffusion.NET;
namespace StableDiffusion.NET;
[PublicAPI]
public sealed class PhotoMakerParameter public sealed class PhotoMakerParameter
{ {
/// <summary> /// <summary>
@ -13,10 +10,12 @@ public sealed class PhotoMakerParameter
/// <summary> /// <summary>
/// strength for keeping input identity (default: 20) /// strength for keeping input identity (default: 20)
/// </summary> /// </summary>
public float StyleRatio { get; set; } = 20f; public float StyleStrength { get; set; } = 20f;
/// <summary> /// <summary>
/// normalize PHOTOMAKER input id images /// normalize PHOTOMAKER input id images
/// </summary> /// </summary>
public bool NormalizeInput { get; set; } = false; public bool NormalizeInput { get; set; } = false;
internal PhotoMakerParameter() { }
} }

27
StableDiffusion.NET/Models/Parameter/SlgParameter.cs Normal file
View File

@ -0,0 +1,27 @@
namespace StableDiffusion.NET;
public sealed class SlgParameter
{
/// <summary>
/// Layers to skip for SLG steps: (default: [7,8,9])
/// </summary>
public int[] Layers { get; set; } = [7, 8, 9];
/// <summary>
/// SLG enabling point: (default: 0.01)
/// </summary>
public float SkipLayerStart { get; set; } = 0.01f;
/// <summary>
/// SLG disabling point: (default: 0.2)
/// </summary>
public float SkipLayerEnd { get; set; } = 0.2f;
/// <summary>
/// skip layer guidance (SLG) scale, only for DiT models: (default: 0)
/// 0 means disabled, a value of 2.5 is nice for sd3.5 medium
/// </summary>
public float Scale { get; set; } = 0f;
internal SlgParameter() { }
}

9
StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs
View File

@ -3,7 +3,7 @@
namespace StableDiffusion.NET; namespace StableDiffusion.NET;
[PublicAPI] [PublicAPI]
public sealed class UpscaleModelParameter : IQuantizedModelParameter public sealed class UpscaleModelParameter : IUpscaleModelParameter
{ {
/// <summary> /// <summary>
/// path to esrgan model. Upscale images after generate, just RealESRGAN_x4plus_anime_6B supported by now /// path to esrgan model. Upscale images after generate, just RealESRGAN_x4plus_anime_6B supported by now
@ -17,7 +17,10 @@ public sealed class UpscaleModelParameter : IQuantizedModelParameter
public int ThreadCount { get; set; } = 1; public int ThreadCount { get; set; } = 1;
/// <summary> /// <summary>
/// /// use Conv2d direct in the diffusion model
/// This might crash if it is not supported by the backend.
/// </summary> /// </summary>
public Quantization Quantization { get; set; } = Quantization.F16; public bool ConvDirect { get; set; } = false;
public static UpscaleModelParameter Create() => new();
} }

21
StableDiffusion.NET/Models/UpscaleModel.cs
View File

@ -13,7 +13,7 @@ public sealed unsafe class UpscaleModel : IDisposable
public UpscaleModelParameter ModelParameter { get; } public UpscaleModelParameter ModelParameter { get; }
private Native.upscaler_ctx_t* _ctx; private Native.Types.upscaler_ctx_t* _ctx;
#endregion #endregion
@ -40,32 +40,19 @@ public sealed unsafe class UpscaleModel : IDisposable
{ {
_ctx = Native.new_upscaler_ctx(ModelParameter.ModelPath, _ctx = Native.new_upscaler_ctx(ModelParameter.ModelPath,
ModelParameter.ThreadCount, ModelParameter.ThreadCount,
ModelParameter.Quantization); ModelParameter.ConvDirect);
if (_ctx == null) throw new NullReferenceException("Failed to initialize upscale-model."); if (_ctx == null) throw new NullReferenceException("Failed to initialize upscale-model.");
} }
public IImage<ColorRGB> Upscale(IImage image, int upscaleFactor) public Image<ColorRGB> Upscale(IImage image, int upscaleFactor)
{ {
ObjectDisposedException.ThrowIf(_disposed, this); ObjectDisposedException.ThrowIf(_disposed, this);
ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(upscaleFactor, 0, nameof(upscaleFactor)); ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(upscaleFactor, 0, nameof(upscaleFactor));
if (_ctx == null) throw new NullReferenceException("The model is not initialized."); if (_ctx == null) throw new NullReferenceException("The model is not initialized.");
if (image is not IImage<ColorRGB> sourceImage) return Native.upscale(_ctx, image, (uint)upscaleFactor);
sourceImage = image.ConvertTo<ColorRGB>();
fixed (byte* imagePtr = sourceImage.AsRefImage())
{
Native.sd_image_t result = Native.upscale(_ctx, sourceImage.ToSdImage(imagePtr), upscaleFactor);
return ImageHelper.ToImage(result);
}
}
private IImage<ColorRGB> Upscale(Native.sd_image_t image, int upscaleFactor)
{
Native.sd_image_t result = Native.upscale(_ctx, image, upscaleFactor);
return ImageHelper.ToImage(result);
} }
public void Dispose() public void Dispose()

86
StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs Normal file
View File

@ -0,0 +1,86 @@
using System.Runtime.InteropServices.Marshalling;
namespace StableDiffusion.NET;
[CustomMarshaller(typeof(DiffusionModelParameter), MarshalMode.ManagedToUnmanagedIn, typeof(DiffusionModelParameterMarshaller))]
[CustomMarshaller(typeof(DiffusionModelParameter), MarshalMode.ManagedToUnmanagedRef, typeof(DiffusionModelParameterMarshaller))]
internal static unsafe class DiffusionModelParameterMarshaller
{
public static Native.Types.sd_ctx_params_t ConvertToUnmanaged(DiffusionModelParameter managed)
=> new()
{
model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ModelPath),
clip_l_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipLPath),
clip_g_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipGPath),
t5xxl_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.T5xxlPath),
diffusion_model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.DiffusionModelPath),
vae_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.VaePath),
taesd_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.TaesdPath),
control_net_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ControlNetPath),
lora_model_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.LoraModelDirectory),
embedding_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.EmbeddingsDirectory),
stacked_id_embed_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.StackedIdEmbeddingsDirectory),
vae_decode_only = (sbyte)(managed.VaeDecodeOnly ? 1 : 0),
vae_tiling = (sbyte)(managed.VaeTiling ? 1 : 0),
free_params_immediately = 0, // DarthAffe 06.08.2025: Static value
n_threads = managed.ThreadCount,
wtype = managed.Quantization,
rng_type = managed.RngType,
schedule = managed.Schedule,
keep_clip_on_cpu = (sbyte)(managed.KeepClipOnCPU ? 1 : 0),
keep_control_net_on_cpu = (sbyte)(managed.KeepControlNetOnCPU ? 1 : 0),
keep_vae_on_cpu = (sbyte)(managed.KeepVaeOnCPU ? 1 : 0),
diffusion_flash_attn = (sbyte)(managed.FlashAttention ? 1 : 0),
diffusion_conv_direct = (sbyte)(managed.DiffusionConvDirect ? 1 : 0),
vae_conv_direct = (sbyte)(managed.VaeConfDirect ? 1 : 0),
chroma_use_dit_mask = (sbyte)(managed.ChromaUseDitMap ? 1 : 0),
chroma_use_t5_mask = (sbyte)(managed.ChromaEnableT5Map ? 1 : 0),
chroma_t5_mask_pad = managed.ChromaT5MaskPad
};
public static DiffusionModelParameter ConvertToManaged(Native.Types.sd_ctx_params_t unmanaged)
=> new()
{
ModelPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.model_path) ?? string.Empty,
ClipLPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.clip_l_path) ?? string.Empty,
ClipGPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.clip_g_path) ?? string.Empty,
T5xxlPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.t5xxl_path) ?? string.Empty,
DiffusionModelPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.diffusion_model_path) ?? string.Empty,
VaePath = AnsiStringMarshaller.ConvertToManaged(unmanaged.vae_path) ?? string.Empty,
TaesdPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.taesd_path) ?? string.Empty,
ControlNetPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.control_net_path) ?? string.Empty,
LoraModelDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.lora_model_dir) ?? string.Empty,
EmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.embedding_dir) ?? string.Empty,
StackedIdEmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.stacked_id_embed_dir) ?? string.Empty,
VaeDecodeOnly = unmanaged.vae_decode_only == 1,
VaeTiling = unmanaged.vae_tiling == 1,
ThreadCount = unmanaged.n_threads,
Quantization = unmanaged.wtype,
RngType = unmanaged.rng_type,
Schedule = unmanaged.schedule,
KeepClipOnCPU = unmanaged.keep_clip_on_cpu == 1,
KeepControlNetOnCPU = unmanaged.keep_control_net_on_cpu == 1,
KeepVaeOnCPU = unmanaged.keep_vae_on_cpu == 1,
FlashAttention = unmanaged.diffusion_flash_attn == 1,
DiffusionConvDirect = unmanaged.diffusion_conv_direct == 1,
VaeConfDirect = unmanaged.vae_conv_direct == 1,
ChromaUseDitMap = unmanaged.chroma_use_dit_mask == 1,
ChromaEnableT5Map = unmanaged.chroma_use_t5_mask == 1,
ChromaT5MaskPad = unmanaged.chroma_t5_mask_pad
};
public static void Free(Native.Types.sd_ctx_params_t unmanaged)
{
AnsiStringMarshaller.Free(unmanaged.model_path);
AnsiStringMarshaller.Free(unmanaged.clip_l_path);
AnsiStringMarshaller.Free(unmanaged.clip_g_path);
AnsiStringMarshaller.Free(unmanaged.t5xxl_path);
AnsiStringMarshaller.Free(unmanaged.diffusion_model_path);
AnsiStringMarshaller.Free(unmanaged.vae_path);
AnsiStringMarshaller.Free(unmanaged.taesd_path);
AnsiStringMarshaller.Free(unmanaged.control_net_path);
AnsiStringMarshaller.Free(unmanaged.lora_model_dir);
AnsiStringMarshaller.Free(unmanaged.embedding_dir);
AnsiStringMarshaller.Free(unmanaged.stacked_id_embed_dir);
}
}

197
StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs Normal file
View File

@ -0,0 +1,197 @@
// ReSharper disable MemberCanBeMadeStatic.Global
using System;
using System.Runtime.InteropServices;
using System.Runtime.InteropServices.Marshalling;
namespace StableDiffusion.NET;
[CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedIn, typeof(ImageGenerationParameterMarshallerIn))]
[CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedOut, typeof(ImageGenerationParameterMarshaller))]
[CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedRef, typeof(ImageGenerationParameterMarshallerRef))]
internal static class ImageGenerationParameterMarshaller
{
public static unsafe ImageGenerationParameter ConvertToManaged(Native.Types.sd_img_gen_params_t unmanaged)
{
ImageGenerationParameter parameter = new()
{
Prompt = AnsiStringMarshaller.ConvertToManaged(unmanaged.prompt) ?? string.Empty,
NegativePrompt = AnsiStringMarshaller.ConvertToManaged(unmanaged.negative_prompt) ?? string.Empty,
ClipSkip = unmanaged.clip_skip,
Guidance =
{
TxtCfg = unmanaged.guidance.txt_cfg,
ImgCfg = unmanaged.guidance.img_cfg,
MinCfg = unmanaged.guidance.min_cfg,
DistilledGuidance = unmanaged.guidance.distilled_guidance,
Slg =
{
Layers = new int[unmanaged.guidance.slg.layer_count],
SkipLayerStart = unmanaged.guidance.slg.layer_start,
SkipLayerEnd = unmanaged.guidance.slg.layer_end,
Scale = unmanaged.guidance.slg.scale
}
},
InitImage = unmanaged.init_image.data == null ? null : unmanaged.init_image.ToImage(),
RefImages = unmanaged.ref_images == null ? null : ImageHelper.ToImageArrayIFace(unmanaged.ref_images, unmanaged.ref_images_count),
MaskImage = unmanaged.mask_image.data == null ? null : unmanaged.mask_image.ToImage(),
Width = unmanaged.width,
Height = unmanaged.height,
SampleMethod = unmanaged.sample_method,
SampleSteps = unmanaged.sample_steps,
Eta = unmanaged.eta,
Strength = unmanaged.strength,
Seed = unmanaged.seed,
ControlNet =
{
Image = unmanaged.control_cond == null ? null : ImageHelper.GetImage(unmanaged.control_cond, 0),
Strength = unmanaged.control_strength,
},
PhotoMaker =
{
StyleStrength = unmanaged.style_strength,
NormalizeInput = unmanaged.normalize_input == 1,
InputIdImageDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.input_id_images_path) ?? string.Empty,
}
};
if (unmanaged.guidance.slg.layers != null)
new Span<int>(unmanaged.guidance.slg.layers, (int)unmanaged.guidance.slg.layer_count).CopyTo(parameter.Guidance.Slg.Layers);
return parameter;
}
public static unsafe void Free(Native.Types.sd_img_gen_params_t unmanaged)
{
AnsiStringMarshaller.Free(unmanaged.prompt);
AnsiStringMarshaller.Free(unmanaged.negative_prompt);
AnsiStringMarshaller.Free(unmanaged.input_id_images_path);
unmanaged.init_image.Free();
unmanaged.mask_image.Free();
if (unmanaged.ref_images != null)
ImageHelper.Free(unmanaged.ref_images, unmanaged.ref_images_count);
if (unmanaged.control_cond != null)
ImageHelper.Free(unmanaged.control_cond, 1);
if (unmanaged.guidance.slg.layers != null)
NativeMemory.Free(unmanaged.guidance.slg.layers);
}
internal unsafe ref struct ImageGenerationParameterMarshallerIn
{
private Native.Types.sd_img_gen_params_t _imgGenParams;
private Native.Types.sd_image_t _initImage;
private Native.Types.sd_image_t _maskImage;
private Native.Types.sd_image_t* _refImages;
private Native.Types.sd_image_t* _controlNetImage;
private int* _slgLayers;
public void FromManaged(ImageGenerationParameter managed)
{
_initImage = managed.InitImage?.ToSdImage() ?? new Native.Types.sd_image_t();
_refImages = managed.RefImages == null ? null : managed.RefImages.ToSdImage();
_controlNetImage = managed.ControlNet.Image == null ? null : managed.ControlNet.Image.ToSdImagePtr();
_slgLayers = (int*)NativeMemory.Alloc((nuint)managed.Guidance.Slg.Layers.Length, (nuint)Marshal.SizeOf<int>());
managed.Guidance.Slg.Layers.AsSpan().CopyTo(new Span<int>(_slgLayers, managed.Guidance.Slg.Layers.Length));
if (managed.MaskImage != null)
_maskImage = managed.MaskImage.ToSdImage(true);
else if (managed.InitImage != null)
{
// DarthAffe 16.08.2025: Mask needs to be a 1 channel all max value image when it's not used - I really don't like this concept as it adds unnecessary allocations, but that's how it is :(
uint maskImageByteSize = _initImage.width * _initImage.height;
_maskImage = new Native.Types.sd_image_t
{
width = _initImage.width,
height = _initImage.height,
channel = 1,
data = (byte*)NativeMemory.Alloc(maskImageByteSize)
};
new Span<byte>(_maskImage.data, (int)maskImageByteSize).Fill(byte.MaxValue);
}
Native.Types.sd_slg_params_t slg = new()
{
layers = _slgLayers,
layer_count = (uint)managed.Guidance.Slg.Layers.Length,
layer_start = managed.Guidance.Slg.SkipLayerStart,
layer_end = managed.Guidance.Slg.SkipLayerEnd,
scale = managed.Guidance.Slg.Scale,
};
Native.Types.sd_guidance_params_t guidance = new()
{
txt_cfg = managed.Guidance.TxtCfg,
img_cfg = managed.Guidance.ImgCfg,
min_cfg = managed.Guidance.MinCfg,
distilled_guidance = managed.Guidance.DistilledGuidance,
slg = slg
};
_imgGenParams = new Native.Types.sd_img_gen_params_t
{
prompt = AnsiStringMarshaller.ConvertToUnmanaged(managed.Prompt),
negative_prompt = AnsiStringMarshaller.ConvertToUnmanaged(managed.NegativePrompt),
clip_skip = managed.ClipSkip,
guidance = guidance,
init_image = _initImage,
ref_images = _refImages,
ref_images_count = managed.RefImages?.Length ?? 0,
mask_image = _maskImage,
width = managed.Width,
height = managed.Height,
sample_method = managed.SampleMethod,
sample_steps = managed.SampleSteps,
eta = managed.Eta,
strength = managed.Strength,
seed = managed.Seed,
batch_count = 1,
control_cond = _controlNetImage,
control_strength = managed.ControlNet.Strength,
style_strength = managed.PhotoMaker.StyleStrength,
normalize_input = (sbyte)(managed.PhotoMaker.NormalizeInput ? 1 : 0),
input_id_images_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.PhotoMaker.InputIdImageDirectory),
};
}
public Native.Types.sd_img_gen_params_t ToUnmanaged() => _imgGenParams;
public void Free()
{
AnsiStringMarshaller.Free(_imgGenParams.prompt);
AnsiStringMarshaller.Free(_imgGenParams.negative_prompt);
AnsiStringMarshaller.Free(_imgGenParams.input_id_images_path);
_initImage.Free();
_maskImage.Free();
if (_refImages != null)
ImageHelper.Free(_refImages, _imgGenParams.ref_images_count);
if (_controlNetImage != null)
ImageHelper.Free(_controlNetImage, 1);
if (_slgLayers != null)
NativeMemory.Free(_slgLayers);
}
}
internal ref struct ImageGenerationParameterMarshallerRef()
{
private ImageGenerationParameterMarshallerIn _inMarshaller = new();
private ImageGenerationParameter _parameter;
public void FromManaged(ImageGenerationParameter managed) => _inMarshaller.FromManaged(managed);
public Native.Types.sd_img_gen_params_t ToUnmanaged() => _inMarshaller.ToUnmanaged();
public void FromUnmanaged(Native.Types.sd_img_gen_params_t unmanaged) => _parameter = ConvertToManaged(unmanaged);
public ImageGenerationParameter ToManaged() => _parameter;
public void Free() => _inMarshaller.Free();
}
}

24
StableDiffusion.NET/Native/Marshaller/ImageMarshaller.cs Normal file
View File

@ -0,0 +1,24 @@
using HPPH;
using System.Runtime.InteropServices.Marshalling;
namespace StableDiffusion.NET;
[CustomMarshaller(typeof(IImage), MarshalMode.ManagedToUnmanagedIn, typeof(ImageMarshallerIn))]
[CustomMarshaller(typeof(Image<ColorRGB>), MarshalMode.ManagedToUnmanagedOut, typeof(ImageMarshaller))]
internal static class ImageMarshaller
{
public static Image<ColorRGB> ConvertToManaged(Native.Types.sd_image_t unmanaged) => unmanaged.ToImage();
public static void Free(Native.Types.sd_image_t unmanaged) => unmanaged.Free();
internal ref struct ImageMarshallerIn
{
private Native.Types.sd_image_t _image;
public void FromManaged(IImage managed) => _image = managed.ToSdImage();
public Native.Types.sd_image_t ToUnmanaged() => _image;
public void Free() => _image.Free();
}
}

2
StableDiffusion.NET/Native/Native.Load.cs
View File

@ -38,7 +38,7 @@ internal static partial class Native
return false; return false;
} }
private static nint ResolveDllImport(string libraryname, Assembly assembly, DllImportSearchPath? searchpath) private static nint ResolveDllImport(string libraryname, Assembly _, DllImportSearchPath? __)
{ {
if (libraryname != LIB_NAME) return nint.Zero; if (libraryname != LIB_NAME) return nint.Zero;
if (_loadedLibraryHandle != nint.Zero) return _loadedLibraryHandle; if (_loadedLibraryHandle != nint.Zero) return _loadedLibraryHandle;

409
StableDiffusion.NET/Native/Native.cs
View File

@ -2,8 +2,11 @@
#pragma warning disable IDE1006 #pragma warning disable IDE1006
// ReSharper disable InconsistentNaming // ReSharper disable InconsistentNaming
// ReSharper disable ArrangeTypeMemberModifiers // ReSharper disable ArrangeTypeMemberModifiers
// ReSharper disable UseSymbolAlias
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using System.Runtime.InteropServices.Marshalling;
using HPPH;
namespace StableDiffusion.NET; namespace StableDiffusion.NET;
@ -12,6 +15,17 @@ using sample_method_t = Sampler;
using schedule_t = Schedule; using schedule_t = Schedule;
using sd_type_t = Quantization; using sd_type_t = Quantization;
using sd_log_level_t = LogLevel; using sd_log_level_t = LogLevel;
using uint32_t = uint;
using uint8_t = byte;
using int64_t = long;
using size_t = nuint;
using int32_t = int;
using sd_ctx_params_t = DiffusionModelParameter;
using sd_img_gen_params_t = ImageGenerationParameter;
using sd_vid_gen_params_t = Native.Types.sd_vid_gen_params_t;
using sd_image_t = Native.Types.sd_image_t;
using sd_ctx_t = Native.Types.sd_ctx_t;
using upscaler_ctx_t = Native.Types.upscaler_ctx_t;
internal unsafe partial class Native internal unsafe partial class Native
{ {
@ -21,6 +35,116 @@ internal unsafe partial class Native
#endregion #endregion
internal static class Types
{
[StructLayout(LayoutKind.Sequential)]
internal struct sd_ctx_params_t
{
public byte* model_path;
public byte* clip_l_path;
public byte* clip_g_path;
public byte* t5xxl_path;
public byte* diffusion_model_path;
public byte* vae_path;
public byte* taesd_path;
public byte* control_net_path;
public byte* lora_model_dir;
public byte* embedding_dir;
public byte* stacked_id_embed_dir;
public sbyte vae_decode_only;
public sbyte vae_tiling;
public sbyte free_params_immediately;
public int n_threads;
public sd_type_t wtype;
public rng_type_t rng_type;
public schedule_t schedule;
public sbyte keep_clip_on_cpu;
public sbyte keep_control_net_on_cpu;
public sbyte keep_vae_on_cpu;
public sbyte diffusion_flash_attn;
public sbyte diffusion_conv_direct;
public sbyte vae_conv_direct;
public sbyte chroma_use_dit_mask;
public sbyte chroma_use_t5_mask;
public int chroma_t5_mask_pad;
}
[StructLayout(LayoutKind.Sequential)]
internal struct sd_image_t
{
public uint32_t width;
public uint32_t height;
public uint32_t channel;
public uint8_t* data;
}
[StructLayout(LayoutKind.Sequential)]
internal struct sd_slg_params_t
{
public int* layers;
public size_t layer_count;
public float layer_start;
public float layer_end;
public float scale;
}
[StructLayout(LayoutKind.Sequential)]
internal struct sd_guidance_params_t
{
public float txt_cfg;
public float img_cfg;
public float min_cfg;
public float distilled_guidance;
public sd_slg_params_t slg;
}
[StructLayout(LayoutKind.Sequential)]
internal struct sd_img_gen_params_t
{
public byte* prompt;
public byte* negative_prompt;
public int clip_skip;
public sd_guidance_params_t guidance;
public sd_image_t init_image;
public sd_image_t* ref_images;
public int ref_images_count;
public sd_image_t mask_image;
public int width;
public int height;
public sample_method_t sample_method;
public int sample_steps;
public float eta;
public float strength;
public int64_t seed;
public int batch_count;
public sd_image_t* control_cond;
public float control_strength;
public float style_strength;
public sbyte normalize_input;
public byte* input_id_images_path;
}
[StructLayout(LayoutKind.Sequential)]
internal struct sd_vid_gen_params_t
{
public sd_image_t init_image;
public int width;
public int height;
public sd_guidance_params_t guidance;
public sample_method_t sample_method;
public int sample_steps;
public float strength;
public int64_t seed;
public int video_frames;
public int motion_bucket_id;
public int fps;
public float augmentation_level;
}
internal struct sd_ctx_t;
internal struct upscaler_ctx_t;
}
#region Delegates #region Delegates
internal delegate void sd_log_cb_t(sd_log_level_t level, [MarshalAs(UnmanagedType.LPStr)] string text, void* data); internal delegate void sd_log_cb_t(sd_log_level_t level, [MarshalAs(UnmanagedType.LPStr)] string text, void* data);
@ -28,179 +152,7 @@ internal unsafe partial class Native
#endregion #endregion
#region DLL-Import #region Methods
internal struct sd_ctx_t;
internal struct upscaler_ctx_t;
[StructLayout(LayoutKind.Sequential)]
internal struct sd_image_t
{
internal uint width;
internal uint height;
internal uint channel;
internal byte* data;
}
[LibraryImport(LIB_NAME, EntryPoint = "get_num_physical_cores")]
internal static partial int get_num_physical_cores();
[LibraryImport(LIB_NAME, EntryPoint = "sd_get_system_info")]
internal static partial void* sd_get_system_info();
[LibraryImport(LIB_NAME, EntryPoint = "new_sd_ctx")]
internal static partial sd_ctx_t* new_sd_ctx([MarshalAs(UnmanagedType.LPStr)] string model_path,
[MarshalAs(UnmanagedType.LPStr)] string clip_l_path,
[MarshalAs(UnmanagedType.LPStr)] string clip_g_path,
[MarshalAs(UnmanagedType.LPStr)] string t5xxl_path,
[MarshalAs(UnmanagedType.LPStr)] string diffusion_model_path,
[MarshalAs(UnmanagedType.LPStr)] string vae_path,
[MarshalAs(UnmanagedType.LPStr)] string taesd_path,
[MarshalAs(UnmanagedType.LPStr)] string control_net_path_c_str,
[MarshalAs(UnmanagedType.LPStr)] string lora_model_dir,
[MarshalAs(UnmanagedType.LPStr)] string embed_dir_c_str,
[MarshalAs(UnmanagedType.LPStr)] string stacked_id_embed_dir_c_str,
[MarshalAs(UnmanagedType.I1)] bool vae_decode_only,
[MarshalAs(UnmanagedType.I1)] bool vae_tiling,
[MarshalAs(UnmanagedType.I1)] bool free_params_immediately,
int n_threads,
sd_type_t wtype,
rng_type_t rng_type,
schedule_t s,
[MarshalAs(UnmanagedType.I1)] bool keep_clip_on_cpu,
[MarshalAs(UnmanagedType.I1)] bool keep_control_net_cpu,
[MarshalAs(UnmanagedType.I1)] bool keep_vae_on_cpu,
[MarshalAs(UnmanagedType.I1)] bool diffusion_flash_attn,
[MarshalAs(UnmanagedType.I1)] bool chroma_use_dit_mask,
[MarshalAs(UnmanagedType.I1)] bool chroma_use_t5_mask,
int chroma_t5_mask_pad);
[LibraryImport(LIB_NAME, EntryPoint = "free_sd_ctx")]
internal static partial void free_sd_ctx(sd_ctx_t* sd_ctx);
[LibraryImport(LIB_NAME, EntryPoint = "txt2img")]
internal static partial sd_image_t* txt2img(sd_ctx_t* sd_ctx,
[MarshalAs(UnmanagedType.LPStr)] string prompt,
[MarshalAs(UnmanagedType.LPStr)] string negative_prompt,
int clip_skip,
float cfg_scale,
float guidance,
float eta,
int width,
int height,
sample_method_t sample_method,
int sample_steps,
long seed,
int batch_count,
sd_image_t* control_cond,
float control_strength,
float style_strength,
[MarshalAs(UnmanagedType.I1)] bool normalize_input,
[MarshalAs(UnmanagedType.LPStr)] string input_id_images_path,
in int[] skip_layers,
int skip_layers_count,
float slg_scale,
float skip_layer_start,
float skip_layer_end);
[LibraryImport(LIB_NAME, EntryPoint = "img2img")]
internal static partial sd_image_t* img2img(sd_ctx_t* sd_ctx,
sd_image_t init_image,
sd_image_t mask_image,
[MarshalAs(UnmanagedType.LPStr)] string prompt,
[MarshalAs(UnmanagedType.LPStr)] string negative_prompt,
int clip_skip,
float cfg_scale,
float guidance,
int width,
int height,
sample_method_t sample_method,
int sample_steps,
float strength,
long seed,
int batch_count,
sd_image_t* control_cond,
float control_strength,
float style_strength,
[MarshalAs(UnmanagedType.I1)] bool normalize_input,
[MarshalAs(UnmanagedType.LPStr)] string input_id_images_path,
in int[] skip_layers,
int skip_layers_count,
float slg_scale,
float skip_layer_start,
float skip_layer_end);
[LibraryImport(LIB_NAME, EntryPoint = "img2vid")]
internal static partial sd_image_t* img2vid(sd_ctx_t* sd_ctx,
sd_image_t init_image,
int width,
int height,
int video_frames,
int motion_bucket_id,
int fps,
float augmentation_level,
float min_cfg,
float cfg_scale,
sample_method_t sample_method,
int sample_steps,
float strength,
long seed);
[LibraryImport(LIB_NAME, EntryPoint = "edit")]
internal static partial sd_image_t* edit(sd_ctx_t* sd_ctx,
sd_image_t* ref_images,
int ref_images_count,
[MarshalAs(UnmanagedType.LPStr)] string prompt,
[MarshalAs(UnmanagedType.LPStr)] string negative_prompt,
int clip_skip,
float cfg_scale,
float guidance,
float eta,
int width,
int height,
sample_method_t sample_method,
int sample_steps,
float strength,
long seed,
int batch_count,
sd_image_t* control_cond,
float control_strength,
float style_strength,
[MarshalAs(UnmanagedType.I1)] bool normalize_input,
in int[] skip_layers,
int skip_layers_count,
float slg_scale,
float skip_layer_start,
float skip_layer_end);
[LibraryImport(LIB_NAME, EntryPoint = "new_upscaler_ctx")]
internal static partial upscaler_ctx_t* new_upscaler_ctx([MarshalAs(UnmanagedType.LPStr)] string esrgan_path,
int n_threads,
sd_type_t wtype);
[LibraryImport(LIB_NAME, EntryPoint = "free_upscaler_ctx")]
internal static partial void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx);
[LibraryImport(LIB_NAME, EntryPoint = "upscale")]
internal static partial sd_image_t upscale(upscaler_ctx_t* upscaler_ctx,
sd_image_t input_image,
int upscale_factor);
[LibraryImport(LIB_NAME, EntryPoint = "convert")]
internal static partial void convert([MarshalAs(UnmanagedType.LPStr)] string input_path,
[MarshalAs(UnmanagedType.LPStr)] string vae_path,
[MarshalAs(UnmanagedType.LPStr)] string output_path,
sd_type_t output_type);
[LibraryImport(LIB_NAME, EntryPoint = "preprocess_canny")]
internal static partial byte* preprocess_canny(byte* img,
int width,
int height,
float high_threshold,
float low_threshold,
float weak,
float strong,
[MarshalAs(UnmanagedType.I1)] bool inverse);
[LibraryImport(LIB_NAME, EntryPoint = "sd_set_log_callback")] [LibraryImport(LIB_NAME, EntryPoint = "sd_set_log_callback")]
internal static partial void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data); internal static partial void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data);
@ -208,5 +160,116 @@ internal unsafe partial class Native
[LibraryImport(LIB_NAME, EntryPoint = "sd_set_progress_callback")] [LibraryImport(LIB_NAME, EntryPoint = "sd_set_progress_callback")]
internal static partial void sd_set_progress_callback(sd_progress_cb_t cb, void* data); internal static partial void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
[LibraryImport(LIB_NAME, EntryPoint = "get_num_physical_cores")]
internal static partial int32_t get_num_physical_cores();
[LibraryImport(LIB_NAME, EntryPoint = "sd_get_system_info")]
[return: MarshalAs(UnmanagedType.LPStr)]
internal static partial string sd_get_system_info();
//
[LibraryImport(LIB_NAME, EntryPoint = "sd_type_name")]
[return: MarshalAs(UnmanagedType.LPStr)]
internal static partial string sd_type_name(sd_type_t type);
[LibraryImport(LIB_NAME, EntryPoint = "str_to_sd_type")]
internal static partial sd_type_t str_to_sd_type([MarshalAs(UnmanagedType.LPStr)] string str);
[LibraryImport(LIB_NAME, EntryPoint = "sd_rng_type_name")]
[return: MarshalAs(UnmanagedType.LPStr)]
internal static partial string sd_rng_type_name(rng_type_t rng_type);
[LibraryImport(LIB_NAME, EntryPoint = "str_to_rng_type")]
internal static partial rng_type_t str_to_rng_type([MarshalAs(UnmanagedType.LPStr)] string str);
[LibraryImport(LIB_NAME, EntryPoint = "sd_sample_method_name")]
[return: MarshalAs(UnmanagedType.LPStr)]
internal static partial string sd_sample_method_name(sample_method_t sample_method);
[LibraryImport(LIB_NAME, EntryPoint = "str_to_sample_method")]
internal static partial sample_method_t str_to_sample_method([MarshalAs(UnmanagedType.LPStr)] string str);
[LibraryImport(LIB_NAME, EntryPoint = "sd_schedule_name")]
[return: MarshalAs(UnmanagedType.LPStr)]
internal static partial string sd_schedule_name(schedule_t schedule);
[LibraryImport(LIB_NAME, EntryPoint = "str_to_schedule")]
internal static partial schedule_t str_to_schedule([MarshalAs(UnmanagedType.LPStr)] string str);
//
[LibraryImport(LIB_NAME, EntryPoint = "sd_ctx_params_init")]
internal static partial void sd_ctx_params_init([MarshalUsing(typeof(DiffusionModelParameterMarshaller))] ref sd_ctx_params_t sd_ctx_params);
[LibraryImport(LIB_NAME, EntryPoint = "sd_ctx_params_to_str")]
[return: MarshalAs(UnmanagedType.LPStr)]
internal static partial string sd_ctx_params_to_str([MarshalUsing(typeof(DiffusionModelParameterMarshaller))] in sd_ctx_params_t sd_ctx_params);
//
[LibraryImport(LIB_NAME, EntryPoint = "new_sd_ctx")]
internal static partial sd_ctx_t* new_sd_ctx([MarshalUsing(typeof(DiffusionModelParameterMarshaller))] in sd_ctx_params_t sd_ctx_params);
[LibraryImport(LIB_NAME, EntryPoint = "free_sd_ctx")]
internal static partial void free_sd_ctx(sd_ctx_t* sd_ctx);
//
[LibraryImport(LIB_NAME, EntryPoint = "sd_img_gen_params_init")]
internal static partial void sd_img_gen_params_init([MarshalUsing(typeof(ImageGenerationParameterMarshaller))] ref sd_img_gen_params_t sd_img_gen_params);
[LibraryImport(LIB_NAME, EntryPoint = "sd_img_gen_params_to_str")]
[return: MarshalAs(UnmanagedType.LPStr)]
internal static partial string sd_img_gen_params_to_str([MarshalUsing(typeof(ImageGenerationParameterMarshaller))] in sd_img_gen_params_t sd_img_gen_params);
[LibraryImport(LIB_NAME, EntryPoint = "generate_image")]
internal static partial sd_image_t* generate_image(sd_ctx_t* sd_ctx, [MarshalUsing(typeof(ImageGenerationParameterMarshaller))] in sd_img_gen_params_t sd_img_gen_params);
//
[LibraryImport(LIB_NAME, EntryPoint = "sd_vid_gen_params_init")]
internal static partial void sd_vid_gen_params_init(ref sd_vid_gen_params_t sd_vid_gen_params);
[LibraryImport(LIB_NAME, EntryPoint = "generate_video")]
[return: MarshalUsing(typeof(ImageMarshaller))]
internal static partial sd_image_t* generate_video(sd_ctx_t* sd_ctx, in sd_vid_gen_params_t sd_vid_gen_params); // broken
//
[LibraryImport(LIB_NAME, EntryPoint = "new_upscaler_ctx")]
internal static partial upscaler_ctx_t* new_upscaler_ctx([MarshalAs(UnmanagedType.LPStr)] string esrgan_path, int n_threads, [MarshalAs(UnmanagedType.I1)] bool direct);
[LibraryImport(LIB_NAME, EntryPoint = "free_upscaler_ctx")]
internal static partial void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx);
//
[LibraryImport(LIB_NAME, EntryPoint = "upscale")]
[return: MarshalUsing(typeof(ImageMarshaller))]
internal static partial Image<ColorRGB> upscale(upscaler_ctx_t* upscaler_ctx, [MarshalUsing(typeof(ImageMarshaller))] IImage input_image, uint32_t upscale_factor);
//
[LibraryImport(LIB_NAME, EntryPoint = "convert")]
[return: MarshalAs(UnmanagedType.I1)]
internal static partial bool convert([MarshalAs(UnmanagedType.LPStr)] string input_path,
[MarshalAs(UnmanagedType.LPStr)] string vae_path,
[MarshalAs(UnmanagedType.LPStr)] string output_path,
sd_type_t output_type,
[MarshalAs(UnmanagedType.LPStr)] string tensor_type_rules);
//
[LibraryImport(LIB_NAME, EntryPoint = "preprocess_canny")]
internal static partial uint8_t* preprocess_canny(uint8_t* img,
int width,
int height,
float high_threshold,
float low_threshold,
float weak,
float strong,
[MarshalAs(UnmanagedType.I1)] bool inverse);
#endregion #endregion
} }

2
StableDiffusion.NET/StableDiffusion.NET.csproj
View File

@ -56,6 +56,6 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="HPPH" Version="1.0.0" /> <PackageReference Include="HPPH" Version="1.0.0" />
<PackageReference Include="JetBrains.Annotations" Version="2024.2.0" /> <PackageReference Include="JetBrains.Annotations" Version="2025.2.0" />
</ItemGroup> </ItemGroup>
</Project> </Project>

3
StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings
View File

@ -13,4 +13,5 @@
<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cparameter/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cparameter/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cparameter_005Cextensions/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cparameter_005Cextensions/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cparameter_005Cinterfaces/@EntryIndexedValue">True</s:Boolean> <s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cparameter_005Cinterfaces/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=native/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary> <s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=native/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=native_005Cmarshaller/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

57
StableDiffusion.NET/StableDiffusionCpp.cs
View File

@ -1,5 +1,6 @@
using System; using System;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using HPPH;
using JetBrains.Annotations; using JetBrains.Annotations;
namespace StableDiffusion.NET; namespace StableDiffusion.NET;
@ -10,8 +11,8 @@ public static unsafe class StableDiffusionCpp
#region Properties & Fields #region Properties & Fields
// ReSharper disable NotAccessedField.Local - They are important, the delegate can be collected if it's not stored! // ReSharper disable NotAccessedField.Local - They are important, the delegate can be collected if it's not stored!
private static readonly Native.sd_log_cb_t LOG_CALLBACK; private static Native.sd_log_cb_t LOG_CALLBACK;
private static readonly Native.sd_progress_cb_t PROGRESS_CALLBACK; private static Native.sd_progress_cb_t PROGRESS_CALLBACK;
// ReSharper restore NotAccessedField.Local // ReSharper restore NotAccessedField.Local
#endregion #endregion
@ -23,38 +24,60 @@ public static unsafe class StableDiffusionCpp
#endregion #endregion
#region Constructors #region Methods
static StableDiffusionCpp() public static bool LoadNativeLibrary(string libraryPath) => Native.LoadNativeLibrary(libraryPath);
public static void InitializeEvents()
{ {
Native.sd_set_log_callback(LOG_CALLBACK = OnNativeLog, null); Native.sd_set_log_callback(LOG_CALLBACK = OnNativeLog, null);
Native.sd_set_progress_callback(PROGRESS_CALLBACK = OnNativeProgress, null); Native.sd_set_progress_callback(PROGRESS_CALLBACK = OnNativeProgress, null);
} }
#endregion public static void Convert(string modelPath, string vaePath, Quantization quantization, string outputPath, string tensorTypeRules = "")
#region Methods
public static bool LoadNativeLibrary(string libraryPath) => Native.LoadNativeLibrary(libraryPath);
public static void Convert(string modelPath, string vaePath, Quantization quantization, string outputPath)
{ {
ArgumentException.ThrowIfNullOrWhiteSpace(nameof(modelPath)); ArgumentException.ThrowIfNullOrWhiteSpace(nameof(modelPath));
ArgumentException.ThrowIfNullOrWhiteSpace(nameof(outputPath)); ArgumentException.ThrowIfNullOrWhiteSpace(nameof(outputPath));
ArgumentNullException.ThrowIfNull(vaePath); ArgumentNullException.ThrowIfNull(vaePath);
if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization)); if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization));
Native.convert(modelPath, vaePath, outputPath, quantization); Native.convert(modelPath, vaePath, outputPath, quantization, tensorTypeRules);
} }
public static string GetSystemInfo() public static string GetSystemInfo() => Native.sd_get_system_info();
{
void* s = Native.sd_get_system_info();
return Marshal.PtrToStringUTF8((nint)s) ?? "";
}
public static int GetNumPhysicalCores() => Native.get_num_physical_cores(); public static int GetNumPhysicalCores() => Native.get_num_physical_cores();
public static Image<ColorRGB> PreprocessCanny(CannyParameter parameter)
{
parameter.Validate();
IImage<ColorRGB> controlImage = parameter.Image as IImage<ColorRGB> ?? parameter.Image!.ConvertTo<ColorRGB>();
byte[] controlImageData = controlImage.ToRawArray();
fixed (byte* controlImagePtr = controlImageData)
{
byte* result = Native.preprocess_canny(controlImagePtr,
controlImage.Width,
controlImage.Height,
parameter.HighThreshold,
parameter.LowThreshold,
parameter.Weak,
parameter.Strong,
parameter.Inverse);
try
{
return Image<ColorRGB>.Create(new ReadOnlySpan<ColorRGB>(result, controlImageData.Length),
controlImage.Width, controlImage.Height);
}
finally
{
NativeMemory.Free(result);
}
}
}
private static void OnNativeLog(LogLevel level, string text, void* data) private static void OnNativeLog(LogLevel level, string text, void* data)
{ {
try try