diff --git a/README.md b/README.md
index 16eb363..9c48634 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,9 @@
Based on https://github.com/leejet/stable-diffusion.cpp
+> At least for me the current version of stable-diffusion.cpp has really bad tiling issues.
+If you experience them too, I'd recommend using https://github.com/DarthAffe/StableDiffusion.NET/releases/tag/c6071fa until that's fixed.
+
## Usage
### Setup
Run `build.bat` to build the native libs (modify params like CUDA-builds if needed)
diff --git a/StableDiffusion.NET.sln.DotSettings b/StableDiffusion.NET.sln.DotSettings
new file mode 100644
index 0000000..3769d7c
--- /dev/null
+++ b/StableDiffusion.NET.sln.DotSettings
@@ -0,0 +1,2 @@
+
+ CPU
\ No newline at end of file
diff --git a/StableDiffusion.NET/ModelParameter.cs b/StableDiffusion.NET/ModelParameter.cs
index dff426b..22ee931 100644
--- a/StableDiffusion.NET/ModelParameter.cs
+++ b/StableDiffusion.NET/ModelParameter.cs
@@ -11,6 +11,9 @@ public class ModelParameter
public string LoraModelDir { get; set; } = string.Empty;
public RngType RngType { get; set; } = RngType.Standard;
public string VaePath { get; set; } = string.Empty;
+ public string ControlNetPath { get; set; } = string.Empty;
+ public string EmbeddingsDirectory { get; set; } = string.Empty;
+ public bool KeepControlNetOnCPU { get; set; } = false;
//TODO DarthAffe 01.01.2024: K-Quants doesn't seem to work so far
public Quantization Quantization { get; set; } = Quantization.F16;
diff --git a/StableDiffusion.NET/Native.cs b/StableDiffusion.NET/Native.cs
index df8a2b0..0e74e81 100644
--- a/StableDiffusion.NET/Native.cs
+++ b/StableDiffusion.NET/Native.cs
@@ -51,14 +51,17 @@ internal unsafe partial class Native
internal static partial sd_ctx_t* new_sd_ctx([MarshalAs(UnmanagedType.LPStr)] string model_path,
[MarshalAs(UnmanagedType.LPStr)] string vae_path,
[MarshalAs(UnmanagedType.LPStr)] string taesd_path,
+ [MarshalAs(UnmanagedType.LPStr)] string control_net_path_c_str,
[MarshalAs(UnmanagedType.LPStr)] string lora_model_dir,
+ [MarshalAs(UnmanagedType.LPStr)] string embed_dir_c_str,
[MarshalAs(UnmanagedType.I1)] bool vae_decode_only,
[MarshalAs(UnmanagedType.I1)] bool vae_tiling,
[MarshalAs(UnmanagedType.I1)] bool free_params_immediately,
int n_threads,
sd_type_t wtype,
rng_type_t rng_type,
- schedule_t s);
+ schedule_t s,
+ [MarshalAs(UnmanagedType.I1)] bool keep_control_net_cpu);
[LibraryImport(LIB_NAME, EntryPoint = "free_sd_ctx")]
internal static partial void free_sd_ctx(sd_ctx_t* sd_ctx);
@@ -74,7 +77,9 @@ internal unsafe partial class Native
sample_method_t sample_method,
int sample_steps,
long seed,
- int batch_count);
+ int batch_count,
+ sd_image_t* control_cond,
+ float control_strength);
[LibraryImport(LIB_NAME, EntryPoint = "img2img")]
internal static partial sd_image_t* img2img(sd_ctx_t* sd_ctx,
diff --git a/StableDiffusion.NET/StableDiffusionModel.cs b/StableDiffusion.NET/StableDiffusionModel.cs
index 3b18976..e7e579c 100644
--- a/StableDiffusion.NET/StableDiffusionModel.cs
+++ b/StableDiffusion.NET/StableDiffusionModel.cs
@@ -51,14 +51,17 @@ public sealed unsafe class StableDiffusionModel : IDisposable
_ctx = Native.new_sd_ctx(_modelPath,
_parameter.VaePath,
_parameter.TaesdPath,
+ _parameter.ControlNetPath,
_parameter.LoraModelDir,
+ _parameter.EmbeddingsDirectory,
_parameter.VaeDecodeOnly,
_parameter.VaeTiling,
false,
_parameter.ThreadCount,
_parameter.Quantization,
_parameter.RngType,
- _parameter.Schedule);
+ _parameter.Schedule,
+ _parameter.KeepControlNetOnCPU);
if (_ctx == null) throw new NullReferenceException("Failed to initialize Stable Diffusion");
if (_upscalerParameter != null)
@@ -74,23 +77,58 @@ public sealed unsafe class StableDiffusionModel : IDisposable
{
ObjectDisposedException.ThrowIf(_disposed, this);
- Native.sd_image_t* result = Native.txt2img(_ctx,
- prompt,
- parameter.NegativePrompt,
- parameter.ClipSkip,
- parameter.CfgScale,
- parameter.Width,
- parameter.Height,
- parameter.SampleMethod,
- parameter.SampleSteps,
- parameter.Seed,
- 1);
+ Native.sd_image_t* result;
+ if ((parameter.ControlNetImage == null) || (parameter.ControlNetImage.Length == 0))
+ {
+ result = Native.txt2img(_ctx,
+ prompt,
+ parameter.NegativePrompt,
+ parameter.ClipSkip,
+ parameter.CfgScale,
+ parameter.Width,
+ parameter.Height,
+ parameter.SampleMethod,
+ parameter.SampleSteps,
+ parameter.Seed,
+ 1,
+ null,
+ 0);
+ }
+ else
+ {
+ fixed (byte* imagePtr = parameter.ControlNetImage)
+ {
+ Native.sd_image_t controlNetImage = new()
+ {
+ width = (uint)parameter.Width,
+ height = (uint)parameter.Height,
+ channel = 3,
+ data = imagePtr
+ };
+
+ result = Native.txt2img(_ctx,
+ prompt,
+ parameter.NegativePrompt,
+ parameter.ClipSkip,
+ parameter.CfgScale,
+ parameter.Width,
+ parameter.Height,
+ parameter.SampleMethod,
+ parameter.SampleSteps,
+ parameter.Seed,
+ 1,
+ &controlNetImage,
+ parameter.ControlNetStrength);
+ }
+ }
return new StableDiffusionImage(result);
}
public StableDiffusionImage ImageToImage(string prompt, in ReadOnlySpan image, StableDiffusionParameter parameter)
{
+ ObjectDisposedException.ThrowIf(_disposed, this);
+
fixed (byte* imagePtr = image)
{
Native.sd_image_t img = new()
@@ -110,6 +148,8 @@ public sealed unsafe class StableDiffusionModel : IDisposable
private StableDiffusionImage ImageToImage(string prompt, Native.sd_image_t image, StableDiffusionParameter parameter)
{
+ ObjectDisposedException.ThrowIf(_disposed, this);
+
Native.sd_image_t* result = Native.img2img(_ctx,
image,
prompt,
diff --git a/StableDiffusion.NET/StableDiffusionParameter.cs b/StableDiffusion.NET/StableDiffusionParameter.cs
index 886a908..3a512d4 100644
--- a/StableDiffusion.NET/StableDiffusionParameter.cs
+++ b/StableDiffusion.NET/StableDiffusionParameter.cs
@@ -13,6 +13,8 @@ public sealed class StableDiffusionParameter
public long Seed { get; set; } = -1;
public float Strength { get; set; } = 0.7f;
public int ClipSkip { get; set; } = -1;
+ public byte[]? ControlNetImage { get; set; } = null;
+ public float ControlNetStrength { get; set; } = 0.9f;
#endregion
}
\ No newline at end of file
diff --git a/build.bat b/build.bat
index ea528c0..c3ef809 100644
--- a/build.bat
+++ b/build.bat
@@ -4,7 +4,7 @@ if not exist stable-diffusion.cpp (
cd stable-diffusion.cpp
git fetch
-git checkout c6071fa82fb1d0e688f75c9a3d870fe71d3a7a1d
+git checkout 36ec16ac9937d393d0ba8939640352cee430efb7
git submodule init
git submodule update