Updated stablediffusion.cpp to 1ce9470

README.md

@@ -2,9 +2,6 @@
 
 Based on https://github.com/leejet/stable-diffusion.cpp
 
-> At least for me the current version of stable-diffusion.cpp has really bad tiling issues.   
-If you experience them too, I'd recommend using https://github.com/DarthAffe/StableDiffusion.NET/releases/tag/c6071fa until that's fixed.
-
 ## Usage
 ### Setup
 Run `build.bat` to build the native libs (modify params like CUDA-builds if needed)

StableDiffusion.NET/Enums/Quantization.cs

@@ -1,4 +1,5 @@
-namespace StableDiffusion.NET;
+// ReSharper disable InconsistentNaming
+namespace StableDiffusion.NET;
 
 public enum Quantization
 {
@@ -17,7 +18,15 @@ public enum Quantization
     Q4_K = 12,
     Q5_K = 13,
     Q6_K = 14,
-    Q8_K = 15,
+    Q8_K = 15, 
+    IQ2_XXS = 16,
+    IQ2_XS = 17,
+    IQ3_XXS = 18,
+    IQ1_S = 19,
+    IQ4_NL = 20,
+    IQ3_S = 21,
+    IQ2_S = 22,
+    IQ4_XS = 23,
     I8,
     I16,
     I32,

StableDiffusion.NET/EventArgs/StableDiffusionProgressEventArgs.cs

@@ -0,0 +1,15 @@
+namespace StableDiffusion.NET;
+
+public class StableDiffusionProgressEventArgs(int step, int steps, float time)
+{
+    #region Properties & Fields
+
+    public int Step { get; } = step;
+    public int Steps { get; } = steps;
+    public float Time { get; } = time;
+
+    public double Progress => (double)Step / Steps;
+    public float IterationsPerSecond => 1f / Time;
+
+    #endregion
+}

StableDiffusion.NET/Native.cs

@@ -24,6 +24,7 @@ internal unsafe partial class Native
     #region Delegates
 
     internal delegate void sd_log_cb_t(sd_log_level_t level, [MarshalAs(UnmanagedType.LPStr)] string text, void* data);
+    internal delegate void sd_progress_cb_t(int step, int steps, float time, void* data);
 
     #endregion
 
@@ -96,6 +97,22 @@ internal unsafe partial class Native
                                                 long seed,
                                                 int batch_count);
 
+    [LibraryImport(LIB_NAME, EntryPoint = "img2vid")]
+    internal static partial sd_image_t* img2vid(sd_ctx_t* sd_ctx,
+                                                sd_image_t init_image,
+                                                int width,
+                                                int height,
+                                                int video_frames,
+                                                int motion_bucket_id,
+                                                int fps,
+                                                float augmentation_level,
+                                                float min_cfg,
+                                                float cfg_scale,
+                                                sample_method_t sample_method,
+                                                int sample_steps,
+                                                float strength,
+                                                long seed);
+
     [LibraryImport(LIB_NAME, EntryPoint = "new_upscaler_ctx")]
     internal static partial upscaler_ctx_t* new_upscaler_ctx([MarshalAs(UnmanagedType.LPStr)] string esrgan_path,
                                                              int n_threads,
@@ -115,8 +132,21 @@ internal unsafe partial class Native
                                          [MarshalAs(UnmanagedType.LPStr)] string output_path,
                                          sd_type_t output_type);
 
+    [LibraryImport(LIB_NAME, EntryPoint = "convert")]
+    internal static partial byte* preprocess_canny(byte* img,
+                                                   int width,
+                                                   int height,
+                                                   float high_threshold,
+                                                   float low_threshold,
+                                                   float weak,
+                                                   float strong,
+                                                   [MarshalAs(UnmanagedType.I1)] bool inverse);
+
     [LibraryImport(LIB_NAME, EntryPoint = "sd_set_log_callback")]
     internal static partial void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data);
 
+    [LibraryImport(LIB_NAME, EntryPoint = "sd_set_progress_callback")]
+    internal static partial void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
+
     #endregion
 }

StableDiffusion.NET/StableDiffusionModel.cs

@@ -21,6 +21,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable
     #region Events
 
     public static event EventHandler<StableDiffusionLogEventArgs>? Log;
+    public static event EventHandler<StableDiffusionProgressEventArgs>? Progress;
 
     #endregion
 
@@ -29,6 +30,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable
     static StableDiffusionModel()
     {
         Native.sd_set_log_callback(OnNativeLog, null);
+        Native.sd_set_progress_callback(OnNativeProgress, null);
     }
 
     public StableDiffusionModel(string modelPath, ModelParameter parameter, UpscalerModelParameter? upscalerParameter = null)
@@ -78,7 +80,71 @@ public sealed unsafe class StableDiffusionModel : IDisposable
         ObjectDisposedException.ThrowIf(_disposed, this);
 
         Native.sd_image_t* result;
-        if ((parameter.ControlNetImage == null) || (parameter.ControlNetImage.Length == 0))
+        if (parameter.ControlNet.IsEnabled)
+        {
+            fixed (byte* imagePtr = parameter.ControlNet.Image)
+            {
+
+                if (parameter.ControlNet.CannyPreprocess)
+                {
+                    Native.sd_image_t controlNetImage = new()
+                    {
+                        width = (uint)parameter.Width,
+                        height = (uint)parameter.Height,
+                        channel = 3,
+                        data = Native.preprocess_canny(imagePtr,
+                                                       parameter.Width,
+                                                       parameter.Height,
+                                                       parameter.ControlNet.CannyHighThreshold,
+                                                       parameter.ControlNet.CannyLowThreshold,
+                                                       parameter.ControlNet.CannyWeak,
+                                                       parameter.ControlNet.CannyStrong,
+                                                       parameter.ControlNet.CannyInverse)
+                    };
+
+                    result = Native.txt2img(_ctx,
+                                            prompt,
+                                            parameter.NegativePrompt,
+                                            parameter.ClipSkip,
+                                            parameter.CfgScale,
+                                            parameter.Width,
+                                            parameter.Height,
+                                            parameter.SampleMethod,
+                                            parameter.SampleSteps,
+                                            parameter.Seed,
+                                            1,
+                                            &controlNetImage,
+                                            parameter.ControlNet.Strength);
+
+                    Marshal.FreeHGlobal((nint)controlNetImage.data);
+                }
+                else
+                {
+                    Native.sd_image_t controlNetImage = new()
+                    {
+                        width = (uint)parameter.Width,
+                        height = (uint)parameter.Height,
+                        channel = 3,
+                        data = imagePtr
+                    };
+
+                    result = Native.txt2img(_ctx,
+                                            prompt,
+                                            parameter.NegativePrompt,
+                                            parameter.ClipSkip,
+                                            parameter.CfgScale,
+                                            parameter.Width,
+                                            parameter.Height,
+                                            parameter.SampleMethod,
+                                            parameter.SampleSteps,
+                                            parameter.Seed,
+                                            1,
+                                            &controlNetImage,
+                                            parameter.ControlNet.Strength);
+                }
+            }
+        }
+        else
         {
             result = Native.txt2img(_ctx,
                                     prompt,
@@ -94,33 +160,6 @@ public sealed unsafe class StableDiffusionModel : IDisposable
                                     null,
                                     0);
         }
-        else
-        {
-            fixed (byte* imagePtr = parameter.ControlNetImage)
-            {
-                Native.sd_image_t controlNetImage = new()
-                {
-                    width = (uint)parameter.Width,
-                    height = (uint)parameter.Height,
-                    channel = 3,
-                    data = imagePtr
-                };
-
-                result = Native.txt2img(_ctx,
-                                        prompt,
-                                        parameter.NegativePrompt,
-                                        parameter.ClipSkip,
-                                        parameter.CfgScale,
-                                        parameter.Width,
-                                        parameter.Height,
-                                        parameter.SampleMethod,
-                                        parameter.SampleSteps,
-                                        parameter.Seed,
-                                        1,
-                                        &controlNetImage,
-                                        parameter.ControlNetStrength);
-            }
-        }
 
         return new StableDiffusionImage(result);
     }
@@ -240,5 +279,14 @@ public sealed unsafe class StableDiffusionModel : IDisposable
         catch { /**/ }
     }
 
+    private static void OnNativeProgress(int step, int steps, float time, void* data)
+    {
+        try
+        {
+            Progress?.Invoke(null, new StableDiffusionProgressEventArgs(step, steps, time));
+        }
+        catch { /**/ }
+    }
+
     #endregion
 }

StableDiffusion.NET/StableDiffusionParameter.cs

@@ -13,8 +13,22 @@ public sealed class StableDiffusionParameter
     public long Seed { get; set; } = -1;
     public float Strength { get; set; } = 0.7f;
     public int ClipSkip { get; set; } = -1;
-    public byte[]? ControlNetImage { get; set; } = null;
+
-    public float ControlNetStrength { get; set; } = 0.9f;
+    public StableDiffusionControlNetParameter ControlNet { get; } = new();
 
     #endregion
+}
+
+public sealed class StableDiffusionControlNetParameter
+{
+    public bool IsEnabled => Image?.Length > 0;
+
+    public byte[]? Image { get; set; } = null;
+    public float Strength { get; set; } = 0.9f;
+    public bool CannyPreprocess { get; set; } = false;
+    public float CannyHighThreshold { get; set; } = 0.08f;
+    public float CannyLowThreshold { get; set; } = 0.08f;
+    public float CannyWeak { get; set; } = 0.8f;
+    public float CannyStrong { get; set; } = 1.0f;
+    public bool CannyInverse { get; set; } = false;
 }

build.bat

@@ -4,7 +4,7 @@ if not exist stable-diffusion.cpp (
 
 cd stable-diffusion.cpp
 git fetch
-git checkout 36ec16ac9937d393d0ba8939640352cee430efb7
+git checkout 1ce9470f27d480c6aa5d43c0af5b60db99454252
 git submodule init
 git submodule update
 
@@ -14,8 +14,29 @@ if not exist build (
 
 cd build
 
-rem remove -DSD_CUBLAS=ON to disable cuda support
+Rem ----------------------------------------------------------------------------
-cmake .. -DBUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF -DSD_CUBLAS=ON 
+rem Pick one of the builds below.
+
+rem # cuda12 #
+cmake .. -DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF
+
+rem # rocm5.5 #
+rem cmake .. -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS="gfx1100;gfx1102;gfx1030" -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF
+
+rem # avx512 #
+rem cmake .. -DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF
+
+rem # avx2 #
+rem cmake .. -DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF
+
+rem # avx #
+rem cmake .. -DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF
+
+rem # noavx #
+rem cmake .. -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF
+
+Rem ----------------------------------------------------------------------------
+
 cmake --build . --config Release
 
 cd ..\..