diff --git a/README.md b/README.md index 9c48634..16eb363 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,6 @@ Based on https://github.com/leejet/stable-diffusion.cpp -> At least for me the current version of stable-diffusion.cpp has really bad tiling issues. -If you experience them too, I'd recommend using https://github.com/DarthAffe/StableDiffusion.NET/releases/tag/c6071fa until that's fixed. - ## Usage ### Setup Run `build.bat` to build the native libs (modify params like CUDA-builds if needed) diff --git a/StableDiffusion.NET/Enums/Quantization.cs b/StableDiffusion.NET/Enums/Quantization.cs index e32f880..73cea61 100644 --- a/StableDiffusion.NET/Enums/Quantization.cs +++ b/StableDiffusion.NET/Enums/Quantization.cs @@ -1,4 +1,5 @@ -namespace StableDiffusion.NET; +// ReSharper disable InconsistentNaming +namespace StableDiffusion.NET; public enum Quantization { @@ -17,7 +18,15 @@ public enum Quantization Q4_K = 12, Q5_K = 13, Q6_K = 14, - Q8_K = 15, + Q8_K = 15, + IQ2_XXS = 16, + IQ2_XS = 17, + IQ3_XXS = 18, + IQ1_S = 19, + IQ4_NL = 20, + IQ3_S = 21, + IQ2_S = 22, + IQ4_XS = 23, I8, I16, I32, diff --git a/StableDiffusion.NET/EventArgs/StableDiffusionProgressEventArgs.cs b/StableDiffusion.NET/EventArgs/StableDiffusionProgressEventArgs.cs new file mode 100644 index 0000000..538f4a0 --- /dev/null +++ b/StableDiffusion.NET/EventArgs/StableDiffusionProgressEventArgs.cs @@ -0,0 +1,15 @@ +namespace StableDiffusion.NET; + +public class StableDiffusionProgressEventArgs(int step, int steps, float time) +{ + #region Properties & Fields + + public int Step { get; } = step; + public int Steps { get; } = steps; + public float Time { get; } = time; + + public double Progress => (double)Step / Steps; + public float IterationsPerSecond => 1f / Time; + + #endregion +} \ No newline at end of file diff --git a/StableDiffusion.NET/Native.cs b/StableDiffusion.NET/Native.cs index 0e74e81..eff2397 100644 --- a/StableDiffusion.NET/Native.cs +++ b/StableDiffusion.NET/Native.cs @@ -24,6 +24,7 @@ internal unsafe partial class Native #region Delegates internal delegate void sd_log_cb_t(sd_log_level_t level, [MarshalAs(UnmanagedType.LPStr)] string text, void* data); + internal delegate void sd_progress_cb_t(int step, int steps, float time, void* data); #endregion @@ -96,6 +97,22 @@ internal unsafe partial class Native long seed, int batch_count); + [LibraryImport(LIB_NAME, EntryPoint = "img2vid")] + internal static partial sd_image_t* img2vid(sd_ctx_t* sd_ctx, + sd_image_t init_image, + int width, + int height, + int video_frames, + int motion_bucket_id, + int fps, + float augmentation_level, + float min_cfg, + float cfg_scale, + sample_method_t sample_method, + int sample_steps, + float strength, + long seed); + [LibraryImport(LIB_NAME, EntryPoint = "new_upscaler_ctx")] internal static partial upscaler_ctx_t* new_upscaler_ctx([MarshalAs(UnmanagedType.LPStr)] string esrgan_path, int n_threads, @@ -115,8 +132,21 @@ internal unsafe partial class Native [MarshalAs(UnmanagedType.LPStr)] string output_path, sd_type_t output_type); + [LibraryImport(LIB_NAME, EntryPoint = "convert")] + internal static partial byte* preprocess_canny(byte* img, + int width, + int height, + float high_threshold, + float low_threshold, + float weak, + float strong, + [MarshalAs(UnmanagedType.I1)] bool inverse); + [LibraryImport(LIB_NAME, EntryPoint = "sd_set_log_callback")] internal static partial void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data); + [LibraryImport(LIB_NAME, EntryPoint = "sd_set_progress_callback")] + internal static partial void sd_set_progress_callback(sd_progress_cb_t cb, void* data); + #endregion } \ No newline at end of file diff --git a/StableDiffusion.NET/StableDiffusionModel.cs b/StableDiffusion.NET/StableDiffusionModel.cs index e7e579c..bf45786 100644 --- a/StableDiffusion.NET/StableDiffusionModel.cs +++ b/StableDiffusion.NET/StableDiffusionModel.cs @@ -21,6 +21,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable #region Events public static event EventHandler? Log; + public static event EventHandler? Progress; #endregion @@ -29,6 +30,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable static StableDiffusionModel() { Native.sd_set_log_callback(OnNativeLog, null); + Native.sd_set_progress_callback(OnNativeProgress, null); } public StableDiffusionModel(string modelPath, ModelParameter parameter, UpscalerModelParameter? upscalerParameter = null) @@ -78,7 +80,71 @@ public sealed unsafe class StableDiffusionModel : IDisposable ObjectDisposedException.ThrowIf(_disposed, this); Native.sd_image_t* result; - if ((parameter.ControlNetImage == null) || (parameter.ControlNetImage.Length == 0)) + if (parameter.ControlNet.IsEnabled) + { + fixed (byte* imagePtr = parameter.ControlNet.Image) + { + + if (parameter.ControlNet.CannyPreprocess) + { + Native.sd_image_t controlNetImage = new() + { + width = (uint)parameter.Width, + height = (uint)parameter.Height, + channel = 3, + data = Native.preprocess_canny(imagePtr, + parameter.Width, + parameter.Height, + parameter.ControlNet.CannyHighThreshold, + parameter.ControlNet.CannyLowThreshold, + parameter.ControlNet.CannyWeak, + parameter.ControlNet.CannyStrong, + parameter.ControlNet.CannyInverse) + }; + + result = Native.txt2img(_ctx, + prompt, + parameter.NegativePrompt, + parameter.ClipSkip, + parameter.CfgScale, + parameter.Width, + parameter.Height, + parameter.SampleMethod, + parameter.SampleSteps, + parameter.Seed, + 1, + &controlNetImage, + parameter.ControlNet.Strength); + + Marshal.FreeHGlobal((nint)controlNetImage.data); + } + else + { + Native.sd_image_t controlNetImage = new() + { + width = (uint)parameter.Width, + height = (uint)parameter.Height, + channel = 3, + data = imagePtr + }; + + result = Native.txt2img(_ctx, + prompt, + parameter.NegativePrompt, + parameter.ClipSkip, + parameter.CfgScale, + parameter.Width, + parameter.Height, + parameter.SampleMethod, + parameter.SampleSteps, + parameter.Seed, + 1, + &controlNetImage, + parameter.ControlNet.Strength); + } + } + } + else { result = Native.txt2img(_ctx, prompt, @@ -94,33 +160,6 @@ public sealed unsafe class StableDiffusionModel : IDisposable null, 0); } - else - { - fixed (byte* imagePtr = parameter.ControlNetImage) - { - Native.sd_image_t controlNetImage = new() - { - width = (uint)parameter.Width, - height = (uint)parameter.Height, - channel = 3, - data = imagePtr - }; - - result = Native.txt2img(_ctx, - prompt, - parameter.NegativePrompt, - parameter.ClipSkip, - parameter.CfgScale, - parameter.Width, - parameter.Height, - parameter.SampleMethod, - parameter.SampleSteps, - parameter.Seed, - 1, - &controlNetImage, - parameter.ControlNetStrength); - } - } return new StableDiffusionImage(result); } @@ -240,5 +279,14 @@ public sealed unsafe class StableDiffusionModel : IDisposable catch { /**/ } } + private static void OnNativeProgress(int step, int steps, float time, void* data) + { + try + { + Progress?.Invoke(null, new StableDiffusionProgressEventArgs(step, steps, time)); + } + catch { /**/ } + } + #endregion } diff --git a/StableDiffusion.NET/StableDiffusionParameter.cs b/StableDiffusion.NET/StableDiffusionParameter.cs index 3a512d4..74bdc7f 100644 --- a/StableDiffusion.NET/StableDiffusionParameter.cs +++ b/StableDiffusion.NET/StableDiffusionParameter.cs @@ -13,8 +13,22 @@ public sealed class StableDiffusionParameter public long Seed { get; set; } = -1; public float Strength { get; set; } = 0.7f; public int ClipSkip { get; set; } = -1; - public byte[]? ControlNetImage { get; set; } = null; - public float ControlNetStrength { get; set; } = 0.9f; + + public StableDiffusionControlNetParameter ControlNet { get; } = new(); #endregion +} + +public sealed class StableDiffusionControlNetParameter +{ + public bool IsEnabled => Image?.Length > 0; + + public byte[]? Image { get; set; } = null; + public float Strength { get; set; } = 0.9f; + public bool CannyPreprocess { get; set; } = false; + public float CannyHighThreshold { get; set; } = 0.08f; + public float CannyLowThreshold { get; set; } = 0.08f; + public float CannyWeak { get; set; } = 0.8f; + public float CannyStrong { get; set; } = 1.0f; + public bool CannyInverse { get; set; } = false; } \ No newline at end of file diff --git a/build.bat b/build.bat index c3ef809..a52a27c 100644 --- a/build.bat +++ b/build.bat @@ -4,7 +4,7 @@ if not exist stable-diffusion.cpp ( cd stable-diffusion.cpp git fetch -git checkout 36ec16ac9937d393d0ba8939640352cee430efb7 +git checkout 1ce9470f27d480c6aa5d43c0af5b60db99454252 git submodule init git submodule update @@ -14,8 +14,29 @@ if not exist build ( cd build -rem remove -DSD_CUBLAS=ON to disable cuda support -cmake .. -DBUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF -DSD_CUBLAS=ON +Rem ---------------------------------------------------------------------------- +rem Pick one of the builds below. + +rem # cuda12 # +cmake .. -DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF + +rem # rocm5.5 # +rem cmake .. -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS="gfx1100;gfx1102;gfx1030" -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF + +rem # avx512 # +rem cmake .. -DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF + +rem # avx2 # +rem cmake .. -DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF + +rem # avx # +rem cmake .. -DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF + +rem # noavx # +rem cmake .. -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_EXAMPLES=OFF + +Rem ---------------------------------------------------------------------------- + cmake --build . --config Release cd ..\..