From 88e7fb973fc327c58186e7e661f69520655020d6 Mon Sep 17 00:00:00 2001 From: Darth Affe Date: Sun, 25 Aug 2024 14:45:41 +0200 Subject: [PATCH] Updated stablediffusion.cpp with flux flow support, refactored model creation --- .../Enums/DiffusionModelType.cs | 8 + StableDiffusion.NET/Enums/Quantization.cs | 2 + .../Extensions/ParameterExtension.cs | 104 +++++---- .../Models/Builder/ESRGANModelBuilder.cs | 29 +++ .../DiffusionModelBuilderExtension.cs | 122 +++++++++++ .../PhotomakerModelBuilderExtension.cs | 18 ++ .../QuantizedModelBuilderExtension.cs | 36 +++ .../Models/Builder/FluxModelBuilder.cs | 30 +++ .../Interfaces/IDiffusionModelBuilder.cs | 6 + .../Interfaces/IPhotomakerModelBuilder.cs | 6 + .../Interfaces/IQuantizedModelBuilder.cs | 6 + .../Models/Builder/ModelBuilder.cs | 11 + .../Builder/StableDiffusionModelBuilder.cs | 31 +++ .../DiffusionModel.cs} | 206 ++++++------------ .../Models/Parameter/ControlNetParameter.cs | 19 ++ .../Parameter/DiffusionModelParameter.cs} | 43 ++-- .../Models/Parameter/DiffusionParameter.cs | 33 +++ .../Extensions/DiffusionParameterExtension.cs | 114 ++++++++++ .../Interfaces/IDiffusionModelParameter.cs | 22 ++ .../Interfaces/IPhotomakerModelParameter.cs | 6 + .../Interfaces/IQuantizedModelParameter.cs | 8 + .../Models/Parameter/PhotoMakerParameter.cs | 11 + .../Models/Parameter/UpscaleModelParameter.cs | 12 + StableDiffusion.NET/Models/UpscaleModel.cs | 83 +++++++ StableDiffusion.NET/Native/Native.cs | 5 + .../StableDiffusion.NET.csproj.DotSettings | 8 + StableDiffusion.NET/StableDiffusionCpp.cs | 77 +++++++ .../StableDiffusionParameter.cs | 48 ---- StableDiffusion.NET/UpscalerModelParameter.cs | 17 -- 29 files changed, 859 insertions(+), 262 deletions(-) create mode 100644 StableDiffusion.NET/Enums/DiffusionModelType.cs create mode 100644 StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs create mode 100644 StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs create mode 100644 StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs create mode 100644 StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs create mode 100644 StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs create mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs create mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs create mode 100644 StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs create mode 100644 StableDiffusion.NET/Models/Builder/ModelBuilder.cs create mode 100644 StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs rename StableDiffusion.NET/{StableDiffusionModel.cs => Models/DiffusionModel.cs} (62%) create mode 100644 StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs rename StableDiffusion.NET/{ModelParameter.cs => Models/Parameter/DiffusionModelParameter.cs} (54%) create mode 100644 StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs create mode 100644 StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs create mode 100644 StableDiffusion.NET/Models/UpscaleModel.cs create mode 100644 StableDiffusion.NET/StableDiffusionCpp.cs delete mode 100644 StableDiffusion.NET/StableDiffusionParameter.cs delete mode 100644 StableDiffusion.NET/UpscalerModelParameter.cs diff --git a/StableDiffusion.NET/Enums/DiffusionModelType.cs b/StableDiffusion.NET/Enums/DiffusionModelType.cs new file mode 100644 index 0000000..a42e137 --- /dev/null +++ b/StableDiffusion.NET/Enums/DiffusionModelType.cs @@ -0,0 +1,8 @@ +namespace StableDiffusion.NET; + +public enum DiffusionModelType +{ + None = 0, + StableDiffusion = 1, + Flux = 2 +} \ No newline at end of file diff --git a/StableDiffusion.NET/Enums/Quantization.cs b/StableDiffusion.NET/Enums/Quantization.cs index 4507ef5..39e886a 100644 --- a/StableDiffusion.NET/Enums/Quantization.cs +++ b/StableDiffusion.NET/Enums/Quantization.cs @@ -37,4 +37,6 @@ public enum Quantization Q4_0_4_4 = 31, Q4_0_4_8 = 32, Q4_0_8_8 = 33, + + Unspecified } \ No newline at end of file diff --git a/StableDiffusion.NET/Extensions/ParameterExtension.cs b/StableDiffusion.NET/Extensions/ParameterExtension.cs index 13fd6d8..8f33a8c 100644 --- a/StableDiffusion.NET/Extensions/ParameterExtension.cs +++ b/StableDiffusion.NET/Extensions/ParameterExtension.cs @@ -6,65 +6,87 @@ namespace StableDiffusion.NET; public static class ParameterExtension { - public static void Validate(this StableDiffusionParameter parameter) + public static void Validate(this UpscaleModelParameter parameter) { ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); - ArgumentNullException.ThrowIfNull(parameter.ControlNet, nameof(StableDiffusionParameter.ControlNet)); - ArgumentNullException.ThrowIfNull(parameter.PhotoMaker, nameof(StableDiffusionParameter.PhotoMaker)); - ArgumentNullException.ThrowIfNull(parameter.NegativePrompt, nameof(StableDiffusionParameter.NegativePrompt)); + ArgumentNullException.ThrowIfNull(parameter.ModelPath, nameof(UpscaleModelParameter.ModelPath)); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Width, nameof(StableDiffusionParameter.Width)); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Height, nameof(StableDiffusionParameter.Height)); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.SampleSteps, nameof(StableDiffusionParameter.SampleSteps)); + if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(UpscaleModelParameter.Quantization)); + } - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CfgScale, nameof(StableDiffusionParameter.CfgScale)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(StableDiffusionParameter.Strength)); + public static void Validate(this DiffusionModelParameter parameter) + { + ((IQuantizedModelParameter)parameter).Validate(); + ((IDiffusionModelParameter)parameter).Validate(); + ((IPhotomakerModelParameter)parameter).Validate(); + } - if (!Enum.IsDefined(parameter.SampleMethod)) throw new ArgumentOutOfRangeException(nameof(StableDiffusionParameter.SampleMethod)); + public static void Validate(this IQuantizedModelParameter parameter) + { + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.ThreadCount, nameof(IQuantizedModelParameter.ThreadCount)); + + if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(IQuantizedModelParameter.Quantization)); + } + + public static void Validate(this IPhotomakerModelParameter parameter) + { + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); + ArgumentNullException.ThrowIfNull(parameter.StackedIdEmbeddingsDirectory, nameof(IPhotomakerModelParameter.StackedIdEmbeddingsDirectory)); + } + + public static void Validate(this IDiffusionModelParameter parameter) + { + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); + ArgumentNullException.ThrowIfNull(parameter.VaePath, nameof(IDiffusionModelParameter.VaePath)); + ArgumentNullException.ThrowIfNull(parameter.TaesdPath, nameof(IDiffusionModelParameter.TaesdPath)); + ArgumentNullException.ThrowIfNull(parameter.LoraModelDirectory, nameof(IDiffusionModelParameter.LoraModelDirectory)); + ArgumentNullException.ThrowIfNull(parameter.ControlNetPath, nameof(IDiffusionModelParameter.ControlNetPath)); + ArgumentNullException.ThrowIfNull(parameter.EmbeddingsDirectory, nameof(IDiffusionModelParameter.EmbeddingsDirectory)); + + if (!string.IsNullOrWhiteSpace(parameter.VaePath) && !string.IsNullOrWhiteSpace(parameter.TaesdPath)) throw new ArgumentException("VAE and TAESD are mutually exclusive."); + + if (!Enum.IsDefined(parameter.RngType)) throw new ArgumentOutOfRangeException(nameof(IDiffusionModelParameter.RngType)); + if (!Enum.IsDefined(parameter.Schedule)) throw new ArgumentOutOfRangeException(nameof(IDiffusionModelParameter.Schedule)); + } + + public static void Validate(this DiffusionParameter parameter) + { + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); + ArgumentNullException.ThrowIfNull(parameter.ControlNet, nameof(DiffusionParameter.ControlNet)); + ArgumentNullException.ThrowIfNull(parameter.PhotoMaker, nameof(DiffusionParameter.PhotoMaker)); + ArgumentNullException.ThrowIfNull(parameter.NegativePrompt, nameof(DiffusionParameter.NegativePrompt)); + + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Width, nameof(DiffusionParameter.Width)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Height, nameof(DiffusionParameter.Height)); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.SampleSteps, nameof(DiffusionParameter.SampleSteps)); + + ArgumentOutOfRangeException.ThrowIfNegative(parameter.CfgScale, nameof(DiffusionParameter.CfgScale)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.Guidance, nameof(DiffusionParameter.Guidance)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(DiffusionParameter.Strength)); + + if (!Enum.IsDefined(parameter.SampleMethod)) throw new ArgumentOutOfRangeException(nameof(DiffusionParameter.SampleMethod)); parameter.ControlNet.Validate(); parameter.PhotoMaker.Validate(); } - public static void Validate(this StableDiffusionControlNetParameter parameter) + public static void Validate(this ControlNetParameter parameter) { - ArgumentNullException.ThrowIfNull(parameter, nameof(StableDiffusionParameter.ControlNet)); + ArgumentNullException.ThrowIfNull(parameter, nameof(DiffusionParameter.ControlNet)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(StableDiffusionControlNetParameter.Strength)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyHighThreshold, nameof(StableDiffusionControlNetParameter.CannyHighThreshold)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyLowThreshold, nameof(StableDiffusionControlNetParameter.CannyLowThreshold)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyWeak, nameof(StableDiffusionControlNetParameter.CannyWeak)); - ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyStrong, nameof(StableDiffusionControlNetParameter.CannyStrong)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(ControlNetParameter.Strength)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyHighThreshold, nameof(ControlNetParameter.CannyHighThreshold)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyLowThreshold, nameof(ControlNetParameter.CannyLowThreshold)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyWeak, nameof(ControlNetParameter.CannyWeak)); + ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyStrong, nameof(ControlNetParameter.CannyStrong)); } public static void Validate(this PhotoMakerParameter parameter) { - ArgumentNullException.ThrowIfNull(parameter, nameof(StableDiffusionParameter.PhotoMaker)); + ArgumentNullException.ThrowIfNull(parameter, nameof(DiffusionParameter.PhotoMaker)); ArgumentNullException.ThrowIfNull(parameter.InputIdImageDirectory, nameof(PhotoMakerParameter.InputIdImageDirectory)); ArgumentOutOfRangeException.ThrowIfNegative(parameter.StyleRatio, nameof(PhotoMakerParameter.StyleRatio)); } - - public static void Validate(this ModelParameter parameter) - { - ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); - ArgumentNullException.ThrowIfNull(parameter.TaesdPath, nameof(ModelParameter.TaesdPath)); - ArgumentNullException.ThrowIfNull(parameter.LoraModelDir, nameof(ModelParameter.LoraModelDir)); - ArgumentNullException.ThrowIfNull(parameter.VaePath, nameof(ModelParameter.VaePath)); - ArgumentNullException.ThrowIfNull(parameter.ControlNetPath, nameof(ModelParameter.ControlNetPath)); - ArgumentNullException.ThrowIfNull(parameter.EmbeddingsDirectory, nameof(ModelParameter.EmbeddingsDirectory)); - ArgumentNullException.ThrowIfNull(parameter.StackedIdEmbeddingsDirectory, nameof(ModelParameter.StackedIdEmbeddingsDirectory)); - - if (!Enum.IsDefined(parameter.RngType)) throw new ArgumentOutOfRangeException(nameof(ModelParameter.RngType)); - if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(ModelParameter.Quantization)); - if (!Enum.IsDefined(parameter.Schedule)) throw new ArgumentOutOfRangeException(nameof(ModelParameter.Schedule)); - } - - public static void Validate(this UpscalerModelParameter parameter) - { - ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); - ArgumentNullException.ThrowIfNull(parameter.ESRGANPath, nameof(UpscalerModelParameter.ESRGANPath)); - - if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(ModelParameter.Quantization)); - } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs b/StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs new file mode 100644 index 0000000..b66452b --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs @@ -0,0 +1,29 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class ESRGANModelBuilder : IQuantizedModelBuilder +{ + #region Properties & Fields + + public UpscaleModelParameter Parameter { get; } + IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; + + #endregion + + #region Constructors + + public ESRGANModelBuilder(string modelPath) + { + Parameter = new UpscaleModelParameter { ModelPath = modelPath }; + } + + #endregion + + #region Methods + + public UpscaleModel Build() => new(Parameter); + + #endregion +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs new file mode 100644 index 0000000..9f08791 --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs @@ -0,0 +1,122 @@ +using System; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static class DiffusionModelBuilderExtension +{ + public static T WithVae(this T builder, string vaePath) + where T : IDiffusionModelBuilder + { + ArgumentException.ThrowIfNullOrWhiteSpace(vaePath); + + if (!string.IsNullOrWhiteSpace(builder.Parameter.TaesdPath)) throw new ArgumentException("TAESD is already enabled. VAE and TAESD are mutually exclusive.", nameof(vaePath)); + + builder.Parameter.VaePath = vaePath; + + return builder; + } + + public static T WithTaesd(this T builder, string taesdPath) + where T : IDiffusionModelBuilder + { + ArgumentException.ThrowIfNullOrWhiteSpace(taesdPath); + + if (!string.IsNullOrWhiteSpace(builder.Parameter.VaePath)) throw new ArgumentException("VAE is already enabled. TAESD and VAE are mutually exclusive.", nameof(taesdPath)); + + builder.Parameter.TaesdPath = taesdPath; + + return builder; + } + + public static T WithLoraSupport(this T builder, string loraModelDirectory) + where T : IDiffusionModelBuilder + { + ArgumentException.ThrowIfNullOrWhiteSpace(loraModelDirectory); + + builder.Parameter.LoraModelDirectory = loraModelDirectory; + + return builder; + } + + public static T WithEmbeddingSupport(this T builder, string embeddingsDirectory) + where T : IDiffusionModelBuilder + { + ArgumentException.ThrowIfNullOrWhiteSpace(embeddingsDirectory); + + builder.Parameter.EmbeddingsDirectory = embeddingsDirectory; + + return builder; + } + + public static T WithControlNet(this T builder, string controlNetPath) + where T : IDiffusionModelBuilder + { + ArgumentException.ThrowIfNullOrWhiteSpace(controlNetPath); + + builder.Parameter.ControlNetPath = controlNetPath; + + return builder; + } + + public static T WithVaeDecodeOnly(this T builder, bool vaeDecodeOnly = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.VaeDecodeOnly = vaeDecodeOnly; + + return builder; + } + + public static T WithVaeTiling(this T builder, bool vaeTiling = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.VaeTiling = vaeTiling; + + return builder; + } + + public static T KeepVaeOnCpu(this T builder, bool keepVaeOnCpu = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.KeepVaeOnCPU = keepVaeOnCpu; + + return builder; + } + + public static T KeepClipNetOnCpu(this T builder, bool keepClipNetOnCpu = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.KeepClipOnCPU = keepClipNetOnCpu; + + return builder; + } + + public static T KeepControlNetOnCpu(this T builder, bool keepControlNetOnCpu = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.KeepControlNetOnCPU = keepControlNetOnCpu; + + return builder; + } + + public static T WithRngType(this T builder, RngType rngType) + where T : IDiffusionModelBuilder + { + if (!Enum.IsDefined(rngType)) throw new ArgumentOutOfRangeException(nameof(rngType)); + + builder.Parameter.RngType = rngType; + + return builder; + } + + public static T WithSchedule(this T builder, Schedule schedule) + where T : IDiffusionModelBuilder + { + if (!Enum.IsDefined(schedule)) throw new ArgumentOutOfRangeException(nameof(schedule)); + + builder.Parameter.Schedule = schedule; + + return builder; + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs new file mode 100644 index 0000000..57449f4 --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs @@ -0,0 +1,18 @@ +using System; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static class PhotomakerModelBuilderExtension +{ + public static T WithPhotomaker(this T builder, string stackedIdEmbeddingsDirectory) + where T : IPhotomakerModelBuilder + { + ArgumentException.ThrowIfNullOrWhiteSpace(stackedIdEmbeddingsDirectory, nameof(stackedIdEmbeddingsDirectory)); + + builder.Parameter.StackedIdEmbeddingsDirectory = stackedIdEmbeddingsDirectory; + + return builder; + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs new file mode 100644 index 0000000..671c2f1 --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs @@ -0,0 +1,36 @@ +using System; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static class QuantizedModelBuilderExtension +{ + public static T WithoutMultithreading(this T builder) + where T : IQuantizedModelBuilder + { + builder.Parameter.ThreadCount = 1; + + return builder; + } + + public static T WithMultithreading(this T builder, int threadCount = 0) + where T : IQuantizedModelBuilder + { + ArgumentOutOfRangeException.ThrowIfLessThan(threadCount, 0, nameof(threadCount)); + + if (threadCount == 0) threadCount = Environment.ProcessorCount; + + builder.Parameter.ThreadCount = threadCount; + + return builder; + } + + public static T WithQuantization(this T builder, Quantization quantization) + where T : IQuantizedModelBuilder + { + builder.Parameter.Quantization = quantization; + + return builder; + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs b/StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs new file mode 100644 index 0000000..415774f --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs @@ -0,0 +1,30 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class FluxModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder +{ + #region Properties & Fields + + public DiffusionModelParameter Parameter { get; } + IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter; + IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; + + #endregion + + #region Constructors + + public FluxModelBuilder(string diffusionModelPath, string clipLPath, string t5xxlPath) + { + Parameter = new DiffusionModelParameter { DiffusionModelType = DiffusionModelType.Flux, DiffusionModelPath = diffusionModelPath, ClipLPath = clipLPath, T5xxlPath = t5xxlPath }; + } + + #endregion + + #region Methods + + public DiffusionModel Build() => new(Parameter); + + #endregion +} diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs new file mode 100644 index 0000000..cf4aa9f --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs @@ -0,0 +1,6 @@ +namespace StableDiffusion.NET; + +public interface IDiffusionModelBuilder +{ + IDiffusionModelParameter Parameter { get; } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs new file mode 100644 index 0000000..9731fa3 --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs @@ -0,0 +1,6 @@ +namespace StableDiffusion.NET; + +public interface IPhotomakerModelBuilder +{ + IPhotomakerModelParameter Parameter { get; } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs b/StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs new file mode 100644 index 0000000..6aa95ae --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs @@ -0,0 +1,6 @@ +namespace StableDiffusion.NET; + +public interface IQuantizedModelBuilder +{ + IQuantizedModelParameter Parameter { get; } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/ModelBuilder.cs b/StableDiffusion.NET/Models/Builder/ModelBuilder.cs new file mode 100644 index 0000000..d56dfc7 --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/ModelBuilder.cs @@ -0,0 +1,11 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static class ModelBuilder +{ + public static StableDiffusionModelBuilder StableDiffusion(string modelPath) => new(modelPath); + public static FluxModelBuilder Flux(string diffusionModelPath, string clipLPath, string t5xxlPath) => new(diffusionModelPath, clipLPath, t5xxlPath); + public static ESRGANModelBuilder ESRGAN(string modelPath) => new(modelPath); +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs b/StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs new file mode 100644 index 0000000..d1952f2 --- /dev/null +++ b/StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs @@ -0,0 +1,31 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class StableDiffusionModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder, IPhotomakerModelBuilder +{ + #region Properties & Fields + + public DiffusionModelParameter Parameter { get; } + IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter; + IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter; + IPhotomakerModelParameter IPhotomakerModelBuilder.Parameter => Parameter; + + #endregion + + #region Constructors + + public StableDiffusionModelBuilder(string modelPath) + { + Parameter = new DiffusionModelParameter { DiffusionModelType = DiffusionModelType.StableDiffusion, ModelPath = modelPath }; + } + + #endregion + + #region Methods + + public DiffusionModel Build() => new(Parameter); + + #endregion +} \ No newline at end of file diff --git a/StableDiffusion.NET/StableDiffusionModel.cs b/StableDiffusion.NET/Models/DiffusionModel.cs similarity index 62% rename from StableDiffusion.NET/StableDiffusionModel.cs rename to StableDiffusion.NET/Models/DiffusionModel.cs index 05be613..422a06c 100644 --- a/StableDiffusion.NET/StableDiffusionModel.cs +++ b/StableDiffusion.NET/Models/DiffusionModel.cs @@ -1,61 +1,37 @@ using System; -using System.Runtime.InteropServices; using HPPH; +using System.Runtime.InteropServices; using JetBrains.Annotations; namespace StableDiffusion.NET; [PublicAPI] -public sealed unsafe class StableDiffusionModel : IDisposable +public sealed unsafe class DiffusionModel : IDisposable { #region Properties & Fields - // ReSharper disable NotAccessedField.Local - They are important, the delegate can be collected if it's not stored! - private static readonly Native.sd_log_cb_t LOG_CALLBACK; - private static readonly Native.sd_progress_cb_t PROGRESS_CALLBACK; - // ReSharper restore NotAccessedField.Local - private bool _disposed; - private readonly string _modelPath; - private readonly ModelParameter _parameter; - private readonly UpscalerModelParameter? _upscalerParameter; + private readonly DiffusionModelParameter _parameter; private Native.sd_ctx_t* _ctx; - private Native.upscaler_ctx_t* _upscalerCtx; - - #endregion - - #region Events - - public static event EventHandler? Log; - public static event EventHandler? Progress; #endregion #region Constructors - static StableDiffusionModel() + public DiffusionModel(DiffusionModelParameter parameter) { - Native.sd_set_log_callback(LOG_CALLBACK = OnNativeLog, null); - Native.sd_set_progress_callback(PROGRESS_CALLBACK = OnNativeProgress, null); - } - - public StableDiffusionModel(string modelPath, ModelParameter parameter, UpscalerModelParameter? upscalerParameter = null) - { - ArgumentException.ThrowIfNullOrWhiteSpace(modelPath, nameof(modelPath)); + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); parameter.Validate(); - upscalerParameter?.Validate(); - this._modelPath = modelPath; this._parameter = parameter; - this._upscalerParameter = upscalerParameter; Initialize(); } - ~StableDiffusionModel() => Dispose(); + ~DiffusionModel() => Dispose(); #endregion @@ -63,16 +39,19 @@ public sealed unsafe class StableDiffusionModel : IDisposable private void Initialize() { - _ctx = Native.new_sd_ctx(_modelPath, + _ctx = Native.new_sd_ctx(_parameter.ModelPath, + _parameter.ClipLPath, + _parameter.T5xxlPath, + _parameter.DiffusionModelPath, _parameter.VaePath, _parameter.TaesdPath, _parameter.ControlNetPath, - _parameter.LoraModelDir, + _parameter.LoraModelDirectory, _parameter.EmbeddingsDirectory, _parameter.StackedIdEmbeddingsDirectory, _parameter.VaeDecodeOnly, _parameter.VaeTiling, - false, + false, _parameter.ThreadCount, _parameter.Quantization, _parameter.RngType, @@ -80,28 +59,22 @@ public sealed unsafe class StableDiffusionModel : IDisposable _parameter.KeepClipOnCPU, _parameter.KeepControlNetOnCPU, _parameter.KeepVaeOnCPU); - if (_ctx == null) throw new NullReferenceException("Failed to initialize Stable Diffusion"); - if (_upscalerParameter != null) - { - _upscalerCtx = Native.new_upscaler_ctx(_upscalerParameter.ESRGANPath, - _upscalerParameter.ThreadCount, - _upscalerParameter.Quantization); - if (_upscalerCtx == null) throw new NullReferenceException("Failed to initialize Stable Diffusion"); - } + if (_ctx == null) throw new NullReferenceException("Failed to initialize diffusion-model."); } - /// - /// Manually load the native stable diffusion library. - /// Once set, it will continue to be used for all instances. - /// - /// Path to the stable diffusion library. - /// Bool if the library loaded. - public static bool LoadNativeLibrary(string libraryPath) - => Native.LoadNativeLibrary(libraryPath); - - public IImage TextToImage(string prompt, StableDiffusionParameter parameter) + public DiffusionParameter GetDefaultParameter() => _parameter.DiffusionModelType switch { + DiffusionModelType.None => new DiffusionParameter(), + DiffusionModelType.StableDiffusion => DiffusionParameter.StableDiffusionDefault, + DiffusionModelType.Flux => DiffusionParameter.FluxDefault, + _ => throw new ArgumentOutOfRangeException() + }; + + public IImage TextToImage(string prompt, DiffusionParameter? parameter = null) + { + parameter ??= GetDefaultParameter(); + ObjectDisposedException.ThrowIf(_disposed, this); ArgumentNullException.ThrowIfNull(prompt); @@ -110,15 +83,18 @@ public sealed unsafe class StableDiffusionModel : IDisposable Native.sd_image_t* result; if (parameter.ControlNet.IsEnabled) { - fixed (byte* imagePtr = parameter.ControlNet.Image!.ToRawArray()) + if (parameter.ControlNet.Image is not IImage controlNetImage) + controlNetImage = parameter.ControlNet.Image!.ConvertTo(); + + fixed (byte* imagePtr = controlNetImage.ToRawArray()) { if (parameter.ControlNet.CannyPreprocess) { - Native.sd_image_t controlNetImage = new() + Native.sd_image_t nativeControlNetImage = new() { - width = (uint)parameter.ControlNet.Image.Width, - height = (uint)parameter.ControlNet.Image.Height, - channel = (uint)parameter.ControlNet.Image.ColorFormat.BytesPerPixel, + width = (uint)controlNetImage.Width, + height = (uint)controlNetImage.Height, + channel = (uint)controlNetImage.ColorFormat.BytesPerPixel, data = Native.preprocess_canny(imagePtr, parameter.Width, parameter.Height, @@ -134,27 +110,28 @@ public sealed unsafe class StableDiffusionModel : IDisposable parameter.NegativePrompt, parameter.ClipSkip, parameter.CfgScale, + parameter.Guidance, parameter.Width, parameter.Height, parameter.SampleMethod, parameter.SampleSteps, parameter.Seed, 1, - &controlNetImage, + &nativeControlNetImage, parameter.ControlNet.Strength, parameter.PhotoMaker.StyleRatio, parameter.PhotoMaker.NormalizeInput, parameter.PhotoMaker.InputIdImageDirectory); - Marshal.FreeHGlobal((nint)controlNetImage.data); + Marshal.FreeHGlobal((nint)nativeControlNetImage.data); } else { - Native.sd_image_t controlNetImage = new() + Native.sd_image_t nativeControlNetImage = new() { - width = (uint)parameter.ControlNet.Image.Width, - height = (uint)parameter.ControlNet.Image.Height, - channel = (uint)parameter.ControlNet.Image.ColorFormat.BytesPerPixel, + width = (uint)controlNetImage.Width, + height = (uint)controlNetImage.Height, + channel = (uint)controlNetImage.ColorFormat.BytesPerPixel, data = imagePtr }; @@ -163,13 +140,14 @@ public sealed unsafe class StableDiffusionModel : IDisposable parameter.NegativePrompt, parameter.ClipSkip, parameter.CfgScale, + parameter.Guidance, parameter.Width, parameter.Height, parameter.SampleMethod, parameter.SampleSteps, parameter.Seed, 1, - &controlNetImage, + &nativeControlNetImage, parameter.ControlNet.Strength, parameter.PhotoMaker.StyleRatio, parameter.PhotoMaker.NormalizeInput, @@ -184,6 +162,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable parameter.NegativePrompt, parameter.ClipSkip, parameter.CfgScale, + parameter.Guidance, parameter.Width, parameter.Height, parameter.SampleMethod, @@ -200,18 +179,23 @@ public sealed unsafe class StableDiffusionModel : IDisposable return ImageHelper.ToImage(result); } - public IImage ImageToImage(string prompt, IImage image, StableDiffusionParameter parameter) + public IImage ImageToImage(string prompt, IImage image, DiffusionParameter? parameter = null) { + parameter ??= GetDefaultParameter(); + ObjectDisposedException.ThrowIf(_disposed, this); ArgumentNullException.ThrowIfNull(prompt); parameter.Validate(); - fixed (byte* imagePtr = image.AsRefImage()) - return ImageToImage(prompt, image.ToSdImage(imagePtr), parameter); + if (image is not IImage refImage) + refImage = image.ConvertTo(); + + fixed (byte* imagePtr = refImage.AsRefImage()) + return ImageToImage(prompt, refImage.ToSdImage(imagePtr), parameter); } - private IImage ImageToImage(string prompt, Native.sd_image_t image, StableDiffusionParameter parameter) + private IImage ImageToImage(string prompt, Native.sd_image_t image, DiffusionParameter parameter) { ObjectDisposedException.ThrowIf(_disposed, this); ArgumentNullException.ThrowIfNull(prompt); @@ -221,15 +205,18 @@ public sealed unsafe class StableDiffusionModel : IDisposable Native.sd_image_t* result; if (parameter.ControlNet.IsEnabled) { - fixed (byte* imagePtr = parameter.ControlNet.Image!.ToRawArray()) + if (parameter.ControlNet.Image is not IImage controlNetImage) + controlNetImage = parameter.ControlNet.Image!.ConvertTo(); + + fixed (byte* imagePtr = controlNetImage.ToRawArray()) { if (parameter.ControlNet.CannyPreprocess) { - Native.sd_image_t controlNetImage = new() + Native.sd_image_t nativeControlNetImage = new() { - width = (uint)parameter.ControlNet.Image.Width, - height = (uint)parameter.ControlNet.Image.Height, - channel = (uint)parameter.ControlNet.Image.ColorFormat.BytesPerPixel, + width = (uint)controlNetImage.Width, + height = (uint)controlNetImage.Height, + channel = (uint)controlNetImage.ColorFormat.BytesPerPixel, data = Native.preprocess_canny(imagePtr, parameter.Width, parameter.Height, @@ -246,6 +233,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable parameter.NegativePrompt, parameter.ClipSkip, parameter.CfgScale, + parameter.Guidance, parameter.Width, parameter.Height, parameter.SampleMethod, @@ -253,17 +241,17 @@ public sealed unsafe class StableDiffusionModel : IDisposable parameter.Strength, parameter.Seed, 1, - &controlNetImage, + &nativeControlNetImage, parameter.ControlNet.Strength, parameter.PhotoMaker.StyleRatio, parameter.PhotoMaker.NormalizeInput, parameter.PhotoMaker.InputIdImageDirectory); - Marshal.FreeHGlobal((nint)controlNetImage.data); + Marshal.FreeHGlobal((nint)nativeControlNetImage.data); } else { - Native.sd_image_t controlNetImage = new() + Native.sd_image_t nativeControlNetImage = new() { width = (uint)parameter.ControlNet.Image.Width, height = (uint)parameter.ControlNet.Image.Height, @@ -277,6 +265,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable parameter.NegativePrompt, parameter.ClipSkip, parameter.CfgScale, + parameter.Guidance, parameter.Width, parameter.Height, parameter.SampleMethod, @@ -284,7 +273,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable parameter.Strength, parameter.Seed, 1, - &controlNetImage, + &nativeControlNetImage, parameter.ControlNet.Strength, parameter.PhotoMaker.StyleRatio, parameter.PhotoMaker.NormalizeInput, @@ -300,6 +289,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable parameter.NegativePrompt, parameter.ClipSkip, parameter.CfgScale, + parameter.Guidance, parameter.Width, parameter.Height, parameter.SampleMethod, @@ -317,74 +307,16 @@ public sealed unsafe class StableDiffusionModel : IDisposable return ImageHelper.ToImage(result); } - public IImage Upscale(IImage image, int upscaleFactor) - { - ObjectDisposedException.ThrowIf(_disposed, this); - ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(upscaleFactor, 0, nameof(upscaleFactor)); - - if (_upscalerCtx == null) throw new NullReferenceException("The upscaler is not initialized."); - - fixed (byte* imagePtr = image.ConvertTo().AsRefImage()) - { - Native.sd_image_t result = Native.upscale(_upscalerCtx, image.ToSdImage(imagePtr), upscaleFactor); - return ImageHelper.ToImage(&result); - } - } - - private IImage Upscale(Native.sd_image_t image, int upscaleFactor) - { - Native.sd_image_t result = Native.upscale(_upscalerCtx, image, upscaleFactor); - return ImageHelper.ToImage(&result); - } - public void Dispose() { if (_disposed) return; - Native.free_sd_ctx(_ctx); - - if (_upscalerCtx != null) - Native.free_upscaler_ctx(_upscalerCtx); + if (_ctx != null) + Native.free_sd_ctx(_ctx); GC.SuppressFinalize(this); _disposed = true; } - public static void Convert(string modelPath, string vaePath, Quantization quantization, string outputPath) - { - ArgumentException.ThrowIfNullOrWhiteSpace(nameof(modelPath)); - ArgumentException.ThrowIfNullOrWhiteSpace(nameof(outputPath)); - ArgumentNullException.ThrowIfNull(vaePath); - if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization)); - - Native.convert(modelPath, vaePath, outputPath, quantization); - } - - public static string GetSystemInfo() - { - void* s = Native.sd_get_system_info(); - return Marshal.PtrToStringUTF8((nint)s) ?? ""; - } - - public static int GetNumPhysicalCores() => Native.get_num_physical_cores(); - - private static void OnNativeLog(LogLevel level, string text, void* data) - { - try - { - Log?.Invoke(null, new StableDiffusionLogEventArgs(level, text)); - } - catch { /**/ } - } - - private static void OnNativeProgress(int step, int steps, float time, void* data) - { - try - { - Progress?.Invoke(null, new StableDiffusionProgressEventArgs(step, steps, time)); - } - catch { /**/ } - } - #endregion -} +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs b/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs new file mode 100644 index 0000000..9c89dd2 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs @@ -0,0 +1,19 @@ +using HPPH; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class ControlNetParameter +{ + public bool IsEnabled => Image != null; + + public IImage? Image { get; set; } = null; + public float Strength { get; set; } = 0.9f; + public bool CannyPreprocess { get; set; } = false; + public float CannyHighThreshold { get; set; } = 0.08f; + public float CannyLowThreshold { get; set; } = 0.08f; + public float CannyWeak { get; set; } = 0.8f; + public float CannyStrong { get; set; } = 1.0f; + public bool CannyInverse { get; set; } = false; +} \ No newline at end of file diff --git a/StableDiffusion.NET/ModelParameter.cs b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs similarity index 54% rename from StableDiffusion.NET/ModelParameter.cs rename to StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs index 598c080..379750a 100644 --- a/StableDiffusion.NET/ModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs @@ -1,30 +1,35 @@ -using JetBrains.Annotations; +namespace StableDiffusion.NET; -namespace StableDiffusion.NET; - -[PublicAPI] -public class ModelParameter +public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantizedModelParameter, IPhotomakerModelParameter { - #region Properties & Fields + public DiffusionModelType DiffusionModelType { get; set; } = DiffusionModelType.None; + + public string VaePath { get; set; } = string.Empty; + public string TaesdPath { get; set; } = string.Empty; + + public string LoraModelDirectory { get; set; } = string.Empty; + public string EmbeddingsDirectory { get; set; } = string.Empty; + public string ControlNetPath { get; set; } = string.Empty; + + public int ThreadCount { get; set; } = 1; - public int ThreadCount { get; set; } = 8; public bool VaeDecodeOnly { get; set; } = false; public bool VaeTiling { get; set; } = false; - public string TaesdPath { get; set; } = string.Empty; - public string LoraModelDir { get; set; } = string.Empty; - public RngType RngType { get; set; } = RngType.Standard; - public string VaePath { get; set; } = string.Empty; - public string ControlNetPath { get; set; } = string.Empty; - public string EmbeddingsDirectory { get; set; } = string.Empty; - public string StackedIdEmbeddingsDirectory { get; set; } = string.Empty; public bool KeepControlNetOnCPU { get; set; } = false; public bool KeepClipOnCPU { get; set; } = false; public bool KeepVaeOnCPU { get; set; } = false; - //TODO DarthAffe 01.01.2024: K-Quants doesn't seem to work so far - public Quantization Quantization { get; set; } = Quantization.F16; - + public RngType RngType { get; set; } = RngType.Standard; public Schedule Schedule { get; set; } = Schedule.Default; - #endregion -} + public Quantization Quantization { get; set; } = Quantization.Unspecified; + + // Stable Diffusion only + public string ModelPath { get; set; } = string.Empty; + public string StackedIdEmbeddingsDirectory { get; set; } = string.Empty; + + // Flux only + public string DiffusionModelPath { get; set; } = string.Empty; + public string ClipLPath { get; set; } = string.Empty; + public string T5xxlPath { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs b/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs new file mode 100644 index 0000000..b9dc174 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs @@ -0,0 +1,33 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class DiffusionParameter +{ + #region Properties & Fields + + public static DiffusionParameter StableDiffusionDefault => new() { Width = 512, Height = 512, CfgScale = 7.5f, Guidance = 1f, SampleSteps = 25, SampleMethod = Sampler.Euler_A }; + public static DiffusionParameter SDXLDefault => new() { Width = 1024, Height = 1024, CfgScale = 7f, Guidance = 1f, SampleSteps = 30, SampleMethod = Sampler.Euler_A }; + public static DiffusionParameter FluxDefault => new() { Width = 1024, Height = 1024, CfgScale = 1, Guidance = 3.5f, SampleSteps = 20, SampleMethod = Sampler.Euler }; + + public string NegativePrompt { get; set; } = string.Empty; + public int Width { get; set; } = 512; + public int Height { get; set; } = 512; + public Sampler SampleMethod { get; set; } = Sampler.Euler_A; + public int SampleSteps { get; set; } = 25; + public long Seed { get; set; } = -1; + public float Strength { get; set; } = 0.7f; + public int ClipSkip { get; set; } = -1; + + public ControlNetParameter ControlNet { get; } = new(); + + // Stable Diffusion only + public float CfgScale { get; set; } = 7.5f; + public PhotoMakerParameter PhotoMaker { get; } = new(); + + // Flux only + public float Guidance { get; set; } = 3.5f; + + #endregion +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs new file mode 100644 index 0000000..52d4bef --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs @@ -0,0 +1,114 @@ +using HPPH; + +namespace StableDiffusion.NET; + +public static class DiffusionParameterExtension +{ + public static DiffusionParameter WithSize(this DiffusionParameter parameter, int? width = null, int? height = null) + { + if (width != null) + parameter.Width = width.Value; + + if (height != null) + parameter.Height = height.Value; + + return parameter; + } + + public static DiffusionParameter WithSampler(this DiffusionParameter parameter, Sampler sampler) + { + parameter.SampleMethod = sampler; + + return parameter; + } + + public static DiffusionParameter WithSteps(this DiffusionParameter parameter, int steps) + { + parameter.SampleSteps = steps; + + return parameter; + } + + public static DiffusionParameter WithSeed(this DiffusionParameter parameter, long seed) + { + parameter.Seed = seed; + + return parameter; + } + + public static DiffusionParameter WithClipSkip(this DiffusionParameter parameter, int clipSkip) + { + parameter.ClipSkip = clipSkip; + + return parameter; + } + + public static DiffusionParameter WithCfg(this DiffusionParameter parameter, float cfg) + { + parameter.CfgScale = cfg; + + return parameter; + } + + public static DiffusionParameter WithGuidance(this DiffusionParameter parameter, float guidance) + { + parameter.Guidance = guidance; + + return parameter; + } + + public static DiffusionParameter WithNegativePrompt(this DiffusionParameter parameter, string negativePrompt) + { + parameter.NegativePrompt = negativePrompt; + + return parameter; + } + + public static DiffusionParameter WithControlNet(this DiffusionParameter parameter, IImage image, float? strength = null) + { + parameter.ControlNet.Image = image; + + if (strength != null) + parameter.ControlNet.Strength = strength.Value; + + return parameter; + } + + public static DiffusionParameter WithCannyPreprocessing(this DiffusionParameter parameter, + float? cannyHighThreshold = null, float? cannyLowThreshold = null, + float? cannyWeak = null, float? cannyStrong = null, + bool? cannyInverse = null) + { + parameter.ControlNet.CannyPreprocess = true; + + if (cannyHighThreshold != null) + parameter.ControlNet.CannyHighThreshold = cannyHighThreshold.Value; + + if (cannyLowThreshold != null) + parameter.ControlNet.CannyLowThreshold = cannyLowThreshold.Value; + + if (cannyWeak != null) + parameter.ControlNet.CannyWeak = cannyWeak.Value; + + if (cannyStrong != null) + parameter.ControlNet.CannyStrong = cannyStrong.Value; + + if (cannyInverse != null) + parameter.ControlNet.CannyInverse = cannyInverse.Value; + + return parameter; + } + + public static DiffusionParameter WithPhotomaker(this DiffusionParameter parameter, string inputIdImageDirectory, float? styleRatio = null, bool? normalizeInput = null) + { + parameter.PhotoMaker.InputIdImageDirectory = inputIdImageDirectory; + + if (styleRatio != null) + parameter.PhotoMaker.StyleRatio = styleRatio.Value; + + if (normalizeInput != null) + parameter.PhotoMaker.NormalizeInput = normalizeInput.Value; + + return parameter; + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs new file mode 100644 index 0000000..e18ed24 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs @@ -0,0 +1,22 @@ +namespace StableDiffusion.NET; + +public interface IDiffusionModelParameter +{ + DiffusionModelType DiffusionModelType { get; set; } + + string VaePath { get; set; } + string TaesdPath { get; set; } + + string LoraModelDirectory { get; set; } + string EmbeddingsDirectory { get; set; } + string ControlNetPath { get; set; } + + bool VaeDecodeOnly { get; set; } + bool VaeTiling { get; set; } + bool KeepControlNetOnCPU { get; set; } + bool KeepClipOnCPU { get; set; } + bool KeepVaeOnCPU { get; set; } + + RngType RngType { get; set; } + Schedule Schedule { get; set; } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs new file mode 100644 index 0000000..db59a09 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs @@ -0,0 +1,6 @@ +namespace StableDiffusion.NET; + +public interface IPhotomakerModelParameter +{ + string StackedIdEmbeddingsDirectory { get; set; } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs new file mode 100644 index 0000000..bd74727 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs @@ -0,0 +1,8 @@ +namespace StableDiffusion.NET; + +public interface IQuantizedModelParameter +{ + int ThreadCount { get; set; } + + Quantization Quantization { get; set; } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs b/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs new file mode 100644 index 0000000..cab62fd --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs @@ -0,0 +1,11 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class PhotoMakerParameter +{ + public string InputIdImageDirectory { get; set; } = string.Empty; + public float StyleRatio { get; set; } = 20f; + public bool NormalizeInput { get; set; } = false; +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs new file mode 100644 index 0000000..023a817 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs @@ -0,0 +1,12 @@ +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed class UpscaleModelParameter : IQuantizedModelParameter +{ + public string ModelPath { get; set; } = string.Empty; + public int ThreadCount { get; set; } = 1; + + public Quantization Quantization { get; set; } = Quantization.F16; +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/UpscaleModel.cs b/StableDiffusion.NET/Models/UpscaleModel.cs new file mode 100644 index 0000000..8de4d30 --- /dev/null +++ b/StableDiffusion.NET/Models/UpscaleModel.cs @@ -0,0 +1,83 @@ +using HPPH; +using JetBrains.Annotations; +using System; + +namespace StableDiffusion.NET; + +[PublicAPI] +public sealed unsafe class UpscaleModel : IDisposable +{ + #region Properties & Fields + + private bool _disposed; + + private readonly UpscaleModelParameter _parameter; + + private Native.upscaler_ctx_t* _ctx; + + #endregion + + #region Constructors + + public UpscaleModel(UpscaleModelParameter parameter) + { + ArgumentNullException.ThrowIfNull(parameter, nameof(parameter)); + + parameter.Validate(); + + this._parameter = parameter; + + Initialize(); + } + + ~UpscaleModel() => Dispose(); + + #endregion + + #region Methods + + private void Initialize() + { + _ctx = Native.new_upscaler_ctx(_parameter.ModelPath, + _parameter.ThreadCount, + _parameter.Quantization); + + if (_ctx == null) throw new NullReferenceException("Failed to initialize upscale-model."); + } + + public IImage Upscale(IImage image, int upscaleFactor) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(upscaleFactor, 0, nameof(upscaleFactor)); + + if (_ctx == null) throw new NullReferenceException("The model is not initialized."); + + if (image is not IImage sourceImage) + sourceImage = image.ConvertTo(); + + fixed (byte* imagePtr = sourceImage.AsRefImage()) + { + Native.sd_image_t result = Native.upscale(_ctx, sourceImage.ToSdImage(imagePtr), upscaleFactor); + return ImageHelper.ToImage(&result); + } + } + + private IImage Upscale(Native.sd_image_t image, int upscaleFactor) + { + Native.sd_image_t result = Native.upscale(_ctx, image, upscaleFactor); + return ImageHelper.ToImage(&result); + } + + public void Dispose() + { + if (_disposed) return; + + if (_ctx != null) + Native.free_upscaler_ctx(_ctx); + + GC.SuppressFinalize(this); + _disposed = true; + } + + #endregion +} \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Native.cs b/StableDiffusion.NET/Native/Native.cs index fff17d9..8d27155 100644 --- a/StableDiffusion.NET/Native/Native.cs +++ b/StableDiffusion.NET/Native/Native.cs @@ -50,6 +50,9 @@ internal unsafe partial class Native [LibraryImport(LIB_NAME, EntryPoint = "new_sd_ctx")] internal static partial sd_ctx_t* new_sd_ctx([MarshalAs(UnmanagedType.LPStr)] string model_path, + [MarshalAs(UnmanagedType.LPStr)] string clip_l_path, + [MarshalAs(UnmanagedType.LPStr)] string t5xxl_path, + [MarshalAs(UnmanagedType.LPStr)] string diffusion_model_path, [MarshalAs(UnmanagedType.LPStr)] string vae_path, [MarshalAs(UnmanagedType.LPStr)] string taesd_path, [MarshalAs(UnmanagedType.LPStr)] string control_net_path_c_str, @@ -76,6 +79,7 @@ internal unsafe partial class Native [MarshalAs(UnmanagedType.LPStr)] string negative_prompt, int clip_skip, float cfg_scale, + float guidance, int width, int height, sample_method_t sample_method, @@ -95,6 +99,7 @@ internal unsafe partial class Native [MarshalAs(UnmanagedType.LPStr)] string negative_prompt, int clip_skip, float cfg_scale, + float guidance, int width, int height, sample_method_t sample_method, diff --git a/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings b/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings index 13effd6..65e7f6a 100644 --- a/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings +++ b/StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings @@ -1,8 +1,16 @@  True True + True True True True True + True + True + True + True + True + True + True True \ No newline at end of file diff --git a/StableDiffusion.NET/StableDiffusionCpp.cs b/StableDiffusion.NET/StableDiffusionCpp.cs new file mode 100644 index 0000000..cf7e87c --- /dev/null +++ b/StableDiffusion.NET/StableDiffusionCpp.cs @@ -0,0 +1,77 @@ +using System; +using System.Runtime.InteropServices; +using JetBrains.Annotations; + +namespace StableDiffusion.NET; + +[PublicAPI] +public static unsafe class StableDiffusionCpp +{ + #region Properties & Fields + + // ReSharper disable NotAccessedField.Local - They are important, the delegate can be collected if it's not stored! + private static readonly Native.sd_log_cb_t LOG_CALLBACK; + private static readonly Native.sd_progress_cb_t PROGRESS_CALLBACK; + // ReSharper restore NotAccessedField.Local + + #endregion + + #region Events + + public static event EventHandler? Log; + public static event EventHandler? Progress; + + #endregion + + #region Constructors + + static StableDiffusionCpp() + { + Native.sd_set_log_callback(LOG_CALLBACK = OnNativeLog, null); + Native.sd_set_progress_callback(PROGRESS_CALLBACK = OnNativeProgress, null); + } + + #endregion + + #region Methods + + public static bool LoadNativeLibrary(string libraryPath) => Native.LoadNativeLibrary(libraryPath); + + public static void Convert(string modelPath, string vaePath, Quantization quantization, string outputPath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(nameof(modelPath)); + ArgumentException.ThrowIfNullOrWhiteSpace(nameof(outputPath)); + ArgumentNullException.ThrowIfNull(vaePath); + if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization)); + + Native.convert(modelPath, vaePath, outputPath, quantization); + } + + public static string GetSystemInfo() + { + void* s = Native.sd_get_system_info(); + return Marshal.PtrToStringUTF8((nint)s) ?? ""; + } + + public static int GetNumPhysicalCores() => Native.get_num_physical_cores(); + + private static void OnNativeLog(LogLevel level, string text, void* data) + { + try + { + Log?.Invoke(null, new StableDiffusionLogEventArgs(level, text)); + } + catch { /**/ } + } + + private static void OnNativeProgress(int step, int steps, float time, void* data) + { + try + { + Progress?.Invoke(null, new StableDiffusionProgressEventArgs(step, steps, time)); + } + catch { /**/ } + } + + #endregion +} \ No newline at end of file diff --git a/StableDiffusion.NET/StableDiffusionParameter.cs b/StableDiffusion.NET/StableDiffusionParameter.cs deleted file mode 100644 index 4fc8b50..0000000 --- a/StableDiffusion.NET/StableDiffusionParameter.cs +++ /dev/null @@ -1,48 +0,0 @@ -using HPPH; -using JetBrains.Annotations; - -namespace StableDiffusion.NET; - -[PublicAPI] -public sealed class StableDiffusionParameter -{ - #region Properties & Fields - - public string NegativePrompt { get; set; } = string.Empty; - public float CfgScale { get; set; } = 7.5f; - public int Width { get; set; } = 512; - public int Height { get; set; } = 512; - public Sampler SampleMethod { get; set; } = Sampler.Euler_A; - public int SampleSteps { get; set; } = 25; - public long Seed { get; set; } = -1; - public float Strength { get; set; } = 0.7f; - public int ClipSkip { get; set; } = -1; - - public StableDiffusionControlNetParameter ControlNet { get; } = new(); - public PhotoMakerParameter PhotoMaker { get; } = new(); - - #endregion -} - -[PublicAPI] -public sealed class StableDiffusionControlNetParameter -{ - public bool IsEnabled => Image != null; - - public IImage? Image { get; set; } = null; - public float Strength { get; set; } = 0.9f; - public bool CannyPreprocess { get; set; } = false; - public float CannyHighThreshold { get; set; } = 0.08f; - public float CannyLowThreshold { get; set; } = 0.08f; - public float CannyWeak { get; set; } = 0.8f; - public float CannyStrong { get; set; } = 1.0f; - public bool CannyInverse { get; set; } = false; -} - -[PublicAPI] -public sealed class PhotoMakerParameter -{ - public string InputIdImageDirectory { get; set; } = string.Empty; - public float StyleRatio { get; set; } = 20f; - public bool NormalizeInput { get; set; } = false; -} \ No newline at end of file diff --git a/StableDiffusion.NET/UpscalerModelParameter.cs b/StableDiffusion.NET/UpscalerModelParameter.cs deleted file mode 100644 index 264f694..0000000 --- a/StableDiffusion.NET/UpscalerModelParameter.cs +++ /dev/null @@ -1,17 +0,0 @@ -using JetBrains.Annotations; - -namespace StableDiffusion.NET; - -[PublicAPI] -public class UpscalerModelParameter -{ - #region Properties & Fields - - public int ThreadCount { get; set; } = 8; - public string ESRGANPath { get; set; } = string.Empty; - - //TODO DarthAffe 01.01.2024: K-Quants doesn't seem to work so far - public Quantization Quantization { get; set; } = Quantization.F16; - - #endregion -}