diff --git a/Header/stable-diffusion.h b/Header/stable-diffusion.h index e34cdec..d54376b 100644 --- a/Header/stable-diffusion.h +++ b/Header/stable-diffusion.h @@ -150,6 +150,11 @@ typedef struct { float rel_size_y; } sd_tiling_params_t; +typedef struct { + const char* name; + const char* path; +} sd_embedding_t; + typedef struct { const char* model_path; const char* clip_l_path; @@ -164,7 +169,8 @@ typedef struct { const char* taesd_path; const char* control_net_path; const char* lora_model_dir; - const char* embedding_dir; + const sd_embedding_t* embeddings; + uint32_t embedding_count; const char* photo_maker_path; const char* tensor_type_rules; bool vae_decode_only; @@ -219,6 +225,8 @@ typedef struct { int sample_steps; float eta; int shifted_timestep; + float* custom_sigmas; + int custom_sigmas_count; } sd_sample_params_t; typedef struct { @@ -236,6 +244,14 @@ typedef struct { } sd_easycache_params_t; typedef struct { + bool is_high_noise; + float multiplier; + const char* path; +} sd_lora_t; + +typedef struct { + const sd_lora_t* loras; + uint32_t lora_count; const char* prompt; const char* negative_prompt; int clip_skip; @@ -259,6 +275,8 @@ typedef struct { } sd_img_gen_params_t; typedef struct { + const sd_lora_t* loras; + uint32_t lora_count; const char* prompt; const char* negative_prompt; int clip_skip; @@ -331,7 +349,8 @@ typedef struct upscaler_ctx_t upscaler_ctx_t; SD_API upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path, bool offload_params_to_cpu, bool direct, - int n_threads); + int n_threads, + int tile_size); SD_API void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx); SD_API sd_image_t upscale(upscaler_ctx_t* upscaler_ctx, @@ -353,8 +372,11 @@ SD_API bool preprocess_canny(sd_image_t image, float strong, bool inverse); +SD_API const char* sd_commit(void); +SD_API const char* sd_version(void); + #ifdef __cplusplus } #endif -#endif // __STABLE_DIFFUSION_H__ +#endif // __STABLE_DIFFUSION_H__ \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs index dc3a7d4..4a1af18 100644 --- a/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using JetBrains.Annotations; namespace StableDiffusion.NET; @@ -24,8 +25,11 @@ public sealed class DiffusionModelParameter /// /// path to embeddings /// + [Obsolete("Use Embeddings instead")] public string EmbeddingsDirectory { get; set; } = string.Empty; + public List Embeddings { get; } = []; + /// /// path to control net model /// diff --git a/StableDiffusion.NET/Models/Parameter/Embedding.cs b/StableDiffusion.NET/Models/Parameter/Embedding.cs new file mode 100644 index 0000000..b53d5e5 --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Embedding.cs @@ -0,0 +1,18 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StableDiffusion.NET; + +public sealed class Embedding +{ + public required string Name { get; init; } + public required string Path { get; init; } + + public Embedding() { } + + [SetsRequiredMembers] + public Embedding(string name, string path) + { + this.Name = name; + this.Path = path; + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs index 6e398c1..aa53e75 100644 --- a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs +++ b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionModelBuilderExtension.cs @@ -37,6 +37,7 @@ public static class DiffusionModelBuilderExtension return parameter; } + [Obsolete("Use WithEmbedding instead")] public static DiffusionModelParameter WithEmbeddingSupport(this DiffusionModelParameter parameter, string embeddingsDirectory) { ArgumentNullException.ThrowIfNull(embeddingsDirectory); @@ -46,6 +47,15 @@ public static class DiffusionModelBuilderExtension return parameter; } + public static DiffusionModelParameter WithEmbedding(this DiffusionModelParameter parameter, Embedding embedding) + { + ArgumentNullException.ThrowIfNull(embedding); + + parameter.Embeddings.Add(embedding); + + return parameter; + } + public static DiffusionModelParameter WithControlNet(this DiffusionModelParameter parameter, string controlNetPath) { ArgumentNullException.ThrowIfNull(controlNetPath); diff --git a/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs b/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs index 8cfcd41..4e67566 100644 --- a/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/ImageGenerationParameter.cs @@ -1,4 +1,5 @@ -using HPPH; +using System.Collections.Generic; +using HPPH; using JetBrains.Annotations; namespace StableDiffusion.NET; @@ -61,6 +62,8 @@ public sealed class ImageGenerationParameter public EasyCache EasyCache { get; } = new(); + public List Loras { get; } = []; + #endregion public static ImageGenerationParameter Create() => new(); diff --git a/StableDiffusion.NET/Models/Parameter/Lora.cs b/StableDiffusion.NET/Models/Parameter/Lora.cs new file mode 100644 index 0000000..7cd9e5d --- /dev/null +++ b/StableDiffusion.NET/Models/Parameter/Lora.cs @@ -0,0 +1,18 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StableDiffusion.NET; + +public sealed class Lora +{ + public bool IsHighNoise { get; set; } = false; + public float Multiplier { get; set; } = 1f; + public required string Path { get; init; } + + public Lora() { } + + [SetsRequiredMembers] + public Lora(string path) + { + this.Path = path; + } +} \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/SampleParameter.cs b/StableDiffusion.NET/Models/Parameter/SampleParameter.cs index 39e988c..83da2fb 100644 --- a/StableDiffusion.NET/Models/Parameter/SampleParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/SampleParameter.cs @@ -26,5 +26,7 @@ public sealed class SampleParameter public int ShiftedTimestep { get; set; } = 0; + public float[] CustomSigmas { get; set; } = []; + internal SampleParameter() { } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs index b5ae8ff..27419d9 100644 --- a/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs @@ -24,5 +24,7 @@ public sealed class UpscaleModelParameter /// public bool ConvDirect { get; set; } = false; + public int TileSize { get; set; } = 128; + public static UpscaleModelParameter Create() => new(); } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/VideoGenerationParameter.cs b/StableDiffusion.NET/Models/Parameter/VideoGenerationParameter.cs index b6de6af..19549b7 100644 --- a/StableDiffusion.NET/Models/Parameter/VideoGenerationParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/VideoGenerationParameter.cs @@ -1,5 +1,6 @@ using HPPH; using JetBrains.Annotations; +using System.Collections.Generic; namespace StableDiffusion.NET; @@ -40,6 +41,8 @@ public sealed class VideoGenerationParameter public EasyCache EasyCache { get; } = new(); + public List Loras { get; } = []; + #endregion public static VideoGenerationParameter Create() => new(); diff --git a/StableDiffusion.NET/Models/UpscaleModel.cs b/StableDiffusion.NET/Models/UpscaleModel.cs index 9450422..a4bc2cc 100644 --- a/StableDiffusion.NET/Models/UpscaleModel.cs +++ b/StableDiffusion.NET/Models/UpscaleModel.cs @@ -41,7 +41,8 @@ public sealed unsafe class UpscaleModel : IDisposable _ctx = Native.new_upscaler_ctx(ModelParameter.ModelPath, ModelParameter.OffloadParamsToCPU, ModelParameter.ConvDirect, - ModelParameter.ThreadCount); + ModelParameter.ThreadCount, + ModelParameter.TileSize); if (_ctx == null) throw new NullReferenceException("Failed to initialize upscale-model."); } diff --git a/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs index e8ff8b8..ba10023 100644 --- a/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs +++ b/StableDiffusion.NET/Native/Marshaller/DiffusionModelParameterMarshaller.cs @@ -1,55 +1,18 @@ -using System.Runtime.InteropServices.Marshalling; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.Marshalling; namespace StableDiffusion.NET; -[CustomMarshaller(typeof(DiffusionModelParameter), MarshalMode.ManagedToUnmanagedIn, typeof(DiffusionModelParameterMarshaller))] -[CustomMarshaller(typeof(DiffusionModelParameter), MarshalMode.ManagedToUnmanagedRef, typeof(DiffusionModelParameterMarshaller))] +[CustomMarshaller(typeof(DiffusionModelParameter), MarshalMode.ManagedToUnmanagedIn, typeof(DiffusionModelParameterMarshallerIn))] +[CustomMarshaller(typeof(DiffusionModelParameter), MarshalMode.ManagedToUnmanagedOut, typeof(DiffusionModelParameterMarshaller))] +[CustomMarshaller(typeof(DiffusionModelParameter), MarshalMode.ManagedToUnmanagedRef, typeof(DiffusionModelParameterMarshallerRef))] internal static unsafe class DiffusionModelParameterMarshaller { - public static Native.Types.sd_ctx_params_t ConvertToUnmanaged(DiffusionModelParameter managed) - => new() - { - model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ModelPath), - clip_l_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipLPath), - clip_g_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipGPath), - clip_vision_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipVisionPath), - t5xxl_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.T5xxlPath), - llm_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.LLMPath), - llm_vision_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.LLMVisionPath), - diffusion_model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.DiffusionModelPath), - high_noise_diffusion_model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.HighNoiseDiffusionModelPath), - vae_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.VaePath), - taesd_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.TaesdPath), - control_net_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ControlNetPath), - lora_model_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.LoraModelDirectory), - embedding_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.EmbeddingsDirectory), - photo_maker_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.StackedIdEmbeddingsDirectory), - tensor_type_rules = AnsiStringMarshaller.ConvertToUnmanaged(managed.TensorTypeRules), - vae_decode_only = (sbyte)(managed.VaeDecodeOnly ? 1 : 0), - free_params_immediately = (sbyte)(managed.FreeParamsImmediately ? 1 : 0), - n_threads = managed.ThreadCount, - wtype = managed.Quantization, - rng_type = managed.RngType, - sampler_rng_type = managed.SamplerRngType, - prediction = managed.Prediction, - lora_apply_mode = managed.LoraApplyMode, - offload_params_to_cpu = (sbyte)(managed.OffloadParamsToCPU ? 1 : 0), - keep_clip_on_cpu = (sbyte)(managed.KeepClipOnCPU ? 1 : 0), - keep_control_net_on_cpu = (sbyte)(managed.KeepControlNetOnCPU ? 1 : 0), - keep_vae_on_cpu = (sbyte)(managed.KeepVaeOnCPU ? 1 : 0), - diffusion_flash_attn = (sbyte)(managed.FlashAttention ? 1 : 0), - tae_preview_only = (sbyte)(managed.TaePreviewOnly ? 1 : 0), - diffusion_conv_direct = (sbyte)(managed.DiffusionConvDirect ? 1 : 0), - vae_conv_direct = (sbyte)(managed.VaeConvDirect ? 1 : 0), - force_sdxl_vae_conv_scale = (sbyte)(managed.ForceSdxlVaeConvScale ? 1 : 0), - chroma_use_dit_mask = (sbyte)(managed.ChromaUseDitMap ? 1 : 0), - chroma_use_t5_mask = (sbyte)(managed.ChromaEnableT5Map ? 1 : 0), - chroma_t5_mask_pad = managed.ChromaT5MaskPad, - flow_shift = managed.FlowShift - }; - public static DiffusionModelParameter ConvertToManaged(Native.Types.sd_ctx_params_t unmanaged) - => new() + { + DiffusionModelParameter parameter = new() { ModelPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.model_path) ?? string.Empty, ClipLPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.clip_l_path) ?? string.Empty, @@ -64,7 +27,6 @@ internal static unsafe class DiffusionModelParameterMarshaller TaesdPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.taesd_path) ?? string.Empty, ControlNetPath = AnsiStringMarshaller.ConvertToManaged(unmanaged.control_net_path) ?? string.Empty, LoraModelDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.lora_model_dir) ?? string.Empty, - EmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.embedding_dir) ?? string.Empty, StackedIdEmbeddingsDirectory = AnsiStringMarshaller.ConvertToManaged(unmanaged.photo_maker_path) ?? string.Empty, TensorTypeRules = AnsiStringMarshaller.ConvertToManaged(unmanaged.tensor_type_rules) ?? string.Empty, VaeDecodeOnly = unmanaged.vae_decode_only == 1, @@ -90,21 +52,155 @@ internal static unsafe class DiffusionModelParameterMarshaller FlowShift = unmanaged.flow_shift }; - public static void Free(Native.Types.sd_ctx_params_t unmanaged) + for (int i = 0; i < unmanaged.embedding_count; i++) + { + Native.Types.sd_embedding_t embedding = unmanaged.embeddings[i]; + parameter.Embeddings.Add(new Embedding + { + Name = AnsiStringMarshaller.ConvertToManaged(embedding.name) ?? string.Empty, + Path = AnsiStringMarshaller.ConvertToManaged(embedding.path) ?? string.Empty + }); + } + + return parameter; + } + + internal ref struct DiffusionModelParameterMarshallerIn { - AnsiStringMarshaller.Free(unmanaged.model_path); - AnsiStringMarshaller.Free(unmanaged.clip_l_path); - AnsiStringMarshaller.Free(unmanaged.clip_g_path); - AnsiStringMarshaller.Free(unmanaged.t5xxl_path); - AnsiStringMarshaller.Free(unmanaged.llm_path); - AnsiStringMarshaller.Free(unmanaged.llm_vision_path); - AnsiStringMarshaller.Free(unmanaged.diffusion_model_path); - AnsiStringMarshaller.Free(unmanaged.vae_path); - AnsiStringMarshaller.Free(unmanaged.taesd_path); - AnsiStringMarshaller.Free(unmanaged.control_net_path); - AnsiStringMarshaller.Free(unmanaged.lora_model_dir); - AnsiStringMarshaller.Free(unmanaged.embedding_dir); - AnsiStringMarshaller.Free(unmanaged.photo_maker_path); - AnsiStringMarshaller.Free(unmanaged.tensor_type_rules); + private Native.Types.sd_ctx_params_t _ctxParams; + + private Native.Types.sd_embedding_t* _embeddings; + + public DiffusionModelParameterMarshallerIn() { } + + public void FromManaged(DiffusionModelParameter managed) + { + //_embeddings = (Native.Types.sd_embedding_t*)NativeMemory.Alloc((nuint)managed.Embeddings.Count, (nuint)Marshal.SizeOf()); + + //for (int i = 0; i < managed.Embeddings.Count; i++) + //{ + // Embedding embedding = managed.Embeddings[i]; + + // _embeddings[i] = new Native.Types.sd_embedding_t + // { + // name = AnsiStringMarshaller.ConvertToUnmanaged(embedding.Name), + // path = AnsiStringMarshaller.ConvertToUnmanaged(embedding.Path), + // }; + //} + + //HACK DarthAffe 25.12.2025 Workaround to support EmbeddingsDir till the next major release + List embeddings = []; + { + embeddings.AddRange(managed.Embeddings); + + try + { + if (!string.IsNullOrWhiteSpace(managed.EmbeddingsDirectory) && Directory.Exists(managed.EmbeddingsDirectory)) + { + foreach (string file in Directory.GetFiles(managed.EmbeddingsDirectory)) + embeddings.Add(new Embedding(Path.GetFileNameWithoutExtension(file), file)); + } + } + catch { /**/ } + + _embeddings = (Native.Types.sd_embedding_t*)NativeMemory.Alloc((nuint)embeddings.Count, (nuint)Marshal.SizeOf()); + + for (int i = 0; i < embeddings.Count; i++) + { + Embedding embedding = embeddings[i]; + + _embeddings[i] = new Native.Types.sd_embedding_t + { + name = AnsiStringMarshaller.ConvertToUnmanaged(embedding.Name), + path = AnsiStringMarshaller.ConvertToUnmanaged(embedding.Path), + }; + } + } + + _ctxParams = new Native.Types.sd_ctx_params_t + { + model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ModelPath), + clip_l_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipLPath), + clip_g_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipGPath), + clip_vision_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ClipVisionPath), + t5xxl_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.T5xxlPath), + llm_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.LLMPath), + llm_vision_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.LLMVisionPath), + diffusion_model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.DiffusionModelPath), + high_noise_diffusion_model_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.HighNoiseDiffusionModelPath), + vae_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.VaePath), + taesd_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.TaesdPath), + control_net_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.ControlNetPath), + lora_model_dir = AnsiStringMarshaller.ConvertToUnmanaged(managed.LoraModelDirectory), + embeddings = _embeddings, + embedding_count = (uint)embeddings.Count, + photo_maker_path = AnsiStringMarshaller.ConvertToUnmanaged(managed.StackedIdEmbeddingsDirectory), + tensor_type_rules = AnsiStringMarshaller.ConvertToUnmanaged(managed.TensorTypeRules), + vae_decode_only = (sbyte)(managed.VaeDecodeOnly ? 1 : 0), + free_params_immediately = (sbyte)(managed.FreeParamsImmediately ? 1 : 0), + n_threads = managed.ThreadCount, + wtype = managed.Quantization, + rng_type = managed.RngType, + sampler_rng_type = managed.SamplerRngType, + prediction = managed.Prediction, + lora_apply_mode = managed.LoraApplyMode, + offload_params_to_cpu = (sbyte)(managed.OffloadParamsToCPU ? 1 : 0), + keep_clip_on_cpu = (sbyte)(managed.KeepClipOnCPU ? 1 : 0), + keep_control_net_on_cpu = (sbyte)(managed.KeepControlNetOnCPU ? 1 : 0), + keep_vae_on_cpu = (sbyte)(managed.KeepVaeOnCPU ? 1 : 0), + diffusion_flash_attn = (sbyte)(managed.FlashAttention ? 1 : 0), + tae_preview_only = (sbyte)(managed.TaePreviewOnly ? 1 : 0), + diffusion_conv_direct = (sbyte)(managed.DiffusionConvDirect ? 1 : 0), + vae_conv_direct = (sbyte)(managed.VaeConvDirect ? 1 : 0), + force_sdxl_vae_conv_scale = (sbyte)(managed.ForceSdxlVaeConvScale ? 1 : 0), + chroma_use_dit_mask = (sbyte)(managed.ChromaUseDitMap ? 1 : 0), + chroma_use_t5_mask = (sbyte)(managed.ChromaEnableT5Map ? 1 : 0), + chroma_t5_mask_pad = managed.ChromaT5MaskPad, + flow_shift = managed.FlowShift + }; + } + + public Native.Types.sd_ctx_params_t ToUnmanaged() => _ctxParams; + + public void Free() + { + + AnsiStringMarshaller.Free(_ctxParams.model_path); + AnsiStringMarshaller.Free(_ctxParams.clip_l_path); + AnsiStringMarshaller.Free(_ctxParams.clip_g_path); + AnsiStringMarshaller.Free(_ctxParams.t5xxl_path); + AnsiStringMarshaller.Free(_ctxParams.llm_path); + AnsiStringMarshaller.Free(_ctxParams.llm_vision_path); + AnsiStringMarshaller.Free(_ctxParams.diffusion_model_path); + AnsiStringMarshaller.Free(_ctxParams.vae_path); + AnsiStringMarshaller.Free(_ctxParams.taesd_path); + AnsiStringMarshaller.Free(_ctxParams.control_net_path); + AnsiStringMarshaller.Free(_ctxParams.lora_model_dir); + AnsiStringMarshaller.Free(_ctxParams.photo_maker_path); + AnsiStringMarshaller.Free(_ctxParams.tensor_type_rules); + + for (int i = 0; i < _ctxParams.embedding_count; i++) + { + AnsiStringMarshaller.Free(_ctxParams.embeddings[i].name); + AnsiStringMarshaller.Free(_ctxParams.embeddings[i].path); + } + + if (_embeddings != null) + NativeMemory.Free(_embeddings); + } + } + + internal ref struct DiffusionModelParameterMarshallerRef() + { + private DiffusionModelParameterMarshallerIn _inMarshaller = new(); + private DiffusionModelParameter? _parameter; + + public void FromManaged(DiffusionModelParameter managed) => _inMarshaller.FromManaged(managed); + public Native.Types.sd_ctx_params_t ToUnmanaged() => _inMarshaller.ToUnmanaged(); + + public void FromUnmanaged(Native.Types.sd_ctx_params_t unmanaged) => _parameter = ConvertToManaged(unmanaged); + public DiffusionModelParameter ToManaged() => _parameter!; + + public void Free() => _inMarshaller.Free(); } } \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs index 4fbe0d4..b7e2242 100644 --- a/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs +++ b/StableDiffusion.NET/Native/Marshaller/ImageGenerationParameterMarshaller.cs @@ -9,9 +9,9 @@ namespace StableDiffusion.NET; [CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedIn, typeof(ImageGenerationParameterMarshallerIn))] [CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedOut, typeof(ImageGenerationParameterMarshaller))] [CustomMarshaller(typeof(ImageGenerationParameter), MarshalMode.ManagedToUnmanagedRef, typeof(ImageGenerationParameterMarshallerRef))] -internal static class ImageGenerationParameterMarshaller +internal static unsafe class ImageGenerationParameterMarshaller { - public static unsafe ImageGenerationParameter ConvertToManaged(Native.Types.sd_img_gen_params_t unmanaged) + public static ImageGenerationParameter ConvertToManaged(Native.Types.sd_img_gen_params_t unmanaged) { ImageGenerationParameter parameter = new() { @@ -56,28 +56,21 @@ internal static class ImageGenerationParameterMarshaller } }; + for (int i = 0; i < unmanaged.lora_count; i++) + { + Native.Types.sd_lora_t lora = unmanaged.loras[i]; + parameter.Loras.Add(new Lora + { + IsHighNoise = lora.is_high_noise == 1, + Multiplier = lora.multiplier, + Path = AnsiStringMarshaller.ConvertToManaged(lora.path) ?? string.Empty + }); + } + return parameter; } - public static unsafe void Free(Native.Types.sd_img_gen_params_t unmanaged) - { - AnsiStringMarshaller.Free(unmanaged.prompt); - AnsiStringMarshaller.Free(unmanaged.negative_prompt); - - unmanaged.init_image.Free(); - unmanaged.mask_image.Free(); - unmanaged.control_image.Free(); - - if (unmanaged.ref_images != null) - ImageHelper.Free(unmanaged.ref_images, unmanaged.ref_images_count); - - if (unmanaged.pm_params.id_images != null) - ImageHelper.Free(unmanaged.pm_params.id_images, unmanaged.pm_params.id_images_count); - - SampleParameterMarshaller.Free(unmanaged.sample_params); - } - - internal unsafe ref struct ImageGenerationParameterMarshallerIn + internal ref struct ImageGenerationParameterMarshallerIn { private SampleParameterMarshaller.SampleParameterMarshallerIn _sampleParameterMarshaller = new(); private Native.Types.sd_img_gen_params_t _imgGenParams; @@ -87,6 +80,7 @@ internal static class ImageGenerationParameterMarshaller private Native.Types.sd_image_t _controlNetImage; private Native.Types.sd_image_t* _refImages; private Native.Types.sd_image_t* _pmIdImages; + private Native.Types.sd_lora_t* _loras; public ImageGenerationParameterMarshallerIn() { } @@ -99,6 +93,18 @@ internal static class ImageGenerationParameterMarshaller _refImages = managed.RefImages == null ? null : managed.RefImages.ToSdImage(); _pmIdImages = managed.PhotoMaker.IdImages == null ? null : managed.PhotoMaker.IdImages.ToSdImage(); + _loras = (Native.Types.sd_lora_t*)NativeMemory.Alloc((nuint)managed.Loras.Count, (nuint)Marshal.SizeOf()); + for (int i = 0; i < managed.Loras.Count; i++) + { + Lora lora = managed.Loras[i]; + _loras[i] = new Native.Types.sd_lora_t + { + is_high_noise = (sbyte)(lora.IsHighNoise ? 1 : 0), + multiplier = lora.Multiplier, + path = AnsiStringMarshaller.ConvertToUnmanaged(lora.Path) + }; + } + if (managed.MaskImage != null) _maskImage = managed.MaskImage.ToSdImage(true); else if (managed.InitImage != null) @@ -161,7 +167,9 @@ internal static class ImageGenerationParameterMarshaller control_strength = managed.ControlNet.Strength, pm_params = photoMakerParams, vae_tiling_params = tilingParams, - easycache = easyCache + easycache = easyCache, + loras = _loras, + lora_count = (uint)managed.Loras.Count }; } @@ -184,6 +192,12 @@ internal static class ImageGenerationParameterMarshaller ImageHelper.Free(_pmIdImages, _imgGenParams.pm_params.id_images_count); _sampleParameterMarshaller.Free(); + + for (int i = 0; i < _imgGenParams.lora_count; i++) + AnsiStringMarshaller.Free(_imgGenParams.loras[i].path); + + if (_loras != null) + NativeMemory.Free(_loras); } } diff --git a/StableDiffusion.NET/Native/Marshaller/SampleParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/SampleParameterMarshaller.cs index 79c79ac..54de39b 100644 --- a/StableDiffusion.NET/Native/Marshaller/SampleParameterMarshaller.cs +++ b/StableDiffusion.NET/Native/Marshaller/SampleParameterMarshaller.cs @@ -9,9 +9,9 @@ namespace StableDiffusion.NET; [CustomMarshaller(typeof(SampleParameter), MarshalMode.ManagedToUnmanagedIn, typeof(SampleParameterMarshallerIn))] [CustomMarshaller(typeof(SampleParameter), MarshalMode.ManagedToUnmanagedOut, typeof(SampleParameterMarshaller))] [CustomMarshaller(typeof(SampleParameter), MarshalMode.ManagedToUnmanagedRef, typeof(SampleParameterMarshallerRef))] -internal static class SampleParameterMarshaller +internal static unsafe class SampleParameterMarshaller { - public static unsafe SampleParameter ConvertToManaged(Native.Types.sd_sample_params_t unmanaged) + public static SampleParameter ConvertToManaged(Native.Types.sd_sample_params_t unmanaged) { SampleParameter parameter = new() { @@ -33,32 +33,34 @@ internal static class SampleParameterMarshaller SampleMethod = unmanaged.sample_method, SampleSteps = unmanaged.sample_steps, Eta = unmanaged.eta, - ShiftedTimestep = unmanaged.shifted_timestep + ShiftedTimestep = unmanaged.shifted_timestep, + CustomSigmas = new float[unmanaged.custom_sigmas_count] }; if (unmanaged.guidance.slg.layers != null) new Span(unmanaged.guidance.slg.layers, (int)unmanaged.guidance.slg.layer_count).CopyTo(parameter.Guidance.Slg.Layers); + if (unmanaged.custom_sigmas != null) + new Span(unmanaged.custom_sigmas, unmanaged.custom_sigmas_count).CopyTo(parameter.CustomSigmas); + return parameter; } - public static unsafe void Free(Native.Types.sd_sample_params_t unmanaged) - { - if (unmanaged.guidance.slg.layers != null) - NativeMemory.Free(unmanaged.guidance.slg.layers); - } - - internal unsafe ref struct SampleParameterMarshallerIn + internal ref struct SampleParameterMarshallerIn { private Native.Types.sd_sample_params_t _sampleParams; private int* _slgLayers; + private float* _customSigmas; public void FromManaged(SampleParameter managed) { _slgLayers = (int*)NativeMemory.Alloc((nuint)managed.Guidance.Slg.Layers.Length, (nuint)Marshal.SizeOf()); managed.Guidance.Slg.Layers.AsSpan().CopyTo(new Span(_slgLayers, managed.Guidance.Slg.Layers.Length)); + _customSigmas = (float*)NativeMemory.Alloc((nuint)managed.CustomSigmas.Length, (nuint)Marshal.SizeOf()); + managed.CustomSigmas.AsSpan().CopyTo(new Span(_customSigmas, managed.CustomSigmas.Length)); + Native.Types.sd_slg_params_t slg = new() { layers = _slgLayers, @@ -84,7 +86,9 @@ internal static class SampleParameterMarshaller sample_method = managed.SampleMethod, sample_steps = managed.SampleSteps, eta = managed.Eta, - shifted_timestep = managed.ShiftedTimestep + shifted_timestep = managed.ShiftedTimestep, + custom_sigmas = _customSigmas, + custom_sigmas_count = managed.CustomSigmas.Length }; } @@ -94,6 +98,9 @@ internal static class SampleParameterMarshaller { if (_slgLayers != null) NativeMemory.Free(_slgLayers); + + if (_customSigmas != null) + NativeMemory.Free(_customSigmas); } } diff --git a/StableDiffusion.NET/Native/Marshaller/VideoGenerationParameterMarshaller.cs b/StableDiffusion.NET/Native/Marshaller/VideoGenerationParameterMarshaller.cs index c6f77f4..02a10e0 100644 --- a/StableDiffusion.NET/Native/Marshaller/VideoGenerationParameterMarshaller.cs +++ b/StableDiffusion.NET/Native/Marshaller/VideoGenerationParameterMarshaller.cs @@ -1,5 +1,6 @@ // ReSharper disable MemberCanBeMadeStatic.Global +using System.Runtime.InteropServices; using System.Runtime.InteropServices.Marshalling; namespace StableDiffusion.NET; @@ -7,9 +8,9 @@ namespace StableDiffusion.NET; [CustomMarshaller(typeof(VideoGenerationParameter), MarshalMode.ManagedToUnmanagedIn, typeof(VideoGenerationParameterMarshallerIn))] [CustomMarshaller(typeof(VideoGenerationParameter), MarshalMode.ManagedToUnmanagedOut, typeof(VideoGenerationParameterMarshaller))] [CustomMarshaller(typeof(VideoGenerationParameter), MarshalMode.ManagedToUnmanagedRef, typeof(VideoGenerationParameterMarshallerRef))] -internal static class VideoGenerationParameterMarshaller +internal static unsafe class VideoGenerationParameterMarshaller { - public static unsafe VideoGenerationParameter ConvertToManaged(Native.Types.sd_vid_gen_params_t unmanaged) + public static VideoGenerationParameter ConvertToManaged(Native.Types.sd_vid_gen_params_t unmanaged) { VideoGenerationParameter parameter = new() { @@ -37,25 +38,21 @@ internal static class VideoGenerationParameterMarshaller } }; + for (int i = 0; i < unmanaged.lora_count; i++) + { + Native.Types.sd_lora_t lora = unmanaged.loras[i]; + parameter.Loras.Add(new Lora + { + IsHighNoise = lora.is_high_noise == 1, + Multiplier = lora.multiplier, + Path = AnsiStringMarshaller.ConvertToManaged(lora.path) ?? string.Empty + }); + } + return parameter; } - public static unsafe void Free(Native.Types.sd_vid_gen_params_t unmanaged) - { - AnsiStringMarshaller.Free(unmanaged.prompt); - AnsiStringMarshaller.Free(unmanaged.negative_prompt); - - unmanaged.init_image.Free(); - unmanaged.end_image.Free(); - - if (unmanaged.control_frames != null) - ImageHelper.Free(unmanaged.control_frames, unmanaged.control_frames_size); - - SampleParameterMarshaller.Free(unmanaged.sample_params); - SampleParameterMarshaller.Free(unmanaged.high_noise_sample_params); - } - - internal unsafe ref struct VideoGenerationParameterMarshallerIn + internal ref struct VideoGenerationParameterMarshallerIn { private SampleParameterMarshaller.SampleParameterMarshallerIn _sampleParameterMarshaller = new(); private SampleParameterMarshaller.SampleParameterMarshallerIn _highNoiseSampleParameterMarshaller = new(); @@ -64,6 +61,7 @@ internal static class VideoGenerationParameterMarshaller private Native.Types.sd_image_t _initImage; private Native.Types.sd_image_t _endImage; private Native.Types.sd_image_t* _controlFrames; + private Native.Types.sd_lora_t* _loras; public VideoGenerationParameterMarshallerIn() { } @@ -75,7 +73,19 @@ internal static class VideoGenerationParameterMarshaller _initImage = managed.InitImage?.ToSdImage() ?? new Native.Types.sd_image_t(); _endImage = managed.EndImage?.ToSdImage() ?? new Native.Types.sd_image_t(); _controlFrames = managed.ControlFrames == null ? null : managed.ControlFrames.ToSdImage(); - + + _loras = (Native.Types.sd_lora_t*)NativeMemory.Alloc((nuint)managed.Loras.Count, (nuint)Marshal.SizeOf()); + for (int i = 0; i < managed.Loras.Count; i++) + { + Lora lora = managed.Loras[i]; + _loras[i] = new Native.Types.sd_lora_t + { + is_high_noise = (sbyte)(lora.IsHighNoise ? 1 : 0), + multiplier = lora.Multiplier, + path = AnsiStringMarshaller.ConvertToUnmanaged(lora.Path) + }; + } + Native.Types.sd_easycache_params_t easyCache = new() { enabled = (sbyte)(managed.EasyCache.IsEnabled ? 1 : 0), @@ -103,6 +113,8 @@ internal static class VideoGenerationParameterMarshaller video_frames = managed.FrameCount, vace_strength = managed.VaceStrength, easycache = easyCache, + loras = _loras, + lora_count = (uint)managed.Loras.Count }; } @@ -121,6 +133,12 @@ internal static class VideoGenerationParameterMarshaller _sampleParameterMarshaller.Free(); _highNoiseSampleParameterMarshaller.Free(); + + for (int i = 0; i < _vidGenParams.lora_count; i++) + AnsiStringMarshaller.Free(_vidGenParams.loras[i].path); + + if (_loras != null) + NativeMemory.Free(_loras); } } diff --git a/StableDiffusion.NET/Native/Native.cs b/StableDiffusion.NET/Native/Native.cs index ffdd241..a6c0875 100644 --- a/StableDiffusion.NET/Native/Native.cs +++ b/StableDiffusion.NET/Native/Native.cs @@ -42,7 +42,7 @@ internal unsafe partial class Native internal static class Types { [StructLayout(LayoutKind.Sequential)] - public struct sd_tiling_params_t + internal struct sd_tiling_params_t { public sbyte enabled; public int tile_size_x; @@ -52,6 +52,13 @@ internal unsafe partial class Native public float rel_size_y; } + [StructLayout(LayoutKind.Sequential)] + internal struct sd_embedding_t + { + public byte* name; + public byte* path; + } + [StructLayout(LayoutKind.Sequential)] internal struct sd_ctx_params_t { @@ -68,7 +75,8 @@ internal unsafe partial class Native public byte* taesd_path; public byte* control_net_path; public byte* lora_model_dir; - public byte* embedding_dir; + public sd_embedding_t* embeddings; + public uint32_t embedding_count; public byte* photo_maker_path; public byte* tensor_type_rules; public sbyte vae_decode_only; @@ -132,6 +140,8 @@ internal unsafe partial class Native public int sample_steps; public float eta; public int shifted_timestep; + public float* custom_sigmas; + public int custom_sigmas_count; } [StructLayout(LayoutKind.Sequential)] @@ -152,9 +162,19 @@ internal unsafe partial class Native public float end_percent; } + [StructLayout(LayoutKind.Sequential)] + internal struct sd_lora_t + { + public sbyte is_high_noise; + public float multiplier; + public byte* path; + } + [StructLayout(LayoutKind.Sequential)] internal struct sd_img_gen_params_t { + public sd_lora_t* loras; + public uint32_t lora_count; public byte* prompt; public byte* negative_prompt; public int clip_skip; @@ -180,6 +200,8 @@ internal unsafe partial class Native [StructLayout(LayoutKind.Sequential)] internal struct sd_vid_gen_params_t { + public sd_lora_t* loras; + public uint32_t lora_count; public byte* prompt; public byte* negative_prompt; public int clip_skip; @@ -305,7 +327,8 @@ internal unsafe partial class Native [LibraryImport(LIB_NAME, EntryPoint = "sd_sample_params_init")] internal static partial void sd_sample_params_init([MarshalUsing(typeof(SampleParameterMarshaller))] ref sd_sample_params_t sample_params); [LibraryImport(LIB_NAME, EntryPoint = "sd_sample_params_to_str")] - internal static partial char* sd_sample_params_to_str([MarshalUsing(typeof(SampleParameterMarshaller))] in sd_sample_params_t sample_params); + [return: MarshalAs(UnmanagedType.LPStr)] + internal static partial string sd_sample_params_to_str([MarshalUsing(typeof(SampleParameterMarshaller))] in sd_sample_params_t sample_params); // @@ -339,7 +362,8 @@ internal unsafe partial class Native internal static partial upscaler_ctx_t* new_upscaler_ctx([MarshalAs(UnmanagedType.LPStr)] string esrgan_path, [MarshalAs(UnmanagedType.I1)] bool offload_params_to_cpu, [MarshalAs(UnmanagedType.I1)] bool direct, - int n_threads); + int n_threads, + int tile_size); [LibraryImport(LIB_NAME, EntryPoint = "free_upscaler_ctx")] internal static partial void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx); @@ -374,5 +398,11 @@ internal unsafe partial class Native float strong, [MarshalAs(UnmanagedType.I1)] bool inverse); + [LibraryImport(LIB_NAME, EntryPoint = "sd_commit")] + internal static partial byte* sd_commit(); + + [LibraryImport(LIB_NAME, EntryPoint = "sd_version")] + internal static partial byte* sd_version(); + #endregion } \ No newline at end of file diff --git a/StableDiffusion.NET/StableDiffusionCpp.cs b/StableDiffusion.NET/StableDiffusionCpp.cs index 7ff65f2..3b5114c 100644 --- a/StableDiffusion.NET/StableDiffusionCpp.cs +++ b/StableDiffusion.NET/StableDiffusionCpp.cs @@ -1,6 +1,7 @@ using HPPH; using JetBrains.Annotations; using System; +using System.Runtime.InteropServices.Marshalling; namespace StableDiffusion.NET; @@ -15,6 +16,8 @@ public static unsafe class StableDiffusionCpp private static Native.sd_preview_cb_t? _previewCallback; // ReSharper restore NotAccessedField.Local + public static string ExpectedSDCommit => "43a70e8"; + #endregion #region Events @@ -62,6 +65,10 @@ public static unsafe class StableDiffusionCpp public static int GetNumPhysicalCores() => Native.sd_get_num_physical_cores(); + public static string GetSDCommit() => AnsiStringMarshaller.ConvertToManaged(Native.sd_commit()) ?? string.Empty; + + public static string GetSDVersion() => AnsiStringMarshaller.ConvertToManaged(Native.sd_version()) ?? string.Empty; + public static Image PreprocessCanny(CannyParameter parameter) { parameter.Validate();