mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-08-10 06:07:58 +00:00
Compare commits
5 Commits
e31a86ce91
...
db99efdd6d
| Author | SHA1 | Date | |
|---|---|---|---|
|
db99efdd6d | ||
|
|
eb7f35ca49 | ||
|
|
50062a4bba | ||
|
|
8457624101 | ||
|
|
10378f42db |
@ -70,7 +70,7 @@ API and command-line option may change frequently.***
|
||||
- [HunyuanVideo 1.5](./docs/hunyuan_video.md)
|
||||
- [LingBot-Video](./docs/lingbot_video.md)
|
||||
- [PhotoMaker](./docs/photo_maker.md) support.
|
||||
- [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL)
|
||||
- [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL, including Plus)
|
||||
- Control Net support with SD 1.5
|
||||
- [ADetailer](./docs/adetailer.md)
|
||||
- LoRA support, same as [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora)
|
||||
|
||||
@ -11,6 +11,10 @@ through a decoupled cross-attention added to every attn2 layer of the
|
||||
UNet. It composes with Control Net, so a reference image (appearance) and
|
||||
an OpenPose hint (pose) can be combined in a single generation.
|
||||
|
||||
Both the classic adapters and the higher-fidelity **Plus** adapters are
|
||||
supported; see [Plus variants](#plus-variants) below. The variant is
|
||||
detected from the weight file, so the same options work for both.
|
||||
|
||||
## Required weights
|
||||
|
||||
1. A base SD 1.5 or SDXL model.
|
||||
@ -21,6 +25,11 @@ an OpenPose hint (pose) can be combined in a single generation.
|
||||
[h94/IP-Adapter](https://huggingface.co/h94/IP-Adapter):
|
||||
- SD 1.5: `models/ip-adapter_sd15.safetensors`
|
||||
- SDXL: `sdxl_models/ip-adapter_sdxl_vit-h.safetensors`
|
||||
- SD 1.5 Plus: `models/ip-adapter-plus_sd15.safetensors`
|
||||
- SDXL Plus: `sdxl_models/ip-adapter-plus_sdxl_vit-h.safetensors`
|
||||
|
||||
The Plus files (`ip-adapter-plus_*`) are used exactly like the classic
|
||||
ones; see [Plus variants](#plus-variants).
|
||||
|
||||
## Options
|
||||
|
||||
@ -45,6 +54,29 @@ sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safet
|
||||
The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and
|
||||
`--offload-to-cpu`) on GPUs with limited VRAM.
|
||||
|
||||
## Plus variants
|
||||
|
||||
The Plus adapters (`ip-adapter-plus_sd15`, `ip-adapter-plus_sdxl_vit-h`)
|
||||
replace the small linear image projection with a Resampler (a
|
||||
Perceiver-style module with learned latent queries). Instead of pooling the
|
||||
CLIP-Vision output into one vector, the Resampler attends over the full grid
|
||||
of penultimate CLIP-Vision hidden states and emits more image tokens (16
|
||||
instead of 4). The result transfers finer detail and layout from the
|
||||
reference, at a small extra cost in the image-projection step.
|
||||
|
||||
No extra flags are needed. The variant is detected from the weight file (the
|
||||
Resampler's `image_proj.latents` tensor), and every Resampler dimension is
|
||||
read from the tensor shapes, so the same `--ip-adapter`,
|
||||
`--ip-adapter-image`, and `--ip-adapter-strength` options apply. Plus
|
||||
composes with Control Net in the same way as the classic adapters.
|
||||
|
||||
```
|
||||
sd-cli -m ..\models\sd_v1.5.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter-plus_sd15.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 7 --steps 30 --sampling-method dpm++2m --scheduler karras -W 512 -H 512
|
||||
```
|
||||
|
||||
The startup log line `IP-Adapter: 16 image tokens` (versus `4` for the
|
||||
classic adapters) confirms a Plus file was loaded.
|
||||
|
||||
## Combining with Control Net
|
||||
|
||||
Add the usual Control Net options to keep the reference appearance while
|
||||
|
||||
@ -1008,7 +1008,7 @@ ArgOptions SDGenerationParams::get_options() {
|
||||
&hires_upscaler},
|
||||
{"",
|
||||
"--extra-sample-args",
|
||||
"extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware",
|
||||
"extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware; lms supports lms_divisions",
|
||||
(int)',',
|
||||
&extra_sample_args},
|
||||
{"",
|
||||
@ -1538,12 +1538,12 @@ ArgOptions SDGenerationParams::get_options() {
|
||||
on_seed_arg},
|
||||
{"",
|
||||
"--sampling-method",
|
||||
"sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]"
|
||||
"sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp, lms]"
|
||||
"(default: euler for Flux/SD3/Wan, euler_a otherwise)",
|
||||
on_sample_method_arg},
|
||||
{"",
|
||||
"--high-noise-sampling-method",
|
||||
"(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]"
|
||||
"(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp, lms]"
|
||||
" default: euler for Flux/SD3/Wan, euler_a otherwise",
|
||||
on_high_noise_sample_method_arg},
|
||||
{"",
|
||||
|
||||
@ -56,6 +56,7 @@ enum sample_method_t {
|
||||
EULER_GE_SAMPLE_METHOD,
|
||||
DPMPP2M_SDE_SAMPLE_METHOD,
|
||||
DPMPP2M_SDE_BT_SAMPLE_METHOD,
|
||||
LMS_SAMPLE_METHOD,
|
||||
SAMPLE_METHOD_COUNT
|
||||
};
|
||||
|
||||
|
||||
@ -31,8 +31,92 @@ namespace IPAdapter {
|
||||
}
|
||||
};
|
||||
|
||||
struct Resampler : public GGMLBlock {
|
||||
int64_t dim = 1280;
|
||||
int64_t depth = 4;
|
||||
int64_t num_queries = 16;
|
||||
int64_t embed_dim = 1280;
|
||||
int64_t output_dim = 2048;
|
||||
int64_t ff_inner = 5120;
|
||||
int64_t dim_head = 64;
|
||||
int64_t heads = 20;
|
||||
|
||||
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
||||
params["latents"] = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, dim, num_queries, 1);
|
||||
}
|
||||
|
||||
Resampler() {}
|
||||
Resampler(int64_t dim, int64_t depth, int64_t num_queries, int64_t embed_dim, int64_t output_dim, int64_t ff_inner)
|
||||
: dim(dim), depth(depth), num_queries(num_queries), embed_dim(embed_dim), output_dim(output_dim), ff_inner(ff_inner) {
|
||||
heads = dim / dim_head;
|
||||
blocks["proj_in"] = std::shared_ptr<GGMLBlock>(new Linear(embed_dim, dim, true));
|
||||
blocks["proj_out"] = std::shared_ptr<GGMLBlock>(new Linear(dim, output_dim, true));
|
||||
blocks["norm_out"] = std::shared_ptr<GGMLBlock>(new LayerNorm(output_dim));
|
||||
for (int64_t i = 0; i < depth; i++) {
|
||||
std::string p = "layers." + std::to_string(i);
|
||||
blocks[p + ".0.norm1"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
|
||||
blocks[p + ".0.norm2"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
|
||||
blocks[p + ".0.to_q"] = std::shared_ptr<GGMLBlock>(new Linear(dim, dim, false));
|
||||
blocks[p + ".0.to_kv"] = std::shared_ptr<GGMLBlock>(new Linear(dim, dim * 2, false));
|
||||
blocks[p + ".0.to_out"] = std::shared_ptr<GGMLBlock>(new Linear(dim, dim, false));
|
||||
blocks[p + ".1.0"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
|
||||
blocks[p + ".1.1"] = std::shared_ptr<GGMLBlock>(new Linear(dim, ff_inner, false));
|
||||
blocks[p + ".1.3"] = std::shared_ptr<GGMLBlock>(new Linear(ff_inner, dim, false));
|
||||
}
|
||||
}
|
||||
|
||||
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image_embeds) {
|
||||
int64_t N = image_embeds->ne[2];
|
||||
auto proj_in = std::dynamic_pointer_cast<Linear>(blocks["proj_in"]);
|
||||
auto proj_out = std::dynamic_pointer_cast<Linear>(blocks["proj_out"]);
|
||||
auto norm_out = std::dynamic_pointer_cast<LayerNorm>(blocks["norm_out"]);
|
||||
|
||||
ggml_tensor* x = proj_in->forward(ctx, image_embeds);
|
||||
ggml_tensor* latents = params["latents"];
|
||||
if (N > 1) {
|
||||
latents = ggml_repeat(ctx->ggml_ctx, latents, ggml_new_tensor_3d(ctx->ggml_ctx, GGML_TYPE_F32, dim, num_queries, N));
|
||||
}
|
||||
|
||||
for (int64_t i = 0; i < depth; i++) {
|
||||
std::string p = "layers." + std::to_string(i);
|
||||
auto norm1 = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".0.norm1"]);
|
||||
auto norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".0.norm2"]);
|
||||
auto to_q = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_q"]);
|
||||
auto to_kv = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_kv"]);
|
||||
auto to_out = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_out"]);
|
||||
|
||||
ggml_tensor* xn = norm1->forward(ctx, x);
|
||||
ggml_tensor* ln = norm2->forward(ctx, latents);
|
||||
ggml_tensor* q = to_q->forward(ctx, ln);
|
||||
ggml_tensor* kv_in = ggml_concat(ctx->ggml_ctx, xn, ln, 1);
|
||||
ggml_tensor* kv = to_kv->forward(ctx, kv_in);
|
||||
int64_t L = kv->ne[1];
|
||||
ggml_tensor* k = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], 0));
|
||||
ggml_tensor* v = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], dim * kv->nb[0]));
|
||||
ggml_tensor* attn = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, heads, nullptr, false, false);
|
||||
attn = to_out->forward(ctx, attn);
|
||||
latents = ggml_add(ctx->ggml_ctx, latents, attn);
|
||||
|
||||
auto ff_norm = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".1.0"]);
|
||||
auto ff_fc1 = std::dynamic_pointer_cast<Linear>(blocks[p + ".1.1"]);
|
||||
auto ff_fc2 = std::dynamic_pointer_cast<Linear>(blocks[p + ".1.3"]);
|
||||
ggml_tensor* h = ff_norm->forward(ctx, latents);
|
||||
h = ff_fc1->forward(ctx, h);
|
||||
h = ggml_gelu_erf(ctx->ggml_ctx, h);
|
||||
h = ff_fc2->forward(ctx, h);
|
||||
latents = ggml_add(ctx->ggml_ctx, latents, h);
|
||||
}
|
||||
|
||||
latents = proj_out->forward(ctx, latents);
|
||||
latents = norm_out->forward(ctx, latents);
|
||||
return latents;
|
||||
}
|
||||
};
|
||||
|
||||
struct IPAdapterRunner : public GGMLRunner {
|
||||
ImageProjModel image_proj;
|
||||
Resampler resampler;
|
||||
bool is_plus = false;
|
||||
int64_t num_tokens = 4;
|
||||
std::string prefix;
|
||||
|
||||
@ -41,6 +125,38 @@ namespace IPAdapter {
|
||||
const std::string prefix,
|
||||
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
|
||||
: GGMLRunner(backend, weight_manager), prefix(prefix) {
|
||||
is_plus = tensor_storage_map.find(prefix + ".image_proj.latents") != tensor_storage_map.end();
|
||||
if (is_plus) {
|
||||
int64_t dim = 1280;
|
||||
int64_t num_queries = 16;
|
||||
int64_t embed_dim = 1280;
|
||||
int64_t output_dim = 2048;
|
||||
int64_t ff_inner = 5120;
|
||||
auto latents_iter = tensor_storage_map.find(prefix + ".image_proj.latents");
|
||||
if (latents_iter != tensor_storage_map.end()) {
|
||||
dim = latents_iter->second.ne[0];
|
||||
num_queries = latents_iter->second.ne[1];
|
||||
}
|
||||
auto proj_in_iter = tensor_storage_map.find(prefix + ".image_proj.proj_in.weight");
|
||||
if (proj_in_iter != tensor_storage_map.end()) {
|
||||
embed_dim = proj_in_iter->second.ne[0];
|
||||
}
|
||||
auto proj_out_iter = tensor_storage_map.find(prefix + ".image_proj.proj_out.weight");
|
||||
if (proj_out_iter != tensor_storage_map.end()) {
|
||||
output_dim = proj_out_iter->second.ne[1];
|
||||
}
|
||||
auto ff_iter = tensor_storage_map.find(prefix + ".image_proj.layers.0.1.1.weight");
|
||||
if (ff_iter != tensor_storage_map.end()) {
|
||||
ff_inner = ff_iter->second.ne[1];
|
||||
}
|
||||
int64_t depth = 0;
|
||||
while (tensor_storage_map.find(prefix + ".image_proj.layers." + std::to_string(depth) + ".0.to_q.weight") != tensor_storage_map.end()) {
|
||||
depth++;
|
||||
}
|
||||
num_tokens = num_queries;
|
||||
resampler = Resampler(dim, depth, num_queries, embed_dim, output_dim, ff_inner);
|
||||
resampler.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
|
||||
} else {
|
||||
int64_t ctx_dim = 768;
|
||||
int64_t clip_dim = 1024;
|
||||
int64_t out_dim = 3072;
|
||||
@ -57,20 +173,25 @@ namespace IPAdapter {
|
||||
image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim);
|
||||
image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
|
||||
}
|
||||
}
|
||||
|
||||
std::string get_desc() override {
|
||||
return "ip_adapter";
|
||||
}
|
||||
|
||||
void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string = "") {
|
||||
if (is_plus) {
|
||||
resampler.get_param_tensors(tensors, prefix + ".image_proj");
|
||||
} else {
|
||||
image_proj.get_param_tensors(tensors, prefix + ".image_proj");
|
||||
}
|
||||
}
|
||||
|
||||
ggml_cgraph* build_graph(const sd::Tensor<float>& image_embeds_tensor) {
|
||||
ggml_cgraph* gf = new_graph_custom(1024);
|
||||
ggml_tensor* embeds = make_input(image_embeds_tensor);
|
||||
auto runner_ctx = get_context();
|
||||
ggml_tensor* out = image_proj.forward(&runner_ctx, embeds);
|
||||
ggml_tensor* out = is_plus ? resampler.forward(&runner_ctx, embeds) : image_proj.forward(&runner_ctx, embeds);
|
||||
ggml_build_forward_expand(gf, out);
|
||||
return gf;
|
||||
}
|
||||
|
||||
@ -588,6 +588,10 @@ struct LoraModel : public GGMLRunner {
|
||||
const std::string& model_tensor_name) {
|
||||
ggml_tensor* out_diff = nullptr;
|
||||
int index = 0;
|
||||
|
||||
std::vector<std::string> used_tensors;
|
||||
bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D;
|
||||
|
||||
while (true) {
|
||||
std::string key;
|
||||
if (index == 0) {
|
||||
@ -595,7 +599,6 @@ struct LoraModel : public GGMLRunner {
|
||||
} else {
|
||||
key = model_tensor_name + "." + std::to_string(index);
|
||||
}
|
||||
bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D;
|
||||
|
||||
std::string lokr_w1_name = "lora." + key + ".lokr_w1";
|
||||
std::string lokr_w1_a_name = "lora." + key + ".lokr_w1_a";
|
||||
@ -663,7 +666,6 @@ struct LoraModel : public GGMLRunner {
|
||||
if (iter != lora_tensors.end()) {
|
||||
float alpha = ggml_ext_backend_tensor_get_f32(iter->second);
|
||||
scale_value = alpha / rank;
|
||||
applied_lora_tensors.insert(alpha_name);
|
||||
}
|
||||
|
||||
if (rank == 1) {
|
||||
@ -678,19 +680,27 @@ struct LoraModel : public GGMLRunner {
|
||||
out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0);
|
||||
}
|
||||
|
||||
if (lokr_w1)
|
||||
applied_lora_tensors.insert(lokr_w1_name);
|
||||
if (lokr_w1_a)
|
||||
applied_lora_tensors.insert(lokr_w1_a_name);
|
||||
if (lokr_w1_b)
|
||||
applied_lora_tensors.insert(lokr_w1_b_name);
|
||||
if (lokr_w2)
|
||||
applied_lora_tensors.insert(lokr_w2_name);
|
||||
if (lokr_w2_a)
|
||||
applied_lora_tensors.insert(lokr_w2_a_name);
|
||||
if (lokr_w2_b)
|
||||
applied_lora_tensors.insert(lokr_w2_b_name);
|
||||
applied_lora_tensors.insert(alpha_name);
|
||||
if (lokr_w1) {
|
||||
used_tensors.push_back(lokr_w1_name);
|
||||
}
|
||||
if (lokr_w1_a) {
|
||||
used_tensors.push_back(lokr_w1_a_name);
|
||||
}
|
||||
if (lokr_w1_b) {
|
||||
used_tensors.push_back(lokr_w1_b_name);
|
||||
}
|
||||
if (lokr_w2) {
|
||||
used_tensors.push_back(lokr_w2_name);
|
||||
}
|
||||
if (lokr_w2_a) {
|
||||
used_tensors.push_back(lokr_w2_a_name);
|
||||
}
|
||||
if (lokr_w2_b) {
|
||||
used_tensors.push_back(lokr_w2_b_name);
|
||||
}
|
||||
if (iter != lora_tensors.end()) {
|
||||
used_tensors.push_back(alpha_name);
|
||||
}
|
||||
|
||||
index++;
|
||||
continue;
|
||||
@ -740,10 +750,8 @@ struct LoraModel : public GGMLRunner {
|
||||
const int64_t down_in = lora_down->ne[0];
|
||||
const int64_t down_out = lora_down->ne[1];
|
||||
const int64_t up_in = lora_up->ne[0];
|
||||
const int64_t up_out = lora_up->ne[1];
|
||||
|
||||
bool compatible = down_in == model_weight->ne[0] &&
|
||||
up_out == model_weight->ne[1];
|
||||
bool compatible = down_in == model_weight->ne[0];
|
||||
if (lora_mid != nullptr) {
|
||||
compatible = compatible &&
|
||||
lora_mid->ne[0] == down_out &&
|
||||
@ -755,45 +763,43 @@ struct LoraModel : public GGMLRunner {
|
||||
if (!compatible) {
|
||||
skipped_incompatible_lora_tensors.insert(lora_down_name);
|
||||
skipped_incompatible_lora_tensors.insert(lora_up_name);
|
||||
if (lora_mid != nullptr) {
|
||||
skipped_incompatible_lora_tensors.insert(lora_mid_name);
|
||||
}
|
||||
if (lora_tensors.find(scale_name) != lora_tensors.end()) {
|
||||
skipped_incompatible_lora_tensors.insert(scale_name);
|
||||
} else if (lora_tensors.find(alpha_name) != lora_tensors.end()) {
|
||||
skipped_incompatible_lora_tensors.insert(alpha_name);
|
||||
}
|
||||
if (warned_incompatible_model_tensors.insert(model_tensor_name).second) {
|
||||
LOG_WARN("skip incompatible LoRA tensor |%s|: model shape = [%lld, %lld], down shape = [%lld, %lld], up shape = [%lld, %lld]",
|
||||
LOG_WARN("skip incompatible LoRA tensor |%s|: model input dim = %lld, down shape = [%lld, %lld], up shape = [%lld, %lld]",
|
||||
model_tensor_name.c_str(),
|
||||
static_cast<long long>(model_weight->ne[0]),
|
||||
static_cast<long long>(model_weight->ne[1]),
|
||||
static_cast<long long>(down_in),
|
||||
static_cast<long long>(down_out),
|
||||
static_cast<long long>(up_in),
|
||||
static_cast<long long>(up_out));
|
||||
static_cast<long long>(lora_up->ne[1]));
|
||||
}
|
||||
index++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
applied_lora_tensors.insert(lora_up_name);
|
||||
applied_lora_tensors.insert(lora_down_name);
|
||||
|
||||
if (lora_mid) {
|
||||
applied_lora_tensors.insert(lora_mid_name);
|
||||
}
|
||||
|
||||
float scale_value = 1.0f;
|
||||
std::string scale_tensor_name;
|
||||
|
||||
int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1];
|
||||
iter = lora_tensors.find(scale_name);
|
||||
if (iter != lora_tensors.end()) {
|
||||
scale_value = ggml_ext_backend_tensor_get_f32(iter->second);
|
||||
applied_lora_tensors.insert(scale_name);
|
||||
scale_tensor_name = scale_name;
|
||||
} else {
|
||||
iter = lora_tensors.find(alpha_name);
|
||||
if (iter != lora_tensors.end()) {
|
||||
float alpha = ggml_ext_backend_tensor_get_f32(iter->second);
|
||||
scale_value = alpha / rank;
|
||||
scale_tensor_name = alpha_name;
|
||||
// LOG_DEBUG("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value);
|
||||
applied_lora_tensors.insert(alpha_name);
|
||||
}
|
||||
}
|
||||
scale_value *= multiplier;
|
||||
@ -853,15 +859,45 @@ struct LoraModel : public GGMLRunner {
|
||||
}
|
||||
|
||||
auto curr_out_diff = ggml_ext_scale(ctx, lx, scale_value, true);
|
||||
|
||||
if (out_diff == nullptr) {
|
||||
out_diff = curr_out_diff;
|
||||
} else {
|
||||
out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0);
|
||||
out_diff = ggml_concat(ctx, out_diff, curr_out_diff, is_conv2d ? 2 : 0);
|
||||
}
|
||||
|
||||
used_tensors.push_back(lora_up_name);
|
||||
used_tensors.push_back(lora_down_name);
|
||||
if (lora_mid) {
|
||||
used_tensors.push_back(lora_mid_name);
|
||||
}
|
||||
if (!scale_tensor_name.empty()) {
|
||||
used_tensors.push_back(scale_tensor_name);
|
||||
}
|
||||
|
||||
index++;
|
||||
}
|
||||
|
||||
if (out_diff == nullptr)
|
||||
return nullptr;
|
||||
|
||||
int64_t expected_out_dim = is_conv2d ? model_weight->ne[3] : model_weight->ne[1];
|
||||
int64_t actual_out_dim = out_diff->ne[is_conv2d ? 2 : 0];
|
||||
|
||||
if (actual_out_dim != expected_out_dim) {
|
||||
for (const auto& name : used_tensors) {
|
||||
skipped_incompatible_lora_tensors.insert(name);
|
||||
}
|
||||
if (warned_incompatible_model_tensors.insert(model_tensor_name).second) {
|
||||
LOG_WARN("skip incompatible LoRA tensors for |%s|: output dim %lld != model dim %lld",
|
||||
model_tensor_name.c_str(), actual_out_dim, expected_out_dim);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
for (const auto& name : used_tensors) {
|
||||
applied_lora_tensors.insert(name);
|
||||
}
|
||||
|
||||
return out_diff;
|
||||
}
|
||||
|
||||
|
||||
@ -180,6 +180,9 @@ namespace Krea2 {
|
||||
|
||||
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
|
||||
ggml_tensor* scale = params["scale"];
|
||||
if (ctx->weight_adapter) {
|
||||
scale = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, scale, prefix + "scale.weight");
|
||||
}
|
||||
scale = ggml_add(ctx->ggml_ctx, scale, ggml_ext_ones(ctx->ggml_ctx, scale->ne[0], 1, 1, 1));
|
||||
x = ggml_rms_norm(ctx->ggml_ctx, x, eps);
|
||||
x = ggml_mul_inplace(ctx->ggml_ctx, x, scale);
|
||||
@ -295,10 +298,11 @@ namespace Krea2 {
|
||||
class KreaDoubleSharedModulation : public GGMLBlock {
|
||||
protected:
|
||||
int64_t dim;
|
||||
std::string prefix;
|
||||
|
||||
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
||||
GGML_UNUSED(tensor_storage_map);
|
||||
GGML_UNUSED(prefix);
|
||||
this->prefix = prefix;
|
||||
params["lin"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim * 6);
|
||||
}
|
||||
|
||||
@ -307,7 +311,11 @@ namespace Krea2 {
|
||||
: dim(dim) {}
|
||||
|
||||
std::vector<ggml_tensor*> forward(GGMLRunnerContext* ctx, ggml_tensor* vec) {
|
||||
auto lin = ggml_repeat(ctx->ggml_ctx, params["lin"], vec);
|
||||
auto lin = params["lin"];
|
||||
if (ctx->weight_adapter) {
|
||||
lin = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, lin, prefix + "lin.weight");
|
||||
}
|
||||
lin = ggml_repeat(ctx->ggml_ctx, lin, vec);
|
||||
auto out = ggml_add(ctx->ggml_ctx, vec, lin);
|
||||
return ggml_ext_chunk(ctx->ggml_ctx, out, 6, 0);
|
||||
}
|
||||
@ -316,10 +324,11 @@ namespace Krea2 {
|
||||
class KreaFinalModulation : public GGMLBlock {
|
||||
protected:
|
||||
int64_t dim;
|
||||
std::string prefix;
|
||||
|
||||
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
||||
GGML_UNUSED(tensor_storage_map);
|
||||
GGML_UNUSED(prefix);
|
||||
this->prefix = prefix;
|
||||
params["lin"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, dim, 2);
|
||||
}
|
||||
|
||||
@ -328,7 +337,11 @@ namespace Krea2 {
|
||||
: dim(dim) {}
|
||||
|
||||
std::vector<ggml_tensor*> forward(GGMLRunnerContext* ctx, ggml_tensor* vec) {
|
||||
auto out = ggml_add(ctx->ggml_ctx, params["lin"], vec);
|
||||
auto lin = params["lin"];
|
||||
if (ctx->weight_adapter) {
|
||||
lin = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, lin, prefix + "lin.weight");
|
||||
}
|
||||
auto out = ggml_add(ctx->ggml_ctx, lin, vec);
|
||||
return ggml_ext_chunk(ctx->ggml_ctx, out, 2, 1);
|
||||
}
|
||||
};
|
||||
|
||||
@ -1384,6 +1384,8 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
|
||||
{".lora_B.weight", ".weight.lora_up"},
|
||||
{".lora_A.default.weight", ".weight.lora_down"},
|
||||
{".lora_B.default.weight", ".weight.lora_up"},
|
||||
{".lora_A", ".weight.lora_down"},
|
||||
{".lora_B", ".weight.lora_up"},
|
||||
{".lora_linear", ".weight.alpha"},
|
||||
{".alpha", ".weight.alpha"},
|
||||
{".scale", ".weight.scale"},
|
||||
|
||||
@ -2578,6 +2578,88 @@ static sd::Tensor<float> sample_tcd(denoise_cb_t model,
|
||||
return x;
|
||||
}
|
||||
|
||||
static sd::Tensor<float> sample_lms(denoise_cb_t model,
|
||||
sd::Tensor<float> x,
|
||||
const std::vector<float>& sigmas,
|
||||
const SamplerExtraArgs& extra_sample_args) {
|
||||
// Linear Multi-Step from https://github.com/crowsonkb/k-diffusion
|
||||
|
||||
int divisions = 1000;
|
||||
for (const auto& [key, value] : extra_sample_args) {
|
||||
int parsed = 0;
|
||||
if (key == "lms_divisions") {
|
||||
if (!parse_strict_int(value, parsed)) {
|
||||
LOG_WARN("ignoring invalid lms extra sample arg '%s=%s'", key.c_str(), value.c_str());
|
||||
continue;
|
||||
}
|
||||
divisions = parsed; // std::max(1, parsed);
|
||||
// values above 35M produce noise, can be fixed by double precision
|
||||
// values < 1 always produce noise
|
||||
}
|
||||
}
|
||||
LOG_DEBUG("linear multi-step sampler: integrating using %i division%s", divisions, (divisions == 1) ? "" : "s");
|
||||
|
||||
auto linear_multistep_coeff = [=](const int order, const int m, const int j) -> float {
|
||||
if (!divisions)
|
||||
return sigmas[m + 1] - sigmas[m]; // delta / 0 * 0
|
||||
#define LMS_PRECISION float // double
|
||||
const LMS_PRECISION a = sigmas[m], dx = (sigmas[m + 1] - a) / divisions, s = sigmas[m - j];
|
||||
const LMS_PRECISION b0 = a + 0.5f * dx; // using Riemann middle integral
|
||||
LMS_PRECISION sum = 0.0f;
|
||||
for (int h = 0; h < divisions; h++) {
|
||||
const LMS_PRECISION b = h * dx + b0;
|
||||
LMS_PRECISION prod = 1.0f;
|
||||
for (int k = 0; k < j; k++) {
|
||||
const LMS_PRECISION t = sigmas[m - k];
|
||||
prod *= (b - t) / (s - t);
|
||||
}
|
||||
for (int k = j + 1; k < order; k++) {
|
||||
const LMS_PRECISION t = sigmas[m - k];
|
||||
prod *= (b - t) / (s - t);
|
||||
}
|
||||
sum += prod;
|
||||
}
|
||||
return sum * dx;
|
||||
};
|
||||
|
||||
const int max_order = 4;
|
||||
float lms_coeff[max_order];
|
||||
std::vector<sd::Tensor<float>> hist = {};
|
||||
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
const float sigma = sigmas[i];
|
||||
|
||||
auto denoised_opt = model(x, sigma, i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
|
||||
const int order = std::min(max_order, i + 1);
|
||||
for (int c = 0; c < order; c++) // computing coefficients
|
||||
lms_coeff[c] = linear_multistep_coeff(order, i, c);
|
||||
|
||||
sd::Tensor<float> d_cur = (x - denoised) / sigma;
|
||||
switch (order) {
|
||||
case 4: // derivative + 3 history points
|
||||
x += hist[hist.size() - 2] * lms_coeff[3];
|
||||
case 3:
|
||||
x += hist[hist.size() - 1] * lms_coeff[2];
|
||||
case 2:
|
||||
x += hist.back() * lms_coeff[1];
|
||||
case 1:
|
||||
x += d_cur * lms_coeff[0];
|
||||
}
|
||||
|
||||
if (hist.size() == static_cast<size_t>(max_order - 1)) {
|
||||
hist.erase(hist.begin());
|
||||
}
|
||||
hist.push_back(std::move(d_cur));
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
static sd::Tensor<float> sample_euler_cfg_pp(denoise_cb_t model,
|
||||
sd::Tensor<float> x,
|
||||
const std::vector<float>& sigmas) {
|
||||
@ -2739,6 +2821,8 @@ static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
|
||||
return sample_euler_ancestral(model, std::move(x), sigmas, rng, is_flow_denoiser, eta);
|
||||
case TCD_SAMPLE_METHOD:
|
||||
return sample_tcd(model, std::move(x), sigmas, rng, eta);
|
||||
case LMS_SAMPLE_METHOD:
|
||||
return sample_lms(model, std::move(x), sigmas, extra_args);
|
||||
case EULER_CFG_PP_SAMPLE_METHOD:
|
||||
return sample_euler_cfg_pp(model, std::move(x), sigmas);
|
||||
case EULER_A_CFG_PP_SAMPLE_METHOD:
|
||||
|
||||
@ -143,6 +143,7 @@ const char* sampling_methods_str[] = {
|
||||
"Euler GE",
|
||||
"DPM++ (2M) SDE",
|
||||
"DPM++ (2M) SDE BT",
|
||||
"LMS",
|
||||
};
|
||||
|
||||
/*================================================== Helper Functions ================================================*/
|
||||
@ -695,45 +696,11 @@ public:
|
||||
LOG_DEBUG("loaded alphas_cumprod from model file");
|
||||
}
|
||||
|
||||
bool init(const sd_ctx_params_t* sd_ctx_params) {
|
||||
n_threads = sd_ctx_params->n_threads;
|
||||
enable_mmap = sd_ctx_params->enable_mmap;
|
||||
stream_layers = sd_ctx_params->stream_layers;
|
||||
eager_load = sd_ctx_params->eager_load;
|
||||
backend_spec = SAFE_STR(sd_ctx_params->backend);
|
||||
params_backend_spec = SAFE_STR(sd_ctx_params->params_backend);
|
||||
split_mode_spec = SAFE_STR(sd_ctx_params->split_mode);
|
||||
auto_fit_enabled = sd_ctx_params->auto_fit;
|
||||
max_vram_assignment.reset(0.f);
|
||||
{
|
||||
std::string error;
|
||||
if (!max_vram_assignment.parse(SAFE_STR(sd_ctx_params->max_vram), &error)) {
|
||||
LOG_ERROR("%s", error.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::string rpc_servers_spec = SAFE_STR(sd_ctx_params->rpc_servers);
|
||||
add_rpc_devices(rpc_servers_spec);
|
||||
|
||||
bool use_tae = false;
|
||||
bool use_audio_vae = false;
|
||||
bool use_control_net = false;
|
||||
|
||||
rng = get_rng(sd_ctx_params->rng_type);
|
||||
if (sd_ctx_params->sampler_rng_type != RNG_TYPE_COUNT && sd_ctx_params->sampler_rng_type != sd_ctx_params->rng_type) {
|
||||
sampler_rng = get_rng(sd_ctx_params->sampler_rng_type);
|
||||
} else {
|
||||
sampler_rng = rng;
|
||||
}
|
||||
|
||||
ggml_log_set(ggml_log_callback_default, nullptr);
|
||||
|
||||
model_manager = std::make_shared<ModelManager>();
|
||||
model_manager->set_n_threads(n_threads);
|
||||
model_manager->set_enable_mmap(enable_mmap);
|
||||
ModelLoader& model_loader = model_manager->loader();
|
||||
|
||||
bool init_model_loader(ModelLoader& model_loader,
|
||||
const sd_ctx_params_t* sd_ctx_params,
|
||||
bool& use_tae,
|
||||
bool& use_audio_vae,
|
||||
bool& use_control_net) {
|
||||
if (strlen(SAFE_STR(sd_ctx_params->model_path)) > 0) {
|
||||
LOG_INFO("loading model from '%s'", sd_ctx_params->model_path);
|
||||
if (!model_loader.init_from_file(sd_ctx_params->model_path)) {
|
||||
@ -873,24 +840,69 @@ public:
|
||||
|
||||
model_loader.convert_tensors_name();
|
||||
|
||||
version = model_loader.get_sd_version();
|
||||
if (version == VERSION_COUNT) {
|
||||
LOG_ERROR("get sd version from file failed: '%s'", SAFE_STR(sd_ctx_params->model_path));
|
||||
return false;
|
||||
}
|
||||
|
||||
auto& tensor_storage_map = model_loader.get_tensor_storage_map();
|
||||
|
||||
LOG_INFO("Version: %s ", model_version_to_str[version]);
|
||||
ggml_type wtype = sd_type_to_ggml_type(sd_ctx_params->wtype);
|
||||
std::string tensor_type_rules = SAFE_STR(sd_ctx_params->tensor_type_rules);
|
||||
if (wtype != GGML_TYPE_COUNT || tensor_type_rules.size() > 0) {
|
||||
model_loader.set_wtype_override(wtype, tensor_type_rules);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool init(const sd_ctx_params_t* sd_ctx_params) {
|
||||
n_threads = sd_ctx_params->n_threads;
|
||||
enable_mmap = sd_ctx_params->enable_mmap;
|
||||
stream_layers = sd_ctx_params->stream_layers;
|
||||
eager_load = sd_ctx_params->eager_load;
|
||||
backend_spec = SAFE_STR(sd_ctx_params->backend);
|
||||
params_backend_spec = SAFE_STR(sd_ctx_params->params_backend);
|
||||
split_mode_spec = SAFE_STR(sd_ctx_params->split_mode);
|
||||
auto_fit_enabled = sd_ctx_params->auto_fit;
|
||||
max_vram_assignment.reset(0.f);
|
||||
{
|
||||
std::string error;
|
||||
if (!max_vram_assignment.parse(SAFE_STR(sd_ctx_params->max_vram), &error)) {
|
||||
LOG_ERROR("%s", error.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::string rpc_servers_spec = SAFE_STR(sd_ctx_params->rpc_servers);
|
||||
add_rpc_devices(rpc_servers_spec);
|
||||
|
||||
bool use_tae = false;
|
||||
bool use_audio_vae = false;
|
||||
bool use_control_net = false;
|
||||
|
||||
rng = get_rng(sd_ctx_params->rng_type);
|
||||
if (sd_ctx_params->sampler_rng_type != RNG_TYPE_COUNT && sd_ctx_params->sampler_rng_type != sd_ctx_params->rng_type) {
|
||||
sampler_rng = get_rng(sd_ctx_params->sampler_rng_type);
|
||||
} else {
|
||||
sampler_rng = rng;
|
||||
}
|
||||
|
||||
ggml_log_set(ggml_log_callback_default, nullptr);
|
||||
|
||||
model_manager = std::make_shared<ModelManager>();
|
||||
model_manager->set_n_threads(n_threads);
|
||||
model_manager->set_enable_mmap(enable_mmap);
|
||||
ModelLoader& model_loader = model_manager->loader();
|
||||
|
||||
if (!init_model_loader(model_loader, sd_ctx_params, use_tae, use_audio_vae, use_control_net)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
version = model_loader.get_sd_version();
|
||||
if (version == VERSION_COUNT) {
|
||||
LOG_ERROR("get sd version from file failed: '%s'", SAFE_STR(sd_ctx_params->model_path));
|
||||
return false;
|
||||
} else {
|
||||
LOG_INFO("Version: %s ", model_version_to_str[version]);
|
||||
}
|
||||
|
||||
if (auto_fit_enabled) {
|
||||
if (!sd::backend_fit::derive_backend_specs(model_loader,
|
||||
wtype,
|
||||
sd_type_to_ggml_type(sd_ctx_params->wtype),
|
||||
max_vram_assignment,
|
||||
backend_spec,
|
||||
params_backend_spec)) {
|
||||
@ -945,16 +957,12 @@ public:
|
||||
|
||||
if (sd_ctx_params->lora_apply_mode == LORA_APPLY_AUTO) {
|
||||
bool have_quantized_weight = false;
|
||||
if (wtype != GGML_TYPE_COUNT && ggml_is_quantized(wtype)) {
|
||||
have_quantized_weight = true;
|
||||
} else {
|
||||
for (const auto& [type, _] : wtype_stat) {
|
||||
if (ggml_is_quantized(type)) {
|
||||
have_quantized_weight = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Avoid full-model LoRA merge buffers on constrained setups.
|
||||
const bool params_offloaded = params_backend_for(SDBackendModule::DIFFUSION) != backend_for(SDBackendModule::DIFFUSION);
|
||||
const bool streaming_constrained = stream_layers || params_offloaded;
|
||||
@ -996,6 +1004,8 @@ public:
|
||||
use_tae = true;
|
||||
}
|
||||
|
||||
auto& tensor_storage_map = model_loader.get_tensor_storage_map();
|
||||
|
||||
{
|
||||
if (!ensure_backend_pair(SDBackendModule::TE) ||
|
||||
!ensure_backend_pair(SDBackendModule::DIFFUSION)) {
|
||||
@ -2126,7 +2136,9 @@ public:
|
||||
return;
|
||||
}
|
||||
auto image_tensor = sd_image_to_tensor(image);
|
||||
auto embed = get_clip_vision_output(image_tensor, true, -1);
|
||||
auto embed = ip_adapter->is_plus
|
||||
? get_clip_vision_output(image_tensor, false, 2)
|
||||
: get_clip_vision_output(image_tensor, true, -1);
|
||||
if (embed.empty()) {
|
||||
return;
|
||||
}
|
||||
@ -3206,6 +3218,7 @@ const char* sample_method_to_str[] = {
|
||||
"euler_ge",
|
||||
"dpm++2m_sde",
|
||||
"dpm++2m_sde_bt",
|
||||
"lms",
|
||||
};
|
||||
|
||||
const char* sd_sample_method_name(enum sample_method_t sample_method) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user