README.md

@@ -70,7 +70,7 @@ API and command-line option may change frequently.***
     - [HunyuanVideo 1.5](./docs/hunyuan_video.md)
     - [LingBot-Video](./docs/lingbot_video.md)
   - [PhotoMaker](./docs/photo_maker.md) support.
-  - [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL, including Plus)
+  - [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL)
   - Control Net support with SD 1.5
   - [ADetailer](./docs/adetailer.md)
   - LoRA support, same as [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora)

docs/ip_adapter.md

@@ -11,10 +11,6 @@ through a decoupled cross-attention added to every attn2 layer of the
 UNet. It composes with Control Net, so a reference image (appearance) and
 an OpenPose hint (pose) can be combined in a single generation.
 
-Both the classic adapters and the higher-fidelity **Plus** adapters are
-supported; see [Plus variants](#plus-variants) below. The variant is
-detected from the weight file, so the same options work for both.
-
 ## Required weights
 
 1. A base SD 1.5 or SDXL model.
@@ -25,11 +21,6 @@ detected from the weight file, so the same options work for both.
    [h94/IP-Adapter](https://huggingface.co/h94/IP-Adapter):
    - SD 1.5: `models/ip-adapter_sd15.safetensors`
    - SDXL: `sdxl_models/ip-adapter_sdxl_vit-h.safetensors`
-   - SD 1.5 Plus: `models/ip-adapter-plus_sd15.safetensors`
-   - SDXL Plus: `sdxl_models/ip-adapter-plus_sdxl_vit-h.safetensors`
-
-   The Plus files (`ip-adapter-plus_*`) are used exactly like the classic
-   ones; see [Plus variants](#plus-variants).
 
 ## Options
 
@@ -54,29 +45,6 @@ sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safet
 The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and
 `--offload-to-cpu`) on GPUs with limited VRAM.
 
-## Plus variants
-
-The Plus adapters (`ip-adapter-plus_sd15`, `ip-adapter-plus_sdxl_vit-h`)
-replace the small linear image projection with a Resampler (a
-Perceiver-style module with learned latent queries). Instead of pooling the
-CLIP-Vision output into one vector, the Resampler attends over the full grid
-of penultimate CLIP-Vision hidden states and emits more image tokens (16
-instead of 4). The result transfers finer detail and layout from the
-reference, at a small extra cost in the image-projection step.
-
-No extra flags are needed. The variant is detected from the weight file (the
-Resampler's `image_proj.latents` tensor), and every Resampler dimension is
-read from the tensor shapes, so the same `--ip-adapter`,
-`--ip-adapter-image`, and `--ip-adapter-strength` options apply. Plus
-composes with Control Net in the same way as the classic adapters.
-
-```
-sd-cli -m ..\models\sd_v1.5.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter-plus_sd15.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 7 --steps 30 --sampling-method dpm++2m --scheduler karras -W 512 -H 512
-```
-
-The startup log line `IP-Adapter: 16 image tokens` (versus `4` for the
-classic adapters) confirms a Plus file was loaded.
-
 ## Combining with Control Net
 
 Add the usual Control Net options to keep the reference appearance while

examples/common/common.cpp

@@ -1008,7 +1008,7 @@ ArgOptions SDGenerationParams::get_options() {
          &hires_upscaler},
         {"",
          "--extra-sample-args",
-         "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware; lms supports lms_divisions",
+         "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware",
          (int)',',
          &extra_sample_args},
         {"",
@@ -1538,12 +1538,12 @@ ArgOptions SDGenerationParams::get_options() {
          on_seed_arg},
         {"",
          "--sampling-method",
-         "sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp, lms]"
+         "sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]"
          "(default: euler for Flux/SD3/Wan, euler_a otherwise)",
          on_sample_method_arg},
         {"",
          "--high-noise-sampling-method",
-         "(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp, lms]"
+         "(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]"
          " default: euler for Flux/SD3/Wan, euler_a otherwise",
          on_high_noise_sample_method_arg},
         {"",

include/stable-diffusion.h

@@ -56,7 +56,6 @@ enum sample_method_t {
     EULER_GE_SAMPLE_METHOD,
     DPMPP2M_SDE_SAMPLE_METHOD,
     DPMPP2M_SDE_BT_SAMPLE_METHOD,
-    LMS_SAMPLE_METHOD,
     SAMPLE_METHOD_COUNT
 };
 

src/model/adapter/ip_adapter.hpp

@@ -31,92 +31,8 @@ namespace IPAdapter {
         }
     };
 
-    struct Resampler : public GGMLBlock {
-        int64_t dim         = 1280;
-        int64_t depth       = 4;
-        int64_t num_queries = 16;
-        int64_t embed_dim   = 1280;
-        int64_t output_dim  = 2048;
-        int64_t ff_inner    = 5120;
-        int64_t dim_head    = 64;
-        int64_t heads       = 20;
-
-        void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
-            params["latents"] = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, dim, num_queries, 1);
-        }
-
-        Resampler() {}
-        Resampler(int64_t dim, int64_t depth, int64_t num_queries, int64_t embed_dim, int64_t output_dim, int64_t ff_inner)
-            : dim(dim), depth(depth), num_queries(num_queries), embed_dim(embed_dim), output_dim(output_dim), ff_inner(ff_inner) {
-            heads              = dim / dim_head;
-            blocks["proj_in"]  = std::shared_ptr<GGMLBlock>(new Linear(embed_dim, dim, true));
-            blocks["proj_out"] = std::shared_ptr<GGMLBlock>(new Linear(dim, output_dim, true));
-            blocks["norm_out"] = std::shared_ptr<GGMLBlock>(new LayerNorm(output_dim));
-            for (int64_t i = 0; i < depth; i++) {
-                std::string p           = "layers." + std::to_string(i);
-                blocks[p + ".0.norm1"]  = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
-                blocks[p + ".0.norm2"]  = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
-                blocks[p + ".0.to_q"]   = std::shared_ptr<GGMLBlock>(new Linear(dim, dim, false));
-                blocks[p + ".0.to_kv"]  = std::shared_ptr<GGMLBlock>(new Linear(dim, dim * 2, false));
-                blocks[p + ".0.to_out"] = std::shared_ptr<GGMLBlock>(new Linear(dim, dim, false));
-                blocks[p + ".1.0"]      = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
-                blocks[p + ".1.1"]      = std::shared_ptr<GGMLBlock>(new Linear(dim, ff_inner, false));
-                blocks[p + ".1.3"]      = std::shared_ptr<GGMLBlock>(new Linear(ff_inner, dim, false));
-            }
-        }
-
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image_embeds) {
-            int64_t N     = image_embeds->ne[2];
-            auto proj_in  = std::dynamic_pointer_cast<Linear>(blocks["proj_in"]);
-            auto proj_out = std::dynamic_pointer_cast<Linear>(blocks["proj_out"]);
-            auto norm_out = std::dynamic_pointer_cast<LayerNorm>(blocks["norm_out"]);
-
-            ggml_tensor* x       = proj_in->forward(ctx, image_embeds);
-            ggml_tensor* latents = params["latents"];
-            if (N > 1) {
-                latents = ggml_repeat(ctx->ggml_ctx, latents, ggml_new_tensor_3d(ctx->ggml_ctx, GGML_TYPE_F32, dim, num_queries, N));
-            }
-
-            for (int64_t i = 0; i < depth; i++) {
-                std::string p = "layers." + std::to_string(i);
-                auto norm1    = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".0.norm1"]);
-                auto norm2    = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".0.norm2"]);
-                auto to_q     = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_q"]);
-                auto to_kv    = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_kv"]);
-                auto to_out   = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_out"]);
-
-                ggml_tensor* xn    = norm1->forward(ctx, x);
-                ggml_tensor* ln    = norm2->forward(ctx, latents);
-                ggml_tensor* q     = to_q->forward(ctx, ln);
-                ggml_tensor* kv_in = ggml_concat(ctx->ggml_ctx, xn, ln, 1);
-                ggml_tensor* kv    = to_kv->forward(ctx, kv_in);
-                int64_t L          = kv->ne[1];
-                ggml_tensor* k     = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], 0));
-                ggml_tensor* v     = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], dim * kv->nb[0]));
-                ggml_tensor* attn  = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, heads, nullptr, false, false);
-                attn               = to_out->forward(ctx, attn);
-                latents            = ggml_add(ctx->ggml_ctx, latents, attn);
-
-                auto ff_norm   = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".1.0"]);
-                auto ff_fc1    = std::dynamic_pointer_cast<Linear>(blocks[p + ".1.1"]);
-                auto ff_fc2    = std::dynamic_pointer_cast<Linear>(blocks[p + ".1.3"]);
-                ggml_tensor* h = ff_norm->forward(ctx, latents);
-                h              = ff_fc1->forward(ctx, h);
-                h              = ggml_gelu_erf(ctx->ggml_ctx, h);
-                h              = ff_fc2->forward(ctx, h);
-                latents        = ggml_add(ctx->ggml_ctx, latents, h);
-            }
-
-            latents = proj_out->forward(ctx, latents);
-            latents = norm_out->forward(ctx, latents);
-            return latents;
-        }
-    };
-
     struct IPAdapterRunner : public GGMLRunner {
         ImageProjModel image_proj;
-        Resampler resampler;
-        bool is_plus       = false;
         int64_t num_tokens = 4;
         std::string prefix;
 
@@ -125,54 +41,21 @@ namespace IPAdapter {
                         const std::string prefix,
                         std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
             : GGMLRunner(backend, weight_manager), prefix(prefix) {
-            is_plus = tensor_storage_map.find(prefix + ".image_proj.latents") != tensor_storage_map.end();
+            int64_t ctx_dim  = 768;
-            if (is_plus) {
+            int64_t clip_dim = 1024;
-                int64_t dim         = 1280;
+            int64_t out_dim  = 3072;
-                int64_t num_queries = 16;
+            auto norm_iter   = tensor_storage_map.find(prefix + ".image_proj.norm.weight");
-                int64_t embed_dim   = 1280;
+            if (norm_iter != tensor_storage_map.end()) {
-                int64_t output_dim  = 2048;
+                ctx_dim = norm_iter->second.ne[0];
-                int64_t ff_inner    = 5120;
-                auto latents_iter   = tensor_storage_map.find(prefix + ".image_proj.latents");
-                if (latents_iter != tensor_storage_map.end()) {
-                    dim         = latents_iter->second.ne[0];
-                    num_queries = latents_iter->second.ne[1];
-                }
-                auto proj_in_iter = tensor_storage_map.find(prefix + ".image_proj.proj_in.weight");
-                if (proj_in_iter != tensor_storage_map.end()) {
-                    embed_dim = proj_in_iter->second.ne[0];
-                }
-                auto proj_out_iter = tensor_storage_map.find(prefix + ".image_proj.proj_out.weight");
-                if (proj_out_iter != tensor_storage_map.end()) {
-                    output_dim = proj_out_iter->second.ne[1];
-                }
-                auto ff_iter = tensor_storage_map.find(prefix + ".image_proj.layers.0.1.1.weight");
-                if (ff_iter != tensor_storage_map.end()) {
-                    ff_inner = ff_iter->second.ne[1];
-                }
-                int64_t depth = 0;
-                while (tensor_storage_map.find(prefix + ".image_proj.layers." + std::to_string(depth) + ".0.to_q.weight") != tensor_storage_map.end()) {
-                    depth++;
-                }
-                num_tokens = num_queries;
-                resampler  = Resampler(dim, depth, num_queries, embed_dim, output_dim, ff_inner);
-                resampler.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
-            } else {
-                int64_t ctx_dim  = 768;
-                int64_t clip_dim = 1024;
-                int64_t out_dim  = 3072;
-                auto norm_iter   = tensor_storage_map.find(prefix + ".image_proj.norm.weight");
-                if (norm_iter != tensor_storage_map.end()) {
-                    ctx_dim = norm_iter->second.ne[0];
-                }
-                auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight");
-                if (proj_iter != tensor_storage_map.end()) {
-                    clip_dim = proj_iter->second.ne[0];
-                    out_dim  = proj_iter->second.ne[1];
-                }
-                num_tokens = out_dim / ctx_dim;
-                image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim);
-                image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
             }
+            auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight");
+            if (proj_iter != tensor_storage_map.end()) {
+                clip_dim = proj_iter->second.ne[0];
+                out_dim  = proj_iter->second.ne[1];
+            }
+            num_tokens = out_dim / ctx_dim;
+            image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim);
+            image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
         }
 
         std::string get_desc() override {
@@ -180,18 +63,14 @@ namespace IPAdapter {
         }
 
         void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string = "") {
-            if (is_plus) {
+            image_proj.get_param_tensors(tensors, prefix + ".image_proj");
-                resampler.get_param_tensors(tensors, prefix + ".image_proj");
-            } else {
-                image_proj.get_param_tensors(tensors, prefix + ".image_proj");
-            }
         }
 
         ggml_cgraph* build_graph(const sd::Tensor<float>& image_embeds_tensor) {
             ggml_cgraph* gf     = new_graph_custom(1024);
             ggml_tensor* embeds = make_input(image_embeds_tensor);
             auto runner_ctx     = get_context();
-            ggml_tensor* out    = is_plus ? resampler.forward(&runner_ctx, embeds) : image_proj.forward(&runner_ctx, embeds);
+            ggml_tensor* out    = image_proj.forward(&runner_ctx, embeds);
             ggml_build_forward_expand(gf, out);
             return gf;
         }

src/model/adapter/lora.hpp

@@ -588,10 +588,6 @@ struct LoraModel : public GGMLRunner {
                               const std::string& model_tensor_name) {
         ggml_tensor* out_diff = nullptr;
         int index             = 0;
-
-        std::vector<std::string> used_tensors;
-        bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D;
-
         while (true) {
             std::string key;
             if (index == 0) {
@@ -599,6 +595,7 @@ struct LoraModel : public GGMLRunner {
             } else {
                 key = model_tensor_name + "." + std::to_string(index);
             }
+            bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D;
 
             std::string lokr_w1_name   = "lora." + key + ".lokr_w1";
             std::string lokr_w1_a_name = "lora." + key + ".lokr_w1_a";
@@ -666,6 +663,7 @@ struct LoraModel : public GGMLRunner {
                 if (iter != lora_tensors.end()) {
                     float alpha = ggml_ext_backend_tensor_get_f32(iter->second);
                     scale_value = alpha / rank;
+                    applied_lora_tensors.insert(alpha_name);
                 }
 
                 if (rank == 1) {
@@ -680,27 +678,19 @@ struct LoraModel : public GGMLRunner {
                     out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0);
                 }
 
-                if (lokr_w1) {
+                if (lokr_w1)
-                    used_tensors.push_back(lokr_w1_name);
+                    applied_lora_tensors.insert(lokr_w1_name);
-                }
+                if (lokr_w1_a)
-                if (lokr_w1_a) {
+                    applied_lora_tensors.insert(lokr_w1_a_name);
-                    used_tensors.push_back(lokr_w1_a_name);
+                if (lokr_w1_b)
-                }
+                    applied_lora_tensors.insert(lokr_w1_b_name);
-                if (lokr_w1_b) {
+                if (lokr_w2)
-                    used_tensors.push_back(lokr_w1_b_name);
+                    applied_lora_tensors.insert(lokr_w2_name);
-                }
+                if (lokr_w2_a)
-                if (lokr_w2) {
+                    applied_lora_tensors.insert(lokr_w2_a_name);
-                    used_tensors.push_back(lokr_w2_name);
+                if (lokr_w2_b)
-                }
+                    applied_lora_tensors.insert(lokr_w2_b_name);
-                if (lokr_w2_a) {
+                applied_lora_tensors.insert(alpha_name);
-                    used_tensors.push_back(lokr_w2_a_name);
-                }
-                if (lokr_w2_b) {
-                    used_tensors.push_back(lokr_w2_b_name);
-                }
-                if (iter != lora_tensors.end()) {
-                    used_tensors.push_back(alpha_name);
-                }
 
                 index++;
                 continue;
@@ -750,8 +740,10 @@ struct LoraModel : public GGMLRunner {
                 const int64_t down_in  = lora_down->ne[0];
                 const int64_t down_out = lora_down->ne[1];
                 const int64_t up_in    = lora_up->ne[0];
+                const int64_t up_out   = lora_up->ne[1];
 
-                bool compatible = down_in == model_weight->ne[0];
+                bool compatible = down_in == model_weight->ne[0] &&
+                                  up_out == model_weight->ne[1];
                 if (lora_mid != nullptr) {
                     compatible = compatible &&
                                  lora_mid->ne[0] == down_out &&
@@ -763,43 +755,45 @@ struct LoraModel : public GGMLRunner {
                 if (!compatible) {
                     skipped_incompatible_lora_tensors.insert(lora_down_name);
                     skipped_incompatible_lora_tensors.insert(lora_up_name);
-                    if (lora_mid != nullptr) {
+                    skipped_incompatible_lora_tensors.insert(lora_mid_name);
-                        skipped_incompatible_lora_tensors.insert(lora_mid_name);
+                    skipped_incompatible_lora_tensors.insert(scale_name);
-                    }
+                    skipped_incompatible_lora_tensors.insert(alpha_name);
-                    if (lora_tensors.find(scale_name) != lora_tensors.end()) {
-                        skipped_incompatible_lora_tensors.insert(scale_name);
-                    } else if (lora_tensors.find(alpha_name) != lora_tensors.end()) {
-                        skipped_incompatible_lora_tensors.insert(alpha_name);
-                    }
                     if (warned_incompatible_model_tensors.insert(model_tensor_name).second) {
-                        LOG_WARN("skip incompatible LoRA tensor |%s|: model input dim = %lld, down shape = [%lld, %lld], up shape = [%lld, %lld]",
+                        LOG_WARN("skip incompatible LoRA tensor |%s|: model shape = [%lld, %lld], down shape = [%lld, %lld], up shape = [%lld, %lld]",
                                  model_tensor_name.c_str(),
                                  static_cast<long long>(model_weight->ne[0]),
+                                 static_cast<long long>(model_weight->ne[1]),
                                  static_cast<long long>(down_in),
                                  static_cast<long long>(down_out),
                                  static_cast<long long>(up_in),
-                                 static_cast<long long>(lora_up->ne[1]));
+                                 static_cast<long long>(up_out));
                     }
                     index++;
                     continue;
                 }
             }
 
+            applied_lora_tensors.insert(lora_up_name);
+            applied_lora_tensors.insert(lora_down_name);
+
+            if (lora_mid) {
+                applied_lora_tensors.insert(lora_mid_name);
+            }
+
             float scale_value = 1.0f;
-            std::string scale_tensor_name;
 
             int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1];
             iter         = lora_tensors.find(scale_name);
             if (iter != lora_tensors.end()) {
-                scale_value       = ggml_ext_backend_tensor_get_f32(iter->second);
+                scale_value = ggml_ext_backend_tensor_get_f32(iter->second);
-                scale_tensor_name = scale_name;
+                applied_lora_tensors.insert(scale_name);
             } else {
                 iter = lora_tensors.find(alpha_name);
                 if (iter != lora_tensors.end()) {
-                    float alpha       = ggml_ext_backend_tensor_get_f32(iter->second);
+                    float alpha = ggml_ext_backend_tensor_get_f32(iter->second);
-                    scale_value       = alpha / rank;
+                    scale_value = alpha / rank;
-                    scale_tensor_name = alpha_name;
                     // LOG_DEBUG("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value);
+                    applied_lora_tensors.insert(alpha_name);
                 }
             }
             scale_value *= multiplier;
@@ -859,45 +853,15 @@ struct LoraModel : public GGMLRunner {
             }
 
             auto curr_out_diff = ggml_ext_scale(ctx, lx, scale_value, true);
+
             if (out_diff == nullptr) {
                 out_diff = curr_out_diff;
             } else {
-                out_diff = ggml_concat(ctx, out_diff, curr_out_diff, is_conv2d ? 2 : 0);
+                out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0);
-            }
-
-            used_tensors.push_back(lora_up_name);
-            used_tensors.push_back(lora_down_name);
-            if (lora_mid) {
-                used_tensors.push_back(lora_mid_name);
-            }
-            if (!scale_tensor_name.empty()) {
-                used_tensors.push_back(scale_tensor_name);
             }
 
             index++;
         }
-
-        if (out_diff == nullptr)
-            return nullptr;
-
-        int64_t expected_out_dim = is_conv2d ? model_weight->ne[3] : model_weight->ne[1];
-        int64_t actual_out_dim   = out_diff->ne[is_conv2d ? 2 : 0];
-
-        if (actual_out_dim != expected_out_dim) {
-            for (const auto& name : used_tensors) {
-                skipped_incompatible_lora_tensors.insert(name);
-            }
-            if (warned_incompatible_model_tensors.insert(model_tensor_name).second) {
-                LOG_WARN("skip incompatible LoRA tensors for |%s|: output dim %lld != model dim %lld",
-                         model_tensor_name.c_str(), actual_out_dim, expected_out_dim);
-            }
-            return nullptr;
-        }
-
-        for (const auto& name : used_tensors) {
-            applied_lora_tensors.insert(name);
-        }
-
         return out_diff;
     }
 

src/model/diffusion/krea2.hpp

@@ -180,12 +180,9 @@ namespace Krea2 {
 
         ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
             ggml_tensor* scale = params["scale"];
-            if (ctx->weight_adapter) {
+            scale              = ggml_add(ctx->ggml_ctx, scale, ggml_ext_ones(ctx->ggml_ctx, scale->ne[0], 1, 1, 1));
-                scale = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, scale, prefix + "scale.weight");
+            x                  = ggml_rms_norm(ctx->ggml_ctx, x, eps);
-            }
+            x                  = ggml_mul_inplace(ctx->ggml_ctx, x, scale);
-            scale = ggml_add(ctx->ggml_ctx, scale, ggml_ext_ones(ctx->ggml_ctx, scale->ne[0], 1, 1, 1));
-            x     = ggml_rms_norm(ctx->ggml_ctx, x, eps);
-            x     = ggml_mul_inplace(ctx->ggml_ctx, x, scale);
             return x;
         }
     };
@@ -298,11 +295,10 @@ namespace Krea2 {
     class KreaDoubleSharedModulation : public GGMLBlock {
     protected:
         int64_t dim;
-        std::string prefix;
 
         void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
             GGML_UNUSED(tensor_storage_map);
-            this->prefix  = prefix;
+            GGML_UNUSED(prefix);
             params["lin"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim * 6);
         }
 
@@ -311,11 +307,7 @@ namespace Krea2 {
             : dim(dim) {}
 
         std::vector<ggml_tensor*> forward(GGMLRunnerContext* ctx, ggml_tensor* vec) {
-            auto lin = params["lin"];
+            auto lin = ggml_repeat(ctx->ggml_ctx, params["lin"], vec);
-            if (ctx->weight_adapter) {
-                lin = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, lin, prefix + "lin.weight");
-            }
-            lin      = ggml_repeat(ctx->ggml_ctx, lin, vec);
             auto out = ggml_add(ctx->ggml_ctx, vec, lin);
             return ggml_ext_chunk(ctx->ggml_ctx, out, 6, 0);
         }
@@ -324,11 +316,10 @@ namespace Krea2 {
     class KreaFinalModulation : public GGMLBlock {
     protected:
         int64_t dim;
-        std::string prefix;
 
         void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
             GGML_UNUSED(tensor_storage_map);
-            this->prefix  = prefix;
+            GGML_UNUSED(prefix);
             params["lin"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, dim, 2);
         }
 
@@ -337,11 +328,7 @@ namespace Krea2 {
             : dim(dim) {}
 
         std::vector<ggml_tensor*> forward(GGMLRunnerContext* ctx, ggml_tensor* vec) {
-            auto lin = params["lin"];
+            auto out = ggml_add(ctx->ggml_ctx, params["lin"], vec);
-            if (ctx->weight_adapter) {
-                lin = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, lin, prefix + "lin.weight");
-            }
-            auto out = ggml_add(ctx->ggml_ctx, lin, vec);
             return ggml_ext_chunk(ctx->ggml_ctx, out, 2, 1);
         }
     };

src/name_conversion.cpp

@@ -1384,8 +1384,6 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
             {".lora_B.weight", ".weight.lora_up"},
             {".lora_A.default.weight", ".weight.lora_down"},
             {".lora_B.default.weight", ".weight.lora_up"},
-            {".lora_A", ".weight.lora_down"},
-            {".lora_B", ".weight.lora_up"},
             {".lora_linear", ".weight.alpha"},
             {".alpha", ".weight.alpha"},
             {".scale", ".weight.scale"},

src/runtime/denoiser.hpp

@@ -2578,88 +2578,6 @@ static sd::Tensor<float> sample_tcd(denoise_cb_t model,
     return x;
 }
 
-static sd::Tensor<float> sample_lms(denoise_cb_t model,
-                                    sd::Tensor<float> x,
-                                    const std::vector<float>& sigmas,
-                                    const SamplerExtraArgs& extra_sample_args) {
-    // Linear Multi-Step from https://github.com/crowsonkb/k-diffusion
-
-    int divisions = 1000;
-    for (const auto& [key, value] : extra_sample_args) {
-        int parsed = 0;
-        if (key == "lms_divisions") {
-            if (!parse_strict_int(value, parsed)) {
-                LOG_WARN("ignoring invalid lms extra sample arg '%s=%s'", key.c_str(), value.c_str());
-                continue;
-            }
-            divisions = parsed;  // std::max(1, parsed);
-            // values above 35M produce noise, can be fixed by double precision
-            // values < 1 always produce noise
-        }
-    }
-    LOG_DEBUG("linear multi-step sampler: integrating using %i division%s", divisions, (divisions == 1) ? "" : "s");
-
-    auto linear_multistep_coeff = [=](const int order, const int m, const int j) -> float {
-        if (!divisions)
-            return sigmas[m + 1] - sigmas[m];  // delta / 0 * 0
-#define LMS_PRECISION float                    // double
-        const LMS_PRECISION a = sigmas[m], dx = (sigmas[m + 1] - a) / divisions, s = sigmas[m - j];
-        const LMS_PRECISION b0 = a + 0.5f * dx;  // using Riemann middle integral
-        LMS_PRECISION sum      = 0.0f;
-        for (int h = 0; h < divisions; h++) {
-            const LMS_PRECISION b = h * dx + b0;
-            LMS_PRECISION prod    = 1.0f;
-            for (int k = 0; k < j; k++) {
-                const LMS_PRECISION t = sigmas[m - k];
-                prod *= (b - t) / (s - t);
-            }
-            for (int k = j + 1; k < order; k++) {
-                const LMS_PRECISION t = sigmas[m - k];
-                prod *= (b - t) / (s - t);
-            }
-            sum += prod;
-        }
-        return sum * dx;
-    };
-
-    const int max_order = 4;
-    float lms_coeff[max_order];
-    std::vector<sd::Tensor<float>> hist = {};
-
-    int steps = static_cast<int>(sigmas.size()) - 1;
-    for (int i = 0; i < steps; i++) {
-        const float sigma = sigmas[i];
-
-        auto denoised_opt = model(x, sigma, i + 1);
-        if (denoised_opt.pred.empty()) {
-            return {};
-        }
-        sd::Tensor<float> denoised = std::move(denoised_opt.pred);
-
-        const int order = std::min(max_order, i + 1);
-        for (int c = 0; c < order; c++)  // computing coefficients
-            lms_coeff[c] = linear_multistep_coeff(order, i, c);
-
-        sd::Tensor<float> d_cur = (x - denoised) / sigma;
-        switch (order) {
-            case 4:  // derivative + 3 history points
-                x += hist[hist.size() - 2] * lms_coeff[3];
-            case 3:
-                x += hist[hist.size() - 1] * lms_coeff[2];
-            case 2:
-                x += hist.back() * lms_coeff[1];
-            case 1:
-                x += d_cur * lms_coeff[0];
-        }
-
-        if (hist.size() == static_cast<size_t>(max_order - 1)) {
-            hist.erase(hist.begin());
-        }
-        hist.push_back(std::move(d_cur));
-    }
-    return x;
-}
-
 static sd::Tensor<float> sample_euler_cfg_pp(denoise_cb_t model,
                                              sd::Tensor<float> x,
                                              const std::vector<float>& sigmas) {
@@ -2821,8 +2739,6 @@ static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
             return sample_euler_ancestral(model, std::move(x), sigmas, rng, is_flow_denoiser, eta);
         case TCD_SAMPLE_METHOD:
             return sample_tcd(model, std::move(x), sigmas, rng, eta);
-        case LMS_SAMPLE_METHOD:
-            return sample_lms(model, std::move(x), sigmas, extra_args);
         case EULER_CFG_PP_SAMPLE_METHOD:
             return sample_euler_cfg_pp(model, std::move(x), sigmas);
         case EULER_A_CFG_PP_SAMPLE_METHOD:

src/stable-diffusion.cpp

@@ -143,7 +143,6 @@ const char* sampling_methods_str[] = {
     "Euler GE",
     "DPM++ (2M) SDE",
     "DPM++ (2M) SDE BT",
-    "LMS",
 };
 
 /*================================================== Helper Functions ================================================*/
@@ -696,11 +695,45 @@ public:
         LOG_DEBUG("loaded alphas_cumprod from model file");
     }
 
-    bool init_model_loader(ModelLoader& model_loader,
+    bool init(const sd_ctx_params_t* sd_ctx_params) {
-                           const sd_ctx_params_t* sd_ctx_params,
+        n_threads           = sd_ctx_params->n_threads;
-                           bool& use_tae,
+        enable_mmap         = sd_ctx_params->enable_mmap;
-                           bool& use_audio_vae,
+        stream_layers       = sd_ctx_params->stream_layers;
-                           bool& use_control_net) {
+        eager_load          = sd_ctx_params->eager_load;
+        backend_spec        = SAFE_STR(sd_ctx_params->backend);
+        params_backend_spec = SAFE_STR(sd_ctx_params->params_backend);
+        split_mode_spec     = SAFE_STR(sd_ctx_params->split_mode);
+        auto_fit_enabled    = sd_ctx_params->auto_fit;
+        max_vram_assignment.reset(0.f);
+        {
+            std::string error;
+            if (!max_vram_assignment.parse(SAFE_STR(sd_ctx_params->max_vram), &error)) {
+                LOG_ERROR("%s", error.c_str());
+                return false;
+            }
+        }
+
+        std::string rpc_servers_spec = SAFE_STR(sd_ctx_params->rpc_servers);
+        add_rpc_devices(rpc_servers_spec);
+
+        bool use_tae         = false;
+        bool use_audio_vae   = false;
+        bool use_control_net = false;
+
+        rng = get_rng(sd_ctx_params->rng_type);
+        if (sd_ctx_params->sampler_rng_type != RNG_TYPE_COUNT && sd_ctx_params->sampler_rng_type != sd_ctx_params->rng_type) {
+            sampler_rng = get_rng(sd_ctx_params->sampler_rng_type);
+        } else {
+            sampler_rng = rng;
+        }
+
+        ggml_log_set(ggml_log_callback_default, nullptr);
+
+        model_manager = std::make_shared<ModelManager>();
+        model_manager->set_n_threads(n_threads);
+        model_manager->set_enable_mmap(enable_mmap);
+        ModelLoader& model_loader = model_manager->loader();
+
         if (strlen(SAFE_STR(sd_ctx_params->model_path)) > 0) {
             LOG_INFO("loading model from '%s'", sd_ctx_params->model_path);
             if (!model_loader.init_from_file(sd_ctx_params->model_path)) {
@@ -840,69 +873,24 @@ public:
 
         model_loader.convert_tensors_name();
 
+        version = model_loader.get_sd_version();
+        if (version == VERSION_COUNT) {
+            LOG_ERROR("get sd version from file failed: '%s'", SAFE_STR(sd_ctx_params->model_path));
+            return false;
+        }
+
+        auto& tensor_storage_map = model_loader.get_tensor_storage_map();
+
+        LOG_INFO("Version: %s ", model_version_to_str[version]);
         ggml_type wtype               = sd_type_to_ggml_type(sd_ctx_params->wtype);
         std::string tensor_type_rules = SAFE_STR(sd_ctx_params->tensor_type_rules);
         if (wtype != GGML_TYPE_COUNT || tensor_type_rules.size() > 0) {
             model_loader.set_wtype_override(wtype, tensor_type_rules);
         }
 
-        return true;
-    }
-
-    bool init(const sd_ctx_params_t* sd_ctx_params) {
-        n_threads           = sd_ctx_params->n_threads;
-        enable_mmap         = sd_ctx_params->enable_mmap;
-        stream_layers       = sd_ctx_params->stream_layers;
-        eager_load          = sd_ctx_params->eager_load;
-        backend_spec        = SAFE_STR(sd_ctx_params->backend);
-        params_backend_spec = SAFE_STR(sd_ctx_params->params_backend);
-        split_mode_spec     = SAFE_STR(sd_ctx_params->split_mode);
-        auto_fit_enabled    = sd_ctx_params->auto_fit;
-        max_vram_assignment.reset(0.f);
-        {
-            std::string error;
-            if (!max_vram_assignment.parse(SAFE_STR(sd_ctx_params->max_vram), &error)) {
-                LOG_ERROR("%s", error.c_str());
-                return false;
-            }
-        }
-
-        std::string rpc_servers_spec = SAFE_STR(sd_ctx_params->rpc_servers);
-        add_rpc_devices(rpc_servers_spec);
-
-        bool use_tae         = false;
-        bool use_audio_vae   = false;
-        bool use_control_net = false;
-
-        rng = get_rng(sd_ctx_params->rng_type);
-        if (sd_ctx_params->sampler_rng_type != RNG_TYPE_COUNT && sd_ctx_params->sampler_rng_type != sd_ctx_params->rng_type) {
-            sampler_rng = get_rng(sd_ctx_params->sampler_rng_type);
-        } else {
-            sampler_rng = rng;
-        }
-
-        ggml_log_set(ggml_log_callback_default, nullptr);
-
-        model_manager = std::make_shared<ModelManager>();
-        model_manager->set_n_threads(n_threads);
-        model_manager->set_enable_mmap(enable_mmap);
-        ModelLoader& model_loader = model_manager->loader();
-
-        if (!init_model_loader(model_loader, sd_ctx_params, use_tae, use_audio_vae, use_control_net)) {
-            return false;
-        }
-
-        version = model_loader.get_sd_version();
-        if (version == VERSION_COUNT) {
-            LOG_ERROR("get sd version from file failed: '%s'", SAFE_STR(sd_ctx_params->model_path));
-            return false;
-        } else {
-            LOG_INFO("Version: %s ", model_version_to_str[version]);
-        }
-
         if (auto_fit_enabled) {
             if (!sd::backend_fit::derive_backend_specs(model_loader,
-                                                       sd_type_to_ggml_type(sd_ctx_params->wtype),
+                                                       wtype,
                                                        max_vram_assignment,
                                                        backend_spec,
                                                        params_backend_spec)) {
@@ -957,10 +945,14 @@ public:
 
         if (sd_ctx_params->lora_apply_mode == LORA_APPLY_AUTO) {
             bool have_quantized_weight = false;
-            for (const auto& [type, _] : wtype_stat) {
+            if (wtype != GGML_TYPE_COUNT && ggml_is_quantized(wtype)) {
-                if (ggml_is_quantized(type)) {
+                have_quantized_weight = true;
-                    have_quantized_weight = true;
+            } else {
-                    break;
+                for (const auto& [type, _] : wtype_stat) {
+                    if (ggml_is_quantized(type)) {
+                        have_quantized_weight = true;
+                        break;
+                    }
                 }
             }
             // Avoid full-model LoRA merge buffers on constrained setups.
@@ -1004,8 +996,6 @@ public:
             use_tae          = true;
         }
 
-        auto& tensor_storage_map = model_loader.get_tensor_storage_map();
-
         {
             if (!ensure_backend_pair(SDBackendModule::TE) ||
                 !ensure_backend_pair(SDBackendModule::DIFFUSION)) {
@@ -2136,9 +2126,7 @@ public:
             return;
         }
         auto image_tensor = sd_image_to_tensor(image);
-        auto embed        = ip_adapter->is_plus
+        auto embed        = get_clip_vision_output(image_tensor, true, -1);
-                                ? get_clip_vision_output(image_tensor, false, 2)
-                                : get_clip_vision_output(image_tensor, true, -1);
         if (embed.empty()) {
             return;
         }
@@ -3218,7 +3206,6 @@ const char* sample_method_to_str[] = {
     "euler_ge",
     "dpm++2m_sde",
     "dpm++2m_sde_bt",
-    "lms",
 };
 
 const char* sd_sample_method_name(enum sample_method_t sample_method) {