feat: support Diffusers-format MiniMax-H3 LoRAs (#2159)

src/model/adapter/lora.hpp

@@ -1,6 +1,7 @@
 #ifndef __SD_MODEL_ADAPTER_LORA_HPP__
 #define __SD_MODEL_ADAPTER_LORA_HPP__
 
+#include <cmath>
 #include <mutex>
 #include "core/ggml_extend.h"
 #include "core/ggml_extend_backend.h"
@@ -11,15 +12,18 @@
 #include "model/adapter/lora_ops.h"
 #include "model_loader.h"
 #include "model_manager.h"
+#include "name_conversion.h"
 
 #define LORA_GRAPH_BASE_SIZE 10240
 
 struct LoraModel : public GGMLRunner {
     std::string lora_id;
-    float multiplier = 1.0f;
+    float multiplier     = 1.0f;
+    float metadata_alpha = -1.f;
     std::unordered_map<std::string, ggml_tensor*> lora_tensors;
     std::map<ggml_tensor*, ggml_tensor*> original_tensor_to_final_tensor;
     std::set<std::string> applied_lora_tensors;
+    std::set<std::string> swapped_lora_up_tensors;
     std::set<std::string> skipped_incompatible_lora_tensors;
     std::set<std::string> warned_incompatible_model_tensors;
     std::string file_path;
@@ -62,6 +66,31 @@ struct LoraModel : public GGMLRunner {
         if (load_failed || !registered_params.empty())
             return false;
         model_manager->set_n_threads(n_threads);
+        metadata_alpha      = -1.f;
+        const auto metadata = model_manager->loader().file_metadata(source_file);
+        const auto alpha    = metadata.find("alpha");
+        if (alpha != metadata.end()) {
+            try {
+                size_t parsed  = 0;
+                metadata_alpha = std::stof(alpha->second, &parsed);
+                if (parsed != alpha->second.size() || !std::isfinite(metadata_alpha) || metadata_alpha < 0.f) {
+                    throw std::invalid_argument("invalid alpha");
+                }
+            } catch (const std::exception&) {
+                LOG_ERROR("invalid LoRA alpha metadata '%s' in '%s'", alpha->second.c_str(), file_path.c_str());
+                return false;
+            }
+            LOG_VERBOSE("LoRA '%s' uses metadata alpha %.6g when tensor alpha/scale is absent", file_path.c_str(), metadata_alpha);
+        }
+        if (sd_version_is_minimax_h3(source_version)) {
+            for (const auto& entry : model_manager->loader().file_tensors(source_file)) {
+                const auto name = convert_tensor_name(entry.first, VERSION_COUNT);
+                if (ends_with(name, ".ff.net.0.proj.weight.lora_up")) {
+                    // Diffusers SwiGLU stores [value, gate]; MiniMax-H3 uses [gate, value].
+                    swapped_lora_up_tensors.insert(convert_tensor_name(entry.first, source_version));
+                }
+            }
+        }
         const auto sources = model_manager->loader().file_tensors(source_file, source_version);
         std::map<std::string, ggml_tensor*> tensors;
         std::map<std::string, ggml_tensor*> scalars;
@@ -116,6 +145,8 @@ struct LoraModel : public GGMLRunner {
         scalar_values.clear();
         original_tensor_to_final_tensor.clear();
         applied_lora_tensors.clear();
+        swapped_lora_up_tensors.clear();
+        metadata_alpha = -1.f;
         skipped_incompatible_lora_tensors.clear();
         warned_incompatible_model_tensors.clear();
         applied             = false;
@@ -210,6 +241,10 @@ struct LoraModel : public GGMLRunner {
             auto iter = lora_tensors.find(lora_up_name);
             if (iter != lora_tensors.end()) {
                 lora_up = ggml_ext_cast_f32(ctx, backend, iter->second);
+                if (swapped_lora_up_tensors.count(lora_up_name)) {
+                    auto halves = ggml_ext_chunk(ctx, lora_up, 2, 1);
+                    lora_up     = ggml_concat(ctx, halves[1], halves[0], 1);
+                }
             }
 
             iter = lora_tensors.find(lora_mid_name);
@@ -236,6 +271,7 @@ struct LoraModel : public GGMLRunner {
             float scale_value = 1.0f;
 
             int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1];
+            scale_value  = metadata_alpha >= 0.f ? metadata_alpha / rank : 1.f;
             iter         = lora_tensors.find(scale_name);
             if (iter != lora_tensors.end()) {
                 scale_value = scalar_value(iter->second);
@@ -723,6 +759,10 @@ struct LoraModel : public GGMLRunner {
             iter = lora_tensors.find(lora_up_name);
             if (iter != lora_tensors.end()) {
                 lora_up = iter->second;
+                if (swapped_lora_up_tensors.count(lora_up_name)) {
+                    auto halves = ggml_ext_chunk(ctx, lora_up, 2, 1);
+                    lora_up     = ggml_concat(ctx, halves[1], halves[0], 1);
+                }
                 if (is_conv2d && lora_up->type != GGML_TYPE_F16) {
                     lora_up = ggml_cast(ctx, lora_up, GGML_TYPE_F16);
                 }
@@ -791,6 +831,7 @@ struct LoraModel : public GGMLRunner {
             std::string scale_tensor_name;
 
             int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1];
+            scale_value  = metadata_alpha >= 0.f ? metadata_alpha / rank : 1.f;
             iter         = lora_tensors.find(scale_name);
             if (iter != lora_tensors.end()) {
                 scale_value       = scalar_value(iter->second);

src/model_loader.h

@@ -105,6 +105,8 @@ public:
     bool del_file(FileId id);
     uint64_t file_revision(FileId id) const;
     std::string file_path(FileId id) const;
+    std::map<std::string, std::string> file_metadata(FileId id) const;
+    String2TensorStorage file_tensors(FileId id) const;
     String2TensorStorage file_tensors(FileId id, SDVersion version) const;
     bool load_file_tensors(FileId id, SDVersion version, on_new_tensor_cb_t callback, const std::set<std::string>& names, bool use_mmap = false) const;
     bool refresh_files(bool include_isolated = true);

src/model_loader_files.cpp

@@ -324,6 +324,24 @@ ModelLoader ModelLoader::file_reader(FileId id, SDVersion version) const {
     return reader;
 }
 
+std::map<std::string, std::string> ModelLoader::file_metadata(FileId id) const {
+    for (const auto& file : files_) {
+        if (file.id == id) {
+            return file.metadata;
+        }
+    }
+    return {};
+}
+
+String2TensorStorage ModelLoader::file_tensors(FileId id) const {
+    for (const auto& file : files_) {
+        if (file.id == id) {
+            return file.tensors;
+        }
+    }
+    return {};
+}
+
 String2TensorStorage ModelLoader::file_tensors(FileId id, SDVersion version) const {
     return file_reader(id, version).tensor_storage_map;
 }

src/name_conversion.cpp

@@ -1024,6 +1024,24 @@ static std::string convert_ming_image_dit_name(std::string name) {
     return name;
 }
 
+std::string convert_diffusers_dit_to_original_minimax_h3(std::string name) {
+    static const std::vector<std::pair<std::string, std::string>> prefix_map = {
+        {"transformer_blocks.", "blocks."},
+        {"token_refiner.refiner_blocks.", "token_refiner.blocks."},
+    };
+    static const std::vector<std::pair<std::string, std::string>> name_map = {
+        {".attn.to_q.weight", ".attn.qkv_proj.weight"},
+        {".attn.to_k.weight", ".attn.qkv_proj.weight.1"},
+        {".attn.to_v.weight", ".attn.qkv_proj.weight.2"},
+        {".attn.to_out.0.", ".attn.out_proj."},
+        {".ff.net.0.proj.", ".mlp.fc1."},
+        {".ff.net.2.", ".mlp.fc2."},
+    };
+    replace_with_prefix_map(name, prefix_map);
+    replace_with_name_map(name, name_map);
+    return name;
+}
+
 std::string convert_diffusion_model_name(std::string name, std::string prefix, SDVersion version) {
     if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
         name = convert_diffusers_unet_to_original_sd1(name);
@@ -1047,6 +1065,8 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S
         name = convert_diffusers_dit_to_original_krea2(name);
     } else if (sd_version_is_pixart(version)) {
         name = convert_diffusers_dit_to_original_pixart(name);
+    } else if (sd_version_is_minimax_h3(version)) {
+        name = convert_diffusers_dit_to_original_minimax_h3(name);
     }
     return name;
 }