DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-14 12:56:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: unify text encoder selection across H3 variants (#2340)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-11-13 00:23:54 +08:00
1 parent 8b1e9df27d
commit b9d7176bf8
14 files changed
+123 -57
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/conditioning/conditioner.hpp
+16 -13
View File
@@ -2030,20 +2030,23 @@ struct LLMEmbedder : public Conditioner {
"text_encoders.llm",
enable_vision,
weight_manager);
// MiniMax-H3-Image also accepts the P-H3-Image text encoder in place of the MiniMax-H3 Qwen3-VL-32B encoder.
const bool has_ph3_connector = tensor_storage_map.find(std::string(PH3ImageTextConnectorRunner::prefix) + ".proj.weight") !=
tensor_storage_map.end();
if (version == VERSION_P_H3_IMAGE || (version == VERSION_MINIMAX_H3_IMAGE && has_ph3_connector)) {
if (llm->config.num_layers != 36 || llm->config.hidden_size != 2560) {
throw std::runtime_error(version == VERSION_P_H3_IMAGE
? "P-H3-Image requires the Qwen3-VL-4B text encoder"
: "The P-H3-Image text connector requires the Qwen3-VL-4B text encoder");
if (sd_version_is_minimax_h3(version)) {
const bool has_ph3_connector = tensor_storage_map.count(std::string(PH3ImageTextConnectorRunner::prefix) + ".proj.weight") != 0;
if (has_ph3_connector) {
if (llm->config.num_layers != 36 || llm->config.hidden_size != 2560) {
throw std::runtime_error("The P-H3-Image text connector requires the Qwen3-VL-4B text encoder");
}
ph3_connector = std::make_shared<PH3ImageTextConnectorRunner>(backend, tensor_storage_map, weight_manager);
} else if (llm->config.hidden_size == 2560) {
throw std::runtime_error(
sd_format("%s with Qwen3-VL-4B requires the P-H3-Image text connector; "
"pass it with --embeddings-connectors or use the MiniMax-H3 Qwen3-VL-32B text encoder",
sd_version_name(version)));
} else if (llm->config.hidden_size != 5120 || (llm->config.num_layers != 50 && llm->config.num_layers != 64)) {
throw std::runtime_error(
sd_format("%s requires the MiniMax-H3 Qwen3-VL-32B text encoder or Qwen3-VL-4B with the P-H3-Image text connector",
sd_version_name(version)));
}
ph3_connector = std::make_shared<PH3ImageTextConnectorRunner>(backend, tensor_storage_map, weight_manager);
} else if (version == VERSION_MINIMAX_H3_IMAGE && llm->config.hidden_size == 2560) {
throw std::runtime_error(
"MiniMax-H3-Image with Qwen3-VL-4B requires the P-H3-Image text connector; "
"pass it with --embeddings-connectors or use the MiniMax-H3 Qwen3-VL-32B text encoder");
}
int pad_id = 151643;
if (arch == LLM::LLMArch::MISTRAL_SMALL_3_2 || arch == LLM::LLMArch::MINISTRAL_3_3B) {
src/model/diffusion/fibo2.hpp
+2 -2
View File
@@ -131,8 +131,8 @@ namespace Fibo2 {
v = reshape(project("to_v", kv));
}
}
q = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_q"])->forward(ctx, q);
k = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_k"])->forward(ctx, k);
q = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_q"])->forward(ctx, q);
k = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_k"])->forward(ctx, k);
auto prepare = [&](ggml_tensor* h) {
if (pe) {
return Rope::apply_rope(ctx, h, pe);
src/model/diffusion/iris.hpp
+4 -4
View File
@@ -91,10 +91,10 @@ namespace Iris {
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe = nullptr, ggml_tensor* mask = nullptr) {
auto c = ctx->ggml_ctx;
auto head_dim = dim / heads;
auto qkv = std::dynamic_pointer_cast<Linear>(blocks["qkv"])->forward(ctx, x);
auto qkv_dim = qkv->ne[0] / 3;
auto c = ctx->ggml_ctx;
auto head_dim = dim / heads;
auto qkv = std::dynamic_pointer_cast<Linear>(blocks["qkv"])->forward(ctx, x);
auto qkv_dim = qkv->ne[0] / 3;
auto split_qkv = [&](int i) {
return ggml_view_4d(c, qkv, head_dim, qkv_dim / head_dim, qkv->ne[1], qkv->ne[2] * qkv->ne[3],
head_dim * qkv->nb[0], qkv->nb[1], qkv->nb[2], i * qkv_dim * qkv->nb[0]);
src/model/diffusion/mmdit.hpp
+1 -1
View File
@@ -373,7 +373,7 @@ public:
ggml_tensor* x) {
auto qkv = pre_attention(ctx, x);
x = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, nullptr, true, ctx->flash_attn_enabled); // [N, n_token, dim]
x = post_attention(ctx, x); // [N, n_token, dim]
x = post_attention(ctx, x); // [N, n_token, dim]
return x;
}
};
src/model/diffusion/seedvr2.hpp
+8 -8
View File
@@ -194,17 +194,17 @@ namespace SeedVR2 {
return x;
return std::dynamic_pointer_cast<AdaSingle>(blocks["ada." + branch(text)])->forward(ctx, x, emb, layer, gate);
};
auto vqkv = linear("attn.proj_qkv", false, ada(ggml_rms_norm(c, vid, 1e-5f), false, "attn"));
auto tqkv = linear("attn.proj_qkv", true, ada(ggml_rms_norm(c, txt, 1e-5f), true, "attn"));
vqkv = ggml_get_rows(c, vqkv, inputs.input(ctx, plan.order));
auto vpe = rotation(ctx, inputs.input(ctx, plan.positions));
auto tpe = rotation(ctx, inputs.input(ctx, plan.text_positions));
auto vqkv = linear("attn.proj_qkv", false, ada(ggml_rms_norm(c, vid, 1e-5f), false, "attn"));
auto tqkv = linear("attn.proj_qkv", true, ada(ggml_rms_norm(c, txt, 1e-5f), true, "attn"));
vqkv = ggml_get_rows(c, vqkv, inputs.input(ctx, plan.order));
auto vpe = rotation(ctx, inputs.input(ctx, plan.positions));
auto tpe = rotation(ctx, inputs.input(ctx, plan.text_positions));
auto split_qkv = [&](ggml_tensor* x, int i) {
int64_t qkv_dim = x->ne[0] / 3;
return ggml_view_4d(c, x, 128, qkv_dim / 128, x->ne[1], x->ne[2] * x->ne[3],
128 * x->nb[0], x->nb[1], x->nb[2], i * qkv_dim * x->nb[0]);
};
auto qkv = [&](ggml_tensor* x, bool text, ggml_tensor* pe) {
auto qkv = [&](ggml_tensor* x, bool text, ggml_tensor* pe) {
std::vector<ggml_tensor*> parts{nullptr, nullptr, nullptr};
for (int i = 0; i < 2; ++i) {
auto h = std::dynamic_pointer_cast<RMSNorm>(blocks[std::string(i == 0 ? "attn.norm_q." : "attn.norm_k.") + branch(text)])->forward(ctx, split_qkv(x, i));
@@ -212,8 +212,8 @@ namespace SeedVR2 {
parts[i] = VSR::apply_rope(ctx, h, pe, heads);
}
// v must stay 3-D [dim, L, N]: the window loop slices dim 1 as tokens.
auto v = split_qkv(x, 2);
parts[2] = ggml_view_3d(c, v, dim, x->ne[1], x->ne[2], v->nb[2], v->nb[3], 0);
auto v = split_qkv(x, 2);
parts[2] = ggml_view_3d(c, v, dim, x->ne[1], x->ne[2], v->nb[2], v->nb[3], 0);
return parts;
};
auto vp = qkv(vqkv, false, vpe), tp = qkv(tqkv, true, tpe);
src/model_loader.cpp
+1 -12
View File
@@ -581,7 +581,7 @@ SDVersion ModelLoader::get_sd_version() const {
if (tensor_storage_map.count("model.diffusion_model.audio_patch_proj.weight") == 0) {
return VERSION_MINIMAX_H3_IMAGE;
}
// P-H3 uses the same DiT architecture as H3; its VAE or connector identifies the image pipeline.
// The connector is shared with H3 video; the single-frame VAE identifies P-H3.
auto decoder = tensor_storage_map.find("first_stage_model.decoder.proj_out.weight");
if (decoder == tensor_storage_map.end()) {
decoder = tensor_storage_map.find("vae.decoder.proj_out.weight");
@@ -589,17 +589,6 @@ SDVersion ModelLoader::get_sd_version() const {
if (decoder != tensor_storage_map.end() && decoder->second.ne[1] == 768) {
return VERSION_P_H3_IMAGE;
}
for (const std::string prefix : {"", "embeddings_connectors."}) {
if (tensor_storage_map.count(prefix + "text_encoders.ph3_image_connector.proj.weight") ||
tensor_storage_map.count(prefix + "text_encoders.llm.text_connector.proj.weight")) {
return VERSION_P_H3_IMAGE;
}
}
auto connector = tensor_storage_map.find("embeddings_connectors.proj.weight");
if (connector != tensor_storage_map.end() && connector->second.logical_ne0() == 12800 &&
connector->second.ne[1] == 5120 && tensor_storage_map.count("embeddings_connectors.sink_embedding")) {
return VERSION_P_H3_IMAGE;
}
return VERSION_MINIMAX_H3;
}
if (name == "model.diffusion_model.y_embedder.layer_pool.weight" &&
src/name_conversion.cpp
+1 -1
View File
@@ -1741,7 +1741,7 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
const std::vector<std::pair<std::string, std::string>> connector_prefix_map = {
{"embeddings_connectors.text_encoders.", "text_encoders."},
{"embeddings_connectors.", sd_version_is_minimax_h3_image(version) ? "text_encoders.ph3_image_connector." : ""},
{"embeddings_connectors.", sd_version_is_minimax_h3(version) ? "text_encoders.ph3_image_connector." : ""},
};
replace_with_prefix_map(name, connector_prefix_map);