DarthAffe/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-09-24 20:20:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: validate vision projector output dim against LLM hidden size (#1918)

Browse Source
This commit is contained in:
fszontagh 2026-09-13 17:42:46 +02:00 committed by GitHub
parent 0bd72f075a
commit ca37fad89a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 20 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/model/te/llm.hpp
View File

@ -139,7 +139,8 @@ namespace LLM {
static LLMConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, static LLMConfig detect_from_weights(const String2TensorStorage& tensor_storage_map,
const std::string& prefix, const std::string& prefix,
LLMArch arch) { LLMArch arch,
bool& enable_vision) {
LLMConfig config; LLMConfig config;
config.arch = arch; config.arch = arch;
if (arch == LLMArch::MISTRAL_SMALL_3_2 || arch == LLMArch::MINISTRAL_3_3B) { if (arch == LLMArch::MISTRAL_SMALL_3_2 || arch == LLMArch::MINISTRAL_3_3B) {
@ -230,8 +231,9 @@ namespace LLM {
config.num_experts_per_tok = 4; config.num_experts_per_tok = 4;
} }
config.num_layers = 0; config.num_layers = 0;
int detected_vision_layers = 0; int detected_vision_layers = 0;
bool out_hidden_size_detected = false;
for (const auto& [name, tensor_storage] : tensor_storage_map) { for (const auto& [name, tensor_storage] : tensor_storage_map) {
if (!starts_with(name, prefix)) { if (!starts_with(name, prefix)) {
continue; continue;
@ -277,6 +279,7 @@ namespace LLM {
if (ends_with(name, "visual.merger.linear_fc2.weight") || if (ends_with(name, "visual.merger.linear_fc2.weight") ||
ends_with(name, "visual.merger.mlp.2.weight")) { ends_with(name, "visual.merger.mlp.2.weight")) {
config.vision.out_hidden_size = tensor_storage.ne[1]; config.vision.out_hidden_size = tensor_storage.ne[1];
out_hidden_size_detected = true;
} }
continue; continue;
} }
@ -330,6 +333,19 @@ namespace LLM {
config.vocab_size, config.vocab_size,
config.hidden_size, config.hidden_size,
config.intermediate_size); config.intermediate_size);
if (enable_vision && !config.have_vision_weight) {
LOG_WARN("no vision weights detected, vision disabled");
enable_vision = false;
}
// The default would reject valid models, so only compare a detected dim.
if (enable_vision && out_hidden_size_detected &&
config.vision.out_hidden_size != config.hidden_size) {
LOG_ERROR("vision projector output size (%" PRId64 ") does not match LLM hidden size (%" PRId64 "), "
"the vision weights (mmproj) likely belong to a different LLM variant, vision disabled",
config.vision.out_hidden_size,
config.hidden_size);
enable_vision = false;
}
return config; return config;
} }
}; };
@ -1886,12 +1902,8 @@ namespace LLM {
bool enable_vision_ = false, bool enable_vision_ = false,
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr) std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
: GGMLRunner(backend, weight_manager), : GGMLRunner(backend, weight_manager),
config(LLMConfig::detect_from_weights(tensor_storage_map, prefix, arch)), config(LLMConfig::detect_from_weights(tensor_storage_map, prefix, arch, enable_vision_)),
enable_vision(enable_vision_) { enable_vision(enable_vision_) {
if (enable_vision && !config.have_vision_weight) {
LOG_WARN("no vision weights detected, vision disabled");
enable_vision = false;
}
if (enable_vision) { if (enable_vision) {
LOG_VERBOSE("enable llm vision"); LOG_VERBOSE("enable llm vision");
if (config.llama_cpp_style) { if (config.llama_cpp_style) {