mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-11-11 03:24:30 +00:00
feat: support external Hugging Face tokenizer JSON files (#1973)
This commit is contained in:
leejet authored and GitHub committed
1 parent
42d6c0ab92
commit
4964abdfc5
58 files changed
+97390
-180
No files matched your search
@@ -73,12 +73,13 @@ namespace sd::model_builders {
|
||||
}
|
||||
}
|
||||
|
||||
bool build_core_runners(const Context& ctx, CoreRunners& runners) {
|
||||
bool build_core_runners(const Context& ctx, CoreRunners& runners) try {
|
||||
const auto* sd_ctx_params = &ctx.params;
|
||||
const auto& tensor_storage_map = ctx.tensor_storage_map;
|
||||
const auto version = ctx.version;
|
||||
const auto& weight_manager = ctx.weight_manager;
|
||||
CoreRunners result;
|
||||
TokenizerConfig tokenizers(sd_ctx_params->tokenizer);
|
||||
if (!ensure_backend_pair(ctx.backends, SDBackendModule::TE) ||
|
||||
!ensure_backend_pair(ctx.backends, SDBackendModule::DIFFUSION)) {
|
||||
return false;
|
||||
@@ -87,7 +88,8 @@ namespace sd::model_builders {
|
||||
if (sd_version_is_sd3(version)) {
|
||||
result.conditioner = std::make_shared<SD3CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
|
||||
tensor_storage_map,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<MMDiTRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -98,7 +100,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
false,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<Pid::PiDRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model.net",
|
||||
@@ -109,7 +112,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
false,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<Ideogram4::Ideogram4Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -120,7 +124,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
true,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<Krea2::Krea2Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -147,11 +152,13 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
false,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
} else {
|
||||
result.conditioner = std::make_shared<FluxCLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
|
||||
tensor_storage_map,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
}
|
||||
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
@@ -166,7 +173,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
false,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -178,7 +186,8 @@ namespace sd::model_builders {
|
||||
tensor_storage_map,
|
||||
"text_encoders.llm",
|
||||
"text_embedding_projection",
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<LTXV::LTXAVRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -189,7 +198,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
true,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<MiniMaxH3::MiniMaxH3Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -200,7 +210,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
false,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<Hunyuan::HunyuanVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -258,7 +269,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
enable_vision,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<LingBotVideo::LingBotVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -271,7 +283,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
enable_vision,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<Qwen::QwenImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -284,7 +297,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
true,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<MageFlow::MageFlowRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -295,7 +309,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
true,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -305,7 +320,8 @@ namespace sd::model_builders {
|
||||
} else if (version == VERSION_HIDREAM_O1) {
|
||||
result.conditioner = std::make_shared<HiDreamO1::HiDreamO1Conditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
|
||||
tensor_storage_map,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<HiDreamO1::HiDreamO1Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model",
|
||||
@@ -327,7 +343,8 @@ namespace sd::model_builders {
|
||||
} else if (sd_version_is_anima(version)) {
|
||||
result.conditioner = std::make_shared<AnimaConditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
|
||||
tensor_storage_map,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<Anima::AnimaRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -338,7 +355,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
false,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<ZImage::ZImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -350,7 +368,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
true,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<Boogu::BooguImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -362,7 +381,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
false,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<ErnieImage::ErnieImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -373,7 +393,8 @@ namespace sd::model_builders {
|
||||
version,
|
||||
"",
|
||||
false,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<Lens::LensRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -387,7 +408,8 @@ namespace sd::model_builders {
|
||||
tensor_storage_map,
|
||||
embbeding_map,
|
||||
version,
|
||||
weight_manager);
|
||||
weight_manager,
|
||||
tokenizers);
|
||||
result.diffusion = std::make_shared<UNetModelRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
|
||||
tensor_storage_map,
|
||||
"model.diffusion_model",
|
||||
@@ -429,8 +451,12 @@ namespace sd::model_builders {
|
||||
if (result.ip_adapter) {
|
||||
result.ip_adapter->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
|
||||
}
|
||||
tokenizers.validate_usage();
|
||||
runners = std::move(result);
|
||||
return true;
|
||||
} catch (const std::exception& error) {
|
||||
LOG_ERROR("failed to build model runners: %s", error.what());
|
||||
return false;
|
||||
}
|
||||
|
||||
bool build_vae_runners(const Context& ctx, const VAEOptions& options, VAERunners& runners) {
|
||||
|
||||
Reference in new issue
Block a user