DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-12 03:49:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support external Hugging Face tokenizer JSON files (#1973)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-09-15 00:27:39 +08:00
1 parent 42d6c0ab92
commit 4964abdfc5
58 files changed
+97390 -180
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/pipeline/diffusion_engine.h
+1
View File
@@ -128,6 +128,7 @@ public:
&sd_ctx_params_t::clip_g_path, &sd_ctx_params_t::clip_vision_path,
&sd_ctx_params_t::t5xxl_path, &sd_ctx_params_t::llm_path,
&sd_ctx_params_t::llm_vision_path, &sd_ctx_params_t::diffusion_model_path,
&sd_ctx_params_t::tokenizer,
&sd_ctx_params_t::high_noise_diffusion_model_path, &sd_ctx_params_t::uncond_diffusion_model_path,
&sd_ctx_params_t::embeddings_connectors_path, &sd_ctx_params_t::vae_path,
&sd_ctx_params_t::audio_vae_path, &sd_ctx_params_t::taesd_path,
src/pipeline/model_builders.cpp
+48 -22
View File
@@ -73,12 +73,13 @@ namespace sd::model_builders {
}
}
bool build_core_runners(const Context& ctx, CoreRunners& runners) {
bool build_core_runners(const Context& ctx, CoreRunners& runners) try {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
CoreRunners result;
TokenizerConfig tokenizers(sd_ctx_params->tokenizer);
if (!ensure_backend_pair(ctx.backends, SDBackendModule::TE) ||
!ensure_backend_pair(ctx.backends, SDBackendModule::DIFFUSION)) {
return false;
@@ -87,7 +88,8 @@ namespace sd::model_builders {
if (sd_version_is_sd3(version)) {
result.conditioner = std::make_shared<SD3CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<MMDiTRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -98,7 +100,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Pid::PiDRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model.net",
@@ -109,7 +112,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Ideogram4::Ideogram4Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -120,7 +124,8 @@ namespace sd::model_builders {
version,
"",
true,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Krea2::Krea2Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -147,11 +152,13 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
} else {
result.conditioner = std::make_shared<FluxCLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager);
weight_manager,
tokenizers);
}
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
@@ -166,7 +173,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -178,7 +186,8 @@ namespace sd::model_builders {
tensor_storage_map,
"text_encoders.llm",
"text_embedding_projection",
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<LTXV::LTXAVRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -189,7 +198,8 @@ namespace sd::model_builders {
version,
"",
true,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<MiniMaxH3::MiniMaxH3Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -200,7 +210,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Hunyuan::HunyuanVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -258,7 +269,8 @@ namespace sd::model_builders {
version,
"",
enable_vision,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<LingBotVideo::LingBotVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -271,7 +283,8 @@ namespace sd::model_builders {
version,
"",
enable_vision,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Qwen::QwenImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -284,7 +297,8 @@ namespace sd::model_builders {
version,
"",
true,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<MageFlow::MageFlowRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -295,7 +309,8 @@ namespace sd::model_builders {
version,
"",
true,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -305,7 +320,8 @@ namespace sd::model_builders {
} else if (version == VERSION_HIDREAM_O1) {
result.conditioner = std::make_shared<HiDreamO1::HiDreamO1Conditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<HiDreamO1::HiDreamO1Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model",
@@ -327,7 +343,8 @@ namespace sd::model_builders {
} else if (sd_version_is_anima(version)) {
result.conditioner = std::make_shared<AnimaConditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Anima::AnimaRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -338,7 +355,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<ZImage::ZImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -350,7 +368,8 @@ namespace sd::model_builders {
version,
"",
true,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Boogu::BooguImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -362,7 +381,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<ErnieImage::ErnieImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -373,7 +393,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Lens::LensRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -387,7 +408,8 @@ namespace sd::model_builders {
tensor_storage_map,
embbeding_map,
version,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<UNetModelRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -429,8 +451,12 @@ namespace sd::model_builders {
if (result.ip_adapter) {
result.ip_adapter->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
}
tokenizers.validate_usage();
runners = std::move(result);
return true;
} catch (const std::exception& error) {
LOG_ERROR("failed to build model runners: %s", error.what());
return false;
}
bool build_vae_runners(const Context& ctx, const VAEOptions& options, VAERunners& runners) {
src/pipeline/video.cpp
+25 -13
View File
@@ -1139,10 +1139,10 @@ namespace sd::pipeline {
return latents;
}
static ImageGenerationEmbeds prepare_video_generation_embeds(StableDiffusionGGML* sd,
const sd_vid_gen_params_t* sd_vid_gen_params,
const GenerationRequest& request,
const ImageGenerationLatents& latents) {
static std::optional<ImageGenerationEmbeds> prepare_video_generation_embeds(StableDiffusionGGML* sd,
const sd_vid_gen_params_t* sd_vid_gen_params,
const GenerationRequest& request,
const ImageGenerationLatents& latents) {
ConditionerRunnerEndOnExit conditioner_runner_end{sd->cond_stage_model.get()};
ImageGenerationEmbeds embeds;
@@ -1159,8 +1159,12 @@ namespace sd::pipeline {
int64_t prepare_start_ms = ggml_time_ms();
embeds.cond = sd->cond_stage_model->get_learned_condition(sd->n_threads,
condition_params);
embeds.cond.c_concat = latents.concat_latent;
embeds.cond.c_vector = latents.clip_vision_output;
if (embeds.cond.empty()) {
LOG_ERROR("failed to encode video prompt");
return std::nullopt;
}
embeds.cond.c_concat = latents.concat_latent;
embeds.cond.c_vector = latents.clip_vision_output;
if (sd_version_is_minimax_h3(sd->version)) {
embeds.cond.c_ref_images = latents.ref_latents;
embeds.cond.c_ref_audios = latents.reference_audio_latents;
@@ -1178,9 +1182,13 @@ namespace sd::pipeline {
}
}
if (request.use_uncond) {
condition_params.text = request.negative_prompt;
embeds.uncond = sd->cond_stage_model->get_learned_condition(sd->n_threads,
condition_params);
condition_params.text = request.negative_prompt;
embeds.uncond = sd->cond_stage_model->get_learned_condition(sd->n_threads,
condition_params);
if (embeds.uncond.empty()) {
LOG_ERROR("failed to encode negative video prompt");
return std::nullopt;
}
embeds.uncond.c_concat = latents.concat_latent;
embeds.uncond.c_vector = latents.clip_vision_output;
if (sd_version_is_minimax_h3(sd->version)) {
@@ -1574,10 +1582,14 @@ namespace sd::pipeline {
}
ImageGenerationLatents latents = std::move(*latent_inputs_opt);
ImageGenerationEmbeds embeds = prepare_video_generation_embeds(sd,
sd_vid_gen_params,
request,
latents);
auto embeds_opt = prepare_video_generation_embeds(sd,
sd_vid_gen_params,
request,
latents);
if (!embeds_opt) {
return false;
}
ImageGenerationEmbeds embeds = std::move(*embeds_opt);
if (latent_upscale_enabled) {
LOG_INFO("generate_video %dx%dx%d -> LTX latent spatial upscale",
request.width,