DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-12 03:49:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: centralize model factories and defaults (#2198)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-10-24 01:03:05 +08:00
1 parent baf45b49e2
commit cd7f986058
50 files changed
+2233 -943
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/pipeline/diffusion_engine.cpp
+33 -253
View File
@@ -21,6 +21,7 @@
#include "core/probe/conv_direct.h"
#include "core/probe/flash_attn.h"
#include "model.h"
#include "model_definition.h"
#include "core/rng.hpp"
#include "core/rng_mt19937.hpp"
@@ -55,70 +56,6 @@
#include <atomic>
const char* model_version_to_str[] = {
"SD 1.x",
"SD 1.x Inpaint",
"Instruct-Pix2Pix",
"SD 1.x Tiny UNet",
"SD 2.x",
"SD 2.x Inpaint",
"SD 2.x Tiny UNet",
"SDXS (512-DS)",
"SDXS (09)",
"SDXL",
"SDXL Inpaint",
"SDXL Instruct-Pix2Pix",
"SDXL (Vega)",
"SDXL (SSD1B)",
"SVD",
"SD3.x",
"Flux",
"Flux Fill",
"Flux Control",
"Flex.2",
"Chroma Radiance",
"Wan 2.x",
"Wan 2.2 I2V",
"Wan 2.2 TI2V",
"Wan 2.2 S2V",
"LingBot Video",
"Qwen Image",
"Qwen Image Layered",
"Qwen Image 2.1",
"Hunyuan Video",
"Anima",
"Flux.2",
"Flux.2 klein",
"LTXAV",
"MiniMax-H3",
"HiDream O1",
"Z-Image",
"Boogu Image",
"Ovis Image",
"Ernie Image",
"Lens",
"MiniT2I",
"Longcat-Image",
"PiD",
"Ideogram 4",
"SeFi-Image",
"Krea2",
"Mage Flow",
"SenseNova U1.5",
"LLaDA-Image",
"ESRGAN",
"PixArt",
"Ming-Image",
"Z-Image L2P",
"Iris-3B",
"Flux.3 Image Klein",
"MOSS-Image",
"Fibo-2.0",
};
static_assert(VERSION_COUNT == sizeof(model_version_to_str) / sizeof(model_version_to_str[0]),
"\nnumber of elements in model_version_to_str[] != VERSION_COUNT");
void calculate_alphas_cumprod(float* alphas_cumprod,
float linear_start = 0.00085f,
float linear_end = 0.0120f,
@@ -1014,7 +951,7 @@ bool StableDiffusionGGML::init(const sd_ctx_params_t* sd_ctx_params) {
LOG_ERROR("get sd version from file failed: '%s'", SAFE_STR(sd_ctx_params->model_path));
return false;
} else {
LOG_INFO("Version: %s ", model_version_to_str[version]);
LOG_INFO("Version: %s ", sd_version_name(version));
}
if (auto_fit_enabled) {
@@ -1470,98 +1407,16 @@ bool StableDiffusionGGML::build_denoiser() {
prediction_t pred_type = sd_ctx_params->prediction;
if (pred_type == PREDICTION_COUNT) {
if (sd_version_is_sd2(version)) {
const auto* definition = find_model_definition(version);
if (!definition)
return false;
const auto& policy = definition->sampling;
pred_type = policy.default_prediction;
default_flow_shift = policy.flow_shift_from_weights ? policy.flow_shift_from_weights(tensor_storage_map) : policy.default_flow_shift;
if (policy.probe_v_prediction) {
pred_type = is_using_v_parameterization_for_sd2(sd_version_is_inpaint(version)) ? V_PRED : EPS_PRED;
} else if (sd_version_is_sdxl(version)) {
if (tensor_storage_map.find("edm_vpred.sigma_max") != tensor_storage_map.end()) {
// CosXL models
// TODO: get sigma_min and sigma_max values from file
pred_type = EDM_V_PRED;
} else if (tensor_storage_map.find("v_pred") != tensor_storage_map.end()) {
pred_type = V_PRED;
} else {
pred_type = EPS_PRED;
}
} else if (sd_version_is_sd3(version) ||
sd_version_is_wan(version) ||
sd_version_is_hunyuan_video(version) ||
sd_version_is_lingbot_video(version) ||
sd_version_is_minimax_h3(version) ||
sd_version_is_qwen_image(version) ||
sd_version_is_mage_flow(version) ||
version == VERSION_HIDREAM_O1 ||
sd_version_is_anima(version) ||
sd_version_is_ernie_image(version) ||
sd_version_is_z_image(version) ||
sd_version_is_z_image_l2p(version) ||
version == VERSION_MING_IMAGE ||
sd_version_is_llada_image(version) ||
sd_version_is_boogu_image(version) ||
sd_version_is_pid(version) ||
sd_version_is_ideogram4(version) ||
version == VERSION_IRIS ||
version == VERSION_MOSS_IMAGE ||
version == VERSION_FIBO2) {
pred_type = FLOW_PRED;
if (sd_version_is_wan(version) || version == VERSION_MOSS_IMAGE) {
default_flow_shift = 5.f;
} else if (sd_version_is_hunyuan_video(version)) {
default_flow_shift = 7.f;
} else if (sd_version_is_minimax_h3(version)) {
default_flow_shift = 12.f;
} else if (sd_version_is_ernie_image(version) || version == VERSION_IRIS) {
default_flow_shift = 4.f;
} else if (sd_version_is_pid(version)) {
default_flow_shift = 1.5f;
} else if (sd_version_is_ideogram4(version)) {
default_flow_shift = 1.0f;
} else if (sd_version_is_boogu_image(version)) {
default_flow_shift = 3.16f;
} else if (sd_version_is_mage_flow(version)) {
default_flow_shift = 6.f;
} else if (version == VERSION_MING_IMAGE) {
default_flow_shift = INFINITY;
} else if (sd_version_is_llada_image(version)) {
default_flow_shift = 1.0f; // unused: LLADA_IMAGE_SCHEDULER builds a fixed grid
} else {
default_flow_shift = 3.f;
}
} else if (sd_version_is_flux3(version)) {
pred_type = FLUX_FLOW_PRED;
default_flow_shift = 6.93f;
} else if (sd_version_is_flux(version) ||
sd_version_is_flux2(version) ||
sd_version_is_longcat(version) ||
sd_version_is_lens(version) ||
sd_version_is_ltxav(version) ||
sd_version_is_krea2(version)) {
pred_type = FLUX_FLOW_PRED;
default_flow_shift = 1.0f; // TODO: validate
for (const auto& [name, tensor_storage] : tensor_storage_map) {
if (starts_with(name, "model.diffusion_model.guidance_in.in_layer.weight")) {
default_flow_shift = 1.15f;
break;
}
}
if (sd_version_is_longcat(version)) {
default_flow_shift = 3.0f;
} else if (sd_version_is_lens(version)) {
default_flow_shift = 1.83f;
} else if (sd_version_is_ltxav(version)) {
default_flow_shift = 2.37f;
} else if (sd_version_is_krea2(version)) {
default_flow_shift = 1.15f;
}
} else if (sd_version_is_sefi_image(version)) {
pred_type = SEFI_FLOW_PRED;
} else if (sd_version_is_minit2i(version)) {
pred_type = MINIT2I_FLOW_PRED;
} else if (sd_version_is_sensenova_u1(version)) {
pred_type = SENSENOVA_U1_FLOW_PRED;
default_flow_shift = 3.f;
} else {
pred_type = EPS_PRED;
} else if (policy.prediction_from_weights) {
pred_type = policy.prediction_from_weights(tensor_storage_map);
}
}
@@ -1578,27 +1433,28 @@ bool StableDiffusionGGML::build_denoiser() {
denoiser = std::make_shared<EDMVDenoiser>();
break;
case FLOW_PRED: {
if (sd_version_is_ltxav(version)) {
LOG_INFO("running in LTXAV FLOW mode");
denoiser = std::make_shared<FluxFlowDenoiser>();
} else if (sd_version_is_minimax_h3(version)) {
LOG_INFO("running in MiniMax H3 AV FLOW mode");
denoiser = std::make_shared<H3AVFlowDenoiser>(default_flow_shift, 3.f, get_latent_channel());
} else if (version == VERSION_MING_IMAGE) {
denoiser = std::make_shared<MingImageFlowDenoiser>();
} else {
LOG_INFO("running in FLOW mode");
denoiser = std::make_shared<DiscreteFlowDenoiser>();
}
LOG_INFO("running in FLOW mode");
denoiser = std::make_shared<DiscreteFlowDenoiser>();
break;
}
case FLUX_FLOW_PRED: {
LOG_INFO("running in Flux FLOW mode");
if (sd_version_is_flux3(version)) {
denoiser = std::make_shared<Flux3FlowDenoiser>();
} else {
denoiser = std::make_shared<FluxFlowDenoiser>();
}
denoiser = std::make_shared<FluxFlowDenoiser>();
break;
}
case FLUX3_FLOW_PRED: {
LOG_INFO("running in Flux.3 FLOW mode");
denoiser = std::make_shared<Flux3FlowDenoiser>();
break;
}
case H3_AV_FLOW_PRED: {
LOG_INFO("running in MiniMax H3 AV FLOW mode");
denoiser = std::make_shared<H3AVFlowDenoiser>(default_flow_shift, 3.f, get_latent_channel());
break;
}
case MING_IMAGE_FLOW_PRED: {
LOG_INFO("running in Ming-Image FLOW mode");
denoiser = std::make_shared<MingImageFlowDenoiser>();
break;
}
case SEFI_FLOW_PRED: {
@@ -2999,63 +2855,15 @@ int StableDiffusionGGML::get_vae_scale_factor() {
}
int StableDiffusionGGML::get_diffusion_model_down_factor() {
int down_factor = 8; // unet
if (sd_version_is_dit(version)) {
if (sd_version_is_sensenova_u1(version)) {
down_factor = 32;
} else if (sd_version_is_z_image_l2p(version) || version == VERSION_IRIS) {
down_factor = 16;
} else if (version == VERSION_QWEN_IMAGE_2_1 || version == VERSION_MING_IMAGE || sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_minimax_h3(version) || sd_version_is_pixart(version)) {
down_factor = 2;
} else {
down_factor = 1;
}
}
return down_factor;
return find_model_definition(version)->geometry.diffusion_down_factor;
}
int StableDiffusionGGML::get_latent_channel() {
int latent_channel = 4;
if (sd_version_is_dit(version)) {
if (sd_version_is_ltxav(version)) {
latent_channel = 128;
} else if (sd_version_is_minimax_h3(version)) {
latent_channel = 24;
} else if (version == VERSION_QWEN_IMAGE_2_1) {
latent_channel = 64;
} else if (version == VERSION_WAN2_2_TI2V) {
latent_channel = 48;
} else if (sd_version_is_hunyuan_video(version)) {
latent_channel = 32;
} else if (version == VERSION_HIDREAM_O1) {
latent_channel = 3;
} else if (version == VERSION_CHROMA_RADIANCE) {
latent_channel = 3;
} else if (sd_version_is_minit2i(version)) {
latent_channel = 3;
} else if (sd_version_is_sensenova_u1(version)) {
latent_channel = 3;
} else if (sd_version_is_z_image_l2p(version)) {
latent_channel = 3;
} else if (sd_version_is_pid(version) || version == VERSION_IRIS) {
latent_channel = 3;
} else if (sd_version_is_sefi_image(version)) {
latent_channel = 144;
} else if (sd_version_uses_flux2_vae(version)) {
latent_channel = 128;
} else if (sd_version_is_mage_flow(version)) {
latent_channel = 128;
} else if (sd_version_is_pixart(version)) {
latent_channel = 4;
} else {
latent_channel = 16;
}
}
return latent_channel;
return find_model_definition(version)->geometry.latent_channels;
}
int StableDiffusionGGML::get_image_channels() const {
return version == VERSION_QWEN_IMAGE_LAYERED || version == VERSION_QWEN_IMAGE_2_1 || version == VERSION_MING_IMAGE ? 4 : 3;
return find_model_definition(version)->geometry.image_channels;
}
int StableDiffusionGGML::get_image_seq_len(int h, int w) {
@@ -3207,35 +3015,7 @@ bool StableDiffusionGGML::is_flow_denoiser() {
}
std::string StableDiffusionGGML::get_default_ref_image_preset(SDVersion version) const {
if (version == VERSION_FIBO2) {
return "fibo2";
} else if (sd_version_is_longcat(version)) {
return "longcat";
} else if (sd_version_is_flux(version)) {
return "flux_kontext";
} else if (sd_version_is_flux3(version)) {
return "flux2";
} else if (sd_version_is_flux2(version) || sd_version_is_sefi_image(version)) {
return "flux2";
} else if (version == VERSION_QWEN_IMAGE_LAYERED) {
return "qwen_layered";
} else if (sd_version_is_qwen_image(version)) {
return "qwen";
} else if (sd_version_is_mage_flow(version)) {
return "mage_flow";
} else if (sd_version_is_boogu_image(version)) {
return "z_image_omni";
} else if (sd_version_is_llada_image(version)) {
return "llada_image";
} else if (sd_version_is_krea2(version)) {
// have to make a choice between "krea2_edit" mode (for lbouaraba/krea2edit)
// and "krea2_ostris_edit" (for krea2 ostris edit)
// since krea2 ostris edit support predates, it should probably be default
return "krea2_ostris_edit";
} else if (sd_version_is_anima(version)) {
return "cosmos_reference";
}
return "default";
return find_model_definition(version)->ref_image_preset;
}
RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_image_args) const {
src/pipeline/diffusion_engine.h
-2
View File
@@ -34,8 +34,6 @@ namespace Wav2Vec2 {
class Wav2Vec2ModelRunner;
}
extern const char* model_version_to_str[];
static inline bool sd_version_supports_ref_latent_img_cfg(SDVersion version) {
return version == VERSION_FLUX ||
sd_version_is_flux2(version) ||
src/pipeline/image.cpp
+1 -1
View File
@@ -804,7 +804,7 @@ namespace sd::pipeline {
if (!sd_version_supports_image_generation(sd->version)) {
LOG_ERROR("%s cannot be run with generate_image(); use generate_video() or --mode vid_gen in the CLI",
model_version_to_str[sd->version]);
sd_version_name(sd->version));
return false;
}
src/pipeline/model_builders.cpp
+19 -473
View File
@@ -5,44 +5,13 @@
#include <utility>
#include "conditioning/conditioner.hpp"
#include "conditioning/fibo2.hpp"
#include "core/ggml_extend_backend.h"
#include "core/util.h"
#include "extensions/generation_extension.h"
#include "model/adapter/ip_adapter.hpp"
#include "model/audio/wav2vec2.hpp"
#include "model/diffusion/anima.hpp"
#include "model/diffusion/boogu.hpp"
#include "model/diffusion/control.hpp"
#include "model/diffusion/ernie_image.hpp"
#include "model/diffusion/fibo2.hpp"
#include "model/diffusion/flux.hpp"
#include "model/diffusion/flux3.hpp"
#include "model/diffusion/hidream_o1.hpp"
#include "model/diffusion/hunyuan.hpp"
#include "model/diffusion/ideogram4.hpp"
#include "model/diffusion/iris.hpp"
#include "model/diffusion/krea2.hpp"
#include "model/diffusion/lens.hpp"
#include "model/diffusion/lingbot_video.hpp"
#include "model/diffusion/llada_image.hpp"
#include "model/diffusion/ltxv.hpp"
#include "model/diffusion/mage_flow.hpp"
#include "model/diffusion/ming_image.hpp"
#include "model/diffusion/minimax_h3.hpp"
#include "model/diffusion/minit2i.hpp"
#include "model/diffusion/mmdit.hpp"
#include "model/diffusion/model.hpp"
#include "model/diffusion/moss_image.hpp"
#include "model/diffusion/pid.hpp"
#include "model/diffusion/pixart.hpp"
#include "model/diffusion/qwen_image.hpp"
#include "model/diffusion/qwen_image_2_1.hpp"
#include "model/diffusion/sensenova_u1.h"
#include "model/diffusion/unet.hpp"
#include "model/diffusion/wan.hpp"
#include "model/diffusion/z_image.hpp"
#include "model/diffusion/z_image_l2p.hpp"
#include "model/vae/auto_encoder_kl.hpp"
#include "model/vae/hunyuan_vae.hpp"
#include "model/vae/ltx_audio_vae.hpp"
@@ -53,10 +22,11 @@
#include "model/vae/tae.hpp"
#include "model/vae/vae.hpp"
#include "model/vae/wan_vae.hpp"
#include "model_definition.h"
namespace sd::model_builders {
static bool ensure_backend_pair(SDBackendManager& backends, SDBackendModule module) {
bool ensure_backend_pair(SDBackendManager& backends, SDBackendModule module) {
if (backends.runtime_backend(module) == nullptr) {
LOG_ERROR("failed to initialize %s backend", sd_backend_module_name(module));
return false;
@@ -96,407 +66,10 @@ namespace sd::model_builders {
return false;
}
if (version == VERSION_FIBO2) {
result.conditioner = std::make_shared<Fibo2Conditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map, weight_manager, tokenizers);
result.diffusion = std::make_shared<Fibo2::Fibo2Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map, "model.diffusion_model", weight_manager);
} else if (sd_version_is_sd3(version)) {
result.conditioner = std::make_shared<SD3CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<MMDiTRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager);
} else if (sd_version_is_pid(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
false,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Pid::PiDRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model.net",
weight_manager);
} else if (sd_version_is_ideogram4(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
false,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Ideogram4::Ideogram4Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager);
} else if (sd_version_is_krea2(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
true,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Krea2::Krea2Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager);
} else if (sd_version_is_flux(version)) {
bool is_chroma = false;
for (auto pair : tensor_storage_map) {
if (pair.first.find("distilled_guidance_layer.in_proj.weight") != std::string::npos) {
is_chroma = true;
break;
}
}
if (is_chroma) {
result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
false,
1,
false,
weight_manager,
sd_ctx_params->model_args);
} else if (version == VERSION_OVIS_IMAGE) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
false,
weight_manager,
tokenizers);
} else {
result.conditioner = std::make_shared<FluxCLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager,
tokenizers);
}
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
version,
weight_manager,
sd_ctx_params->model_args);
} else if (sd_version_is_flux3(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map, version, "", false, weight_manager, tokenizers);
result.diffusion = std::make_shared<Flux3::Flux3Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map, "model.diffusion_model", weight_manager);
} else if (sd_version_is_flux2(version) || sd_version_is_sefi_image(version)) {
bool is_chroma = false;
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
false,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
version,
weight_manager,
sd_ctx_params->model_args);
} else if (sd_version_is_ltxav(version)) {
result.conditioner = std::make_shared<LTXAVEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
"text_encoders.llm",
"text_embedding_projection",
weight_manager,
tokenizers);
result.diffusion = std::make_shared<LTXV::LTXAVRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager);
} else if (sd_version_is_minimax_h3(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
true,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<MiniMaxH3::MiniMaxH3Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager);
} else if (sd_version_is_hunyuan_video(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
false,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Hunyuan::HunyuanVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
version,
weight_manager);
} else if (sd_version_is_wan(version)) {
result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
true,
0,
true,
weight_manager);
result.diffusion = std::make_shared<WAN::WanRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
version,
weight_manager);
if (strlen(SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path)) > 0) {
result.high_noise_diffusion = std::make_shared<WAN::WanRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.high_noise_diffusion_model",
version,
weight_manager);
}
if (result.diffusion->get_desc() == "Wan2.1-I2V-14B" ||
result.diffusion->get_desc() == "Wan2.1-FLF2V-14B" ||
result.diffusion->get_desc() == "Wan2.1-I2V-1.3B") {
if (!ensure_backend_pair(ctx.backends, SDBackendModule::CLIP_VISION)) {
return false;
}
result.clip_vision = std::make_shared<FrozenCLIPVisionEmbedder>(ctx.backends.runtime_backend(SDBackendModule::CLIP_VISION),
tensor_storage_map,
weight_manager);
}
if (version == VERSION_WAN2_2_S2V &&
tensor_storage_map.count("wav2vec2.encoder.layer_norm.bias") > 0) {
if (!ensure_backend_pair(ctx.backends, SDBackendModule::AUDIO_ENCODER)) {
return false;
}
result.audio_encoder = std::make_shared<Wav2Vec2::Wav2Vec2ModelRunner>(ctx.backends.runtime_backend(SDBackendModule::AUDIO_ENCODER),
tensor_storage_map,
"wav2vec2.",
weight_manager);
}
} else if (sd_version_is_lingbot_video(version)) {
bool enable_vision = false;
for (const auto& [name, _] : tensor_storage_map) {
if (starts_with(name, "text_encoders.llm.visual.")) {
enable_vision = true;
break;
}
}
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
enable_vision,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<LingBotVideo::LingBotVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager,
sd_ctx_params->model_args);
} else if (sd_version_is_qwen_image(version)) {
bool enable_vision = version != VERSION_QWEN_IMAGE_LAYERED;
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
enable_vision,
weight_manager,
tokenizers);
if (version == VERSION_QWEN_IMAGE_2_1) {
result.diffusion = std::make_shared<Qwen::QwenImage21Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager,
sd_ctx_params->model_args);
} else {
result.diffusion = std::make_shared<Qwen::QwenImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
version,
weight_manager,
sd_ctx_params->model_args);
}
} else if (version == VERSION_PIXART) {
result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
true,
0,
false,
weight_manager,
sd_ctx_params->model_args);
result.diffusion = std::make_shared<PixArt::PixArtRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager,
sd_ctx_params->model_args);
} else if (sd_version_is_mage_flow(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
true,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<MageFlow::MageFlowRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager);
} else if (sd_version_is_longcat(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
true,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
version,
weight_manager,
sd_ctx_params->model_args);
} else if (version == VERSION_HIDREAM_O1) {
result.conditioner = std::make_shared<HiDreamO1::HiDreamO1Conditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<HiDreamO1::HiDreamO1Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model",
weight_manager);
} else if (sd_version_is_minit2i(version)) {
result.conditioner = std::make_shared<MiniT2IConditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager);
result.diffusion = std::make_shared<MiniT2I::MiniT2IRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model.model.net",
weight_manager);
} else if (sd_version_is_sensenova_u1(version)) {
result.conditioner = std::make_shared<SenseNovaU1Conditioner>();
result.diffusion = std::make_shared<SenseNovaU1::SenseNovaU1Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"",
weight_manager);
} else if (sd_version_is_anima(version)) {
result.conditioner = std::make_shared<AnimaConditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Anima::AnimaRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager);
} else if (version == VERSION_MING_IMAGE) {
result.conditioner = std::make_shared<MingImageEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map, weight_manager, tokenizers);
result.diffusion = std::make_shared<MingImage::MingImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map, "model.diffusion_model", weight_manager);
} else if (sd_version_is_z_image(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
false,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<ZImage::ZImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
version,
weight_manager);
} else if (sd_version_is_z_image_l2p(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
false,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<ZImageL2P::ZImageL2PRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager);
} else if (sd_version_is_llada_image(version)) {
result.conditioner = std::make_shared<LLaDAImageEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
"text_encoders.llm",
"queryformer",
"text_projection",
"sigvq",
weight_manager,
tokenizers);
result.diffusion = std::make_shared<LLaDAImage::LLaDAImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager);
} else if (sd_version_is_boogu_image(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
true,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Boogu::BooguImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
version,
weight_manager);
} else if (sd_version_is_ernie_image(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
false,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<ErnieImage::ErnieImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager);
} else if (sd_version_is_lens(version)) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
version,
"",
false,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Lens::LensRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
weight_manager);
} else if (version == VERSION_IRIS) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map, version, "", false, weight_manager, tokenizers);
result.diffusion = std::make_shared<Iris::IrisRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map, "model.diffusion_model", weight_manager);
} else if (version == VERSION_MOSS_IMAGE) {
result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map, version, "", false, weight_manager, tokenizers);
result.diffusion = std::make_shared<MOSSImage::MOSSImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map, "model.diffusion_model", weight_manager);
} else { // SD1.x SD2.x SDXL
std::map<std::string, std::string> embbeding_map;
for (uint32_t i = 0; i < sd_ctx_params->embedding_count; i++) {
embbeding_map.emplace(SAFE_STR(sd_ctx_params->embeddings[i].name), SAFE_STR(sd_ctx_params->embeddings[i].path));
}
result.conditioner = std::make_shared<FrozenCLIPEmbedderWithCustomWords>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
embbeding_map,
version,
weight_manager,
tokenizers);
result.diffusion = std::make_shared<UNetModelRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
version,
weight_manager);
const auto* definition = find_model_definition(version);
if (!definition || !definition->build || !definition->build(ctx, tokenizers, result)) {
LOG_ERROR("failed to build model runners for %s", sd_version_name(version));
return false;
}
if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0 && result.clip_vision == nullptr) {
@@ -548,7 +121,7 @@ namespace sd::model_builders {
}
auto create_tae = [&](bool decode_only) -> std::shared_ptr<VAE> {
if ((sd_version_uses_wan_vae(version) && version != VERSION_QWEN_IMAGE_2_1) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version) || sd_version_is_minimax_h3(version)) {
if (find_model_traits(version)->video_tae) {
return std::make_shared<TinyVideoAutoEncoder>(ctx.backends.runtime_backend(SDBackendModule::VAE),
tensor_storage_map,
"decoder",
@@ -573,36 +146,36 @@ namespace sd::model_builders {
vae_format = SD_VAE_FORMAT_AUTO;
}
SDVersion vae_version = version;
if (sd_version_is_pid(version) && vae_format != SD_VAE_FORMAT_AUTO) {
if (find_model_traits(version)->vae_format_override && vae_format != SD_VAE_FORMAT_AUTO) {
vae_version = sd_vae_format_to_version(vae_format, vae_version);
}
auto create_vae = [&]() -> std::shared_ptr<VAE> {
if (sd_version_is_ltxav(version)) {
if (find_model_traits(vae_version)->vae_kind == ModelVAEKind::LTXAV) {
return std::make_shared<LTXVideoVAE>(ctx.backends.runtime_backend(SDBackendModule::VAE),
tensor_storage_map,
"first_stage_model",
false,
version,
weight_manager);
} else if (sd_version_is_minimax_h3(version)) {
} else if (find_model_traits(vae_version)->vae_kind == ModelVAEKind::MINIMAX_H3) {
return std::make_shared<MiniMaxH3VAE::MiniMaxH3VideoVAERunner>(ctx.backends.runtime_backend(SDBackendModule::VAE),
tensor_storage_map,
"first_stage_model",
weight_manager);
} else if (sd_version_is_mage_flow(vae_version)) {
} else if (find_model_traits(vae_version)->vae_kind == ModelVAEKind::MAGE) {
return std::make_shared<MageVAE::MageVAERunner>(ctx.backends.runtime_backend(SDBackendModule::VAE),
tensor_storage_map,
"first_stage_model",
weight_manager);
} else if (sd_version_uses_hunyuan_video_vae(vae_version)) {
} else if (find_model_traits(vae_version)->vae_kind == ModelVAEKind::HUNYUAN) {
return std::make_shared<Hunyuan::HunyuanVideoVAERunner>(ctx.backends.runtime_backend(SDBackendModule::VAE),
tensor_storage_map,
"first_stage_model",
false,
vae_version,
weight_manager);
} else if (sd_version_uses_wan_vae(vae_version)) {
} else if (find_model_traits(vae_version)->vae_kind == ModelVAEKind::WAN) {
return std::make_shared<WAN::WanVAERunner>(ctx.backends.runtime_backend(SDBackendModule::VAE),
tensor_storage_map,
"first_stage_model",
@@ -610,49 +183,22 @@ namespace sd::model_builders {
vae_version,
weight_manager);
} else {
auto model = std::make_shared<AutoEncoderKL>(ctx.backends.runtime_backend(SDBackendModule::VAE),
auto model = std::make_shared<AutoEncoderKL>(ctx.backends.runtime_backend(SDBackendModule::VAE),
tensor_storage_map,
"first_stage_model",
false,
false,
vae_version,
weight_manager);
if (sd_version_is_pixart(version)) {
// Alpha-512 and Sigma share tensor layouts; Alpha-512 needs an explicit scale override.
if (tensor_storage_map.count("model.diffusion_model.csize_embedder.mlp.0.weight") != 0) {
model->scale_factor = 0.18215f;
}
for (const auto& [key, value] : parse_key_value_args(sd_ctx_params->model_args, "model arg")) {
if (key == "pixart_vae_scale_factor") {
float parsed = 0.f;
if (parse_strict_float(value, parsed) && std::isfinite(parsed) && parsed > 0.f) {
model->scale_factor = parsed;
} else {
LOG_WARN("ignoring invalid PixArt model arg '%s=%s'", key.c_str(), value.c_str());
}
}
}
LOG_VERBOSE("pixart: VAE scale factor = %.5f", model->scale_factor);
}
if (sd_version_is_sdxl(version) &&
(strlen(SAFE_STR(sd_ctx_params->vae_path)) == 0 || sd_ctx_params->force_sdxl_vae_conv_scale || options.external_vae_is_invalid)) {
float vae_conv_2d_scale = 1.f / 32.f;
LOG_WARN(
"No valid VAE specified with --vae or --force-sdxl-vae-conv-scale flag set, "
"using Conv2D scale %.3f",
vae_conv_2d_scale);
model->set_conv2d_scale(vae_conv_2d_scale);
const auto* definition = find_model_definition(version);
if (definition && definition->configure_vae) {
definition->configure_vae(ctx, options, *model);
}
return model;
}
};
if (version == VERSION_CHROMA_RADIANCE ||
version == VERSION_HIDREAM_O1 ||
sd_version_is_minit2i(version) ||
sd_version_is_sensenova_u1(version) ||
sd_version_is_z_image_l2p(version) ||
version == VERSION_IRIS) {
if (find_model_traits(version)->fake_vae) {
LOG_INFO("using FakeVAE");
result.vae = std::make_shared<FakeVAE>(version,
ctx.backends.runtime_backend(SDBackendModule::VAE),
@@ -670,7 +216,7 @@ namespace sd::model_builders {
}
if (options.use_audio_vae) {
if (sd_version_is_minimax_h3(version)) {
if (find_model_traits(version)->vae_kind == ModelVAEKind::MINIMAX_H3) {
result.audio = std::make_shared<MiniMaxH3::AudioVAERunner>(ctx.backends.runtime_backend(SDBackendModule::VAE),
tensor_storage_map,
"",
src/pipeline/model_builders.h
+3
View File
@@ -4,6 +4,7 @@
#include <memory>
#include <vector>
#include "core/ggml_extend_backend.h"
#include "model.h"
#include "stable-diffusion.h"
@@ -56,6 +57,8 @@ namespace sd::model_builders {
std::shared_ptr<AudioVAERunner> audio;
};
bool ensure_backend_pair(SDBackendManager& backends, SDBackendModule module);
bool build_core_runners(const Context& ctx, CoreRunners& runners);
bool build_vae_runners(const Context& ctx, const VAEOptions& options, VAERunners& runners);
bool build_control_net_runner(const Context& ctx, std::shared_ptr<ControlNet>& runner);
src/pipeline/request.cpp
+7 -20
View File
@@ -6,6 +6,7 @@
#include <ctime>
#include "diffusion_engine.h"
#include "model_definition.h"
#include "runtime/denoiser.hpp"
namespace sd::pipeline {
@@ -43,15 +44,8 @@ namespace sd::pipeline {
}
enum sample_method_t default_sample_method(const StableDiffusionGGML* sd) {
if (sd != nullptr) {
if (sd_version_is_pid(sd->version)) {
return LCM_SAMPLE_METHOD;
}
if (sd_version_is_dit(sd->version)) {
return EULER_SAMPLE_METHOD;
}
}
return EULER_A_SAMPLE_METHOD;
const auto* definition = sd ? find_model_definition(sd->version) : nullptr;
return definition ? definition->sampling.default_sample_method : EULER_A_SAMPLE_METHOD;
}
enum scheduler_t default_scheduler(const StableDiffusionGGML* sd, enum sample_method_t sample_method) {
@@ -65,16 +59,9 @@ namespace sd::pipeline {
return LCM_SCHEDULER;
} else if (sample_method == DDIM_TRAILING_SAMPLE_METHOD) {
return SIMPLE_SCHEDULER;
} else if (sd != nullptr && (sd_version_is_flux(sd->version) || sd->version == VERSION_QWEN_IMAGE_2_1 || sd->version == VERSION_FIBO2)) {
return FLUX_SCHEDULER;
} else if (sd != nullptr && sd_version_is_flux2(sd->version)) {
return FLUX2_SCHEDULER;
} else if (sd != nullptr && sd_version_is_ltxav(sd->version)) {
return LTX2_SCHEDULER;
} else if (sd != nullptr && sd_version_is_ideogram4(sd->version)) {
return LOGIT_NORMAL_SCHEDULER;
} else if (sd != nullptr && sd_version_is_llada_image(sd->version)) {
return LLADA_IMAGE_SCHEDULER;
}
if (const auto* definition = sd ? find_model_definition(sd->version) : nullptr) {
return definition->sampling.default_scheduler;
}
return DISCRETE_SCHEDULER;
}
@@ -184,7 +171,7 @@ namespace sd::pipeline {
LOG_WARN("align video frames from %d to %d for %s",
requested_frames,
frames,
model_version_to_str[sd->version]);
sd_version_name(sd->version));
}
}