mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-08-10 14:16:50 +00:00
Compare commits
No commits in common. "8caa3f908ae6d4a4bef531e73b9a969f266a3d1f" and "f440ad9c29dd8bc34e5d1f4b863832b96d6ea05f" have entirely different histories.
8caa3f908a
...
f440ad9c29
@ -15,7 +15,6 @@ API and command-line option may change frequently.***
|
|||||||
|
|
||||||
## 🔥Important News
|
## 🔥Important News
|
||||||
|
|
||||||
* **2026/06/25** 🚀 stable-diffusion.cpp now supports **Krea2**
|
|
||||||
* **2026/06/04** 🚀 stable-diffusion.cpp now supports **Ideogram4**
|
* **2026/06/04** 🚀 stable-diffusion.cpp now supports **Ideogram4**
|
||||||
* **2026/05/31** 🚀 stable-diffusion.cpp now supports **PiD**
|
* **2026/05/31** 🚀 stable-diffusion.cpp now supports **PiD**
|
||||||
* **2026/05/27** 🚀 stable-diffusion.cpp now supports **Lens**
|
* **2026/05/27** 🚀 stable-diffusion.cpp now supports **Lens**
|
||||||
@ -52,7 +51,6 @@ API and command-line option may change frequently.***
|
|||||||
- [Anima](./docs/anima.md)
|
- [Anima](./docs/anima.md)
|
||||||
- [ERNIE-Image](./docs/ernie_image.md)
|
- [ERNIE-Image](./docs/ernie_image.md)
|
||||||
- [Boogu Image](./docs/boogu_image.md)
|
- [Boogu Image](./docs/boogu_image.md)
|
||||||
- [Krea2](./docs/krea2.md)
|
|
||||||
- [HiDream-O1-Image](./docs/hidream_o1_image.md)
|
- [HiDream-O1-Image](./docs/hidream_o1_image.md)
|
||||||
- [Ideogram4](./docs/ideogram4.md)
|
- [Ideogram4](./docs/ideogram4.md)
|
||||||
- Image Edit Models
|
- Image Edit Models
|
||||||
|
|||||||
Binary file not shown.
|
Before Width: | Height: | Size: 289 KiB |
@ -1,27 +0,0 @@
|
|||||||
# How to Use
|
|
||||||
|
|
||||||
Krea2 uses a Krea2 diffusion transformer, the Wan2.1 VAE, and Qwen3-VL 4B as the LLM text encoder.
|
|
||||||
|
|
||||||
## Download weights
|
|
||||||
|
|
||||||
- Download Krea2 Raw
|
|
||||||
- safetensors: https://huggingface.co/krea/Krea-2-Raw/tree/main
|
|
||||||
- gguf: https://huggingface.co/realrebelai/KREA-2_GGUFs/tree/main/BASE
|
|
||||||
- Download Krea2 Turbo
|
|
||||||
- safetensors: https://huggingface.co/krea/Krea-2-Turbo/tree/main
|
|
||||||
- gguf: https://huggingface.co/realrebelai/KREA-2_GGUFs/tree/main/TURBO
|
|
||||||
- Download vae
|
|
||||||
- safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/vae/wan_2.1_vae.safetensors
|
|
||||||
- Download Qwen3-VL 4B
|
|
||||||
- safetensors: https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders
|
|
||||||
- gguf: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
### Krea2
|
|
||||||
|
|
||||||
```
|
|
||||||
.\bin\Release\sd-cli.exe --diffusion-model ..\..\ComfyUI\models\diffusion_models\Krea-2-Raw-Q8_0.gguf --llm ..\..\ComfyUI\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'" --diffusion-fa -v --offload-to-cpu
|
|
||||||
```
|
|
||||||
|
|
||||||
<img width="256" alt="Krea2 Raw example" src="../assets/krea2/example.png" />
|
|
||||||
@ -960,7 +960,7 @@ ArgOptions SDGenerationParams::get_options() {
|
|||||||
&hires_upscaler},
|
&hires_upscaler},
|
||||||
{"",
|
{"",
|
||||||
"--extra-sample-args",
|
"--extra-sample-args",
|
||||||
"extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma;; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware",
|
"extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma;",
|
||||||
(int)',',
|
(int)',',
|
||||||
&extra_sample_args},
|
&extra_sample_args},
|
||||||
{"",
|
{"",
|
||||||
@ -1475,7 +1475,7 @@ ArgOptions SDGenerationParams::get_options() {
|
|||||||
on_high_noise_sample_method_arg},
|
on_high_noise_sample_method_arg},
|
||||||
{"",
|
{"",
|
||||||
"--scheduler",
|
"--scheduler",
|
||||||
"denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent, ltx2, logit_normal], default: model-specific",
|
"denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent, ltx2], default: model-specific",
|
||||||
on_scheduler_arg},
|
on_scheduler_arg},
|
||||||
{"",
|
{"",
|
||||||
"--sigmas",
|
"--sigmas",
|
||||||
|
|||||||
@ -70,7 +70,6 @@ enum scheduler_t {
|
|||||||
LCM_SCHEDULER,
|
LCM_SCHEDULER,
|
||||||
BONG_TANGENT_SCHEDULER,
|
BONG_TANGENT_SCHEDULER,
|
||||||
LTX2_SCHEDULER,
|
LTX2_SCHEDULER,
|
||||||
LOGIT_NORMAL_SCHEDULER,
|
|
||||||
SCHEDULER_COUNT
|
SCHEDULER_COUNT
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@ -1518,7 +1518,7 @@ struct LLMEmbedder : public Conditioner {
|
|||||||
arch = LLM::LLMArch::GPT_OSS_20B;
|
arch = LLM::LLMArch::GPT_OSS_20B;
|
||||||
} else if (sd_version_is_pid(version)) {
|
} else if (sd_version_is_pid(version)) {
|
||||||
arch = LLM::LLMArch::GEMMA2_2B;
|
arch = LLM::LLMArch::GEMMA2_2B;
|
||||||
} else if (sd_version_is_ideogram4(version) || sd_version_is_boogu_image(version) || sd_version_is_krea2(version)) {
|
} else if (sd_version_is_ideogram4(version) || sd_version_is_boogu_image(version)) {
|
||||||
arch = LLM::LLMArch::QWEN3_VL;
|
arch = LLM::LLMArch::QWEN3_VL;
|
||||||
} else if (sd_version_is_z_image(version) || version == VERSION_OVIS_IMAGE || version == VERSION_FLUX2_KLEIN) {
|
} else if (sd_version_is_z_image(version) || version == VERSION_OVIS_IMAGE || version == VERSION_FLUX2_KLEIN) {
|
||||||
arch = LLM::LLMArch::QWEN3;
|
arch = LLM::LLMArch::QWEN3;
|
||||||
@ -1837,17 +1837,6 @@ struct LLMEmbedder : public Conditioner {
|
|||||||
prompt_attn_range.second = static_cast<int>(prompt.size());
|
prompt_attn_range.second = static_cast<int>(prompt.size());
|
||||||
prompt += "<|im_end|>\n";
|
prompt += "<|im_end|>\n";
|
||||||
}
|
}
|
||||||
} else if (sd_version_is_krea2(version)) {
|
|
||||||
prompt_template_encode_start_idx = 34;
|
|
||||||
out_layers = {2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 32, 35};
|
|
||||||
|
|
||||||
prompt = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n";
|
|
||||||
|
|
||||||
prompt_attn_range.first = static_cast<int>(prompt.size());
|
|
||||||
prompt += conditioner_params.text;
|
|
||||||
prompt_attn_range.second = static_cast<int>(prompt.size());
|
|
||||||
|
|
||||||
prompt += "<|im_end|>\n<|im_start|>assistant\n";
|
|
||||||
} else if (sd_version_is_longcat(version)) {
|
} else if (sd_version_is_longcat(version)) {
|
||||||
spell_quotes = true;
|
spell_quotes = true;
|
||||||
|
|
||||||
|
|||||||
@ -1382,16 +1382,7 @@ __STATIC_INLINE__ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
|
|||||||
if (!ggml_backend_supports_op(backend, kqv)) {
|
if (!ggml_backend_supports_op(backend, kqv)) {
|
||||||
kqv = nullptr;
|
kqv = nullptr;
|
||||||
} else {
|
} else {
|
||||||
kqv = ggml_view_4d(ctx,
|
kqv = ggml_view_3d(ctx, kqv, d_head, n_head, L_q, kqv->nb[1], kqv->nb[2], 0);
|
||||||
kqv,
|
|
||||||
d_head,
|
|
||||||
n_head,
|
|
||||||
L_q,
|
|
||||||
N,
|
|
||||||
kqv->nb[1],
|
|
||||||
kqv->nb[2],
|
|
||||||
kqv->nb[1] * n_head,
|
|
||||||
0);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
11
src/model.h
11
src/model.h
@ -49,7 +49,6 @@ enum SDVersion {
|
|||||||
VERSION_LONGCAT,
|
VERSION_LONGCAT,
|
||||||
VERSION_PID,
|
VERSION_PID,
|
||||||
VERSION_IDEOGRAM4,
|
VERSION_IDEOGRAM4,
|
||||||
VERSION_KREA2,
|
|
||||||
VERSION_ESRGAN,
|
VERSION_ESRGAN,
|
||||||
VERSION_COUNT,
|
VERSION_COUNT,
|
||||||
};
|
};
|
||||||
@ -187,13 +186,6 @@ static inline bool sd_version_is_ideogram4(SDVersion version) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline bool sd_version_is_krea2(SDVersion version) {
|
|
||||||
if (version == VERSION_KREA2) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline bool sd_version_uses_flux_vae(SDVersion version) {
|
static inline bool sd_version_uses_flux_vae(SDVersion version) {
|
||||||
if (sd_version_is_flux(version) || sd_version_is_z_image(version) || sd_version_is_boogu_image(version) || sd_version_is_longcat(version)) {
|
if (sd_version_is_flux(version) || sd_version_is_z_image(version) || sd_version_is_boogu_image(version) || sd_version_is_longcat(version)) {
|
||||||
return true;
|
return true;
|
||||||
@ -234,8 +226,7 @@ static inline bool sd_version_is_dit(SDVersion version) {
|
|||||||
sd_version_is_lens(version) ||
|
sd_version_is_lens(version) ||
|
||||||
sd_version_is_longcat(version) ||
|
sd_version_is_longcat(version) ||
|
||||||
sd_version_is_pid(version) ||
|
sd_version_is_pid(version) ||
|
||||||
sd_version_is_ideogram4(version) ||
|
sd_version_is_ideogram4(version)) {
|
||||||
sd_version_is_krea2(version)) {
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@ -1,683 +0,0 @@
|
|||||||
#ifndef __SD_MODEL_DIFFUSION_KREA2_HPP__
|
|
||||||
#define __SD_MODEL_DIFFUSION_KREA2_HPP__
|
|
||||||
|
|
||||||
#include <inttypes.h>
|
|
||||||
#include <algorithm>
|
|
||||||
#include <cmath>
|
|
||||||
#include <cstdint>
|
|
||||||
#include <cstdlib>
|
|
||||||
#include <memory>
|
|
||||||
#include <set>
|
|
||||||
#include <string>
|
|
||||||
#include <utility>
|
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
#include "core/ggml_extend.hpp"
|
|
||||||
#include "core/ggml_graph_cut.h"
|
|
||||||
#include "model/common/rope.hpp"
|
|
||||||
#include "model/diffusion/dit.hpp"
|
|
||||||
#include "model/diffusion/flux.hpp"
|
|
||||||
#include "model/diffusion/model.hpp"
|
|
||||||
#include "model_loader.h"
|
|
||||||
|
|
||||||
namespace Krea2 {
|
|
||||||
constexpr int KREA2_GRAPH_SIZE = 65536;
|
|
||||||
|
|
||||||
struct Krea2Config {
|
|
||||||
int patch_size = 2;
|
|
||||||
int64_t in_channels = 16;
|
|
||||||
int64_t out_channels = 16;
|
|
||||||
int64_t features = 6144;
|
|
||||||
int64_t timestep_dim = 256;
|
|
||||||
int64_t text_dim = 2560;
|
|
||||||
int64_t text_layers = 12;
|
|
||||||
int64_t layers = 28;
|
|
||||||
int64_t heads = 48;
|
|
||||||
int64_t kv_heads = 12;
|
|
||||||
int64_t text_heads = 20;
|
|
||||||
int64_t text_kv_heads = 20;
|
|
||||||
int64_t mlp_multiplier = 4;
|
|
||||||
float theta = 1000.f;
|
|
||||||
float norm_eps = 1e-5f;
|
|
||||||
std::vector<int> axes_dim = {32, 48, 48};
|
|
||||||
int axes_dim_sum = 128;
|
|
||||||
|
|
||||||
int64_t head_dim() const {
|
|
||||||
return features / heads;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int64_t count_blocks(const String2TensorStorage& tensor_storage_map,
|
|
||||||
const std::string& prefix,
|
|
||||||
const std::string& block_prefix) {
|
|
||||||
int64_t count = 0;
|
|
||||||
std::string full_prefix = prefix.empty() ? block_prefix : prefix + "." + block_prefix;
|
|
||||||
for (const auto& [name, _] : tensor_storage_map) {
|
|
||||||
if (!starts_with(name, full_prefix)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
std::string tail = name.substr(full_prefix.size());
|
|
||||||
size_t dot = tail.find('.');
|
|
||||||
if (dot == std::string::npos) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
int block_index = std::atoi(tail.substr(0, dot).c_str());
|
|
||||||
count = std::max<int64_t>(count, block_index + 1);
|
|
||||||
}
|
|
||||||
return count;
|
|
||||||
}
|
|
||||||
|
|
||||||
void update_axes_dim() {
|
|
||||||
int64_t dim_head = head_dim();
|
|
||||||
int64_t unit = dim_head / 16;
|
|
||||||
axes_dim = {
|
|
||||||
static_cast<int>(dim_head - 12 * unit),
|
|
||||||
static_cast<int>(6 * unit),
|
|
||||||
static_cast<int>(6 * unit),
|
|
||||||
};
|
|
||||||
axes_dim_sum = axes_dim[0] + axes_dim[1] + axes_dim[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
static Krea2Config detect_from_weights(const String2TensorStorage& tensor_storage_map,
|
|
||||||
const std::string& prefix) {
|
|
||||||
Krea2Config config;
|
|
||||||
int64_t detected_head_dim = 0;
|
|
||||||
int64_t detected_text_head_dim = 0;
|
|
||||||
|
|
||||||
for (const auto& [name, tensor_storage] : tensor_storage_map) {
|
|
||||||
if (!starts_with(name, prefix)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (ends_with(name, "first.weight") && tensor_storage.n_dims == 2) {
|
|
||||||
config.in_channels = tensor_storage.ne[0] / (config.patch_size * config.patch_size);
|
|
||||||
config.out_channels = config.in_channels;
|
|
||||||
config.features = tensor_storage.ne[1];
|
|
||||||
} else if (ends_with(name, "blocks.0.attn.qknorm.qnorm.scale") && tensor_storage.n_dims == 1) {
|
|
||||||
detected_head_dim = tensor_storage.ne[0];
|
|
||||||
} else if (ends_with(name, "blocks.0.attn.wq.weight") && tensor_storage.n_dims == 2) {
|
|
||||||
if (detected_head_dim > 0) {
|
|
||||||
config.heads = tensor_storage.ne[1] / detected_head_dim;
|
|
||||||
}
|
|
||||||
} else if (ends_with(name, "blocks.0.attn.wk.weight") && tensor_storage.n_dims == 2) {
|
|
||||||
if (detected_head_dim > 0) {
|
|
||||||
config.kv_heads = tensor_storage.ne[1] / detected_head_dim;
|
|
||||||
}
|
|
||||||
} else if (ends_with(name, "txtfusion.projector.weight") && tensor_storage.n_dims == 2) {
|
|
||||||
config.text_layers = tensor_storage.ne[0];
|
|
||||||
} else if (ends_with(name, "txtfusion.layerwise_blocks.0.prenorm.scale") && tensor_storage.n_dims == 1) {
|
|
||||||
config.text_dim = tensor_storage.ne[0];
|
|
||||||
} else if (ends_with(name, "txtfusion.layerwise_blocks.0.attn.qknorm.qnorm.scale") && tensor_storage.n_dims == 1) {
|
|
||||||
detected_text_head_dim = tensor_storage.ne[0];
|
|
||||||
} else if (ends_with(name, "txtfusion.layerwise_blocks.0.attn.wq.weight") && tensor_storage.n_dims == 2) {
|
|
||||||
if (detected_text_head_dim > 0) {
|
|
||||||
config.text_heads = tensor_storage.ne[1] / detected_text_head_dim;
|
|
||||||
}
|
|
||||||
} else if (ends_with(name, "txtfusion.layerwise_blocks.0.attn.wk.weight") && tensor_storage.n_dims == 2) {
|
|
||||||
if (detected_text_head_dim > 0) {
|
|
||||||
config.text_kv_heads = tensor_storage.ne[1] / detected_text_head_dim;
|
|
||||||
}
|
|
||||||
} else if (ends_with(name, "last.linear.weight") && tensor_storage.n_dims == 2) {
|
|
||||||
config.out_channels = tensor_storage.ne[1] / (config.patch_size * config.patch_size);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
config.layers = std::max<int64_t>(1, count_blocks(tensor_storage_map, prefix, "blocks."));
|
|
||||||
if (detected_head_dim > 0 && config.features > 0) {
|
|
||||||
config.heads = config.features / detected_head_dim;
|
|
||||||
}
|
|
||||||
if (detected_head_dim > 0) {
|
|
||||||
std::string wk_name = prefix.empty() ? "blocks.0.attn.wk.weight" : prefix + ".blocks.0.attn.wk.weight";
|
|
||||||
auto it = tensor_storage_map.find(wk_name);
|
|
||||||
if (it != tensor_storage_map.end() && it->second.n_dims == 2) {
|
|
||||||
config.kv_heads = it->second.ne[1] / detected_head_dim;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (detected_text_head_dim > 0 && config.text_dim > 0) {
|
|
||||||
config.text_heads = config.text_dim / detected_text_head_dim;
|
|
||||||
}
|
|
||||||
if (detected_text_head_dim > 0) {
|
|
||||||
std::string wk_name = prefix.empty() ? "txtfusion.layerwise_blocks.0.attn.wk.weight" : prefix + ".txtfusion.layerwise_blocks.0.attn.wk.weight";
|
|
||||||
auto it = tensor_storage_map.find(wk_name);
|
|
||||||
if (it != tensor_storage_map.end() && it->second.n_dims == 2) {
|
|
||||||
config.text_kv_heads = it->second.ne[1] / detected_text_head_dim;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
config.update_axes_dim();
|
|
||||||
|
|
||||||
LOG_DEBUG("krea2: layers=%" PRId64 ", features=%" PRId64 ", heads=%" PRId64 ", kv_heads=%" PRId64 ", text_dim=%" PRId64 ", text_layers=%" PRId64 ", text_heads=%" PRId64 ", text_kv_heads=%" PRId64 ", channels=%" PRId64,
|
|
||||||
config.layers,
|
|
||||||
config.features,
|
|
||||||
config.heads,
|
|
||||||
config.kv_heads,
|
|
||||||
config.text_dim,
|
|
||||||
config.text_layers,
|
|
||||||
config.text_heads,
|
|
||||||
config.text_kv_heads,
|
|
||||||
config.in_channels);
|
|
||||||
return config;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
__STATIC_INLINE__ int64_t ceil_to_multiple(int64_t value, int64_t multiple) {
|
|
||||||
return ((value + multiple - 1) / multiple) * multiple;
|
|
||||||
}
|
|
||||||
|
|
||||||
class KreaRMSNorm : public UnaryBlock {
|
|
||||||
protected:
|
|
||||||
int64_t hidden_size;
|
|
||||||
float eps;
|
|
||||||
std::string prefix;
|
|
||||||
|
|
||||||
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
|
||||||
GGML_UNUSED(tensor_storage_map);
|
|
||||||
this->prefix = prefix;
|
|
||||||
params["scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size);
|
|
||||||
}
|
|
||||||
|
|
||||||
public:
|
|
||||||
KreaRMSNorm(int64_t hidden_size, float eps = 1e-5f)
|
|
||||||
: hidden_size(hidden_size),
|
|
||||||
eps(eps) {}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
|
|
||||||
ggml_tensor* scale = params["scale"];
|
|
||||||
scale = ggml_add(ctx->ggml_ctx, scale, ggml_ext_ones(ctx->ggml_ctx, scale->ne[0], 1, 1, 1));
|
|
||||||
x = ggml_rms_norm(ctx->ggml_ctx, x, eps);
|
|
||||||
x = ggml_mul_inplace(ctx->ggml_ctx, x, scale);
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class KreaSwiGLU : public UnaryBlock {
|
|
||||||
public:
|
|
||||||
KreaSwiGLU(int64_t features, int64_t multiplier) {
|
|
||||||
int64_t mlp_dim = ceil_to_multiple(((2 * features) / 3) * multiplier, 128);
|
|
||||||
blocks["gate"] = std::make_shared<Linear>(features, mlp_dim, false);
|
|
||||||
blocks["up"] = std::make_shared<Linear>(features, mlp_dim, false);
|
|
||||||
blocks["down"] = std::make_shared<Linear>(mlp_dim, features, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
|
|
||||||
auto gate = std::dynamic_pointer_cast<Linear>(blocks["gate"]);
|
|
||||||
auto up = std::dynamic_pointer_cast<Linear>(blocks["up"]);
|
|
||||||
auto down = std::dynamic_pointer_cast<Linear>(blocks["down"]);
|
|
||||||
|
|
||||||
auto gated = ggml_silu(ctx->ggml_ctx, gate->forward(ctx, x));
|
|
||||||
auto up_x = up->forward(ctx, x);
|
|
||||||
x = ggml_mul(ctx->ggml_ctx, gated, up_x);
|
|
||||||
return down->forward(ctx, x);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class KreaAttention : public GGMLBlock {
|
|
||||||
protected:
|
|
||||||
int64_t features;
|
|
||||||
int64_t heads;
|
|
||||||
int64_t kv_heads;
|
|
||||||
int64_t head_dim_;
|
|
||||||
|
|
||||||
ggml_tensor* attention_no_rope(GGMLRunnerContext* ctx,
|
|
||||||
ggml_tensor* q,
|
|
||||||
ggml_tensor* k,
|
|
||||||
ggml_tensor* v,
|
|
||||||
ggml_tensor* mask) {
|
|
||||||
int64_t Lq = q->ne[2];
|
|
||||||
int64_t Lk = k->ne[2];
|
|
||||||
int64_t N = q->ne[3];
|
|
||||||
q = ggml_reshape_3d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, q), head_dim_ * heads, Lq, N);
|
|
||||||
k = ggml_reshape_3d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, k), head_dim_ * kv_heads, Lk, N);
|
|
||||||
v = ggml_reshape_3d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, v), head_dim_ * kv_heads, Lk, N);
|
|
||||||
return ggml_ext_attention_ext(ctx->ggml_ctx,
|
|
||||||
ctx->backend,
|
|
||||||
q,
|
|
||||||
k,
|
|
||||||
v,
|
|
||||||
heads,
|
|
||||||
mask,
|
|
||||||
false,
|
|
||||||
ctx->flash_attn_enabled);
|
|
||||||
}
|
|
||||||
|
|
||||||
public:
|
|
||||||
KreaAttention(int64_t features,
|
|
||||||
int64_t heads,
|
|
||||||
int64_t kv_heads,
|
|
||||||
float eps = 1e-5f)
|
|
||||||
: features(features),
|
|
||||||
heads(heads),
|
|
||||||
kv_heads(kv_heads),
|
|
||||||
head_dim_(features / heads) {
|
|
||||||
blocks["wq"] = std::make_shared<Linear>(features, heads * head_dim_, false);
|
|
||||||
blocks["wk"] = std::make_shared<Linear>(features, kv_heads * head_dim_, false);
|
|
||||||
blocks["wv"] = std::make_shared<Linear>(features, kv_heads * head_dim_, false);
|
|
||||||
blocks["gate"] = std::make_shared<Linear>(features, features, false);
|
|
||||||
blocks["qknorm.qnorm"] = std::make_shared<KreaRMSNorm>(head_dim_, eps);
|
|
||||||
blocks["qknorm.knorm"] = std::make_shared<KreaRMSNorm>(head_dim_, eps);
|
|
||||||
blocks["wo"] = std::make_shared<Linear>(features, features, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx,
|
|
||||||
ggml_tensor* x,
|
|
||||||
ggml_tensor* pe = nullptr,
|
|
||||||
ggml_tensor* mask = nullptr) {
|
|
||||||
auto wq = std::dynamic_pointer_cast<Linear>(blocks["wq"]);
|
|
||||||
auto wk = std::dynamic_pointer_cast<Linear>(blocks["wk"]);
|
|
||||||
auto wv = std::dynamic_pointer_cast<Linear>(blocks["wv"]);
|
|
||||||
auto gate = std::dynamic_pointer_cast<Linear>(blocks["gate"]);
|
|
||||||
auto qnorm = std::dynamic_pointer_cast<KreaRMSNorm>(blocks["qknorm.qnorm"]);
|
|
||||||
auto knorm = std::dynamic_pointer_cast<KreaRMSNorm>(blocks["qknorm.knorm"]);
|
|
||||||
auto wo = std::dynamic_pointer_cast<Linear>(blocks["wo"]);
|
|
||||||
|
|
||||||
if (sd_backend_is(ctx->backend, "Vulkan")) {
|
|
||||||
wo->set_force_prec_f32(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
int64_t L = x->ne[1];
|
|
||||||
int64_t N = x->ne[2];
|
|
||||||
|
|
||||||
auto q = wq->forward(ctx, x);
|
|
||||||
q = ggml_reshape_4d(ctx->ggml_ctx, q, head_dim_, heads, L, N);
|
|
||||||
auto k = wk->forward(ctx, x);
|
|
||||||
k = ggml_reshape_4d(ctx->ggml_ctx, k, head_dim_, kv_heads, L, N);
|
|
||||||
auto v = wv->forward(ctx, x);
|
|
||||||
v = ggml_reshape_4d(ctx->ggml_ctx, v, head_dim_, kv_heads, L, N);
|
|
||||||
|
|
||||||
q = qnorm->forward(ctx, q);
|
|
||||||
k = knorm->forward(ctx, k);
|
|
||||||
|
|
||||||
auto out = pe != nullptr ? Rope::attention(ctx, q, k, v, pe, mask)
|
|
||||||
: attention_no_rope(ctx, q, k, v, mask);
|
|
||||||
out = ggml_mul(ctx->ggml_ctx, out, ggml_sigmoid(ctx->ggml_ctx, gate->forward(ctx, x)));
|
|
||||||
out = wo->forward(ctx, out);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class KreaDoubleSharedModulation : public GGMLBlock {
|
|
||||||
protected:
|
|
||||||
int64_t dim;
|
|
||||||
|
|
||||||
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
|
||||||
GGML_UNUSED(tensor_storage_map);
|
|
||||||
GGML_UNUSED(prefix);
|
|
||||||
params["lin"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim * 6);
|
|
||||||
}
|
|
||||||
|
|
||||||
public:
|
|
||||||
KreaDoubleSharedModulation(int64_t dim)
|
|
||||||
: dim(dim) {}
|
|
||||||
|
|
||||||
std::vector<ggml_tensor*> forward(GGMLRunnerContext* ctx, ggml_tensor* vec) {
|
|
||||||
auto lin = ggml_repeat(ctx->ggml_ctx, params["lin"], vec);
|
|
||||||
auto out = ggml_add(ctx->ggml_ctx, vec, lin);
|
|
||||||
return ggml_ext_chunk(ctx->ggml_ctx, out, 6, 0);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class KreaFinalModulation : public GGMLBlock {
|
|
||||||
protected:
|
|
||||||
int64_t dim;
|
|
||||||
|
|
||||||
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
|
|
||||||
GGML_UNUSED(tensor_storage_map);
|
|
||||||
GGML_UNUSED(prefix);
|
|
||||||
params["lin"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, dim, 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
public:
|
|
||||||
KreaFinalModulation(int64_t dim)
|
|
||||||
: dim(dim) {}
|
|
||||||
|
|
||||||
std::vector<ggml_tensor*> forward(GGMLRunnerContext* ctx, ggml_tensor* vec) {
|
|
||||||
auto out = ggml_add(ctx->ggml_ctx, params["lin"], vec);
|
|
||||||
return ggml_ext_chunk(ctx->ggml_ctx, out, 2, 1);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class KreaTextFusionBlock : public UnaryBlock {
|
|
||||||
public:
|
|
||||||
KreaTextFusionBlock(int64_t dim,
|
|
||||||
int64_t heads,
|
|
||||||
int64_t kv_heads,
|
|
||||||
int64_t multiplier,
|
|
||||||
float eps) {
|
|
||||||
blocks["prenorm"] = std::make_shared<KreaRMSNorm>(dim, eps);
|
|
||||||
blocks["postnorm"] = std::make_shared<KreaRMSNorm>(dim, eps);
|
|
||||||
blocks["attn"] = std::make_shared<KreaAttention>(dim, heads, kv_heads, eps);
|
|
||||||
blocks["mlp"] = std::make_shared<KreaSwiGLU>(dim, multiplier);
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
|
|
||||||
auto prenorm = std::dynamic_pointer_cast<KreaRMSNorm>(blocks["prenorm"]);
|
|
||||||
auto postnorm = std::dynamic_pointer_cast<KreaRMSNorm>(blocks["postnorm"]);
|
|
||||||
auto attn = std::dynamic_pointer_cast<KreaAttention>(blocks["attn"]);
|
|
||||||
auto mlp = std::dynamic_pointer_cast<KreaSwiGLU>(blocks["mlp"]);
|
|
||||||
|
|
||||||
x = ggml_add(ctx->ggml_ctx, x, attn->forward(ctx, prenorm->forward(ctx, x)));
|
|
||||||
x = ggml_add(ctx->ggml_ctx, x, mlp->forward(ctx, postnorm->forward(ctx, x)));
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class KreaTextFusionTransformer : public UnaryBlock {
|
|
||||||
protected:
|
|
||||||
Krea2Config config;
|
|
||||||
|
|
||||||
public:
|
|
||||||
explicit KreaTextFusionTransformer(Krea2Config config)
|
|
||||||
: config(std::move(config)) {
|
|
||||||
for (int i = 0; i < 2; ++i) {
|
|
||||||
blocks["layerwise_blocks." + std::to_string(i)] = std::make_shared<KreaTextFusionBlock>(this->config.text_dim,
|
|
||||||
this->config.text_heads,
|
|
||||||
this->config.text_kv_heads,
|
|
||||||
this->config.mlp_multiplier,
|
|
||||||
this->config.norm_eps);
|
|
||||||
blocks["refiner_blocks." + std::to_string(i)] = std::make_shared<KreaTextFusionBlock>(this->config.text_dim,
|
|
||||||
this->config.text_heads,
|
|
||||||
this->config.text_kv_heads,
|
|
||||||
this->config.mlp_multiplier,
|
|
||||||
this->config.norm_eps);
|
|
||||||
}
|
|
||||||
blocks["projector"] = std::make_shared<Linear>(this->config.text_layers, 1, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* context) override {
|
|
||||||
int64_t text_tokens = context->ne[1];
|
|
||||||
int64_t batch = context->ne[2];
|
|
||||||
|
|
||||||
context = ggml_reshape_3d(ctx->ggml_ctx,
|
|
||||||
context,
|
|
||||||
config.text_dim,
|
|
||||||
config.text_layers,
|
|
||||||
text_tokens * batch);
|
|
||||||
|
|
||||||
for (int i = 0; i < 2; ++i) {
|
|
||||||
auto block = std::dynamic_pointer_cast<KreaTextFusionBlock>(blocks["layerwise_blocks." + std::to_string(i)]);
|
|
||||||
context = block->forward(ctx, context);
|
|
||||||
}
|
|
||||||
|
|
||||||
context = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, context, 1, 0, 2, 3));
|
|
||||||
auto projector = std::dynamic_pointer_cast<Linear>(blocks["projector"]);
|
|
||||||
context = projector->forward(ctx, context);
|
|
||||||
context = ggml_reshape_3d(ctx->ggml_ctx, context, config.text_dim, text_tokens, batch);
|
|
||||||
|
|
||||||
for (int i = 0; i < 2; ++i) {
|
|
||||||
auto block = std::dynamic_pointer_cast<KreaTextFusionBlock>(blocks["refiner_blocks." + std::to_string(i)]);
|
|
||||||
context = block->forward(ctx, context);
|
|
||||||
}
|
|
||||||
return context;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class KreaSingleStreamBlock : public UnaryBlock {
|
|
||||||
public:
|
|
||||||
explicit KreaSingleStreamBlock(Krea2Config config) {
|
|
||||||
blocks["mod"] = std::make_shared<KreaDoubleSharedModulation>(config.features);
|
|
||||||
blocks["prenorm"] = std::make_shared<KreaRMSNorm>(config.features, config.norm_eps);
|
|
||||||
blocks["postnorm"] = std::make_shared<KreaRMSNorm>(config.features, config.norm_eps);
|
|
||||||
blocks["attn"] = std::make_shared<KreaAttention>(config.features, config.heads, config.kv_heads, config.norm_eps);
|
|
||||||
blocks["mlp"] = std::make_shared<KreaSwiGLU>(config.features, config.mlp_multiplier);
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx,
|
|
||||||
ggml_tensor* x,
|
|
||||||
ggml_tensor* vec,
|
|
||||||
ggml_tensor* pe) {
|
|
||||||
auto mod = std::dynamic_pointer_cast<KreaDoubleSharedModulation>(blocks["mod"]);
|
|
||||||
auto prenorm = std::dynamic_pointer_cast<KreaRMSNorm>(blocks["prenorm"]);
|
|
||||||
auto postnorm = std::dynamic_pointer_cast<KreaRMSNorm>(blocks["postnorm"]);
|
|
||||||
auto attn = std::dynamic_pointer_cast<KreaAttention>(blocks["attn"]);
|
|
||||||
auto mlp = std::dynamic_pointer_cast<KreaSwiGLU>(blocks["mlp"]);
|
|
||||||
|
|
||||||
auto mods = mod->forward(ctx, vec);
|
|
||||||
auto attn_input = Flux::modulate(ctx->ggml_ctx,
|
|
||||||
prenorm->forward(ctx, x),
|
|
||||||
mods[1],
|
|
||||||
mods[0],
|
|
||||||
true);
|
|
||||||
auto attn_out = attn->forward(ctx, attn_input, pe);
|
|
||||||
x = ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, attn_out, mods[2]));
|
|
||||||
|
|
||||||
auto mlp_input = Flux::modulate(ctx->ggml_ctx,
|
|
||||||
postnorm->forward(ctx, x),
|
|
||||||
mods[4],
|
|
||||||
mods[3],
|
|
||||||
true);
|
|
||||||
auto mlp_out = mlp->forward(ctx, mlp_input);
|
|
||||||
x = ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, mlp_out, mods[5]));
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
|
|
||||||
GGML_UNUSED(ctx);
|
|
||||||
GGML_UNUSED(x);
|
|
||||||
GGML_ABORT("KreaSingleStreamBlock requires conditioning");
|
|
||||||
return nullptr;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class KreaTimeMLP : public UnaryBlock {
|
|
||||||
public:
|
|
||||||
explicit KreaTimeMLP(Krea2Config config) {
|
|
||||||
blocks["0"] = std::make_shared<Linear>(config.timestep_dim, config.features, true);
|
|
||||||
blocks["2"] = std::make_shared<Linear>(config.features, config.features, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
|
|
||||||
auto linear_0 = std::dynamic_pointer_cast<Linear>(blocks["0"]);
|
|
||||||
auto linear_2 = std::dynamic_pointer_cast<Linear>(blocks["2"]);
|
|
||||||
x = linear_0->forward(ctx, x);
|
|
||||||
x = ggml_ext_gelu(ctx->ggml_ctx, x, false);
|
|
||||||
x = linear_2->forward(ctx, x);
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class KreaTProj : public UnaryBlock {
|
|
||||||
public:
|
|
||||||
explicit KreaTProj(Krea2Config config) {
|
|
||||||
blocks["1"] = std::make_shared<Linear>(config.features, config.features * 6, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
|
|
||||||
auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["1"]);
|
|
||||||
x = ggml_ext_gelu(ctx->ggml_ctx, x, false);
|
|
||||||
x = linear_1->forward(ctx, x);
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class KreaTextMLP : public UnaryBlock {
|
|
||||||
public:
|
|
||||||
explicit KreaTextMLP(Krea2Config config) {
|
|
||||||
blocks["0"] = std::make_shared<KreaRMSNorm>(config.text_dim, config.norm_eps);
|
|
||||||
blocks["1"] = std::make_shared<Linear>(config.text_dim, config.features, true);
|
|
||||||
blocks["3"] = std::make_shared<Linear>(config.features, config.features, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
|
|
||||||
auto norm = std::dynamic_pointer_cast<KreaRMSNorm>(blocks["0"]);
|
|
||||||
auto linear_1 = std::dynamic_pointer_cast<Linear>(blocks["1"]);
|
|
||||||
auto linear_3 = std::dynamic_pointer_cast<Linear>(blocks["3"]);
|
|
||||||
x = norm->forward(ctx, x);
|
|
||||||
x = linear_1->forward(ctx, x);
|
|
||||||
x = ggml_ext_gelu(ctx->ggml_ctx, x, true);
|
|
||||||
x = linear_3->forward(ctx, x);
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class KreaLastLayer : public GGMLBlock {
|
|
||||||
public:
|
|
||||||
explicit KreaLastLayer(Krea2Config config) {
|
|
||||||
blocks["norm"] = std::make_shared<KreaRMSNorm>(config.features, config.norm_eps);
|
|
||||||
blocks["linear"] = std::make_shared<Linear>(config.features, config.patch_size * config.patch_size * config.out_channels, true);
|
|
||||||
blocks["modulation"] = std::make_shared<KreaFinalModulation>(config.features);
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* vec) {
|
|
||||||
auto norm = std::dynamic_pointer_cast<KreaRMSNorm>(blocks["norm"]);
|
|
||||||
auto linear = std::dynamic_pointer_cast<Linear>(blocks["linear"]);
|
|
||||||
auto modulation = std::dynamic_pointer_cast<KreaFinalModulation>(blocks["modulation"]);
|
|
||||||
|
|
||||||
auto mods = modulation->forward(ctx, vec);
|
|
||||||
x = Flux::modulate(ctx->ggml_ctx,
|
|
||||||
norm->forward(ctx, x),
|
|
||||||
mods[1],
|
|
||||||
mods[0],
|
|
||||||
true);
|
|
||||||
x = linear->forward(ctx, x);
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class Krea2Model : public GGMLBlock {
|
|
||||||
protected:
|
|
||||||
Krea2Config config;
|
|
||||||
|
|
||||||
public:
|
|
||||||
Krea2Model() = default;
|
|
||||||
explicit Krea2Model(Krea2Config config)
|
|
||||||
: config(std::move(config)) {
|
|
||||||
blocks["first"] = std::make_shared<Linear>(this->config.patch_size * this->config.patch_size * this->config.in_channels,
|
|
||||||
this->config.features,
|
|
||||||
true);
|
|
||||||
blocks["tmlp"] = std::make_shared<KreaTimeMLP>(this->config);
|
|
||||||
blocks["txtfusion"] = std::make_shared<KreaTextFusionTransformer>(this->config);
|
|
||||||
blocks["txtmlp"] = std::make_shared<KreaTextMLP>(this->config);
|
|
||||||
blocks["tproj"] = std::make_shared<KreaTProj>(this->config);
|
|
||||||
for (int i = 0; i < this->config.layers; ++i) {
|
|
||||||
blocks["blocks." + std::to_string(i)] = std::make_shared<KreaSingleStreamBlock>(this->config);
|
|
||||||
}
|
|
||||||
blocks["last"] = std::make_shared<KreaLastLayer>(this->config);
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_tensor* forward(GGMLRunnerContext* ctx,
|
|
||||||
ggml_tensor* x,
|
|
||||||
ggml_tensor* timestep,
|
|
||||||
ggml_tensor* context,
|
|
||||||
ggml_tensor* pe) {
|
|
||||||
int64_t W = x->ne[0];
|
|
||||||
int64_t H = x->ne[1];
|
|
||||||
int64_t N = x->ne[3];
|
|
||||||
GGML_ASSERT(N == 1);
|
|
||||||
|
|
||||||
auto first = std::dynamic_pointer_cast<Linear>(blocks["first"]);
|
|
||||||
auto tmlp = std::dynamic_pointer_cast<KreaTimeMLP>(blocks["tmlp"]);
|
|
||||||
auto txtfusion = std::dynamic_pointer_cast<KreaTextFusionTransformer>(blocks["txtfusion"]);
|
|
||||||
auto txtmlp = std::dynamic_pointer_cast<KreaTextMLP>(blocks["txtmlp"]);
|
|
||||||
auto tproj = std::dynamic_pointer_cast<KreaTProj>(blocks["tproj"]);
|
|
||||||
auto last = std::dynamic_pointer_cast<KreaLastLayer>(blocks["last"]);
|
|
||||||
|
|
||||||
auto img = DiT::pad_and_patchify(ctx, x, config.patch_size, config.patch_size, true);
|
|
||||||
int64_t img_len = img->ne[1];
|
|
||||||
img = first->forward(ctx, img);
|
|
||||||
|
|
||||||
auto t = ggml_ext_timestep_embedding(ctx->ggml_ctx, timestep, static_cast<int>(config.timestep_dim), 10000, 1000.f);
|
|
||||||
t = tmlp->forward(ctx, t);
|
|
||||||
t = ggml_reshape_3d(ctx->ggml_ctx, t, t->ne[0], 1, t->ne[1]);
|
|
||||||
auto tvec = tproj->forward(ctx, t);
|
|
||||||
|
|
||||||
auto txt = txtfusion->forward(ctx, context);
|
|
||||||
txt = txtmlp->forward(ctx, txt);
|
|
||||||
int64_t txt_len = txt->ne[1];
|
|
||||||
|
|
||||||
auto hidden_states = ggml_concat(ctx->ggml_ctx, txt, img, 1);
|
|
||||||
for (int i = 0; i < config.layers; ++i) {
|
|
||||||
auto block = std::dynamic_pointer_cast<KreaSingleStreamBlock>(blocks["blocks." + std::to_string(i)]);
|
|
||||||
hidden_states = block->forward(ctx, hidden_states, tvec, pe);
|
|
||||||
sd::ggml_graph_cut::mark_graph_cut(hidden_states, "krea2.blocks." + std::to_string(i), "hidden_states");
|
|
||||||
}
|
|
||||||
|
|
||||||
hidden_states = last->forward(ctx, hidden_states, t);
|
|
||||||
hidden_states = ggml_ext_slice(ctx->ggml_ctx, hidden_states, 1, txt_len, txt_len + img_len);
|
|
||||||
hidden_states = DiT::unpatchify_and_crop(ctx->ggml_ctx, hidden_states, H, W, config.patch_size, config.patch_size, true);
|
|
||||||
return hidden_states;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
__STATIC_INLINE__ std::vector<float> gen_krea2_pe(int h,
|
|
||||||
int w,
|
|
||||||
int patch_size,
|
|
||||||
int bs,
|
|
||||||
int context_len,
|
|
||||||
float theta,
|
|
||||||
const std::vector<int>& axes_dim) {
|
|
||||||
auto txt_ids = Rope::gen_flux_txt_ids(bs, context_len, 3, {});
|
|
||||||
auto img_ids = Rope::gen_flux_img_ids(h, w, patch_size, bs, 3, 0, 0, 0, false);
|
|
||||||
auto ids = Rope::concat_ids(txt_ids, img_ids, bs);
|
|
||||||
return Rope::embed_nd(ids, bs, theta, axes_dim);
|
|
||||||
}
|
|
||||||
|
|
||||||
struct Krea2Runner : public DiffusionModelRunner {
|
|
||||||
Krea2Config config;
|
|
||||||
Krea2Model model;
|
|
||||||
std::vector<float> pe_vec;
|
|
||||||
|
|
||||||
Krea2Runner(ggml_backend_t backend,
|
|
||||||
const String2TensorStorage& tensor_storage_map = {},
|
|
||||||
const std::string prefix = "",
|
|
||||||
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
|
|
||||||
: DiffusionModelRunner(backend, prefix, weight_manager),
|
|
||||||
config(Krea2Config::detect_from_weights(tensor_storage_map, prefix)) {
|
|
||||||
model = Krea2Model(config);
|
|
||||||
model.init(params_ctx, tensor_storage_map, prefix);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string get_desc() override {
|
|
||||||
return "krea2";
|
|
||||||
}
|
|
||||||
|
|
||||||
void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
|
|
||||||
model.get_param_tensors(tensors, prefix);
|
|
||||||
}
|
|
||||||
|
|
||||||
ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
|
|
||||||
const sd::Tensor<float>& timesteps_tensor,
|
|
||||||
const sd::Tensor<float>& context_tensor) {
|
|
||||||
ggml_cgraph* gf = new_graph_custom(KREA2_GRAPH_SIZE);
|
|
||||||
ggml_tensor* x = make_input(x_tensor);
|
|
||||||
ggml_tensor* timesteps = make_input(timesteps_tensor);
|
|
||||||
GGML_ASSERT(x->ne[3] == 1);
|
|
||||||
GGML_ASSERT(!context_tensor.empty());
|
|
||||||
ggml_tensor* context = make_input(context_tensor);
|
|
||||||
|
|
||||||
pe_vec = gen_krea2_pe(static_cast<int>(x->ne[1]),
|
|
||||||
static_cast<int>(x->ne[0]),
|
|
||||||
config.patch_size,
|
|
||||||
static_cast<int>(x->ne[3]),
|
|
||||||
static_cast<int>(context->ne[1]),
|
|
||||||
config.theta,
|
|
||||||
config.axes_dim);
|
|
||||||
int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
|
|
||||||
auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
|
|
||||||
set_backend_tensor_data(pe, pe_vec.data());
|
|
||||||
|
|
||||||
auto runner_ctx = get_context();
|
|
||||||
ggml_tensor* out = model.forward(&runner_ctx, x, timesteps, context, pe);
|
|
||||||
ggml_build_forward_expand(gf, out);
|
|
||||||
return gf;
|
|
||||||
}
|
|
||||||
|
|
||||||
sd::Tensor<float> compute(int n_threads,
|
|
||||||
const sd::Tensor<float>& x,
|
|
||||||
const sd::Tensor<float>& timesteps,
|
|
||||||
const sd::Tensor<float>& context) {
|
|
||||||
auto get_graph = [&]() -> ggml_cgraph* {
|
|
||||||
return build_graph(x, timesteps, context);
|
|
||||||
};
|
|
||||||
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
|
|
||||||
}
|
|
||||||
|
|
||||||
sd::Tensor<float> compute(int n_threads,
|
|
||||||
const DiffusionParams& diffusion_params) override {
|
|
||||||
GGML_ASSERT(diffusion_params.x != nullptr);
|
|
||||||
GGML_ASSERT(diffusion_params.timesteps != nullptr);
|
|
||||||
return compute(n_threads,
|
|
||||||
*diffusion_params.x,
|
|
||||||
*diffusion_params.timesteps,
|
|
||||||
tensor_or_empty(diffusion_params.context));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
} // namespace Krea2
|
|
||||||
|
|
||||||
#endif // __SD_MODEL_DIFFUSION_KREA2_HPP__
|
|
||||||
@ -453,10 +453,6 @@ SDVersion ModelLoader::get_sd_version() {
|
|||||||
if (tensor_storage.name.find("embed_image_indicator.weight") != std::string::npos) {
|
if (tensor_storage.name.find("embed_image_indicator.weight") != std::string::npos) {
|
||||||
return VERSION_IDEOGRAM4;
|
return VERSION_IDEOGRAM4;
|
||||||
}
|
}
|
||||||
if (tensor_storage.name.find("model.diffusion_model.txtfusion.projector.weight") != std::string::npos ||
|
|
||||||
tensor_storage.name.find("model.diffusion_model.text_fusion.projector.weight") != std::string::npos) {
|
|
||||||
return VERSION_KREA2;
|
|
||||||
}
|
|
||||||
if (tensor_storage.name.find("model.diffusion_model.nerf_final_layer_conv.") != std::string::npos) {
|
if (tensor_storage.name.find("model.diffusion_model.nerf_final_layer_conv.") != std::string::npos) {
|
||||||
return VERSION_CHROMA_RADIANCE;
|
return VERSION_CHROMA_RADIANCE;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -704,38 +704,6 @@ std::string convert_other_dit_to_original_anima(std::string name) {
|
|||||||
return name;
|
return name;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string convert_diffusers_dit_to_original_krea2(std::string name) {
|
|
||||||
static const std::vector<std::pair<std::string, std::string>> prefix_map = {
|
|
||||||
{"img_in.", "first."},
|
|
||||||
{"time_embed.linear_1.", "tmlp.0."},
|
|
||||||
{"time_embed.linear_2.", "tmlp.2."},
|
|
||||||
{"time_mod_proj.", "tproj.1."},
|
|
||||||
{"txt_in.linear_1.", "txtmlp.1."},
|
|
||||||
{"txt_in.linear_2.", "txtmlp.3."},
|
|
||||||
{"text_fusion.", "txtfusion."},
|
|
||||||
{"transformer_blocks.", "blocks."},
|
|
||||||
{"final_layer.", "last."},
|
|
||||||
};
|
|
||||||
static const std::vector<std::pair<std::string, std::string>> name_map = {
|
|
||||||
{"attn.to_out.0.", "attn.wo."},
|
|
||||||
{"attn.to_out.", "attn.wo."},
|
|
||||||
{"attn.to_gate.", "attn.gate."},
|
|
||||||
{"attn.to_q.", "attn.wq."},
|
|
||||||
{"attn.to_k.", "attn.wk."},
|
|
||||||
{"attn.to_v.", "attn.wv."},
|
|
||||||
{"ff.gate.", "mlp.gate."},
|
|
||||||
{"ff.up.", "mlp.up."},
|
|
||||||
{"ff.down.", "mlp.down."},
|
|
||||||
{"txt_in.norm.", "txtmlp.0."},
|
|
||||||
{"last.norm.weight", "last.norm.scale"},
|
|
||||||
{"last.modulation.weight", "last.modulation.lin"},
|
|
||||||
};
|
|
||||||
|
|
||||||
replace_with_prefix_map(name, prefix_map);
|
|
||||||
replace_with_name_map(name, name_map);
|
|
||||||
return name;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string convert_diffusion_model_name(std::string name, std::string prefix, SDVersion version) {
|
std::string convert_diffusion_model_name(std::string name, std::string prefix, SDVersion version) {
|
||||||
if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
|
if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
|
||||||
name = convert_diffusers_unet_to_original_sd1(name);
|
name = convert_diffusers_unet_to_original_sd1(name);
|
||||||
@ -749,8 +717,6 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S
|
|||||||
name = convert_diffusers_dit_to_original_lumina2(name);
|
name = convert_diffusers_dit_to_original_lumina2(name);
|
||||||
} else if (sd_version_is_anima(version)) {
|
} else if (sd_version_is_anima(version)) {
|
||||||
name = convert_other_dit_to_original_anima(name);
|
name = convert_other_dit_to_original_anima(name);
|
||||||
} else if (sd_version_is_krea2(version)) {
|
|
||||||
name = convert_diffusers_dit_to_original_krea2(name);
|
|
||||||
}
|
}
|
||||||
return name;
|
return name;
|
||||||
}
|
}
|
||||||
@ -1209,7 +1175,7 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
|
|||||||
|
|
||||||
replace_with_prefix_map(name, prefix_map);
|
replace_with_prefix_map(name, prefix_map);
|
||||||
|
|
||||||
if ((sd_version_is_boogu_image(version) || sd_version_is_krea2(version)) && starts_with(name, "text_encoders.llm.visual.")) {
|
if (sd_version_is_boogu_image(version) && starts_with(name, "text_encoders.llm.visual.")) {
|
||||||
name = convert_qwen3_vl_vision_name(std::move(name));
|
name = convert_qwen3_vl_vision_name(std::move(name));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -559,203 +559,6 @@ struct LTX2Scheduler : SigmaScheduler {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
/*
|
|
||||||
* Logit-Normal Scheduler
|
|
||||||
* Based on: https://github.com/ideogram-oss/ideogram4/blob/main/src/ideogram4/scheduler.py
|
|
||||||
*/
|
|
||||||
struct LogitNormalScheduler : SigmaScheduler {
|
|
||||||
float mean = 0.0f;
|
|
||||||
float std = 1.75f;
|
|
||||||
float logsnr_min = -15.0f;
|
|
||||||
float logsnr_max = 18.0f;
|
|
||||||
|
|
||||||
bool resolution_aware = true;
|
|
||||||
|
|
||||||
float one_minus_t_min, one_minus_t_max;
|
|
||||||
|
|
||||||
void parse_extra_sample_args(int image_seq_len = 0, const char* extra_sample_args = nullptr) {
|
|
||||||
const int known_seq_len = (512 * 512) / (16 * 16);
|
|
||||||
if (extra_sample_args) {
|
|
||||||
for (const auto& [key, value] : parse_key_value_args(extra_sample_args, "logit-normal scheduler arg")) {
|
|
||||||
if (key == "mu") {
|
|
||||||
if (!parse_strict_float(value, mean)) {
|
|
||||||
LOG_WARN("ignoring invalid logit-normal scheduler arg '%s=%s'", key.c_str(), value.c_str());
|
|
||||||
}
|
|
||||||
} else if (key == "std") {
|
|
||||||
if (!parse_strict_float(value, std)) {
|
|
||||||
LOG_WARN("ignoring invalid logit-normal scheduler arg '%s=%s'", key.c_str(), value.c_str());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (key == "logsnr_min") {
|
|
||||||
if (!parse_strict_float(value, logsnr_min)) {
|
|
||||||
LOG_WARN("ignoring invalid logit-normal scheduler arg '%s=%s'", key.c_str(), value.c_str());
|
|
||||||
}
|
|
||||||
} else if (key == "logsnr_max") {
|
|
||||||
if (!parse_strict_float(value, logsnr_max)) {
|
|
||||||
LOG_WARN("ignoring invalid logit-normal scheduler arg '%s=%s'", key.c_str(), value.c_str());
|
|
||||||
}
|
|
||||||
} else if (key == "resolution_aware") {
|
|
||||||
if (!parse_strict_bool(value, resolution_aware)) {
|
|
||||||
LOG_WARN("ignoring invalid logit-normal scheduler arg '%s=%s'", key.c_str(), value.c_str());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (image_seq_len > 0 && resolution_aware) {
|
|
||||||
mean += 0.5 * std::log(static_cast<float>(image_seq_len) / static_cast<float>(known_seq_len));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
float sigmoid(float x) {
|
|
||||||
return 1.0f / (1.0f + std::exp(-x));
|
|
||||||
}
|
|
||||||
|
|
||||||
LogitNormalScheduler(float mean = 0.0f, float std = 1.75f, float logsnr_min = -18.0f, float logsnr_max = 15.0f)
|
|
||||||
: mean(mean), std(std), logsnr_min(logsnr_min), logsnr_max(logsnr_max) {
|
|
||||||
// t_min = 1.0f / (1.0f + std::exp(0.5f * logsnr_max));
|
|
||||||
one_minus_t_min = sigmoid(0.5f * logsnr_max);
|
|
||||||
// t_max = 1.0f / (1.0f + std::exp(0.5f * logsnr_min));
|
|
||||||
one_minus_t_max = sigmoid(0.5f * logsnr_min);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
LogitNormalScheduler(int image_seq_len = 0, const char* extra_sample_args = nullptr) {
|
|
||||||
mean = 0.0f;
|
|
||||||
std = 1.75f;
|
|
||||||
logsnr_min = -15.0f;
|
|
||||||
logsnr_max = 18.0f;
|
|
||||||
|
|
||||||
parse_extra_sample_args(image_seq_len, extra_sample_args);
|
|
||||||
// t_min = 1.0f / (1.0f + std::exp(0.5f * logsnr_max));
|
|
||||||
one_minus_t_min = sigmoid(0.5f * logsnr_max);
|
|
||||||
// t_max = 1.0f / (1.0f + std::exp(0.5f * logsnr_min));
|
|
||||||
one_minus_t_max = sigmoid(0.5f * logsnr_min);
|
|
||||||
}
|
|
||||||
|
|
||||||
// https://stackedboxes.org/2017/05/01/acklams-normal-quantile-function/
|
|
||||||
double ndtri(double p) {
|
|
||||||
if (p <= 0.0) {
|
|
||||||
return -std::numeric_limits<double>::infinity();
|
|
||||||
} else if (p >= 1.0) {
|
|
||||||
return std::numeric_limits<double>::infinity();
|
|
||||||
}
|
|
||||||
|
|
||||||
static const double p_low = 0.02425;
|
|
||||||
static const double p_high = 1.0 - p_low;
|
|
||||||
|
|
||||||
static const double c[6] = {-7.784894002430293e-03,
|
|
||||||
-3.223964580411365e-01,
|
|
||||||
-2.400758277161838e+00,
|
|
||||||
-2.549732539343734e+00,
|
|
||||||
4.374664141464968e+00,
|
|
||||||
2.938163982698783e+00};
|
|
||||||
|
|
||||||
static const double d[5] = {7.784695709041462e-03,
|
|
||||||
3.224671290700398e-01,
|
|
||||||
2.445134137142996e+00,
|
|
||||||
3.754408661907416e+00,
|
|
||||||
1.0};
|
|
||||||
|
|
||||||
// Coefficients for the central region
|
|
||||||
static const double a[6] = {-3.969683028665376e+01,
|
|
||||||
2.209460984245205e+02,
|
|
||||||
-2.759285104469687e+02,
|
|
||||||
1.383577518672690e+02,
|
|
||||||
-3.066479806614716e+01,
|
|
||||||
2.506628277459239e+00};
|
|
||||||
|
|
||||||
static const double b[6] = {-5.447609879822406e+01,
|
|
||||||
1.615858368580409e+02,
|
|
||||||
-1.556989798598866e+02,
|
|
||||||
6.680131188771972e+01,
|
|
||||||
-1.328068155288572e+01,
|
|
||||||
1.0};
|
|
||||||
|
|
||||||
double x = 0.0;
|
|
||||||
|
|
||||||
if (p < p_low) {
|
|
||||||
// Lower region
|
|
||||||
double q = std::sqrt(-2.0 * std::log(p));
|
|
||||||
|
|
||||||
// Numerator: c[0]*q^5 + c[1]*q^4 + ... + c[5]
|
|
||||||
double numerator = c[0];
|
|
||||||
for (int i = 1; i < 6; ++i) {
|
|
||||||
numerator = numerator * q + c[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
// Denominator: d[0]*q^4 + d[1]*q^3 + ... + d[3]*q + 1
|
|
||||||
double denominator = d[0];
|
|
||||||
for (int i = 1; i < 5; ++i) {
|
|
||||||
denominator = denominator * q + d[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
x = numerator / denominator;
|
|
||||||
} else if (p > p_high) {
|
|
||||||
// Upper region
|
|
||||||
double q = std::sqrt(-2.0 * std::log(1.0 - p));
|
|
||||||
|
|
||||||
double numerator = c[0];
|
|
||||||
for (int i = 1; i < 6; ++i) {
|
|
||||||
numerator = numerator * q + c[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
double denominator = d[0];
|
|
||||||
for (int i = 1; i < 5; ++i) {
|
|
||||||
denominator = denominator * q + d[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
x = -(numerator / denominator);
|
|
||||||
} else {
|
|
||||||
// Central region
|
|
||||||
double q = p - 0.5;
|
|
||||||
double r = q * q;
|
|
||||||
|
|
||||||
// Numerator: (a[0]*r^5 + a[1]*r^4 + ... + a[5])*q
|
|
||||||
double numerator = a[0];
|
|
||||||
for (int i = 1; i < 6; ++i) {
|
|
||||||
numerator = numerator * r + a[i];
|
|
||||||
}
|
|
||||||
numerator *= q;
|
|
||||||
|
|
||||||
// Denominator: b[0]*r^4 + b[1]*r^3 + ... + b[4]*r + 1
|
|
||||||
double denominator = b[0];
|
|
||||||
for (int i = 1; i < 6; ++i) {
|
|
||||||
denominator = denominator * r + b[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
x = numerator / denominator;
|
|
||||||
}
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<float> get_sigmas(uint32_t n, float /*sigma_min*/, float /*sigma_max*/, t_to_sigma_t /*t_to_sigma*/) override {
|
|
||||||
std::vector<float> sigmas;
|
|
||||||
LOG_INFO("LOGIT_NORMAL_SCHEDULER using mean=%.4f, std=%.4f, logsnr_min=%.4f, logsnr_max=%.4f", mean, std, logsnr_min, logsnr_max);
|
|
||||||
sigmas.reserve(n + 1);
|
|
||||||
for (uint32_t i = 0; i <= n; ++i) {
|
|
||||||
float t = static_cast<float>(i) / static_cast<float>(n);
|
|
||||||
|
|
||||||
// ndtri(1-t) == -ndtri(t)
|
|
||||||
float z = -ndtri(t);
|
|
||||||
|
|
||||||
float y = mean + std * z;
|
|
||||||
|
|
||||||
float timestep = sigmoid(y);
|
|
||||||
|
|
||||||
if (timestep > one_minus_t_min)
|
|
||||||
timestep = one_minus_t_min;
|
|
||||||
if (timestep < one_minus_t_max)
|
|
||||||
timestep = one_minus_t_max;
|
|
||||||
|
|
||||||
float sigma = timestep;
|
|
||||||
|
|
||||||
sigmas.push_back(sigma);
|
|
||||||
}
|
|
||||||
sigmas[n] = 0.0f;
|
|
||||||
return sigmas;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
struct Denoiser {
|
struct Denoiser {
|
||||||
virtual float sigma_min() = 0;
|
virtual float sigma_min() = 0;
|
||||||
virtual float sigma_max() = 0;
|
virtual float sigma_max() = 0;
|
||||||
@ -820,11 +623,6 @@ struct Denoiser {
|
|||||||
LOG_INFO("get_sigmas with LTX2 scheduler");
|
LOG_INFO("get_sigmas with LTX2 scheduler");
|
||||||
scheduler = std::make_shared<LTX2Scheduler>(image_seq_len, extra_sample_args);
|
scheduler = std::make_shared<LTX2Scheduler>(image_seq_len, extra_sample_args);
|
||||||
break;
|
break;
|
||||||
case LOGIT_NORMAL_SCHEDULER: {
|
|
||||||
LOG_INFO("get_sigmas with Logit-Normal scheduler");
|
|
||||||
scheduler = std::make_shared<LogitNormalScheduler>(image_seq_len, extra_sample_args);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
default:
|
default:
|
||||||
LOG_INFO("get_sigmas with discrete scheduler (default)");
|
LOG_INFO("get_sigmas with discrete scheduler (default)");
|
||||||
scheduler = std::make_shared<DiscreteScheduler>();
|
scheduler = std::make_shared<DiscreteScheduler>();
|
||||||
|
|||||||
@ -26,7 +26,6 @@
|
|||||||
#include "model/diffusion/flux.hpp"
|
#include "model/diffusion/flux.hpp"
|
||||||
#include "model/diffusion/hidream_o1.hpp"
|
#include "model/diffusion/hidream_o1.hpp"
|
||||||
#include "model/diffusion/ideogram4.hpp"
|
#include "model/diffusion/ideogram4.hpp"
|
||||||
#include "model/diffusion/krea2.hpp"
|
|
||||||
#include "model/diffusion/lens.hpp"
|
#include "model/diffusion/lens.hpp"
|
||||||
#include "model/diffusion/ltxv.hpp"
|
#include "model/diffusion/ltxv.hpp"
|
||||||
#include "model/diffusion/mmdit.hpp"
|
#include "model/diffusion/mmdit.hpp"
|
||||||
@ -96,7 +95,6 @@ const char* model_version_to_str[] = {
|
|||||||
"Longcat-Image",
|
"Longcat-Image",
|
||||||
"PiD",
|
"PiD",
|
||||||
"Ideogram 4",
|
"Ideogram 4",
|
||||||
"Krea2",
|
|
||||||
"ESRGAN",
|
"ESRGAN",
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -647,17 +645,6 @@ public:
|
|||||||
tensor_storage_map,
|
tensor_storage_map,
|
||||||
"model.diffusion_model",
|
"model.diffusion_model",
|
||||||
model_manager);
|
model_manager);
|
||||||
} else if (sd_version_is_krea2(version)) {
|
|
||||||
cond_stage_model = std::make_shared<LLMEmbedder>(backend_for(SDBackendModule::TE),
|
|
||||||
tensor_storage_map,
|
|
||||||
version,
|
|
||||||
"",
|
|
||||||
false,
|
|
||||||
model_manager);
|
|
||||||
diffusion_model = std::make_shared<Krea2::Krea2Runner>(backend_for(SDBackendModule::DIFFUSION),
|
|
||||||
tensor_storage_map,
|
|
||||||
"model.diffusion_model",
|
|
||||||
model_manager);
|
|
||||||
} else if (sd_version_is_flux(version)) {
|
} else if (sd_version_is_flux(version)) {
|
||||||
bool is_chroma = false;
|
bool is_chroma = false;
|
||||||
for (auto pair : tensor_storage_map) {
|
for (auto pair : tensor_storage_map) {
|
||||||
@ -894,7 +881,6 @@ public:
|
|||||||
auto create_tae = [&](bool decode_only) -> std::shared_ptr<VAE> {
|
auto create_tae = [&](bool decode_only) -> std::shared_ptr<VAE> {
|
||||||
if (sd_version_is_wan(version) ||
|
if (sd_version_is_wan(version) ||
|
||||||
sd_version_is_qwen_image(version) ||
|
sd_version_is_qwen_image(version) ||
|
||||||
sd_version_is_krea2(version) ||
|
|
||||||
sd_version_is_anima(version) ||
|
sd_version_is_anima(version) ||
|
||||||
sd_version_is_ltxav(version)) {
|
sd_version_is_ltxav(version)) {
|
||||||
return std::make_shared<TinyVideoAutoEncoder>(backend_for(SDBackendModule::VAE),
|
return std::make_shared<TinyVideoAutoEncoder>(backend_for(SDBackendModule::VAE),
|
||||||
@ -935,7 +921,6 @@ public:
|
|||||||
model_manager);
|
model_manager);
|
||||||
} else if (sd_version_is_wan(version) ||
|
} else if (sd_version_is_wan(version) ||
|
||||||
sd_version_is_qwen_image(version) ||
|
sd_version_is_qwen_image(version) ||
|
||||||
sd_version_is_krea2(version) ||
|
|
||||||
sd_version_is_anima(version)) {
|
sd_version_is_anima(version)) {
|
||||||
return std::make_shared<WAN::WanVAERunner>(backend_for(SDBackendModule::VAE),
|
return std::make_shared<WAN::WanVAERunner>(backend_for(SDBackendModule::VAE),
|
||||||
tensor_storage_map,
|
tensor_storage_map,
|
||||||
@ -1282,8 +1267,7 @@ public:
|
|||||||
} else if (sd_version_is_flux(version) ||
|
} else if (sd_version_is_flux(version) ||
|
||||||
sd_version_is_longcat(version) ||
|
sd_version_is_longcat(version) ||
|
||||||
sd_version_is_lens(version) ||
|
sd_version_is_lens(version) ||
|
||||||
sd_version_is_ltxav(version) ||
|
sd_version_is_ltxav(version)) {
|
||||||
sd_version_is_krea2(version)) {
|
|
||||||
pred_type = FLUX_FLOW_PRED;
|
pred_type = FLUX_FLOW_PRED;
|
||||||
|
|
||||||
default_flow_shift = 1.0f; // TODO: validate
|
default_flow_shift = 1.0f; // TODO: validate
|
||||||
@ -1299,8 +1283,6 @@ public:
|
|||||||
default_flow_shift = 1.83f;
|
default_flow_shift = 1.83f;
|
||||||
} else if (sd_version_is_ltxav(version)) {
|
} else if (sd_version_is_ltxav(version)) {
|
||||||
default_flow_shift = 2.37f;
|
default_flow_shift = 2.37f;
|
||||||
} else if (sd_version_is_krea2(version)) {
|
|
||||||
default_flow_shift = 1.15f;
|
|
||||||
}
|
}
|
||||||
} else if (sd_version_is_flux2(version)) {
|
} else if (sd_version_is_flux2(version)) {
|
||||||
pred_type = FLUX2_FLOW_PRED;
|
pred_type = FLUX2_FLOW_PRED;
|
||||||
@ -1742,7 +1724,7 @@ public:
|
|||||||
} else if (sd_version_uses_flux_vae(version)) {
|
} else if (sd_version_uses_flux_vae(version)) {
|
||||||
latent_rgb_proj = flux_latent_rgb_proj;
|
latent_rgb_proj = flux_latent_rgb_proj;
|
||||||
latent_rgb_bias = flux_latent_rgb_bias;
|
latent_rgb_bias = flux_latent_rgb_bias;
|
||||||
} else if (sd_version_is_wan(version) || sd_version_is_qwen_image(version) || sd_version_is_anima(version) || sd_version_is_krea2(version)) {
|
} else if (sd_version_is_wan(version) || sd_version_is_qwen_image(version) || sd_version_is_anima(version)) {
|
||||||
latent_rgb_proj = wan_21_latent_rgb_proj;
|
latent_rgb_proj = wan_21_latent_rgb_proj;
|
||||||
latent_rgb_bias = wan_21_latent_rgb_bias;
|
latent_rgb_bias = wan_21_latent_rgb_bias;
|
||||||
} else {
|
} else {
|
||||||
@ -2553,7 +2535,6 @@ const char* scheduler_to_str[] = {
|
|||||||
"lcm",
|
"lcm",
|
||||||
"bong_tangent",
|
"bong_tangent",
|
||||||
"ltx2",
|
"ltx2",
|
||||||
"logit_normal",
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const char* sd_scheduler_name(enum scheduler_t scheduler) {
|
const char* sd_scheduler_name(enum scheduler_t scheduler) {
|
||||||
@ -3156,8 +3137,6 @@ enum scheduler_t sd_get_default_scheduler(const sd_ctx_t* sd_ctx, enum sample_me
|
|||||||
return SIMPLE_SCHEDULER;
|
return SIMPLE_SCHEDULER;
|
||||||
} else if (sd_ctx != nullptr && sd_ctx->sd != nullptr && sd_version_is_ltxav(sd_ctx->sd->version)) {
|
} else if (sd_ctx != nullptr && sd_ctx->sd != nullptr && sd_version_is_ltxav(sd_ctx->sd->version)) {
|
||||||
return LTX2_SCHEDULER;
|
return LTX2_SCHEDULER;
|
||||||
} else if(sd_ctx != nullptr && sd_ctx->sd != nullptr && sd_version_is_ideogram4(sd_ctx->sd->version)) {
|
|
||||||
return LOGIT_NORMAL_SCHEDULER;
|
|
||||||
}
|
}
|
||||||
return DISCRETE_SCHEDULER;
|
return DISCRETE_SCHEDULER;
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user