DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 11:26:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add MiniMax-H3-Image support (#2324)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-11-11 01:24:25 +08:00
1 parent f8417d02e1
commit 32d516c019
15 files changed
+436 -118
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/pipeline/image.cpp
+34 -6
View File
@@ -172,6 +172,17 @@ namespace sd::pipeline {
return packed;
}
static sd::Tensor<float> resize_h3_image_reference(const sd::Tensor<float>& image, int64_t max_short_edge) {
const int64_t short_edge = std::min(image.shape()[0], image.shape()[1]);
const double scale = static_cast<double>(std::clamp<int64_t>(short_edge, 1024, max_short_edge)) / short_edge;
const int64_t width = std::max<int64_t>(32, static_cast<int64_t>(std::nearbyint(image.shape()[0] * scale / 32)) * 32);
const int64_t height = std::max<int64_t>(32, static_cast<int64_t>(std::nearbyint(image.shape()[1] * scale / 32)) * 32);
if (width == image.shape()[0] && height == image.shape()[1]) {
return image;
}
return sd::ops::interpolate(image, {width, height, image.shape()[2], image.shape()[3]}, sd::ops::InterpolateMode::Lanczos);
}
static std::optional<ImageGenerationLatents> prepare_image_generation_latents(StableDiffusionGGML* sd,
const sd_img_gen_params_t* sd_img_gen_params,
GenerationRequest* request,
@@ -354,7 +365,20 @@ namespace sd::pipeline {
continue;
}
sd::Tensor<float> ref_latent;
if (sd->version == VERSION_P_H3_IMAGE) {
if (sd->version == VERSION_MINIMAX_H3_IMAGE) {
ref_latent = sd->encode_first_stage(resize_h3_image_reference(ref_images[i], 2048));
if (!ref_latent.empty()) {
const int64_t width = ref_latent.shape()[0];
const int64_t height = ref_latent.shape()[1];
const int64_t channels = ref_latent.shape()[3];
auto condition_rng = std::make_shared<MT19937RNG>(static_cast<uint64_t>(request->seed));
// Upstream augments patch rows, so the RNG follows token/channel/patch order.
auto noise = sd::randn<float>({2, 2, channels, width / 2, height / 2}, condition_rng)
.permute({0, 3, 1, 4, 2})
.reshape(ref_latent.shape());
ref_latent = ref_latent * 0.999f + noise * (1.f - 0.999f);
}
} else if (sd->version == VERSION_P_H3_IMAGE) {
auto resized = resize_ph3_reference(ref_images[i], int64_t(request->width) * request->height);
ref_latent = sd->encode_first_stage(resized);
} else if (ref_image_params.resize_before_vae && !sd_version_is_pid(sd->version)) {
@@ -484,14 +508,18 @@ namespace sd::pipeline {
}
ImageGenerationLatents latents;
if (sd->version == VERSION_P_H3_IMAGE) {
init_latent = pack_ph3_image_latent(init_latent);
if (sd_version_is_minimax_h3_image(sd->version)) {
if (sd->version == VERSION_P_H3_IMAGE) {
init_latent = pack_ph3_image_latent(init_latent);
} else if (init_latent.dim() == 4) {
init_latent = init_latent.unsqueeze(2);
}
latent_mask = latent_mask.unsqueeze(2);
for (size_t i = 0; i < ref_latents.size(); ++i) {
latents.minimax_reference_blocks.push_back({MiniMaxH3ReferenceKind::IMAGE, static_cast<int32_t>(i), -1});
}
for (auto& image : ref_images) {
image = resize_ph3_reference(image, 1000000);
image = sd->version == VERSION_P_H3_IMAGE ? resize_ph3_reference(image, 1000000) : resize_h3_image_reference(image, 1024);
}
}
latents.init_latent = std::move(init_latent);
@@ -548,7 +576,7 @@ namespace sd::pipeline {
if (cond.c_concat.empty() && ref_image_params.pass_to_dit) {
cond.c_concat = latents->concat_latent; // TODO: optimize
}
if (sd->version == VERSION_P_H3_IMAGE && ref_image_params.pass_to_dit) {
if (sd_version_is_minimax_h3_image(sd->version) && ref_image_params.pass_to_dit) {
cond.c_reference_blocks = latents->minimax_reference_blocks;
}
@@ -596,7 +624,7 @@ namespace sd::pipeline {
if (uncond.c_concat.empty() && ref_image_params.pass_to_dit) {
uncond.c_concat = latents->concat_latent; // TODO: optimize
}
if (sd->version == VERSION_P_H3_IMAGE && ref_image_params.pass_to_dit) {
if (sd_version_is_minimax_h3_image(sd->version) && ref_image_params.pass_to_dit) {
uncond.c_reference_blocks = latents->minimax_reference_blocks;
}
}