DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 11:26:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: forward ref_image_args to the video pipeline (#2177)

Browse Source
This commit is contained in:
Philip Sorst authored and GitHub committed 2026-10-24 00:57:24 +08:00
1 parent fd21ecfcee
commit baf45b49e2
12 files changed
+152 -22
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/pipeline/video.cpp
+65 -17
View File
@@ -4,6 +4,7 @@
#include <cmath>
#include <cstdlib>
#include <optional>
#include <utility>
#include "conditioning/wan_audio.h"
#include "core/rng.hpp"
@@ -461,9 +462,35 @@ namespace sd::pipeline {
return window;
}
// ComfyUI-compatible reference sizing: per-axis rounding to 32 px keeps the reference
// divisible by VAE 16 x patch 2 without padding in the H3 graph. Both modes round rather
// than floor: flooring would under-size in-limit references and diverge from the
// upstream node, which rounds in both branches.
static std::pair<int, int> minimax_h3_reference_size(int source_w,
int source_h,
int target_w,
int target_h,
RefImageSizeMode mode) {
constexpr double kCanvasMultiple = 32.0;
constexpr double kRefImageShortEdge = 2048.0;
double scale = 1.0;
if (mode == RefImageSizeMode::MAX) {
scale = std::min(1.0, kRefImageShortEdge / std::min(source_w, source_h));
} else {
double source_area = static_cast<double>(source_w) * source_h;
double target_area = static_cast<double>(target_w) * target_h;
scale = std::min(1.0, std::sqrt(target_area / source_area));
}
auto align32 = [kCanvasMultiple](double value) {
return std::max(32, static_cast<int>(std::round(value / kCanvasMultiple)) * 32);
};
return {align32(source_w * scale), align32(source_h * scale)};
}
static std::optional<ImageGenerationLatents> prepare_video_generation_latents(StableDiffusionGGML* sd,
const sd_vid_gen_params_t* sd_vid_gen_params,
GenerationRequest* request) {
GenerationRequest* request,
const RefImageParams& ref_image_params) {
ImageGenerationLatents latents;
int64_t prepare_start_ms = ggml_time_ms();
@@ -576,6 +603,7 @@ namespace sd::pipeline {
sd_vid_gen_params->ref_videos_count,
sd_vid_gen_params->ref_audios_count);
int64_t ref_image_tokens = 0;
for (int i = 0; i < sd_vid_gen_params->ref_images_count; ++i) {
auto image = ensure_image_tensor_channels(
sd_image_to_tensor(sd_vid_gen_params->ref_images[i]),
@@ -584,15 +612,18 @@ namespace sd::pipeline {
LOG_ERROR("failed to load MiniMax-H3 reference image %d", i + 1);
return std::nullopt;
}
int source_w = static_cast<int>(image.shape()[0]);
int source_h = static_cast<int>(image.shape()[1]);
double source_area = static_cast<double>(source_w) * source_h;
double target_area = static_cast<double>(request->width) * request->height;
double scale = std::min(1.0, std::sqrt(target_area / source_area));
int width = std::max(32, static_cast<int>(std::round(source_w * scale / 32.f)) * 32);
int height = std::max(32, static_cast<int>(std::round(source_h * scale / 32.f)) * 32);
image = resize_reference(image, width, height);
auto latent = sd->encode_first_stage(image);
int source_w = static_cast<int>(image.shape()[0]);
int source_h = static_cast<int>(image.shape()[1]);
auto [width, height] =
minimax_h3_reference_size(source_w, source_h, request->width, request->height, ref_image_params.ref_image_size);
LOG_INFO("MiniMax-H3 reference image %d: %dx%d -> %dx%d",
i + 1,
source_w,
source_h,
width,
height);
image = resize_reference(image, width, height);
auto latent = sd->encode_first_stage(image);
if (latent.empty()) {
LOG_ERROR("failed to encode MiniMax-H3 reference image %d", i + 1);
return std::nullopt;
@@ -602,6 +633,10 @@ namespace sd::pipeline {
latents.minimax_reference_blocks.push_back({MiniMaxH3ReferenceKind::IMAGE,
video_index,
-1});
// The DiT patchifies latents 2 x 2, so the token count is a quarter of the
// latent grid.
ref_image_tokens += static_cast<int64_t>(latents.ref_latents.back().shape()[0] / 2) *
(latents.ref_latents.back().shape()[1] / 2);
MiniMaxH3PresentationItem item;
item.kind = MiniMaxH3PresentationKind::IMAGE;
item.frames.push_back(std::move(image));
@@ -711,6 +746,18 @@ namespace sd::pipeline {
-1,
encoded_index});
}
if (ref_image_tokens > 0) {
int64_t target_tokens = sd->get_image_seq_len(request->width, request->height) / 4 *
sd->video_frames_to_latent_frames(request->frames);
if (target_tokens > 0 && ref_image_tokens > target_tokens * 4) {
LOG_WARN("MiniMax-H3 reference images add %" PRId64
" tokens versus %" PRId64
" for the generated video; sampling cost scales with every attended token",
ref_image_tokens,
target_tokens);
}
}
}
if (!has_references && (!start_image.empty() || !end_image.empty())) {
@@ -1147,7 +1194,8 @@ namespace sd::pipeline {
static std::optional<ImageGenerationEmbeds> prepare_video_generation_embeds(StableDiffusionGGML* sd,
const sd_vid_gen_params_t* sd_vid_gen_params,
const GenerationRequest& request,
const ImageGenerationLatents& latents) {
const ImageGenerationLatents& latents,
const RefImageParams& ref_image_params) {
ConditionerRunnerEndOnExit conditioner_runner_end{sd->cond_stage_model.get()};
ImageGenerationEmbeds embeds;
@@ -1157,9 +1205,7 @@ namespace sd::pipeline {
condition_params.zero_out_masked = true;
condition_params.ref_images = &latents.ref_images;
condition_params.minimax_h3_references = &latents.minimax_presentation_refs;
if (sd_version_is_lingbot_video(sd->version) || sd_version_is_minimax_h3(sd->version)) {
condition_params.ref_image_params.vlm_resize_mode = RefImageResizeMode::AREA;
}
condition_params.ref_image_params = ref_image_params;
int64_t prepare_start_ms = ggml_time_ms();
embeds.cond = sd->get_learned_condition(condition_params);
@@ -1526,6 +1572,7 @@ namespace sd::pipeline {
img_gen_params.circular_x = sd_vid_gen_params->circular_x;
img_gen_params.circular_y = sd_vid_gen_params->circular_y;
img_gen_params.image_preprocess = sd_vid_gen_params->image_preprocess;
img_gen_params.ref_image_args = sd_vid_gen_params->ref_image_args;
sd->animatediff_num_frames = n_frames;
bool ok = generate_image(sd, &img_gen_params, frames_out, num_frames_out);
@@ -1550,7 +1597,7 @@ namespace sd::pipeline {
sd->reset_cancel_flag();
const RefImageParams ref_image_params;
const RefImageParams ref_image_params = sd->resolve_ref_image_params(sd_vid_gen_params->ref_image_args);
int64_t t0 = ggml_time_ms();
sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
@@ -1589,7 +1636,7 @@ namespace sd::pipeline {
sd->reset_generation_extensions();
SamplePlan plan(sd, sd_vid_gen_params, request);
auto latent_inputs_opt = prepare_video_generation_latents(sd, sd_vid_gen_params, &request);
auto latent_inputs_opt = prepare_video_generation_latents(sd, sd_vid_gen_params, &request, ref_image_params);
if (!latent_inputs_opt.has_value()) {
return false;
}
@@ -1598,7 +1645,8 @@ namespace sd::pipeline {
auto embeds_opt = prepare_video_generation_embeds(sd,
sd_vid_gen_params,
request,
latents);
latents,
ref_image_params);
if (!embeds_opt) {
return false;
}