fix: forward ref_image_args to the video pipeline (#2177)

src/conditioning/conditioning_cache.h

@@ -48,7 +48,8 @@ class ConditioningCache {
             return std::tie(p.text, p.clip_skip, p.width, p.height, p.zero_out_masked,
                             r.pass_to_vlm, r.pass_to_dit, r.ref_index_mode,
                             r.force_ref_timestep_zero, r.resize_before_vae, r.vae_input_max_pixels,
-                            r.vlm_resize_mode, r.vlm_min_size, r.vlm_max_size, r.resize_vae_to_target);
+                            r.vlm_resize_mode, r.vlm_min_size, r.vlm_max_size, r.resize_vae_to_target,
+                            r.ref_image_size);
         };
         if (fields(a) != fields(b) ||
             (a.ref_images == nullptr) != (b.ref_images == nullptr) ||

src/model/diffusion/model.hpp

@@ -16,6 +16,11 @@ enum class RefImageResizeMode {
     AREA,
 };
 
+enum class RefImageSizeMode {
+    MATCH,
+    MAX,
+};
+
 struct RefImageParams {
     bool pass_to_vlm                   = false;
     bool pass_to_dit                   = true;
@@ -31,6 +36,7 @@ struct RefImageParams {
     bool vae_size_buckets              = false;
     bool vae_center_crop               = false;
     bool vlm_use_vae_input             = false;
+    RefImageSizeMode ref_image_size    = RefImageSizeMode::MATCH;
 };
 
 const std::unordered_map<std::string, RefImageParams> REF_IMAGE_PRESETS = {

src/pipeline/diffusion_engine.cpp

@@ -3241,6 +3241,9 @@ std::string StableDiffusionGGML::get_default_ref_image_preset(SDVersion version)
 RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_image_args) const {
     RefImageParams params;
     std::string preset_name = get_default_ref_image_preset(version);
+    // Keys whose value was rejected. They are already reported, so per-model "no effect"
+    // warnings must not repeat them.
+    std::set<std::string> rejected_keys;
 
     for (const auto& [key, value] : parse_key_value_args(ref_image_args, "reference image args")) {
         if (key == "preset") {
@@ -3253,6 +3256,7 @@ RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_ima
                     valid_list += (valid_list.empty() ? "" : ", ") + name;
                 }
                 LOG_WARN("ignoring invalid reference image preset '%s'. Valid options: [%s]", value.c_str(), valid_list.c_str());
+                rejected_keys.insert(key);
             }
             break;
         }
@@ -3269,10 +3273,12 @@ RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_ima
         if (key == "pass_to_vlm") {
             if (!parse_strict_bool(value, params.pass_to_vlm)) {
                 LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
+                rejected_keys.insert(key);
             }
         } else if (key == "pass_to_dit") {
             if (!parse_strict_bool(value, params.pass_to_dit)) {
                 LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
+                rejected_keys.insert(key);
             }
         } else if (key == "ref_index_mode") {
             if (value == "fixed") {
@@ -3283,14 +3289,17 @@ RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_ima
                 params.ref_index_mode = Rope::RefIndexMode::DECREASE;
             } else {
                 LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
+                rejected_keys.insert(key);
             }
         } else if (key == "force_ref_timestep_zero") {
             if (!parse_strict_bool(value, params.force_ref_timestep_zero)) {
                 LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
+                rejected_keys.insert(key);
             }
         } else if (key == "resize_before_vae") {
             if (!parse_strict_bool(value, params.resize_before_vae)) {
                 LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
+                rejected_keys.insert(key);
             }
         } else if (key == "resize_vae_to_target") {
             if (!parse_strict_bool(value, params.resize_vae_to_target)) {
@@ -3315,6 +3324,7 @@ RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_ima
         } else if (key == "vae_input_max_pixels") {
             if (!parse_strict_int(value, params.vae_input_max_pixels)) {
                 LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
+                rejected_keys.insert(key);
             }
         } else if (key == "vlm_resize_mode") {
             if (value == "longest_side") {
@@ -3325,14 +3335,31 @@ RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_ima
                 params.vlm_resize_mode = RefImageResizeMode::NONE;
             } else {
                 LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
+                rejected_keys.insert(key);
             }
         } else if (key == "vlm_max_size") {
             if (!parse_strict_int(value, params.vlm_max_size)) {
                 LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
+                rejected_keys.insert(key);
             }
         } else if (key == "vlm_min_size") {
             if (!parse_strict_int(value, params.vlm_min_size)) {
                 LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
+                rejected_keys.insert(key);
+            }
+        } else if (key == "ref_image_size") {
+            bool valid = true;
+            if (value == "match") {
+                params.ref_image_size = RefImageSizeMode::MATCH;
+            } else if (value == "max") {
+                params.ref_image_size = RefImageSizeMode::MAX;
+            } else {
+                LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
+                rejected_keys.insert(key);
+                valid = false;
+            }
+            if (valid && !sd_version_is_minimax_h3(version)) {
+                LOG_WARN("ref_image_size is only supported by MiniMax-H3 for now");
             }
         } else if (key != "preset" && key != "vlm_size") {
             LOG_WARN("ignoring unknown reference image arg '%s'", key.c_str());
@@ -3343,6 +3370,7 @@ RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_ima
             int vlm_size;
             if (!parse_strict_int(value, vlm_size)) {
                 LOG_WARN("ignoring invalid reference image arg '%s=%s'", key.c_str(), value.c_str());
+                rejected_keys.insert(key);
             } else {
                 LOG_INFO("vlm_size override: setting both min and max size to %ld", (long)vlm_size);
                 params.vlm_min_size = vlm_size;
@@ -3354,6 +3382,29 @@ RefImageParams StableDiffusionGGML::resolve_ref_image_params(const char* ref_ima
     if (params.force_ref_timestep_zero && !sd_version_is_krea2(version)) {
         LOG_WARN("force_ref_timestep_zero is only supported by Krea2 architecture for now");
     }
+    if (sd_version_is_minimax_h3(version)) {
+        // H3 consumes references as VAE latents in its own packed sequence rather than
+        // through the generic conditioner/DiT reference path, so ref_image_size is the
+        // only arg it acts on. Everything else is reported as ineffective rather than
+        // enumerated, so newly added RefImageParams members cannot silently do nothing.
+        static const std::set<std::string> h3_effective_keys = {"ref_image_size"};
+        // Keys that already have their own, more specific warning above.
+        static const std::set<std::string> h3_separately_reported_keys = {
+            "resize_before_vae",
+            "force_ref_timestep_zero",
+        };
+        for (const auto& arg : parse_key_value_args(ref_image_args, "reference image args")) {
+            if (h3_effective_keys.count(arg.first) == 0 &&
+                h3_separately_reported_keys.count(arg.first) == 0 &&
+                rejected_keys.count(arg.first) == 0) {
+                LOG_WARN("reference image arg '%s' has no effect for MiniMax-H3", arg.first.c_str());
+            }
+        }
+
+        if (!params.resize_before_vae) {
+            LOG_WARN("MiniMax-H3 requires 32-pixel-aligned reference images and cannot skip resizing before VAE encoding; ignoring resize_before_vae=false");
+        }
+    }
     return params;
 }
 

src/pipeline/video.cpp

@@ -4,6 +4,7 @@
 #include <cmath>
 #include <cstdlib>
 #include <optional>
+#include <utility>
 
 #include "conditioning/wan_audio.h"
 #include "core/rng.hpp"
@@ -461,9 +462,35 @@ namespace sd::pipeline {
         return window;
     }
 
+    // ComfyUI-compatible reference sizing: per-axis rounding to 32 px keeps the reference
+    // divisible by VAE 16 x patch 2 without padding in the H3 graph. Both modes round rather
+    // than floor: flooring would under-size in-limit references and diverge from the
+    // upstream node, which rounds in both branches.
+    static std::pair<int, int> minimax_h3_reference_size(int source_w,
+                                                         int source_h,
+                                                         int target_w,
+                                                         int target_h,
+                                                         RefImageSizeMode mode) {
+        constexpr double kCanvasMultiple    = 32.0;
+        constexpr double kRefImageShortEdge = 2048.0;
+        double scale                        = 1.0;
+        if (mode == RefImageSizeMode::MAX) {
+            scale = std::min(1.0, kRefImageShortEdge / std::min(source_w, source_h));
+        } else {
+            double source_area = static_cast<double>(source_w) * source_h;
+            double target_area = static_cast<double>(target_w) * target_h;
+            scale              = std::min(1.0, std::sqrt(target_area / source_area));
+        }
+        auto align32 = [kCanvasMultiple](double value) {
+            return std::max(32, static_cast<int>(std::round(value / kCanvasMultiple)) * 32);
+        };
+        return {align32(source_w * scale), align32(source_h * scale)};
+    }
+
     static std::optional<ImageGenerationLatents> prepare_video_generation_latents(StableDiffusionGGML* sd,
                                                                                   const sd_vid_gen_params_t* sd_vid_gen_params,
-                                                                                  GenerationRequest* request) {
+                                                                                  GenerationRequest* request,
+                                                                                  const RefImageParams& ref_image_params) {
         ImageGenerationLatents latents;
         int64_t prepare_start_ms = ggml_time_ms();
 
@@ -576,6 +603,7 @@ namespace sd::pipeline {
                          sd_vid_gen_params->ref_videos_count,
                          sd_vid_gen_params->ref_audios_count);
 
+                int64_t ref_image_tokens = 0;
                 for (int i = 0; i < sd_vid_gen_params->ref_images_count; ++i) {
                     auto image = ensure_image_tensor_channels(
                         sd_image_to_tensor(sd_vid_gen_params->ref_images[i]),
@@ -584,15 +612,18 @@ namespace sd::pipeline {
                         LOG_ERROR("failed to load MiniMax-H3 reference image %d", i + 1);
                         return std::nullopt;
                     }
-                    int source_w       = static_cast<int>(image.shape()[0]);
-                    int source_h       = static_cast<int>(image.shape()[1]);
-                    double source_area = static_cast<double>(source_w) * source_h;
-                    double target_area = static_cast<double>(request->width) * request->height;
-                    double scale       = std::min(1.0, std::sqrt(target_area / source_area));
-                    int width          = std::max(32, static_cast<int>(std::round(source_w * scale / 32.f)) * 32);
-                    int height         = std::max(32, static_cast<int>(std::round(source_h * scale / 32.f)) * 32);
-                    image              = resize_reference(image, width, height);
-                    auto latent        = sd->encode_first_stage(image);
+                    int source_w = static_cast<int>(image.shape()[0]);
+                    int source_h = static_cast<int>(image.shape()[1]);
+                    auto [width, height] =
+                        minimax_h3_reference_size(source_w, source_h, request->width, request->height, ref_image_params.ref_image_size);
+                    LOG_INFO("MiniMax-H3 reference image %d: %dx%d -> %dx%d",
+                             i + 1,
+                             source_w,
+                             source_h,
+                             width,
+                             height);
+                    image       = resize_reference(image, width, height);
+                    auto latent = sd->encode_first_stage(image);
                     if (latent.empty()) {
                         LOG_ERROR("failed to encode MiniMax-H3 reference image %d", i + 1);
                         return std::nullopt;
@@ -602,6 +633,10 @@ namespace sd::pipeline {
                     latents.minimax_reference_blocks.push_back({MiniMaxH3ReferenceKind::IMAGE,
                                                                 video_index,
                                                                 -1});
+                    // The DiT patchifies latents 2 x 2, so the token count is a quarter of the
+                    // latent grid.
+                    ref_image_tokens += static_cast<int64_t>(latents.ref_latents.back().shape()[0] / 2) *
+                                        (latents.ref_latents.back().shape()[1] / 2);
                     MiniMaxH3PresentationItem item;
                     item.kind = MiniMaxH3PresentationKind::IMAGE;
                     item.frames.push_back(std::move(image));
@@ -711,6 +746,18 @@ namespace sd::pipeline {
                                                                 -1,
                                                                 encoded_index});
                 }
+
+                if (ref_image_tokens > 0) {
+                    int64_t target_tokens = sd->get_image_seq_len(request->width, request->height) / 4 *
+                                            sd->video_frames_to_latent_frames(request->frames);
+                    if (target_tokens > 0 && ref_image_tokens > target_tokens * 4) {
+                        LOG_WARN("MiniMax-H3 reference images add %" PRId64
+                                 " tokens versus %" PRId64
+                                 " for the generated video; sampling cost scales with every attended token",
+                                 ref_image_tokens,
+                                 target_tokens);
+                    }
+                }
             }
 
             if (!has_references && (!start_image.empty() || !end_image.empty())) {
@@ -1147,7 +1194,8 @@ namespace sd::pipeline {
     static std::optional<ImageGenerationEmbeds> prepare_video_generation_embeds(StableDiffusionGGML* sd,
                                                                                 const sd_vid_gen_params_t* sd_vid_gen_params,
                                                                                 const GenerationRequest& request,
-                                                                                const ImageGenerationLatents& latents) {
+                                                                                const ImageGenerationLatents& latents,
+                                                                                const RefImageParams& ref_image_params) {
         ConditionerRunnerEndOnExit conditioner_runner_end{sd->cond_stage_model.get()};
 
         ImageGenerationEmbeds embeds;
@@ -1157,9 +1205,7 @@ namespace sd::pipeline {
         condition_params.zero_out_masked       = true;
         condition_params.ref_images            = &latents.ref_images;
         condition_params.minimax_h3_references = &latents.minimax_presentation_refs;
-        if (sd_version_is_lingbot_video(sd->version) || sd_version_is_minimax_h3(sd->version)) {
-            condition_params.ref_image_params.vlm_resize_mode = RefImageResizeMode::AREA;
-        }
+        condition_params.ref_image_params      = ref_image_params;
 
         int64_t prepare_start_ms = ggml_time_ms();
         embeds.cond              = sd->get_learned_condition(condition_params);
@@ -1526,6 +1572,7 @@ namespace sd::pipeline {
         img_gen_params.circular_x        = sd_vid_gen_params->circular_x;
         img_gen_params.circular_y        = sd_vid_gen_params->circular_y;
         img_gen_params.image_preprocess  = sd_vid_gen_params->image_preprocess;
+        img_gen_params.ref_image_args    = sd_vid_gen_params->ref_image_args;
 
         sd->animatediff_num_frames = n_frames;
         bool ok                    = generate_image(sd, &img_gen_params, frames_out, num_frames_out);
@@ -1550,7 +1597,7 @@ namespace sd::pipeline {
 
         sd->reset_cancel_flag();
 
-        const RefImageParams ref_image_params;
+        const RefImageParams ref_image_params = sd->resolve_ref_image_params(sd_vid_gen_params->ref_image_args);
 
         int64_t t0            = ggml_time_ms();
         sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
@@ -1589,7 +1636,7 @@ namespace sd::pipeline {
         sd->reset_generation_extensions();
 
         SamplePlan plan(sd, sd_vid_gen_params, request);
-        auto latent_inputs_opt = prepare_video_generation_latents(sd, sd_vid_gen_params, &request);
+        auto latent_inputs_opt = prepare_video_generation_latents(sd, sd_vid_gen_params, &request, ref_image_params);
         if (!latent_inputs_opt.has_value()) {
             return false;
         }
@@ -1598,7 +1645,8 @@ namespace sd::pipeline {
         auto embeds_opt = prepare_video_generation_embeds(sd,
                                                           sd_vid_gen_params,
                                                           request,
-                                                          latents);
+                                                          latents,
+                                                          ref_image_params);
         if (!embeds_opt) {
             return false;
         }

src/stable-diffusion.cpp

@@ -633,6 +633,7 @@ void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params) {
     sd_vid_gen_params->hires.custom_sigmas_count             = 0;
     sd_vid_gen_params->circular_x                            = false;
     sd_vid_gen_params->circular_y                            = false;
+    sd_vid_gen_params->ref_image_args                        = "";
     sd_cache_params_init(&sd_vid_gen_params->cache);
 }