fix: add frame dimension for Hunyuan IMG2VID encoding (#1816)

src/stable-diffusion.cpp

@@ -224,6 +224,7 @@ public:
     std::shared_ptr<ControlNet> control_net;
     std::shared_ptr<IPAdapter::IPAdapterRunner> ip_adapter;
     sd::Tensor<float> ip_adapter_tokens;
+    sd::Tensor<float> ip_adapter_uncond_tokens;
     float ip_adapter_strength = 1.0f;
     std::vector<std::shared_ptr<GenerationExtension>> generation_extensions;
     std::vector<std::shared_ptr<LoraModel>> runtime_lora_models;
@@ -2101,6 +2102,7 @@ public:
 
     void compute_ip_adapter_tokens(const sd_image_t& image, float strength) {
         ip_adapter_tokens        = {};
+        ip_adapter_uncond_tokens = {};
         ip_adapter_strength      = strength;
         if (ip_adapter == nullptr || clip_vision == nullptr || image.data == nullptr) {
             return;
@@ -2111,11 +2113,20 @@ public:
             return;
         }
         ip_adapter_tokens = ip_adapter->compute(n_threads, embed);
-        if (!ip_adapter_tokens.empty()) {
+        if (ip_adapter_tokens.empty()) {
+            LOG_ERROR("IP-Adapter conditional image projection failed");
+            return;
+        }
+        auto uncond_embed        = sd::Tensor<float>::zeros_like(embed);
+        ip_adapter_uncond_tokens = ip_adapter->compute(n_threads, uncond_embed);
+        if (ip_adapter_uncond_tokens.empty()) {
+            LOG_ERROR("IP-Adapter unconditional image projection failed");
+            ip_adapter_tokens = {};
+            return;
+        }
         LOG_INFO("IP-Adapter: %lld image tokens, strength %.2f",
                  (long long)ip_adapter_tokens.shape()[1], strength);
     }
-    }
 
     std::vector<float> process_timesteps(const std::vector<float>& timesteps,
                                          const sd::Tensor<float>& init_latent,
@@ -2606,7 +2617,7 @@ public:
                                      const sd::Tensor<float>* c_concat_override                 = nullptr,
                                      const std::vector<int>* local_skip_layers                  = nullptr,
                                      const std::vector<sd::Tensor<float>>* ref_latents_override = nullptr,
-                                     bool apply_ip                                              = true) -> sd::Tensor<float> {
+                                     bool use_uncond_ip                                         = false) -> sd::Tensor<float> {
                 diffusion_params.context     = condition.c_crossattn.empty() ? nullptr : &condition.c_crossattn;
                 diffusion_params.c_concat    = c_concat_override != nullptr ? c_concat_override : (condition.c_concat.empty() ? nullptr : &condition.c_concat);
                 diffusion_params.y           = condition.c_vector.empty() ? nullptr : &condition.c_vector;
@@ -2618,8 +2629,9 @@ public:
                         nvf = static_cast<int>(noised_input.shape()[3]);
                     }
                     UNetDiffusionExtra unet_extra{nvf, &controls, control_strength};
-                    if (apply_ip && !ip_adapter_tokens.empty()) {
+                    const auto& ip_tokens = use_uncond_ip ? ip_adapter_uncond_tokens : ip_adapter_tokens;
-                        unet_extra.ip_context = &ip_adapter_tokens;
+                    if (!ip_tokens.empty()) {
+                        unet_extra.ip_context = &ip_tokens;
                         unet_extra.ip_scale   = ip_adapter_strength;
                     }
                     diffusion_params.extra = unet_extra;
@@ -2715,7 +2727,7 @@ public:
                                            uncond.c_concat.empty() ? nullptr : &uncond.c_concat,
                                            uncond_skip_layers,
                                            nullptr,
-                                           false);
+                                           true);
                 if (uncond_out.empty()) {
                     return {};
                 }
@@ -2725,7 +2737,7 @@ public:
                                                img_uncond.c_concat.empty() ? nullptr : &img_uncond.c_concat,
                                                nullptr,
                                                uncond_without_ref_latents ? &empty_ref_latents : nullptr,
-                                               false);
+                                               true);
                 if (img_uncond_out.empty()) {
                     return {};
                 }
@@ -3582,6 +3594,7 @@ void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params) {
     sd_img_gen_params->seed                = -1;
     sd_img_gen_params->batch_count         = 1;
     sd_img_gen_params->control_strength    = 0.9f;
+    sd_img_gen_params->ip_adapter_strength = 1.0f;
     sd_img_gen_params->qwen_image_layers   = 3;
     sd_img_gen_params->circular_x          = false;
     sd_img_gen_params->circular_y          = false;
@@ -5776,7 +5789,7 @@ static std::optional<ImageGenerationLatents> prepare_video_generation_latents(sd
         auto encode_condition_frame = [&](const sd::Tensor<float>& image,
                                           int64_t latent_frame,
                                           const char* name) -> bool {
-            auto encoded = sd_ctx->sd->encode_first_stage(image);
+            auto encoded = sd_ctx->sd->encode_first_stage(image.unsqueeze(2));
             if (encoded.empty()) {
                 LOG_ERROR("failed to encode Hunyuan Video %s conditioning frame", name);
                 return false;