src/stable-diffusion.cpp

@@ -224,7 +224,6 @@ public:
     std::shared_ptr<ControlNet> control_net;
     std::shared_ptr<IPAdapter::IPAdapterRunner> ip_adapter;
     sd::Tensor<float> ip_adapter_tokens;
-    sd::Tensor<float> ip_adapter_uncond_tokens;
     float ip_adapter_strength = 1.0f;
     std::vector<std::shared_ptr<GenerationExtension>> generation_extensions;
     std::vector<std::shared_ptr<LoraModel>> runtime_lora_models;
@@ -2102,7 +2101,6 @@ public:
 
     void compute_ip_adapter_tokens(const sd_image_t& image, float strength) {
         ip_adapter_tokens   = {};
-        ip_adapter_uncond_tokens = {};
         ip_adapter_strength = strength;
         if (ip_adapter == nullptr || clip_vision == nullptr || image.data == nullptr) {
             return;
@@ -2113,20 +2111,11 @@ public:
             return;
         }
         ip_adapter_tokens = ip_adapter->compute(n_threads, embed);
-        if (ip_adapter_tokens.empty()) {
+        if (!ip_adapter_tokens.empty()) {
-            LOG_ERROR("IP-Adapter conditional image projection failed");
-            return;
-        }
-        auto uncond_embed        = sd::Tensor<float>::zeros_like(embed);
-        ip_adapter_uncond_tokens = ip_adapter->compute(n_threads, uncond_embed);
-        if (ip_adapter_uncond_tokens.empty()) {
-            LOG_ERROR("IP-Adapter unconditional image projection failed");
-            ip_adapter_tokens = {};
-            return;
-        }
             LOG_INFO("IP-Adapter: %lld image tokens, strength %.2f",
                      (long long)ip_adapter_tokens.shape()[1], strength);
         }
+    }
 
     std::vector<float> process_timesteps(const std::vector<float>& timesteps,
                                          const sd::Tensor<float>& init_latent,
@@ -2617,7 +2606,7 @@ public:
                                      const sd::Tensor<float>* c_concat_override                 = nullptr,
                                      const std::vector<int>* local_skip_layers                  = nullptr,
                                      const std::vector<sd::Tensor<float>>* ref_latents_override = nullptr,
-                                     bool use_uncond_ip                                         = false) -> sd::Tensor<float> {
+                                     bool apply_ip                                              = true) -> sd::Tensor<float> {
                 diffusion_params.context     = condition.c_crossattn.empty() ? nullptr : &condition.c_crossattn;
                 diffusion_params.c_concat    = c_concat_override != nullptr ? c_concat_override : (condition.c_concat.empty() ? nullptr : &condition.c_concat);
                 diffusion_params.y           = condition.c_vector.empty() ? nullptr : &condition.c_vector;
@@ -2629,9 +2618,8 @@ public:
                         nvf = static_cast<int>(noised_input.shape()[3]);
                     }
                     UNetDiffusionExtra unet_extra{nvf, &controls, control_strength};
-                    const auto& ip_tokens = use_uncond_ip ? ip_adapter_uncond_tokens : ip_adapter_tokens;
+                    if (apply_ip && !ip_adapter_tokens.empty()) {
-                    if (!ip_tokens.empty()) {
+                        unet_extra.ip_context = &ip_adapter_tokens;
-                        unet_extra.ip_context = &ip_tokens;
                         unet_extra.ip_scale   = ip_adapter_strength;
                     }
                     diffusion_params.extra = unet_extra;
@@ -2727,7 +2715,7 @@ public:
                                            uncond.c_concat.empty() ? nullptr : &uncond.c_concat,
                                            uncond_skip_layers,
                                            nullptr,
-                                           true);
+                                           false);
                 if (uncond_out.empty()) {
                     return {};
                 }
@@ -2737,7 +2725,7 @@ public:
                                                img_uncond.c_concat.empty() ? nullptr : &img_uncond.c_concat,
                                                nullptr,
                                                uncond_without_ref_latents ? &empty_ref_latents : nullptr,
-                                               true);
+                                               false);
                 if (img_uncond_out.empty()) {
                     return {};
                 }
@@ -3594,7 +3582,6 @@ void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params) {
     sd_img_gen_params->seed              = -1;
     sd_img_gen_params->batch_count       = 1;
     sd_img_gen_params->control_strength  = 0.9f;
-    sd_img_gen_params->ip_adapter_strength = 1.0f;
     sd_img_gen_params->qwen_image_layers = 3;
     sd_img_gen_params->circular_x        = false;
     sd_img_gen_params->circular_y        = false;
@@ -5789,7 +5776,7 @@ static std::optional<ImageGenerationLatents> prepare_video_generation_latents(sd
         auto encode_condition_frame = [&](const sd::Tensor<float>& image,
                                           int64_t latent_frame,
                                           const char* name) -> bool {
-            auto encoded = sd_ctx->sd->encode_first_stage(image.unsqueeze(2));
+            auto encoded = sd_ctx->sd->encode_first_stage(image);
             if (encoded.empty()) {
                 LOG_ERROR("failed to encode Hunyuan Video %s conditioning frame", name);
                 return false;