feat: ControlNet Union support for SDXL (#2284)

src/pipeline/diffusion_engine.cpp

@@ -2270,6 +2270,10 @@ static sd::Tensor<float> batch_two_condition_tensors(const sd::Tensor<float>& a,
     return sd::ops::concat(a, b, static_cast<size_t>(batch_dim));
 }
 
+bool StableDiffusionGGML::set_control_type(sd_control_type_t type) {
+    return control_net == nullptr || control_net->set_control_type(type);
+}
+
 void StableDiffusionGGML::compute_sample_controls(const sd::Tensor<float>& control_image,
                                                   const sd::Tensor<float>& noised_input,
                                                   const sd::Tensor<float>& timesteps_tensor,

src/pipeline/diffusion_engine.h

@@ -440,6 +440,10 @@ public:
                                 bool terminal_sigma_is_zero,
                                 int64_t* last_progress_us);
 
+    // set the kind of the control image of the next generations (ControlNet Union),
+    // false (after saying why) when the loaded ControlNet can't use it
+    bool set_control_type(sd_control_type_t type);
+
     void compute_sample_controls(const sd::Tensor<float>& control_image,
                                  const sd::Tensor<float>& noised_input,
                                  const sd::Tensor<float>& timesteps_tensor,

src/pipeline/image.cpp

@@ -253,6 +253,9 @@ namespace sd::pipeline {
 
         if (sd_img_gen_params->control_image.data != nullptr) {
             control_image_tensor = sd_image_to_tensor(sd_img_gen_params->control_image, request->width, request->height);
+            if (!sd->set_control_type(request->control_type)) {
+                return std::nullopt;
+            }
         }
 
         if (init_image_tensor.empty() || mask_image_tensor.empty()) {

src/pipeline/request.cpp

@@ -135,6 +135,7 @@ namespace sd::pipeline {
         shifted_timestep            = sd_img_gen_params->sample_params.shifted_timestep;
         strength                    = sd_img_gen_params->strength;
         control_strength            = sd_img_gen_params->control_strength;
+        control_type                = sd_img_gen_params->control_type;
         eta                         = sd_img_gen_params->sample_params.eta;
         has_ref_images              = sd_img_gen_params->ref_images_count > 0;
         guidance                    = sd_img_gen_params->sample_params.guidance;

src/pipeline/request.h

@@ -47,6 +47,7 @@ namespace sd::pipeline {
         int shifted_timestep                        = 0;
         float strength                              = 1.f;
         float control_strength                      = 0.f;
+        sd_control_type_t control_type              = SD_CONTROL_TYPE_NONE;
         float eta                                   = 0.f;
         sd_sample_params_t sample_params            = {};
         sd_sample_params_t high_noise_sample_params = {};