added preprocessing inpit id images

examples/cli/main.cpp

@@ -643,6 +643,11 @@ int main(int argc, const char* argv[]) {
                                              (uint32_t)height,
                                              3,
                                              input_image_buffer};
+                input_image = preprocess_id_image(input_image);
+                if(input_image == NULL){
+                    fprintf(stderr, "preprocess input id image from '%s' failed\n", img_file.c_str());
+                    return 1;
+                }
                 input_id_images.push_back(input_image);
             }
         }
@@ -658,7 +663,8 @@ int main(int argc, const char* argv[]) {
                           params.seed,
                           params.batch_count,
                           control_image,
-                          params.control_strength);
+                          params.control_strength,
+                          input_id_images);
     } else {
         sd_image_t input_image = {(uint32_t)params.width,
                                   (uint32_t)params.height,

ggml_extend.hpp

@@ -211,7 +211,8 @@ __STATIC_INLINE__ uint8_t* sd_tensor_to_image(struct ggml_tensor* input) {
 }
 
 __STATIC_INLINE__ void sd_image_to_tensor(const uint8_t* image_data,
-                                          struct ggml_tensor* output) {
+                                          struct ggml_tensor* output,
+                                          float *mean = NULL, float *std = NULL) {
     int64_t width    = output->ne[0];
     int64_t height   = output->ne[1];
     int64_t channels = output->ne[2];
@@ -220,7 +221,31 @@ __STATIC_INLINE__ void sd_image_to_tensor(const uint8_t* image_data,
         for (int ix = 0; ix < width; ix++) {
             for (int k = 0; k < channels; k++) {
                 int value = *(image_data + iy * width * channels + ix * channels + k);
-                ggml_tensor_set_f32(output, value / 255.0f, ix, iy, k);
+                float pixel_val = value / 255.0f;
+                if(mean != NULL && std != NULL)
+                    pixel_val = (pixel_val - mean[k]) / std[k];
+                ggml_tensor_set_f32(output, pixel_val, ix, iy, k);
+            }
+        }
+    }
+}
+
+__STATIC_INLINE__ void sd_mul_images_to_tensor(const uint8_t* image_data,
+                                              struct ggml_tensor* output,
+                                              int idx,
+                                              float *mean = NULL, float *std = NULL) {
+    int64_t width    = output->ne[0];
+    int64_t height   = output->ne[1];
+    int64_t channels = output->ne[2];
+    GGML_ASSERT(channels == 3 && output->type == GGML_TYPE_F32);
+    for (int iy = 0; iy < height; iy++) {
+        for (int ix = 0; ix < width; ix++) {
+            for (int k = 0; k < channels; k++) {
+                int value = *(image_data + iy * width * channels + ix * channels + k);
+                float pixel_val = value / 255.0f;
+                if(mean != NULL && std != NULL)
+                    pixel_val = (pixel_val - mean[k]) / std[k];
+                ggml_tensor_set_f32(output, pixel_val, ix, iy, k, idx);
             }
         }
     }

stable-diffusion.cpp

@@ -1249,7 +1249,8 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx,
                     int64_t seed,
                     int batch_count,
                     const sd_image_t* control_cond,
-                    float control_strength) {
+                    float control_strength,
+                    std::vector<sd_image_t*> &input_id_images) {
     LOG_DEBUG("txt2img %dx%d", width, height);
     if (sd_ctx == NULL) {
         return NULL;
@@ -1294,6 +1295,23 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx,
         srand((int)time(NULL));
         seed = rand();
     }
+    ggml_tensor* init_img = NULL;
+    if(sd_ctx->sd->stacked_id){
+        int32_t width = input_id_images[0]->width;
+        int32_t height = input_id_images[0]->height;
+        int32_t channels = input_id_images[0]->channel;
+        int32_t num_input_images = input_id_images.size();
+        init_img = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, width, height, channels, num_input_images);
+        float mean[] = {0.48145466, 0.4578275, 0.40821073};
+        float std[]  = {0.26862954, 0.26130258, 0.27577711};
+        for(int i = 0; i <  num_input_images; i++)  {
+            sd_image_t* init_image = input_id_images[i];
+             sd_mul_images_to_tensor(init_image->data, init_img, i, mean, std);
+        }
+        int64_t *ne = init_img->ne;
+        fprintf(stderr, "%s: input id image tensor ne [%ld, %ld, %ld, %ld] \n",
+              __func__, ne[0], ne[1], ne[2], ne[3]);
+    }
 
     t0                            = ggml_time_ms();
     auto cond_pair                = sd_ctx->sd->get_learned_condition(work_ctx, prompt, clip_skip, width, height);

stable-diffusion.h

@@ -132,7 +132,8 @@ SD_API sd_image_t* txt2img(sd_ctx_t* sd_ctx,
                            int64_t seed,
                            int batch_count,
                            const sd_image_t* control_cond,
-                           float control_strength);
+                           float control_strength,
+                           std::vector<sd_image_t*> &input_id_images);
 
 SD_API sd_image_t* img2img(sd_ctx_t* sd_ctx,
                            sd_image_t init_image,

util.cpp

@@ -23,6 +23,10 @@
 #include "ggml/ggml.h"
 #include "stable-diffusion.h"
 
+ #define STB_IMAGE_RESIZE_IMPLEMENTATION
+ #include "stb_image_resize.h"
+
+
 bool ends_with(const std::string& str, const std::string& ending) {
     if (str.length() >= ending.length()) {
         return (str.compare(str.length() - ending.length(), ending.length(), ending) == 0);
@@ -226,6 +230,43 @@ std::string path_join(const std::string& p1, const std::string& p2) {
     return p1 + "/" + p2;
 }
 
+
+sd_image_t *preprocess_id_image(sd_image_t *img){
+    int shortest_edge = 224;
+    int size = shortest_edge;
+    sd_image_t * resized = NULL;
+    uint32_t w = img->width;
+    uint32_t h = img->height;
+    uint32_t c = img->channel;
+
+
+    // 1. do resize using stb_resize functions
+
+    unsigned char *buf = (unsigned char*)malloc(sizeof(unsigned char)*3*size*size);
+    if(!stbir_resize_uint8(img->data, w, h , 0,
+                                 buf, size, size, 0,
+                                 c)){
+        fprintf(stderr, "%s: resize operation failed \n ", __func__);
+        return resized;
+    }
+
+    // 2. do center crop (likely unnecessary due to step 1)
+
+    // 3. do rescale
+
+    // 4. do normalize
+
+    // 3 and 4 will need to be done in float format.
+
+
+
+    resized = new sd_image_t{(uint32_t)shortest_edge,
+                             (uint32_t)shortest_edge,
+                              3,
+                             buf};
+    return resized;
+}
+
 void pretty_progress(int step, int steps, float time) {
     std::string progress = "  |";
     int max_progress     = 50;

util.h

@@ -26,6 +26,9 @@ std::u32string unicode_value_to_utf32(int unicode_value);
 
 std::string sd_basename(const std::string& path);
 
+sd_image_t *preprocess_id_image(sd_image_t * img);
+
+
 std::string path_join(const std::string& p1, const std::string& p2);
 
 void pretty_progress(int step, int steps, float time);