more progress; now alloc buffer failed

clip.hpp

@@ -948,7 +948,7 @@ struct CLIPVisionModel {
     struct ggml_tensor* forward(struct ggml_context* ctx0,  
                                 struct ggml_tensor *x,
                                 struct ggml_tensor *cls,
-                                struct ggml_tensor *temp,
+                                struct ggml_tensor *temp,                                
                                 struct ggml_tensor *positions) {
         // input_ids: [N, n_token]
         // GGML_ASSERT(input_ids->ne[0] <= position_ids->ne[0]);
@@ -965,11 +965,15 @@ struct CLIPVisionModel {
         struct ggml_tensor * inp = ggml_conv_2d(ctx0, patch_embeddings, x, patch_size, patch_size, 0, 0, 1, 1);
 
         ggml_set_name(inp, "inp_conv_2d");
+        print_ggml_tensor(inp, true, "inp_conv_2d");
 
         inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size);
         ggml_set_name(inp, "inp_reshape_3d");
+        print_ggml_tensor(inp, true, "inp_reshape_3d");
         // inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
-        inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 3, 1));
+        print_ggml_tensor(ggml_permute(ctx0, inp, 2, 0, 1, 3), true, "inp_permute");
+        inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
+        print_ggml_tensor(inp, true, "inp_cont");
         ggml_set_name(inp, "inp_cont");
         ggml_set_name(class_embedding, "class_embedding");
 
@@ -977,23 +981,36 @@ struct CLIPVisionModel {
         // struct ggml_tensor * embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size);
                
         // struct ggml_tensor * temp = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, 1, batch_size);
-       
+        // ggml_tensor *class_embedding_rep = ggml_add(ctx0, cast_f32_class, class_embedding);
+        // ggml_set_name(class_embedding_rep, "add_class_embedding_to_zero");
+        print_ggml_tensor(class_embedding, true, "model.class_embedding");
+        // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_bef_repeat");
+        print_ggml_tensor(temp, true, "temp");
         ggml_tensor *class_embedding_rep = ggml_repeat(ctx0, class_embedding, temp);
-        ggml_set_name(class_embedding_rep, "class_embedding_rep");
 
+        ggml_set_name(class_embedding_rep, "class_embedding_rep");
+        print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep");
+        class_embedding_rep = ggml_cast(ctx0, class_embedding_rep, inp->type);
+        print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_aft_casting");
         struct ggml_tensor *embeddings =  ggml_concat(ctx0, class_embedding_rep, inp);
         ggml_set_name(embeddings, "embeddings_after_concat");
-        embeddings =  ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 3, 1, 2));
+        print_ggml_tensor(embeddings, true, "embeddings_after_concat");
+        // print_ggml_tensor(ggml_permute(ctx0, embeddings, 0, 3, 1, 2), true, "embeddings_after_concat_permute");
+        embeddings =  ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 2, 1, 3));
         ggml_set_name(embeddings, "embeddings_after_permute");
 
         // embeddings = ggml_acc(ctx0, embeddings, ggml_repeat(ctx0, class_embedding, temp), embeddings->nb[1],
         //                     embeddings->nb[2], embeddings->nb[3], 0);
         
         // struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions);
-
+        // print_ggml_tensor(embeddings, true, "embeddings_bef_add");
+        print_ggml_tensor(ggml_get_rows(ctx0, position_embeddings, positions), true, "embeddings_add_src1");
+        // print_ggml_tensor(ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings), true, "embeddings_add_src1");
+        // embeddings =  ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 1, 3, 2));
         embeddings =
             ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings));
         ggml_set_name(embeddings, "embeddings_after_add");
+        print_ggml_tensor(embeddings, true, "embeddings_after_add");
 
         // pre-layernorm        
         embeddings = ggml_nn_layer_norm(ctx0, embeddings, pre_ln_w, pre_ln_w);
@@ -1023,6 +1040,7 @@ struct CLIPVisionModel {
     }
 
     void init_params(ggml_context* ctx, ggml_backend_t backend, ggml_type wtype, ggml_allocr* alloc) {
+        wtype = GGML_TYPE_F32;
         class_embedding = ggml_new_tensor_1d(ctx, wtype, hidden_size);
 
         patch_embeddings = ggml_new_tensor_4d(ctx, wtype, patch_size, patch_size, 3, hidden_size);
@@ -1142,7 +1160,8 @@ struct FrozenCLIPEmbedderWithCustomWords : public GGMLModule {
                                                   hidden_states2->ne[2],
                                                   hidden_states2->ne[3]);
             hidden_states2      = ggml_cont(ctx0, ggml_permute(ctx0, hidden_states2, 2, 0, 1, 3));
-
+            // print_ggml_tensor(hidden_states, true, "hidden_states");
+            // print_ggml_tensor(hidden_states2, true, "hidden_states2");
             hidden_states = ggml_concat(ctx0, hidden_states, hidden_states2);  // [N, n_token, hidden_size + hidden_size2]
 
             hidden_states = ggml_cont(ctx0, ggml_permute(ctx0, hidden_states, 1, 2, 0, 3));

ggml_extend.hpp

@@ -81,7 +81,7 @@ __STATIC_INLINE__ ggml_fp16_t ggml_tensor_get_f16(const ggml_tensor* tensor, int
 
 
 __STATIC_INLINE__ void print_ggml_tensor(struct ggml_tensor* tensor, bool shape_only = false, const char *mark = "") {
-    printf("%s: shape(%zu, %zu, %zu, %zu)\n", mark, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);
+    printf("%s (%s): shape(%zu, %zu, %zu, %zu)\n", mark, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);
     fflush(stdout);
     if (shape_only) {
         return;

model.cpp

@@ -789,6 +789,8 @@ bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const
             ne[i] = shape[i].get<int64_t>();
         }
         TensorStorage tensor_storage(prefix + name, type, ne, n_dims, file_index, ST_HEADER_SIZE_LEN + header_size_ + begin);
+        // if(strcmp(prefix.c_str(), "pmid") == 0)
+        // printf(" %s %s \n", (prefix + name).c_str(), ggml_type_name(type));
         tensor_storage.reverse_ne();
 
         size_t tensor_data_size = end - begin;
@@ -1126,7 +1128,7 @@ bool ModelLoader::parse_data_pkl(uint8_t* buffer,
                         reader.tensor_storage.reverse_ne();
                         reader.tensor_storage.file_index = file_index;
                         // if(strcmp(prefix.c_str(), "pm") == 0)
-                        printf(" got tensor %s \n ", reader.tensor_storage.name.c_str());
+                        // printf(" got tensor %s \n ", reader.tensor_storage.name.c_str());
                         reader.tensor_storage.name       = prefix + reader.tensor_storage.name;
                         tensor_storages.push_back(reader.tensor_storage);
                         // LOG_DEBUG("%s", reader.tensor_storage.name.c_str());

pmid.hpp

@@ -37,7 +37,7 @@ struct FuseBlock {
         size_t mem_size = 0;
         mem_size += 2 * ggml_row_size(wtype, in_dim);
         mem_size += ggml_row_size(wtype, in_dim*hidden_dim);
-        mem_size += 5 * ggml_row_size(wtype, in_dim);
+        mem_size += ggml_row_size(wtype, out_dim);
         mem_size += ggml_row_size(wtype, hidden_dim*out_dim);
         mem_size += ggml_row_size(wtype, hidden_dim);
         
@@ -261,7 +261,7 @@ struct PhotoMakerIDEncoder : public GGMLModule {
     }
 
     // void init_params(ggml_context* ctx, ggml_backend_t backend, ggml_type wtype, ggml_allocr* alloc) {   
-    void init_params() {       
+    void init_params() {               
 
         ggml_allocr* alloc = ggml_allocr_new_from_buffer(params_buffer);
         
@@ -281,10 +281,15 @@ struct PhotoMakerIDEncoder : public GGMLModule {
 
     size_t calculate_mem_size() {
         size_t mem_size = vision_model.calculate_mem_size(wtype);
-
+        LOG_INFO("pmid vision memory buffer size = %.2fMB ",
+                 vision_model.calculate_mem_size(wtype) / 1024.0 / 1024.0);
         mem_size += fuse_module.calculate_mem_size(wtype);
+        LOG_INFO("pmid fuse memory buffer size = %.2fMB ",
+                 fuse_module.calculate_mem_size(wtype) / 1024.0 / 1024.0);
         
         mem_size += ggml_row_size(wtype, 1280*1024);
+         LOG_INFO("pmid porject buffer size = %.2fMB ",
+                  ggml_row_size(wtype, 1280*1024) / 1024.0 / 1024.0);
               
         return mem_size;
     }
@@ -393,7 +398,7 @@ struct PhotoMakerIDEncoder : public GGMLModule {
 
         int64_t hidden_size = prompt_embeds->ne[0];
         int64_t seq_length  = prompt_embeds->ne[1];
-        ggml_type type = prompt_embeds->type;
+        ggml_type type = GGML_TYPE_F32;
 
         struct ggml_tensor* id_pixel_values_d = ggml_dup_tensor(ctx0, id_pixel_values);
         ggml_allocr_alloc(allocr, id_pixel_values_d);
@@ -441,11 +446,13 @@ struct PhotoMakerIDEncoder : public GGMLModule {
         struct ggml_tensor * cls = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, batch_size);
 
         struct ggml_tensor * class_embedding_temp = ggml_new_tensor_4d(ctx0, type, 
-                                       vision_model.hidden_size, 1, 1, batch_size);
+                                       vision_model.hidden_size, batch_size, 1, 1);
         struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions);
+        struct ggml_tensor * cast_f32_class = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, vision_model.hidden_size);
         ggml_allocr_alloc(allocr, cls);
         ggml_allocr_alloc(allocr, class_embedding_temp);
         ggml_allocr_alloc(allocr, positions);
+        ggml_allocr_alloc(allocr, cast_f32_class);
 
 
 
@@ -467,6 +474,11 @@ struct PhotoMakerIDEncoder : public GGMLModule {
             ggml_backend_tensor_set(cls, cls_h.data(), 0, ggml_nbytes(cls));
             ggml_backend_tensor_set(positions, pos.data(), 0, ggml_nbytes(positions));
             ggml_backend_tensor_set(class_tokens_mask_pos, ctmpos.data(), 0, ggml_nbytes(class_tokens_mask_pos));
+            std::vector<float> zeros;
+            for (int i = 0; i < hidden_size; i++) {
+                zeros.push_back(0.f);
+            }
+            ggml_backend_tensor_set(cast_f32_class, zeros.data(), 0, ggml_nbytes(cast_f32_class));
             if(left){
                 if(type == GGML_TYPE_F16){
                     std::vector<ggml_fp16_t> zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f));

stable-diffusion.cpp

@@ -218,6 +218,8 @@ public:
                 LOG_ERROR(" pmid model params buffer allocation failed");
                 return false;
             }
+            LOG_INFO("pmid param memory buffer size = %.2fMB ",
+                 pmid_model.params_buffer_size / 1024.0 / 1024.0);
         }
 
 
@@ -246,11 +248,14 @@ public:
                 first_stage_model.init_params();
             }
             first_stage_model.map_by_name(tensors, "first_stage_model.");
-
+            
             if(stacked_id){
+               printf("preparing memory for pmid \n");  
                pmid_model.init_params();
                pmid_model.map_by_name(tensors, "pmid.");
+               printf("done preparing memory for pmid \n");  
             }
+
         }
 
         struct ggml_init_params params;
@@ -311,6 +316,8 @@ public:
             cond_stage_model.params_buffer_size +
             diffusion_model.params_buffer_size +
             first_stage_model.params_buffer_size;
+        if(stacked_id)
+            total_params_size += pmid_model.params_buffer_size;
         LOG_INFO("total memory buffer size = %.2fMB (clip %.2fMB, unet %.2fMB, vae %.2fMB)",
                  total_params_size / 1024.0 / 1024.0,
                  cond_stage_model.params_buffer_size / 1024.0 / 1024.0,