diff --git a/clip.hpp b/clip.hpp index 6042e4b3..39e869c8 100644 --- a/clip.hpp +++ b/clip.hpp @@ -948,7 +948,7 @@ struct CLIPVisionModel { struct ggml_tensor* forward(struct ggml_context* ctx0, struct ggml_tensor *x, struct ggml_tensor *cls, - struct ggml_tensor *temp, + struct ggml_tensor *temp, struct ggml_tensor *positions) { // input_ids: [N, n_token] // GGML_ASSERT(input_ids->ne[0] <= position_ids->ne[0]); @@ -965,11 +965,15 @@ struct CLIPVisionModel { struct ggml_tensor * inp = ggml_conv_2d(ctx0, patch_embeddings, x, patch_size, patch_size, 0, 0, 1, 1); ggml_set_name(inp, "inp_conv_2d"); + print_ggml_tensor(inp, true, "inp_conv_2d"); inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size); ggml_set_name(inp, "inp_reshape_3d"); + print_ggml_tensor(inp, true, "inp_reshape_3d"); // inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); - inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 3, 1)); + print_ggml_tensor(ggml_permute(ctx0, inp, 2, 0, 1, 3), true, "inp_permute"); + inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3)); + print_ggml_tensor(inp, true, "inp_cont"); ggml_set_name(inp, "inp_cont"); ggml_set_name(class_embedding, "class_embedding"); @@ -977,23 +981,36 @@ struct CLIPVisionModel { // struct ggml_tensor * embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size); // struct ggml_tensor * temp = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, 1, batch_size); - + // ggml_tensor *class_embedding_rep = ggml_add(ctx0, cast_f32_class, class_embedding); + // ggml_set_name(class_embedding_rep, "add_class_embedding_to_zero"); + print_ggml_tensor(class_embedding, true, "model.class_embedding"); + // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_bef_repeat"); + print_ggml_tensor(temp, true, "temp"); ggml_tensor *class_embedding_rep = ggml_repeat(ctx0, class_embedding, temp); - ggml_set_name(class_embedding_rep, "class_embedding_rep"); + ggml_set_name(class_embedding_rep, "class_embedding_rep"); + print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep"); + class_embedding_rep = ggml_cast(ctx0, class_embedding_rep, inp->type); + print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_aft_casting"); struct ggml_tensor *embeddings = ggml_concat(ctx0, class_embedding_rep, inp); ggml_set_name(embeddings, "embeddings_after_concat"); - embeddings = ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 3, 1, 2)); + print_ggml_tensor(embeddings, true, "embeddings_after_concat"); + // print_ggml_tensor(ggml_permute(ctx0, embeddings, 0, 3, 1, 2), true, "embeddings_after_concat_permute"); + embeddings = ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 2, 1, 3)); ggml_set_name(embeddings, "embeddings_after_permute"); // embeddings = ggml_acc(ctx0, embeddings, ggml_repeat(ctx0, class_embedding, temp), embeddings->nb[1], // embeddings->nb[2], embeddings->nb[3], 0); // struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions); - + // print_ggml_tensor(embeddings, true, "embeddings_bef_add"); + print_ggml_tensor(ggml_get_rows(ctx0, position_embeddings, positions), true, "embeddings_add_src1"); + // print_ggml_tensor(ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings), true, "embeddings_add_src1"); + // embeddings = ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 1, 3, 2)); embeddings = ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings)); ggml_set_name(embeddings, "embeddings_after_add"); + print_ggml_tensor(embeddings, true, "embeddings_after_add"); // pre-layernorm embeddings = ggml_nn_layer_norm(ctx0, embeddings, pre_ln_w, pre_ln_w); @@ -1023,6 +1040,7 @@ struct CLIPVisionModel { } void init_params(ggml_context* ctx, ggml_backend_t backend, ggml_type wtype, ggml_allocr* alloc) { + wtype = GGML_TYPE_F32; class_embedding = ggml_new_tensor_1d(ctx, wtype, hidden_size); patch_embeddings = ggml_new_tensor_4d(ctx, wtype, patch_size, patch_size, 3, hidden_size); @@ -1142,7 +1160,8 @@ struct FrozenCLIPEmbedderWithCustomWords : public GGMLModule { hidden_states2->ne[2], hidden_states2->ne[3]); hidden_states2 = ggml_cont(ctx0, ggml_permute(ctx0, hidden_states2, 2, 0, 1, 3)); - + // print_ggml_tensor(hidden_states, true, "hidden_states"); + // print_ggml_tensor(hidden_states2, true, "hidden_states2"); hidden_states = ggml_concat(ctx0, hidden_states, hidden_states2); // [N, n_token, hidden_size + hidden_size2] hidden_states = ggml_cont(ctx0, ggml_permute(ctx0, hidden_states, 1, 2, 0, 3)); diff --git a/ggml_extend.hpp b/ggml_extend.hpp index 7bdf319c..a9e5c19b 100644 --- a/ggml_extend.hpp +++ b/ggml_extend.hpp @@ -81,7 +81,7 @@ __STATIC_INLINE__ ggml_fp16_t ggml_tensor_get_f16(const ggml_tensor* tensor, int __STATIC_INLINE__ void print_ggml_tensor(struct ggml_tensor* tensor, bool shape_only = false, const char *mark = "") { - printf("%s: shape(%zu, %zu, %zu, %zu)\n", mark, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); + printf("%s (%s): shape(%zu, %zu, %zu, %zu)\n", mark, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); fflush(stdout); if (shape_only) { return; diff --git a/model.cpp b/model.cpp index b39ad95d..db60c22d 100644 --- a/model.cpp +++ b/model.cpp @@ -789,6 +789,8 @@ bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const ne[i] = shape[i].get(); } TensorStorage tensor_storage(prefix + name, type, ne, n_dims, file_index, ST_HEADER_SIZE_LEN + header_size_ + begin); + // if(strcmp(prefix.c_str(), "pmid") == 0) + // printf(" %s %s \n", (prefix + name).c_str(), ggml_type_name(type)); tensor_storage.reverse_ne(); size_t tensor_data_size = end - begin; @@ -1126,7 +1128,7 @@ bool ModelLoader::parse_data_pkl(uint8_t* buffer, reader.tensor_storage.reverse_ne(); reader.tensor_storage.file_index = file_index; // if(strcmp(prefix.c_str(), "pm") == 0) - printf(" got tensor %s \n ", reader.tensor_storage.name.c_str()); + // printf(" got tensor %s \n ", reader.tensor_storage.name.c_str()); reader.tensor_storage.name = prefix + reader.tensor_storage.name; tensor_storages.push_back(reader.tensor_storage); // LOG_DEBUG("%s", reader.tensor_storage.name.c_str()); diff --git a/pmid.hpp b/pmid.hpp index 8dac2c3a..6d4bed85 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -37,7 +37,7 @@ struct FuseBlock { size_t mem_size = 0; mem_size += 2 * ggml_row_size(wtype, in_dim); mem_size += ggml_row_size(wtype, in_dim*hidden_dim); - mem_size += 5 * ggml_row_size(wtype, in_dim); + mem_size += ggml_row_size(wtype, out_dim); mem_size += ggml_row_size(wtype, hidden_dim*out_dim); mem_size += ggml_row_size(wtype, hidden_dim); @@ -261,7 +261,7 @@ struct PhotoMakerIDEncoder : public GGMLModule { } // void init_params(ggml_context* ctx, ggml_backend_t backend, ggml_type wtype, ggml_allocr* alloc) { - void init_params() { + void init_params() { ggml_allocr* alloc = ggml_allocr_new_from_buffer(params_buffer); @@ -281,10 +281,15 @@ struct PhotoMakerIDEncoder : public GGMLModule { size_t calculate_mem_size() { size_t mem_size = vision_model.calculate_mem_size(wtype); - + LOG_INFO("pmid vision memory buffer size = %.2fMB ", + vision_model.calculate_mem_size(wtype) / 1024.0 / 1024.0); mem_size += fuse_module.calculate_mem_size(wtype); + LOG_INFO("pmid fuse memory buffer size = %.2fMB ", + fuse_module.calculate_mem_size(wtype) / 1024.0 / 1024.0); mem_size += ggml_row_size(wtype, 1280*1024); + LOG_INFO("pmid porject buffer size = %.2fMB ", + ggml_row_size(wtype, 1280*1024) / 1024.0 / 1024.0); return mem_size; } @@ -393,7 +398,7 @@ struct PhotoMakerIDEncoder : public GGMLModule { int64_t hidden_size = prompt_embeds->ne[0]; int64_t seq_length = prompt_embeds->ne[1]; - ggml_type type = prompt_embeds->type; + ggml_type type = GGML_TYPE_F32; struct ggml_tensor* id_pixel_values_d = ggml_dup_tensor(ctx0, id_pixel_values); ggml_allocr_alloc(allocr, id_pixel_values_d); @@ -441,11 +446,13 @@ struct PhotoMakerIDEncoder : public GGMLModule { struct ggml_tensor * cls = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, batch_size); struct ggml_tensor * class_embedding_temp = ggml_new_tensor_4d(ctx0, type, - vision_model.hidden_size, 1, 1, batch_size); + vision_model.hidden_size, batch_size, 1, 1); struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions); + struct ggml_tensor * cast_f32_class = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, vision_model.hidden_size); ggml_allocr_alloc(allocr, cls); ggml_allocr_alloc(allocr, class_embedding_temp); ggml_allocr_alloc(allocr, positions); + ggml_allocr_alloc(allocr, cast_f32_class); @@ -467,6 +474,11 @@ struct PhotoMakerIDEncoder : public GGMLModule { ggml_backend_tensor_set(cls, cls_h.data(), 0, ggml_nbytes(cls)); ggml_backend_tensor_set(positions, pos.data(), 0, ggml_nbytes(positions)); ggml_backend_tensor_set(class_tokens_mask_pos, ctmpos.data(), 0, ggml_nbytes(class_tokens_mask_pos)); + std::vector zeros; + for (int i = 0; i < hidden_size; i++) { + zeros.push_back(0.f); + } + ggml_backend_tensor_set(cast_f32_class, zeros.data(), 0, ggml_nbytes(cast_f32_class)); if(left){ if(type == GGML_TYPE_F16){ std::vector zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f)); diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 45875215..bbce9318 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -218,6 +218,8 @@ public: LOG_ERROR(" pmid model params buffer allocation failed"); return false; } + LOG_INFO("pmid param memory buffer size = %.2fMB ", + pmid_model.params_buffer_size / 1024.0 / 1024.0); } @@ -246,11 +248,14 @@ public: first_stage_model.init_params(); } first_stage_model.map_by_name(tensors, "first_stage_model."); - + if(stacked_id){ + printf("preparing memory for pmid \n"); pmid_model.init_params(); pmid_model.map_by_name(tensors, "pmid."); + printf("done preparing memory for pmid \n"); } + } struct ggml_init_params params; @@ -311,6 +316,8 @@ public: cond_stage_model.params_buffer_size + diffusion_model.params_buffer_size + first_stage_model.params_buffer_size; + if(stacked_id) + total_params_size += pmid_model.params_buffer_size; LOG_INFO("total memory buffer size = %.2fMB (clip %.2fMB, unet %.2fMB, vae %.2fMB)", total_params_size / 1024.0 / 1024.0, cond_stage_model.params_buffer_size / 1024.0 / 1024.0,