DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 19:36:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

photomaker working properly now after merging and adapting to GGMLBlock API

Browse Source
This commit is contained in:
bssrdf committed 2024-02-26 13:32:19 -05:00
1 parent 4ac3df145e
commit 77b6a42bc3
4 files changed
+63 -69
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

clip.hpp
+8 -29
View File
@@ -548,16 +548,7 @@ public:
auto layer_norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm2"]); auto layer_norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm2"]);
auto mlp = std::dynamic_pointer_cast<CLIPMLP>(blocks["mlp"]); auto mlp = std::dynamic_pointer_cast<CLIPMLP>(blocks["mlp"]);
// struct ggml_tensor* r = x; x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask));
struct ggml_tensor* h = layer_norm1->forward(ctx, x);
h = self_attn->forward(ctx, h, mask);
// x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask));
x = ggml_add(ctx, x, h);
x = ggml_add(ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x))); x = ggml_add(ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x)));
return x; return x;
} }
@@ -567,16 +558,10 @@ public:
auto layer_norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm2"]); auto layer_norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm2"]);
auto mlp = std::dynamic_pointer_cast<CLIPMLP>(blocks["mlp"]); auto mlp = std::dynamic_pointer_cast<CLIPMLP>(blocks["mlp"]);
// struct ggml_tensor* r = x; // struct ggml_tensor* h = layer_norm1->forward(ctx, x);
// h = self_attn->forward(ctx, h, mask);
struct ggml_tensor* h = layer_norm1->forward(ctx, x); x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask));
// x = ggml_add(ctx, x, h);
h = self_attn->forward(ctx, h, mask);
// x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask));
x = ggml_add(ctx, x, h);
x = ggml_add(ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x))); x = ggml_add(ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x)));
return x; return x;
} }
@@ -614,10 +599,8 @@ public:
break; break;
} }
std::string name = "layers." + std::to_string(i); std::string name = "layers." + std::to_string(i);
// printf(" about to do %s\n", name.c_str());
auto layer = std::dynamic_pointer_cast<CLIPLayer>(blocks[name]); auto layer = std::dynamic_pointer_cast<CLIPLayer>(blocks[name]);
x = layer->forward(ctx, x, mask, atten1); // [N, n_token, d_model] x = layer->forward(ctx, x, mask, atten1); // [N, n_token, d_model]
// print_ggml_tensor(x, true, ("layer "+std::to_string(i)).c_str());
// LOG_DEBUG("layer %d", i); // LOG_DEBUG("layer %d", i);
} }
return x; return x;
@@ -924,21 +907,17 @@ public:
auto post_layernorm = std::dynamic_pointer_cast<LayerNorm>(blocks["post_layernorm"]); auto post_layernorm = std::dynamic_pointer_cast<LayerNorm>(blocks["post_layernorm"]);
auto x = embeddings->forward(ctx, pixel_values); // [N, num_positions, embed_dim] auto x = embeddings->forward(ctx, pixel_values); // [N, num_positions, embed_dim]
print_ggml_tensor(x, true, "embedding");
x = pre_layernorm->forward(ctx, x); x = pre_layernorm->forward(ctx, x);
print_ggml_tensor(x, true, "pre_layernorm");
x = encoder->forward(ctx, x, -1, false, false); x = encoder->forward(ctx, x, -1, false, false);
print_ggml_tensor(x, true, "encoder");
x = post_layernorm->forward(ctx, x); // [N, n_token, hidden_size] x = post_layernorm->forward(ctx, x); // [N, n_token, hidden_size]
print_ggml_tensor(x, true, "post_layernorm");
GGML_ASSERT(x->ne[3] == 1); GGML_ASSERT(x->ne[3] == 1);
// int64_t max_token_idx = 0; // int64_t max_token_idx = 0;
// ggml_tensor* pooled = ggml_view_1d(ctx, x, x->ne[0], x->nb[1] * max_token_idx); // assert N == 1 // ggml_tensor* pooled = ggml_view_1d(ctx, x, x->ne[0], x->nb[1] * max_token_idx); // assert N == 1
x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));
// ggml_view_2d(ctx, w, w->ne[0], w->ne[1] / 2, w->nb[1], 0); ggml_tensor* pooled = ggml_view_2d(ctx, x, x->ne[0], x->ne[1], x->nb[0], 0);
ggml_tensor* pooled = ggml_view_2d(ctx, x, x->ne[0], x->ne[1], x->nb[1], 0); pooled = ggml_scale(ctx, ggml_cont(ctx, pooled), 1.f);
print_ggml_tensor(pooled, true, "pooled"); // print_ggml_tensor(pooled, true, "pooled");
return pooled; // [N, projection_dim] return pooled; // [N, projection_dim]
} }
ggml_extend.hpp
-8
View File
@@ -926,11 +926,8 @@ public:
alloc_compute_buffer(get_graph); alloc_compute_buffer(get_graph);
reset_compute_ctx(); reset_compute_ctx();
struct ggml_cgraph* gf = get_graph(); struct ggml_cgraph* gf = get_graph();
GGML_ASSERT(ggml_gallocr_alloc_graph(compute_allocr, gf)); GGML_ASSERT(ggml_gallocr_alloc_graph(compute_allocr, gf));
cpy_data_to_backend_tensor(); cpy_data_to_backend_tensor();
if (ggml_backend_is_cpu(backend)) { if (ggml_backend_is_cpu(backend)) {
ggml_backend_cpu_set_n_threads(backend, n_threads); ggml_backend_cpu_set_n_threads(backend, n_threads);
} }
@@ -1037,16 +1034,11 @@ public:
} }
for (auto& pair : blocks) { for (auto& pair : blocks) {
auto& block = pair.second; auto& block = pair.second;
// if(starts_with(prefix, "pmid"))
// printf("block pair.first: %s, %s \n", prefix.c_str(), pair.first.c_str());
block->get_param_tensors(tensors, prefix + pair.first); block->get_param_tensors(tensors, prefix + pair.first);
} }
for (auto& pair : params) { for (auto& pair : params) {
struct ggml_tensor* param = pair.second; struct ggml_tensor* param = pair.second;
// if(starts_with(prefix, "pmid"))
// printf("params pair.first: %s, %s \n", prefix.c_str(), pair.first.c_str());
tensors[prefix + pair.first] = pair.second; tensors[prefix + pair.first] = pair.second;
} }
} }
pmid.hpp
+48 -29
View File
@@ -225,7 +225,13 @@ public:
struct ggml_tensor * valid_id_embeds = id_embeds; struct ggml_tensor * valid_id_embeds = id_embeds;
// # slice out the image token embeddings // # slice out the image token embeddings
// print_ggml_tensor(class_tokens_mask_pos, false);
ggml_set_name(class_tokens_mask_pos, "class_tokens_mask_pos");
ggml_set_name(prompt_embeds, "prompt_embeds");
// print_ggml_tensor(valid_id_embeds, true, "valid_id_embeds");
// print_ggml_tensor(class_tokens_mask_pos, true, "class_tokens_mask_pos");
struct ggml_tensor * image_token_embeds = ggml_get_rows(ctx, prompt_embeds, class_tokens_mask_pos); struct ggml_tensor * image_token_embeds = ggml_get_rows(ctx, prompt_embeds, class_tokens_mask_pos);
ggml_set_name(image_token_embeds, "image_token_embeds");
struct ggml_tensor *stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds); struct ggml_tensor *stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds);
stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3));
@@ -242,6 +248,7 @@ public:
class_tokens_mask = ggml_repeat(ctx, class_tokens_mask, prompt_embeds); class_tokens_mask = ggml_repeat(ctx, class_tokens_mask, prompt_embeds);
prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask); prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask);
struct ggml_tensor * updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds); struct ggml_tensor * updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds);
ggml_set_name(updated_prompt_embeds, "updated_prompt_embeds");
return updated_prompt_embeds; return updated_prompt_embeds;
} }
@@ -289,6 +296,16 @@ public:
VisualProjection visual_projection_2; VisualProjection visual_projection_2;
float style_strength; float style_strength;
std::vector<float> ctm;
std::vector<ggml_fp16_t> ctmf16;
std::vector<int> ctmpos;
std::vector<ggml_fp16_t> zeros_left_16;
std::vector<float> zeros_left;
std::vector<ggml_fp16_t> zeros_right_16;
std::vector<float> zeros_right;
public: public:
PhotoMakerIDEncoder(ggml_backend_t backend, ggml_type wtype, PhotoMakerIDEncoder(ggml_backend_t backend, ggml_type wtype,
SDVersion version = VERSION_XL, float sty = 20.f) SDVersion version = VERSION_XL, float sty = 20.f)
@@ -382,14 +399,14 @@ public:
// positions // positions
// ); // [batch_size, seq_length, hidden_size] // ); // [batch_size, seq_length, hidden_size]
struct ggml_tensor *shared_id_embeds = vision_model.forward(ctx, id_pixel_values); // [batch_size, seq_length, hidden_size] struct ggml_tensor *shared_id_embeds = vision_model.forward(ctx, id_pixel_values); // [batch_size, seq_length, hidden_size]
print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds"); // print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds");
struct ggml_tensor *id_embeds = vision_model.visual_project(ctx, shared_id_embeds); // [batch_size, seq_length, proj_dim(768)] struct ggml_tensor *id_embeds = vision_model.visual_project(ctx, shared_id_embeds); // [batch_size, seq_length, proj_dim(768)]
print_ggml_tensor(id_embeds, true, "id_embeds"); // print_ggml_tensor(id_embeds, true, "id_embeds");
// struct ggml_tensor *id_embeds_2 = ggml_mul_mat(ctx, visual_projection_2, shared_id_embeds); // [batch_size, seq_length, 1280] // struct ggml_tensor *id_embeds_2 = ggml_mul_mat(ctx, visual_projection_2, shared_id_embeds); // [batch_size, seq_length, 1280]
struct ggml_tensor *id_embeds_2 = visual_projection_2.forward(ctx, shared_id_embeds); // [batch_size, seq_length, 1280] struct ggml_tensor *id_embeds_2 = visual_projection_2.forward(ctx, shared_id_embeds); // [batch_size, seq_length, 1280]
print_ggml_tensor(id_embeds_2, true, "id_embeds_2"); // print_ggml_tensor(id_embeds_2, true, "id_embeds_2");
@@ -399,14 +416,14 @@ public:
id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right
id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3)); id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3));
print_ggml_tensor(id_embeds, true, "id_embeds_after_cont+perm"); // print_ggml_tensor(id_embeds, true, "id_embeds_after_cont+perm");
struct ggml_tensor * updated_prompt_embeds = fuse_module.forward(ctx, struct ggml_tensor * updated_prompt_embeds = fuse_module.forward(ctx,
prompt_embeds, id_embeds, prompt_embeds, id_embeds,
class_tokens_mask, class_tokens_mask,
class_tokens_mask_pos, class_tokens_mask_pos,
left, right); left, right);
print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds"); // print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds");
return updated_prompt_embeds; return updated_prompt_embeds;
@@ -431,6 +448,15 @@ public:
// struct ggml_cgraph* gf = ggml_new_graph(ctx0); // struct ggml_cgraph* gf = ggml_new_graph(ctx0);
ctm.clear();
ctmf16.clear();
ctmpos.clear();
zeros_left.clear();
zeros_left_16.clear();
zeros_right.clear();
zeros_right_16.clear();
ggml_context *ctx0 = compute_ctx; ggml_context *ctx0 = compute_ctx;
struct ggml_cgraph* gf = ggml_new_graph(compute_ctx); struct ggml_cgraph* gf = ggml_new_graph(compute_ctx);
@@ -450,9 +476,6 @@ public:
struct ggml_tensor* id_pixel_values_d = to_backend(id_pixel_values); struct ggml_tensor* id_pixel_values_d = to_backend(id_pixel_values);
struct ggml_tensor* prompt_embeds_d = to_backend(prompt_embeds); struct ggml_tensor* prompt_embeds_d = to_backend(prompt_embeds);
std::vector<float> ctm;
std::vector<ggml_fp16_t> ctmf16;
std::vector<int> ctmpos;
struct ggml_tensor* left = NULL; struct ggml_tensor* left = NULL;
struct ggml_tensor* right = NULL; struct ggml_tensor* right = NULL;
for(int i=0; i < class_tokens_mask.size(); i++){ for(int i=0; i < class_tokens_mask.size(); i++){
@@ -517,24 +540,32 @@ public:
set_backend_tensor_data(class_tokens_mask_pos, ctmpos.data()); set_backend_tensor_data(class_tokens_mask_pos, ctmpos.data());
if(left){ if(left){
if(type == GGML_TYPE_F16){ if(type == GGML_TYPE_F16){
std::vector<ggml_fp16_t> zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f)); // std::vector<ggml_fp16_t> zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f));
for(int i = 0; i < ggml_nelements(left); ++i)
zeros_left_16.push_back(ggml_fp32_to_fp16(0.f));
// ggml_backend_tensor_set(left, zeros.data(), 0, ggml_nbytes(left)); // ggml_backend_tensor_set(left, zeros.data(), 0, ggml_nbytes(left));
set_backend_tensor_data(left, zeros.data()); set_backend_tensor_data(left, zeros_left_16.data());
}else{ }else{
std::vector<float> zeros(ggml_nelements(left), 0.f); // std::vector<float> zeros(ggml_nelements(left), 0.f);
for(int i = 0; i < ggml_nelements(left); ++i)
zeros_left.push_back(0.f);
// ggml_backend_tensor_set(left, zeros.data(), 0, ggml_nbytes(left)); // ggml_backend_tensor_set(left, zeros.data(), 0, ggml_nbytes(left));
set_backend_tensor_data(left, zeros.data()); set_backend_tensor_data(left, zeros_left.data());
} }
} }
if(right){ if(right){
if(type == GGML_TYPE_F16){ if(type == GGML_TYPE_F16){
std::vector<ggml_fp16_t> zeros(ggml_nelements(right), ggml_fp32_to_fp16(0.f)); // std::vector<ggml_fp16_t> zeros(ggml_nelements(right), ggml_fp32_to_fp16(0.f));
// ggml_backend_tensor_set(right, zeros.data(), 0, ggml_nbytes(right)); // ggml_backend_tensor_set(right, zeros.data(), 0, ggml_nbytes(right));
set_backend_tensor_data(right, zeros.data()); for(int i = 0; i < ggml_nelements(right); ++i)
zeros_right_16.push_back(ggml_fp32_to_fp16(0.f));
set_backend_tensor_data(right, zeros_right_16.data());
}else{ }else{
std::vector<float> zeros(ggml_nelements(right), 0.f); // std::vector<float> zeros(ggml_nelements(right), 0.f);
for(int i = 0; i < ggml_nelements(right); ++i)
zeros_right.push_back(0.f);
// ggml_backend_tensor_set(right, zeros.data(), 0, ggml_nbytes(right)); // ggml_backend_tensor_set(right, zeros.data(), 0, ggml_nbytes(right));
set_backend_tensor_data(right, zeros.data()); set_backend_tensor_data(right, zeros_right.data());
} }
} }
} }
@@ -554,18 +585,6 @@ public:
return gf; return gf;
} }
void alloc_compute_buffer(ggml_context* work_ctx,
struct ggml_tensor* id_pixel_values,
struct ggml_tensor* prompt_embeds,
std::vector<bool> &class_tokens_mask) {
auto get_graph = [&]() -> struct ggml_cgraph* {
// return build_graph(compute_allocr, id_pixel_values, prompt_embeds, class_tokens_mask);
return build_graph(id_pixel_values, prompt_embeds, class_tokens_mask);
};
GGMLModule::alloc_compute_buffer(get_graph);
}
void compute(const int n_threads, void compute(const int n_threads,
struct ggml_tensor* id_pixel_values, struct ggml_tensor* id_pixel_values,
struct ggml_tensor* prompt_embeds, struct ggml_tensor* prompt_embeds,
@@ -614,7 +633,7 @@ public:
} }
std::string get_desc() { std::string get_desc() {
return "pmid lora"; return "lora_pmid";
} }
size_t get_params_num() { size_t get_params_num() {
stable-diffusion.cpp
+7 -3
View File
@@ -554,6 +554,10 @@ public:
// for(int i = 0; i < tokens.size(); ++i) // for(int i = 0; i < tokens.size(); ++i)
// printf("%d ", tokens[i]); // printf("%d ", tokens[i]);
// printf("\n"); // printf("\n");
// printf("clsm: \n");
// for(int i = 0; i < clsm.size(); ++i)
// printf("%d ", clsm[i]?1:0);
// printf("\n");
int64_t t0 = ggml_time_ms(); int64_t t0 = ggml_time_ms();
struct ggml_tensor* hidden_states = NULL; // [N, n_token, hidden_size] struct ggml_tensor* hidden_states = NULL; // [N, n_token, hidden_size]
struct ggml_tensor* pooled = NULL; struct ggml_tensor* pooled = NULL;
@@ -1691,16 +1695,16 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx,
else else
sd_mul_images_to_tensor(init_image->data, init_img, i, NULL, NULL); sd_mul_images_to_tensor(init_image->data, init_img, i, NULL, NULL);
} }
t0 = ggml_time_ms();
auto cond_tup = sd_ctx->sd->get_learned_condition_with_trigger(work_ctx, prompt, auto cond_tup = sd_ctx->sd->get_learned_condition_with_trigger(work_ctx, prompt,
clip_skip, width, height, num_input_images ); clip_skip, width, height, num_input_images );
LOG_INFO("get_learned_condition_with_trigger finished");
prompts_embeds = std::get<0>(cond_tup); prompts_embeds = std::get<0>(cond_tup);
pooled_prompts_embeds = std::get<1>(cond_tup); // [adm_in_channels, ] pooled_prompts_embeds = std::get<1>(cond_tup); // [adm_in_channels, ]
class_tokens_mask = std::get<2>(cond_tup); // class_tokens_mask = std::get<2>(cond_tup); //
prompts_embeds = sd_ctx->sd->id_encoder(work_ctx, init_img, prompts_embeds, class_tokens_mask); prompts_embeds = sd_ctx->sd->id_encoder(work_ctx, init_img, prompts_embeds, class_tokens_mask);
LOG_INFO("id_encoder"); t1 = ggml_time_ms();
LOG_INFO("Photomaker ID Stacking, taking %" PRId64 " ms", t1 - t0);
if (sd_ctx->sd->free_params_immediately) { if (sd_ctx->sd->free_params_immediately) {
sd_ctx->sd->pmid_model->free_params_buffer(); sd_ctx->sd->pmid_model->free_params_buffer();
} }