DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 11:26:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed a bug in FuseBlock forward; also remove diag_mask op in for vision transformer; getting better results

Browse Source
This commit is contained in:
bssrdf committed 2024-02-15 18:57:19 -05:00
1 parent 26c5cca043
commit 191c5ec05a
4 files changed
+76 -47
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

clip.hpp
+15 -13
View File
@@ -482,6 +482,7 @@ struct ResidualAttentionBlock {
int32_t d_model;
int32_t hidden_size; // n_head * d_model
int32_t intermediate_size;
bool apply_diag_mask = true;
// attention
struct ggml_tensor* q_w; // [hidden_size, hidden_size]
@@ -596,8 +597,8 @@ struct ResidualAttentionBlock {
v = ggml_reshape_3d(ctx, v, n_token, d_model, n_head * N); // [N * n_head, d_model, n_token]
struct ggml_tensor* kq = ggml_mul_mat(ctx, k, q); // [N * n_head, n_token, n_token]
kq = ggml_diag_mask_inf_inplace(ctx, kq, 0);
if(apply_diag_mask)
kq = ggml_diag_mask_inf_inplace(ctx, kq, 0);
kq = ggml_soft_max_inplace(ctx, kq);
struct ggml_tensor* kqv = ggml_mul_mat(ctx, v, kq); // [N * n_head, n_token, d_model]
@@ -904,6 +905,7 @@ struct CLIPVisionModel {
resblocks[i].n_head = n_head;
resblocks[i].hidden_size = hidden_size;
resblocks[i].intermediate_size = intermediate_size;
resblocks[i].apply_diag_mask = false;
}
}
@@ -961,7 +963,7 @@ struct CLIPVisionModel {
const int d_head = hidden_size / n_head;
// ggml_set_name(x, "id_pixel");
print_ggml_tensor(x, true, "id_pixel");
// print_ggml_tensor(x, true, "id_pixel");
ggml_set_name(temp, "temp_input");
int64_t* ne = patch_embeddings->ne;
// struct ggml_tensor *patch_embeddings_f16 = ggml_reshape_3d(ctx0, patch_embeddings, ne[0], ne[1], ne[2]*ne[3]);
@@ -971,15 +973,15 @@ struct CLIPVisionModel {
struct ggml_tensor * inp = ggml_conv_2d(ctx0, patch_embeddings_f16, x, patch_size, patch_size, 0, 0, 1, 1);
ggml_set_name(inp, "inp_conv_2d");
print_ggml_tensor(inp, true, "inp_conv_2d");
// print_ggml_tensor(inp, true, "inp_conv_2d");
inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size);
ggml_set_name(inp, "inp_reshape_3d");
print_ggml_tensor(inp, true, "inp_reshape_3d");
// print_ggml_tensor(inp, true, "inp_reshape_3d");
// inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
// print_ggml_tensor(ggml_permute(ctx0, inp, 2, 0, 1, 3), true, "inp_permute");
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
print_ggml_tensor(inp, true, "inp_cont");
// print_ggml_tensor(inp, true, "inp_cont");
ggml_set_name(inp, "inp_cont");
// ggml_set_name(class_embedding, "class_embedding");
@@ -997,7 +999,7 @@ struct CLIPVisionModel {
ggml_set_name(class_embedding_rep, "class_embedding_rep");
// print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep");
// class_embedding_rep = ggml_cast(ctx0, class_embedding_rep, inp->type);
print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_aft_casting");
// print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_aft_casting");
struct ggml_tensor *embeddings = ggml_concat(ctx0, class_embedding_rep, inp);
ggml_set_name(embeddings, "embeddings_after_concat");
// print_ggml_tensor(embeddings, true, "embeddings_after_concat");
@@ -1017,7 +1019,7 @@ struct CLIPVisionModel {
// ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings));
ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, position_embeddings, positions));
ggml_set_name(embeddings, "embeddings_after_add");
print_ggml_tensor(embeddings, true, "embeddings_after_add");
// print_ggml_tensor(embeddings, true, "embeddings_after_add");
// pre-layernorm
embeddings = ggml_nn_layer_norm(ctx0, embeddings, pre_ln_w, pre_ln_w);
@@ -1028,12 +1030,12 @@ struct CLIPVisionModel {
embeddings = resblocks[i].forward(ctx0, embeddings); // [N, n_token, hidden_size]
}
ggml_set_name(embeddings, "embeddings_after_transformer");
print_ggml_tensor(embeddings, true, "embeddings_after_encoder");
// print_ggml_tensor(embeddings, true, "embeddings_after_encoder");
// get the output of cls token, e.g., 0th index
embeddings = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, embeddings, hidden_size, num_positions * batch_size), cls);
ggml_set_name(embeddings, "embeddings_after_cls_token");
print_ggml_tensor(embeddings, true, "embeddings_pooled_after_encoder");
// print_ggml_tensor(embeddings, true, "embeddings_pooled_after_encoder");
// post-layernorm
embeddings = ggml_nn_layer_norm(ctx0, embeddings, post_ln_w, post_ln_b);
@@ -1055,13 +1057,13 @@ struct CLIPVisionModel {
}
class_embedding = ggml_new_tensor_1d(ctx, wtype, hidden_size);
ggml_set_name(class_embedding, "class_embedding");
ggml_set_name(class_embedding, "vision.class_embedding");
patch_embeddings = ggml_new_tensor_4d(ctx, wtype, patch_size, patch_size, 3, hidden_size);
ggml_set_name(patch_embeddings, "patch_embeddings");
ggml_set_name(patch_embeddings, "vision.patch_embeddings");
position_embeddings = ggml_new_tensor_2d(ctx, wtype, hidden_size, 257);
ggml_set_name(position_embeddings, "position_embeddings");
ggml_set_name(position_embeddings, "vision.position_embeddings");
pre_ln_w = ggml_new_tensor_1d(ctx, wtype, hidden_size);