turn pmid lora apply back on

clip.hpp

@@ -973,15 +973,15 @@ struct CLIPVisionModel {
         struct ggml_tensor * inp = ggml_conv_2d(ctx0, patch_embeddings_f16, x, patch_size, patch_size, 0, 0, 1, 1);
 
         ggml_set_name(inp, "inp_conv_2d");
-        // print_ggml_tensor(inp, true, "inp_conv_2d");
+        print_ggml_tensor(inp, true, "inp_conv_2d");
 
         inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size);
         ggml_set_name(inp, "inp_reshape_3d");
-        // print_ggml_tensor(inp, true, "inp_reshape_3d");
+        print_ggml_tensor(inp, true, "inp_reshape_3d");
         // inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
         // print_ggml_tensor(ggml_permute(ctx0, inp, 2, 0, 1, 3), true, "inp_permute");
         inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
-        // print_ggml_tensor(inp, true, "inp_cont");
+        print_ggml_tensor(inp, true, "inp_cont_perm+cont");
         ggml_set_name(inp, "inp_cont");
         // ggml_set_name(class_embedding, "class_embedding");
 
@@ -991,18 +991,18 @@ struct CLIPVisionModel {
         // struct ggml_tensor * temp = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, 1, batch_size);
         // ggml_tensor *class_embedding_rep = ggml_add(ctx0, cast_f32_class, class_embedding);
         // ggml_set_name(class_embedding_rep, "add_class_embedding_to_zero");
-        // print_ggml_tensor(class_embedding, true, "model.class_embedding");
+        print_ggml_tensor(class_embedding, true, "model.class_embedding");
         // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_bef_repeat");
-        // print_ggml_tensor(temp, true, "temp");
+        print_ggml_tensor(temp, true, "temp");
         ggml_tensor *class_embedding_rep = ggml_repeat(ctx0, class_embedding, temp);
 
         ggml_set_name(class_embedding_rep, "class_embedding_rep");
-        // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep");
+        print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep");
         // class_embedding_rep = ggml_cast(ctx0, class_embedding_rep, inp->type);
         // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_aft_casting");
         struct ggml_tensor *embeddings =  ggml_concat(ctx0, class_embedding_rep, inp);
         ggml_set_name(embeddings, "embeddings_after_concat");
-        // print_ggml_tensor(embeddings, true, "embeddings_after_concat");
+        print_ggml_tensor(embeddings, true, "embeddings_after_concat");
         // print_ggml_tensor(ggml_permute(ctx0, embeddings, 0, 3, 1, 2), true, "embeddings_after_concat_permute");
         embeddings =  ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 2, 1, 3));
         ggml_set_name(embeddings, "embeddings_after_permute");
@@ -1019,7 +1019,7 @@ struct CLIPVisionModel {
             // ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings));
             ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, position_embeddings, positions));
         ggml_set_name(embeddings, "embeddings_after_add");
-        // print_ggml_tensor(embeddings, true, "embeddings_after_add");
+        print_ggml_tensor(embeddings, true, "embeddings_to_transformer");
 
         // pre-layernorm        
         embeddings = ggml_nn_layer_norm(ctx0, embeddings, pre_ln_w, pre_ln_w);
@@ -1342,12 +1342,18 @@ struct FrozenCLIPEmbedderWithCustomWords : public GGMLModule {
         }
 
          for(uint32_t i = 0; i < tokens.size(); i++){       
-            if(class_token_index[0] <= i && i < class_token_index[0]+num_input_imgs)
+            if(class_token_index[0]+1 <= i && i < class_token_index[0]+1+num_input_imgs)
                 class_token_mask.push_back(true);
             else
                 class_token_mask.push_back(false);
         }
 
+        printf("[");
+        for (int i = 0; i < tokens.size(); i++) {
+            printf("%d, ", class_token_mask[i] ? 1 : 0);
+        }
+        printf("]\n");
+
         // for (int i = 0; i < tokens.size(); i++) {
         //     std::cout << tokens[i] << ":" << weights[i] << ", ";
         // }

pmid.hpp

@@ -211,10 +211,10 @@ struct FuseModule{
         // print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_before_concat");
 
         stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3));
-        // print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute");
+        print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute");
         if(left && right){
-            // print_ggml_tensor(left, true, "left");
+            print_ggml_tensor(left, true, "left");
-            // print_ggml_tensor(right, true, "right");
+            print_ggml_tensor(right, true, "right");
             stacked_id_embeds = ggml_concat(ctx, left, stacked_id_embeds);
             stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right);
         }else if(left){
@@ -474,10 +474,10 @@ struct PhotoMakerIDEncoder : public GGMLModule {
             ggml_backend_tensor_set(cls, cls_h.data(), 0, ggml_nbytes(cls));
             ggml_backend_tensor_set(positions, pos.data(), 0, ggml_nbytes(positions));
             ggml_backend_tensor_set(class_tokens_mask_pos, ctmpos.data(), 0, ggml_nbytes(class_tokens_mask_pos));
-            std::vector<float> zeros;
+            // std::vector<float> zeros;
-            for (int i = 0; i < hidden_size; i++) {
+            // for (int i = 0; i < hidden_size; i++) {
-                zeros.push_back(0.f);
+            //     zeros.push_back(0.f);
-            }
+            // }
             if(left){
                 if(type == GGML_TYPE_F16){
                     std::vector<ggml_fp16_t> zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f));
@@ -752,15 +752,23 @@ struct PhotoMakerLoraModel : public GGMLModule {
                 continue;
             }
 
-            ggml_tensor* lora_up_orig = lora_up;
+            // print_ggml_tensor(lora_down, true, lora_down_name.c_str());
+            // print_ggml_tensor(lora_up, true, lora_up_name.c_str());
+
+            // ggml_tensor* lora_up_orig = lora_up;
 
             applied_lora_tensors.insert(lora_up_name);
             applied_lora_tensors.insert(lora_down_name);
             
             // ggml_mul_mat requires tensor b transposed
+            // lora_down                  = ggml_cont(ctx0, ggml_transpose(ctx0, lora_down));
+            // struct ggml_tensor* updown = ggml_mul_mat(ctx0, lora_down, lora_up);
+            // updown                     = ggml_cont(ctx0, updown);
+            // same as in lora.hpp
             lora_down                  = ggml_cont(ctx0, ggml_transpose(ctx0, lora_down));
-            struct ggml_tensor* updown = ggml_mul_mat(ctx0, lora_down, lora_up);
+            struct ggml_tensor* updown = ggml_mul_mat(ctx0, lora_up, lora_down);
-            updown                     = ggml_cont(ctx0, updown);
+            updown                     = ggml_cont(ctx0, ggml_transpose(ctx0, updown));
+            updown                     = ggml_reshape(ctx0, updown, weight);
             GGML_ASSERT(ggml_nelements(updown) == ggml_nelements(weight));
             updown = ggml_scale_inplace(ctx0, updown, multiplier);
             ggml_tensor* final_weight;

stable-diffusion.cpp

@@ -1471,8 +1471,8 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx,
     LOG_INFO("apply_loras completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
 
     if(sd_ctx->sd->stacked_id){
-        // sd_ctx->sd->pmid_lora.apply(sd_ctx->sd->tensors, sd_ctx->sd->n_threads);
+        sd_ctx->sd->pmid_lora.apply(sd_ctx->sd->tensors, sd_ctx->sd->n_threads);
-        // sd_ctx->sd->pmid_lora.free_compute_buffer();
+        sd_ctx->sd->pmid_lora.free_compute_buffer();
         if (sd_ctx->sd->free_params_immediately) {
             sd_ctx->sd->pmid_lora.free_params_buffer();
         }