DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 19:36:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

merged with master and resolved conflicts; adapted to GGMLBlock API

Browse Source
This commit is contained in:
bssrdf committed 2024-02-25 14:35:37 -05:00
2 parents b7a454049d 193fb620b1
commit 53beb92d2f
20 files changed
+4605 -4288
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pmid.hpp
+285 -152
View File
@@ -6,7 +6,7 @@
#include "clip.hpp"
struct FuseBlock {
struct FuseBlock : public GGMLBlock {
// network hparams
int in_dim;
int out_dim;
@@ -17,73 +17,96 @@ struct FuseBlock {
// in_layers
// layer norm
struct ggml_tensor* ln_w; // [in_dim, ]
struct ggml_tensor* ln_b; // [in_dim, ]
// struct ggml_tensor* ln_w; // [in_dim, ]
// struct ggml_tensor* ln_b; // [in_dim, ]
struct ggml_tensor* fc1_w; // [in_dim, hidden_dim]
struct ggml_tensor* fc1_b; // [in_dim, ]
struct ggml_tensor* fc2_w; // [hidden_dim, out_dim ]
struct ggml_tensor* fc2_b; // [hidden_dim, ]
// struct ggml_tensor* fc1_w; // [in_dim, hidden_dim]
// struct ggml_tensor* fc1_b; // [in_dim, ]
// struct ggml_tensor* fc2_w; // [hidden_dim, out_dim ]
// struct ggml_tensor* fc2_b; // [hidden_dim, ]
public:
FuseBlock(int i_d, int o_d, int h_d, bool use_residue = true)
: in_dim(i_d), out_dim(o_d), hidden_dim(h_d),
use_residue(use_residue){
// blocks["fc1"] = std::shared_ptr<GGMLBlock>(new Linear(d_model, intermediate_size));
// blocks["fc2"] = std::shared_ptr<GGMLBlock>(new Linear(intermediate_size, d_model));
blocks["fc1"] = std::shared_ptr<GGMLBlock>(new Linear(in_dim, hidden_dim, true));
blocks["fc2"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_dim, out_dim, true));
blocks["layer_norm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(in_dim));
}
size_t calculate_mem_size(ggml_type wtype) {
size_t mem_size = 0;
mem_size += 2 * ggml_row_size(wtype, in_dim);
mem_size += ggml_row_size(wtype, in_dim*hidden_dim);
mem_size += ggml_row_size(wtype, out_dim);
mem_size += ggml_row_size(wtype, hidden_dim*out_dim);
mem_size += ggml_row_size(wtype, hidden_dim);
// size_t calculate_mem_size(ggml_type wtype) {
// size_t mem_size = 0;
// mem_size += 2 * ggml_row_size(wtype, in_dim);
// mem_size += ggml_row_size(wtype, in_dim*hidden_dim);
// mem_size += ggml_row_size(wtype, out_dim);
// mem_size += ggml_row_size(wtype, hidden_dim*out_dim);
// mem_size += ggml_row_size(wtype, hidden_dim);
return mem_size;
}
// return mem_size;
// }
void init_params(struct ggml_context* ctx, ggml_type wtype, ggml_allocr* alloc) {
ln_w = ggml_new_tensor_1d(ctx, wtype, in_dim);
ln_b = ggml_new_tensor_1d(ctx, wtype, in_dim);
// void init_params(struct ggml_context* ctx, ggml_type wtype, ggml_allocr* alloc) {
// ln_w = ggml_new_tensor_1d(ctx, wtype, in_dim);
// ln_b = ggml_new_tensor_1d(ctx, wtype, in_dim);
fc1_b = ggml_new_tensor_1d(ctx, wtype, hidden_dim);
fc1_w = ggml_new_tensor_2d(ctx, wtype, in_dim, hidden_dim);
fc2_b = ggml_new_tensor_1d(ctx, wtype, out_dim);
fc2_w = ggml_new_tensor_2d(ctx, wtype, hidden_dim, out_dim);
// alloc all tensors linked to this context
for (struct ggml_tensor* t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
if (t->data == NULL) {
ggml_allocr_alloc(alloc, t);
}
}
// fc1_b = ggml_new_tensor_1d(ctx, wtype, hidden_dim);
// fc1_w = ggml_new_tensor_2d(ctx, wtype, in_dim, hidden_dim);
// fc2_b = ggml_new_tensor_1d(ctx, wtype, out_dim);
// fc2_w = ggml_new_tensor_2d(ctx, wtype, hidden_dim, out_dim);
// // alloc all tensors linked to this context
// for (struct ggml_tensor* t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
// if (t->data == NULL) {
// ggml_allocr_alloc(alloc, t);
// }
// }
}
// }s
// size_t get_num_tensors() {
// return 6;
// }
size_t get_num_tensors() {
return 6;
}
void map_by_name(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
tensors[prefix + "fc1.weight"] = fc1_w;
tensors[prefix + "fc1.bias"] = fc1_b;
tensors[prefix + "fc2.weight"] = fc2_w;
tensors[prefix + "fc2.bias"] = fc2_b;
tensors[prefix + "layernorm.weight"] = ln_w;
tensors[prefix + "layernorm.bias"] = ln_b;
}
// void map_by_name(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
// tensors[prefix + "fc1.weight"] = fc1_w;
// tensors[prefix + "fc1.bias"] = fc1_b;
// tensors[prefix + "fc2.weight"] = fc2_w;
// tensors[prefix + "fc2.bias"] = fc2_b;
// tensors[prefix + "layernorm.weight"] = ln_w;
// tensors[prefix + "layernorm.bias"] = ln_b;
// }
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
// x: [N, channels, h, w]
// auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
// auto fc2 = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
// x = fc1->forward(ctx, x);
// if (use_gelu) {
// x = ggml_gelu_inplace(ctx, x);
// } else {
// x = ggml_gelu_quick_inplace(ctx, x);
// }
// x = fc2->forward(ctx, x);
// return x;
// in_layers
auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
auto fc2 = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
auto layer_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm"]);
struct ggml_tensor* r = x;
x = ggml_nn_layer_norm(ctx, x, ln_w, ln_b);
x = ggml_add(ctx, ggml_mul_mat(ctx, fc1_w, x), fc1_b);
// x = ggml_nn_layer_norm(ctx, x, ln_w, ln_b);
x = layer_norm->forward(ctx, x);
// x = ggml_add(ctx, ggml_mul_mat(ctx, fc1_w, x), fc1_b);
x = fc1->forward(ctx, x);
x = ggml_gelu_inplace(ctx, x);
x = ggml_add(ctx, ggml_mul_mat(ctx, fc2_w, x), fc2_b);
x = fc2->forward(ctx, x);
// x = ggml_add(ctx, ggml_mul_mat(ctx, fc2_w, x), fc2_b);
if(use_residue)
x = ggml_add(ctx, x, r);
return x;
@@ -91,74 +114,94 @@ struct FuseBlock {
};
struct FuseModule{
struct FuseModule : public GGMLBlock{
// network hparams
int embed_dim;
struct FuseBlock mlp1;
struct FuseBlock mlp2;
// struct FuseBlock mlp1;
// struct FuseBlock mlp2;
// layer norm
struct ggml_tensor* ln_w; // [in_dim, ]
struct ggml_tensor* ln_b; // [in_dim, ]
// struct ggml_tensor* ln_w; // [in_dim, ]
// struct ggml_tensor* ln_b; // [in_dim, ]
// FuseModule(int imb_d):
// embed_dim(imb_d),
// mlp1(imb_d*2, imb_d, imb_d, false),
// mlp2(imb_d, imb_d, imb_d, true) {
// }
public:
FuseModule(int imb_d):
embed_dim(imb_d),
mlp1(imb_d*2, imb_d, imb_d, false),
mlp2(imb_d, imb_d, imb_d, true) {
embed_dim(imb_d){
blocks["mlp1"] = std::shared_ptr<GGMLBlock>(new FuseBlock(imb_d*2, imb_d, imb_d, false));
blocks["mlp2"] = std::shared_ptr<GGMLBlock>(new FuseBlock(imb_d, imb_d, imb_d, true));
blocks["layer_norm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(embed_dim));
}
void init_params(struct ggml_context* ctx, ggml_type wtype, ggml_allocr* alloc) {
ln_w = ggml_new_tensor_1d(ctx, wtype, embed_dim);
ln_b = ggml_new_tensor_1d(ctx, wtype, embed_dim);
// alloc all tensors linked to this context
mlp1.init_params(ctx, wtype, alloc);
mlp2.init_params(ctx, wtype, alloc);
for (struct ggml_tensor* t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
if (t->data == NULL) {
ggml_allocr_alloc(alloc, t);
}
}
}
// void init_params(struct ggml_context* ctx, ggml_type wtype, ggml_allocr* alloc) {
// ln_w = ggml_new_tensor_1d(ctx, wtype, embed_dim);
// ln_b = ggml_new_tensor_1d(ctx, wtype, embed_dim);
// // alloc all tensors linked to this context
// mlp1.init_params(ctx, wtype, alloc);
// mlp2.init_params(ctx, wtype, alloc);
// for (struct ggml_tensor* t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
// if (t->data == NULL) {
// ggml_allocr_alloc(alloc, t);
// }
// }
// }
void map_by_name(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
// void map_by_name(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
tensors[prefix + "layer_norm.weight"] = ln_w;
tensors[prefix + "layer_norm.bias"] = ln_b;
mlp1.map_by_name(tensors, prefix + "mlp1.");
mlp2.map_by_name(tensors, prefix + "mlp2.");
// tensors[prefix + "layer_norm.weight"] = ln_w;
// tensors[prefix + "layer_norm.bias"] = ln_b;
// mlp1.map_by_name(tensors, prefix + "mlp1.");
// mlp2.map_by_name(tensors, prefix + "mlp2.");
}
// }
size_t get_num_tensors() {
size_t n = mlp1.get_num_tensors();
n += mlp2.get_num_tensors();
n += 2;
return n;
}
// size_t get_num_tensors() {
// size_t n = mlp1.get_num_tensors();
// n += mlp2.get_num_tensors();
// n += 2;
// return n;
// }
size_t calculate_mem_size(ggml_type wtype) {
size_t mem_size = mlp1.calculate_mem_size(wtype);
mem_size += mlp2.calculate_mem_size(wtype);
mem_size += 2 * ggml_row_size(wtype, embed_dim);
return mem_size;
}
// size_t calculate_mem_size(ggml_type wtype) {
// size_t mem_size = mlp1.calculate_mem_size(wtype);
// mem_size += mlp2.calculate_mem_size(wtype);
// mem_size += 2 * ggml_row_size(wtype, embed_dim);
// return mem_size;
// }
struct ggml_tensor* fuse_fn(struct ggml_context* ctx,
struct ggml_tensor* prompt_embeds,
struct ggml_tensor* id_embeds) {
auto mlp1 = std::dynamic_pointer_cast<FuseBlock>(blocks["mlp1"]);
auto mlp2 = std::dynamic_pointer_cast<FuseBlock>(blocks["mlp2"]);
auto layer_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm"]);
auto prompt_embeds0 = ggml_cont(ctx, ggml_permute(ctx, prompt_embeds, 2, 0, 1, 3));
auto id_embeds0 = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3));
// concat is along dim 2
auto stacked_id_embeds = ggml_concat(ctx, prompt_embeds0, id_embeds0);
stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 1, 2, 0, 3));
stacked_id_embeds = mlp1.forward(ctx, stacked_id_embeds);
// stacked_id_embeds = mlp1.forward(ctx, stacked_id_embeds);
// stacked_id_embeds = ggml_add(ctx, stacked_id_embeds, prompt_embeds);
// stacked_id_embeds = mlp2.forward(ctx, stacked_id_embeds);
// stacked_id_embeds = ggml_nn_layer_norm(ctx, stacked_id_embeds, ln_w, ln_b);
stacked_id_embeds = mlp1->forward(ctx, stacked_id_embeds);
stacked_id_embeds = ggml_add(ctx, stacked_id_embeds, prompt_embeds);
stacked_id_embeds = mlp2.forward(ctx, stacked_id_embeds);
stacked_id_embeds = ggml_nn_layer_norm(ctx, stacked_id_embeds, ln_w, ln_b);
stacked_id_embeds = mlp2->forward(ctx, stacked_id_embeds);
stacked_id_embeds = layer_norm->forward(ctx, stacked_id_embeds);
return stacked_id_embeds;
}
@@ -194,60 +237,116 @@ struct FuseModule{
prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask);
struct ggml_tensor * updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds);
return updated_prompt_embeds;
}
}
};
struct VisualProjection : public GGMLBlock{
public:
int32_t hidden_size = 1024;
int32_t projection_dim = 1280;
VisualProjection(int32_t h, int32_t p) :
hidden_size(h), projection_dim(p){
blocks["visual_projection_2"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, projection_dim, false));
}
struct ggml_tensor* forward(struct ggml_context* ctx,
struct ggml_tensor* pixel_values){
auto visual_projection_2 = std::dynamic_pointer_cast<Linear>(blocks["visual_projection_2"]);
auto x = visual_projection_2->forward(ctx, pixel_values); // [N, projection_dim]
return x; // [N, projection_dim]
}
};
struct PhotoMakerIDEncoder : public GGMLModule {
public:
SDVersion version = VERSION_XL;
CLIPVisionModel vision_model;
FuseModule fuse_module;
struct ggml_tensor* visual_projection_2;
// struct ggml_tensor* visual_projection_2;
VisualProjection visual_projection_2;
float style_strength;
PhotoMakerIDEncoder(SDVersion version = VERSION_XL, float sty = 20.f)
: version(version),
public:
PhotoMakerIDEncoder(ggml_backend_t backend, ggml_type wtype,
SDVersion version = VERSION_XL, float sty = 20.f)
: GGMLModule(backend, wtype),
version(version),
fuse_module(2048),
visual_projection_2(1024, 1280),
style_strength(sty){
vision_model = CLIPVisionModel();
// vision_model = CLIPVisionModel();
vision_model.init(params_ctx, wtype);
}
void init_params(ggml_type wtype) {
ggml_allocr* alloc = ggml_allocr_new_from_buffer(params_buffer);
vision_model.init_params(params_ctx, backend, wtype, alloc);
fuse_module.init_params(params_ctx, wtype, alloc);
visual_projection_2 = ggml_new_tensor_2d(params_ctx, wtype, 1024, 1280); // python [1024, 1280]
ggml_allocr_alloc(alloc, visual_projection_2);
ggml_allocr_free(alloc);
std::string get_desc() {
return "pmid";
}
void map_by_name(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
vision_model.map_by_name(tensors, prefix + "vision_model.", prefix);
fuse_module.map_by_name(tensors, prefix + "fuse_module.");
tensors[prefix + "visual_projection_2.weight"] = visual_projection_2;
}
// void init_params(ggml_type wtype) {
// ggml_allocr* alloc = ggml_allocr_new_from_buffer(params_buffer);
// vision_model.init_params(params_ctx, backend, wtype, alloc);
// fuse_module.init_params(params_ctx, wtype, alloc);
// visual_projection_2 = ggml_new_tensor_2d(params_ctx, wtype, 1024, 1280); // python [1024, 1280]
// ggml_allocr_alloc(alloc, visual_projection_2);
// ggml_allocr_free(alloc);
// }
size_t calculate_mem_size() {
// void map_by_name(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
// vision_model.map_by_name(tensors, prefix + "vision_model.", prefix);
// fuse_module.map_by_name(tensors, prefix + "fuse_module.");
// tensors[prefix + "visual_projection_2.weight"] = visual_projection_2;
// }
wtype = GGML_TYPE_F32;
// size_t calculate_mem_size() {
size_t mem_size = vision_model.calculate_mem_size(wtype);
mem_size += fuse_module.calculate_mem_size(wtype);
// wtype = GGML_TYPE_F32;
// size_t mem_size = vision_model.calculate_mem_size(wtype);
// mem_size += fuse_module.calculate_mem_size(wtype);
mem_size += ggml_row_size(wtype, 1280*1024);
// mem_size += ggml_row_size(wtype, 1280*1024);
return mem_size;
// return mem_size;
// }
// size_t get_num_tensors() {
// size_t num_tensors = (3 + 2 + 37 * vision_model.num_hidden_layers);
// num_tensors += fuse_module.get_num_tensors() + 1;
// return num_tensors;
// }
size_t get_params_mem_size() {
size_t params_mem_size = vision_model.get_params_mem_size();
params_mem_size += fuse_module.get_params_mem_size();
params_mem_size += ggml_row_size(wtype, 1280*1024);
return params_mem_size;
}
size_t get_num_tensors() {
size_t num_tensors = (3 + 2 + 37 * vision_model.num_hidden_layers);
num_tensors += fuse_module.get_num_tensors() + 1;
return num_tensors;
size_t get_params_num() {
size_t params_num = vision_model.get_params_num();
params_num += fuse_module.get_params_num() + 1;
return params_num;
}
void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
vision_model.get_param_tensors(tensors, prefix);
fuse_module.get_param_tensors(tensors, prefix);
visual_projection_2.get_param_tensors(tensors, prefix);
}
struct ggml_tensor* forward(struct ggml_context* ctx,
struct ggml_tensor* id_pixel_values,
struct ggml_tensor* prompt_embeds,
@@ -260,15 +359,22 @@ struct PhotoMakerIDEncoder : public GGMLModule {
struct ggml_tensor* right) {
// x: [N, channels, h, w]
// struct ggml_tensor *shared_id_embeds = vision_model.forward(ctx,
// id_pixel_values,
// cls,
// class_embedding_temp,
// positions
// ); // [batch_size, seq_length, hidden_size]
struct ggml_tensor *shared_id_embeds = vision_model.forward(ctx,
id_pixel_values,
cls,
class_embedding_temp,
positions
); // [batch_size, seq_length, hidden_size]
id_pixel_values, false); // [batch_size, seq_length, hidden_size]
struct ggml_tensor *id_embeds = vision_model.visual_project(ctx, shared_id_embeds); // [batch_size, seq_length, proj_dim(768)]
struct ggml_tensor *id_embeds_2 = ggml_mul_mat(ctx, visual_projection_2, shared_id_embeds); // [batch_size, seq_length, 1280]
// struct ggml_tensor *id_embeds_2 = ggml_mul_mat(ctx, visual_projection_2, shared_id_embeds); // [batch_size, seq_length, 1280]
struct ggml_tensor *id_embeds_2 = visual_projection_2.forward(ctx, shared_id_embeds); // [batch_size, seq_length, 1280]
id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3));
id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3));
@@ -292,32 +398,37 @@ struct PhotoMakerIDEncoder : public GGMLModule {
std::vector<bool> &class_tokens_mask
) {
// since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data
static size_t buf_size = ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
static std::vector<uint8_t> buf(buf_size);
// static size_t buf_size = ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
// static std::vector<uint8_t> buf(buf_size);
struct ggml_init_params params = {
/*.mem_size =*/buf_size,
/*.mem_buffer =*/buf.data(),
/*.no_alloc =*/true, // the tensors will be allocated later by ggml_allocr_alloc_graph()
};
// struct ggml_init_params params = {
// /*.mem_size =*/buf_size,
// /*.mem_buffer =*/buf.data(),
// /*.no_alloc =*/true, // the tensors will be allocated later by ggml_allocr_alloc_graph()
// };
struct ggml_context* ctx0 = ggml_init(params);
// struct ggml_context* ctx0 = ggml_init(params);
struct ggml_cgraph* gf = ggml_new_graph(ctx0);
// struct ggml_cgraph* gf = ggml_new_graph(ctx0);
ggml_context *ctx0 = compute_ctx;
struct ggml_cgraph* gf = ggml_new_graph(compute_ctx);
int64_t hidden_size = prompt_embeds->ne[0];
int64_t seq_length = prompt_embeds->ne[1];
ggml_type type = GGML_TYPE_F32;
struct ggml_tensor* id_pixel_values_d = ggml_dup_tensor(ctx0, id_pixel_values);
ggml_allocr_alloc(allocr, id_pixel_values_d);
struct ggml_tensor* prompt_embeds_d = ggml_dup_tensor(ctx0, prompt_embeds);
ggml_allocr_alloc(allocr, prompt_embeds_d);
// struct ggml_tensor* id_pixel_values_d = ggml_dup_tensor(ctx0, id_pixel_values);
// ggml_allocr_alloc(allocr, id_pixel_values_d);
// struct ggml_tensor* prompt_embeds_d = ggml_dup_tensor(ctx0, prompt_embeds);
// ggml_allocr_alloc(allocr, prompt_embeds_d);
struct ggml_tensor* class_tokens_mask_d = ggml_new_tensor_1d(ctx0, type, class_tokens_mask.size());
ggml_allocr_alloc(allocr, class_tokens_mask_d);
struct ggml_tensor* id_pixel_values_d = to_backend(id_pixel_values);
struct ggml_tensor* prompt_embeds_d = to_backend(prompt_embeds);
std::vector<float> ctm;
std::vector<ggml_fp16_t> ctmf16;
@@ -363,8 +474,8 @@ struct PhotoMakerIDEncoder : public GGMLModule {
if (!ggml_allocr_is_measure(allocr)) {
ggml_backend_tensor_set(id_pixel_values_d, id_pixel_values->data, 0, ggml_nbytes(id_pixel_values));
ggml_backend_tensor_set(prompt_embeds_d, prompt_embeds->data, 0, ggml_nbytes(prompt_embeds));
// ggml_backend_tensor_set(id_pixel_values_d, id_pixel_values->data, 0, ggml_nbytes(id_pixel_values));
// ggml_backend_tensor_set(prompt_embeds_d, prompt_embeds->data, 0, ggml_nbytes(prompt_embeds));
if(type == GGML_TYPE_F16)
ggml_backend_tensor_set(class_tokens_mask_d, ctmf16.data(), 0, ggml_nbytes(class_tokens_mask_d));
else
@@ -410,7 +521,7 @@ struct PhotoMakerIDEncoder : public GGMLModule {
left, right
);
ggml_build_forward_expand(gf, updated_prompt_embeds);
ggml_free(ctx0);
// ggml_free(ctx0);
return gf;
}
@@ -430,13 +541,15 @@ struct PhotoMakerIDEncoder : public GGMLModule {
struct ggml_tensor* id_pixel_values,
struct ggml_tensor* prompt_embeds,
std::vector<bool> &class_tokens_mask,
struct ggml_tensor* updated_prompt_embeds) {
struct ggml_tensor** updated_prompt_embeds,
ggml_context* output_ctx) {
auto get_graph = [&]() -> struct ggml_cgraph* {
return build_graph(compute_allocr, id_pixel_values, prompt_embeds, class_tokens_mask);
};
GGMLModule::compute(get_graph, n_threads, updated_prompt_embeds);
// GGMLModule::compute(get_graph, n_threads, updated_prompt_embeds);
GGMLModule::compute(get_graph, n_threads, true, updated_prompt_embeds, output_ctx);
}
@@ -448,42 +561,61 @@ struct PhotoMakerIDEncoder : public GGMLModule {
struct PhotoMakerLoraModel : public GGMLModule {
float multiplier = 1.f;
SDVersion version;
std::map<std::string, struct ggml_tensor*> lora_tensors;
std::vector<std::string> lora_tensors_to_be_ignored;
std::string file_path;
ModelLoader model_loader;
bool load_failed = false;
PhotoMakerLoraModel(){
}
// PhotoMakerLoraModel()
// {}
PhotoMakerLoraModel(const std::string file_path)
: file_path(file_path) {
name = "photomaker lora";
public:
PhotoMakerLoraModel(ggml_backend_t backend, ggml_type wtype,
SDVersion version = VERSION_XL,
const std::string file_path = "")
: GGMLModule(backend, wtype), version(version), file_path(file_path) {
// name = "photomaker lora";
if (!model_loader.init_from_file(file_path, "pmid.")) {
load_failed = true;
}
}
size_t get_num_tensors() {
std::string get_desc() {
return "pmid lora";
}
size_t get_params_num() {
return PM_LORA_GRAPH_SIZE;
}
size_t calculate_mem_size() {
return model_loader.cal_mem_size(NULL);
size_t get_params_mem_size() {
return model_loader.get_params_mem_size(NULL);
}
// size_t get_num_tensors() {
// return PM_LORA_GRAPH_SIZE;
// }
// size_t calculate_mem_size() {
// return model_loader.cal_mem_size(NULL);
// }
bool load_from_file(ggml_backend_t backend) {
if (!alloc_params_buffer(backend)) {
return false;
}
// if (!alloc_params_buffer(backend)) {
// return false;
// }
LOG_INFO("loading LoRA from '%s'", file_path.c_str());
if (load_failed) {
LOG_ERROR("init lora model loader from file failed: '%s'", file_path.c_str());
return false;
}
alloc_params_buffer();
ggml_allocr* alloc = ggml_allocr_new_from_buffer(params_buffer);
@@ -706,7 +838,8 @@ struct PhotoMakerLoraModel : public GGMLModule {
auto get_graph = [&]() -> struct ggml_cgraph* {
return build_graph(model_tensors);
};
GGMLModule::compute(get_graph, n_threads);
// GGMLModule::compute(get_graph, n_threads);
GGMLModule::compute(get_graph, n_threads, true);
}
};