mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-08-10 06:07:58 +00:00
Compare commits
No commits in common. "e92e86fb11b3028ac9edaf63d93709801d106b12" and "22516991cbdf725e69b0b4a87e52ca16cce07c2d" have entirely different histories.
e92e86fb11
...
22516991cb
@ -117,7 +117,6 @@ public:
|
||||
virtual SDCondition get_learned_condition(int n_threads,
|
||||
const ConditionerParams& conditioner_params) = 0;
|
||||
virtual void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) = 0;
|
||||
virtual void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) {}
|
||||
virtual void set_max_graph_vram_bytes(size_t max_vram_bytes) {}
|
||||
virtual void set_stream_layers_enabled(bool enabled) {}
|
||||
virtual void set_runtime_backends(const std::vector<ggml_backend_t>& backends) {}
|
||||
@ -1665,10 +1664,6 @@ struct AnimaConditioner : public Conditioner {
|
||||
llm->get_param_tensors(tensors, "text_encoders.llm");
|
||||
}
|
||||
|
||||
void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) override {
|
||||
llm->get_param_tensor_ops(tensor_ops);
|
||||
}
|
||||
|
||||
void set_max_graph_vram_bytes(size_t max_vram_bytes) override {
|
||||
llm->set_max_graph_vram_bytes(max_vram_bytes);
|
||||
}
|
||||
@ -1852,10 +1847,6 @@ struct LLMEmbedder : public Conditioner {
|
||||
}
|
||||
}
|
||||
|
||||
void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) override {
|
||||
llm->get_param_tensor_ops(tensor_ops);
|
||||
}
|
||||
|
||||
void set_max_graph_vram_bytes(size_t max_vram_bytes) override {
|
||||
llm->set_max_graph_vram_bytes(max_vram_bytes);
|
||||
if (byt5) {
|
||||
@ -2837,10 +2828,6 @@ struct LTXAVEmbedder : public Conditioner {
|
||||
projector->get_param_tensors(tensors, "text_embedding_projection");
|
||||
}
|
||||
|
||||
void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) override {
|
||||
llm->get_param_tensor_ops(tensor_ops);
|
||||
}
|
||||
|
||||
void set_flash_attention_enabled(bool enabled) override {
|
||||
llm->set_flash_attention_enabled(enabled);
|
||||
projector->set_flash_attention_enabled(enabled);
|
||||
|
||||
@ -1753,7 +1753,7 @@ protected:
|
||||
std::vector<size_t> graph_cut_layer_split_backend_vram_limits_;
|
||||
|
||||
std::vector<ggml_backend_t> extra_runtime_backends; // borrowed (SDBackendManager-owned)
|
||||
ggml_backend_sched_t sched = nullptr; // owned
|
||||
ggml_backend_sched_t sched = nullptr; // owned, multi-device only
|
||||
ggml_backend_t cpu_fallback_backend = nullptr; // owned, sched requires a trailing CPU backend
|
||||
bool multi_device_eval_callback_warned = false;
|
||||
|
||||
@ -2147,22 +2147,8 @@ protected:
|
||||
return !extra_runtime_backends.empty();
|
||||
}
|
||||
|
||||
bool graph_requires_backend_fallback(ggml_cgraph* gf) const {
|
||||
if (gf == nullptr || sd_backend_is_cpu(runtime_backend)) {
|
||||
return false;
|
||||
}
|
||||
const int n_nodes = ggml_graph_n_nodes(gf);
|
||||
for (int i = 0; i < n_nodes; ++i) {
|
||||
ggml_tensor* node = ggml_graph_node(gf, i);
|
||||
if (node != nullptr && !ggml_backend_supports_op(runtime_backend, node)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool alloc_compute_buffer(ggml_cgraph* gf) {
|
||||
if (sched != nullptr || is_multi_device() || graph_requires_backend_fallback(gf)) {
|
||||
if (is_multi_device()) {
|
||||
// The sched replaces the gallocr. Do NOT ggml_backend_sched_reserve
|
||||
// the graph here: reserve runs split_graph, which rewires the
|
||||
// graph's src pointers to sched-internal copy tensors, and the
|
||||
@ -2170,10 +2156,6 @@ protected:
|
||||
// rewired graph, silently corrupting every cross-backend input. A
|
||||
// graph must be split at most once; the alloc in execute_graph
|
||||
// performs the real allocation.
|
||||
if (compute_allocr != nullptr) {
|
||||
ggml_gallocr_free(compute_allocr);
|
||||
compute_allocr = nullptr;
|
||||
}
|
||||
return ensure_sched(gf);
|
||||
}
|
||||
if (compute_allocr != nullptr) {
|
||||
@ -2771,7 +2753,7 @@ protected:
|
||||
};
|
||||
ComputeBufferGuard compute_buffer_guard(this, free_compute_buffer);
|
||||
|
||||
if (sched != nullptr) {
|
||||
if (is_multi_device()) {
|
||||
ggml_backend_sched_reset(sched);
|
||||
pin_multi_device_nodes(gf); // reset clears the pins; re-apply before alloc
|
||||
if (!ggml_backend_sched_alloc_graph(sched, gf)) {
|
||||
@ -2792,9 +2774,9 @@ protected:
|
||||
}
|
||||
|
||||
ggml_status status;
|
||||
if (sched != nullptr) {
|
||||
if (is_multi_device()) {
|
||||
if (sd_get_backend_eval_callback() != nullptr && !multi_device_eval_callback_warned) {
|
||||
LOG_WARN("%s: eval callback is not supported with the backend scheduler; ignoring",
|
||||
LOG_WARN("%s: eval callback is not supported with multiple runtime backends; ignoring",
|
||||
get_desc().c_str());
|
||||
multi_device_eval_callback_warned = true;
|
||||
}
|
||||
@ -3036,9 +3018,12 @@ public:
|
||||
|
||||
// do copy after alloc graph
|
||||
void set_backend_tensor_data(ggml_tensor* tensor, const void* data) {
|
||||
// The scheduler only allocates standalone data tensors when they are
|
||||
// marked as graph inputs. The flag is harmless for single-backend graphs.
|
||||
if (is_multi_device()) {
|
||||
// The sched only assigns a backend (and thus a buffer) to tensors
|
||||
// that participate in the graph; flag standalone data tensors as
|
||||
// inputs so they get one.
|
||||
ggml_set_input(tensor);
|
||||
}
|
||||
backend_tensor_data_map[tensor] = data;
|
||||
}
|
||||
|
||||
@ -3255,11 +3240,6 @@ protected:
|
||||
|
||||
virtual void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") {}
|
||||
|
||||
virtual enum ggml_op param_usage_op(const std::string& name) const {
|
||||
(void)name;
|
||||
return GGML_OP_NONE;
|
||||
}
|
||||
|
||||
public:
|
||||
void init(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") {
|
||||
if (prefix.size() > 0) {
|
||||
@ -3310,18 +3290,6 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) {
|
||||
for (auto& pair : blocks) {
|
||||
pair.second->get_param_tensor_ops(tensor_ops);
|
||||
}
|
||||
for (auto& pair : params) {
|
||||
enum ggml_op op = param_usage_op(pair.first);
|
||||
if (op != GGML_OP_NONE) {
|
||||
tensor_ops[pair.second] = op;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
virtual std::string get_desc() {
|
||||
return "GGMLBlock";
|
||||
}
|
||||
@ -3449,10 +3417,6 @@ protected:
|
||||
params["weight"] = ggml_new_tensor_2d(ctx, wtype, embedding_dim, num_embeddings);
|
||||
}
|
||||
|
||||
enum ggml_op param_usage_op(const std::string& name) const override {
|
||||
return name == "weight" ? GGML_OP_GET_ROWS : GGML_OP_NONE;
|
||||
}
|
||||
|
||||
public:
|
||||
Embedding(int64_t num_embeddings, int64_t embedding_dim)
|
||||
: embedding_dim(embedding_dim),
|
||||
|
||||
@ -342,10 +342,8 @@ struct LoraModel : public GGMLRunner {
|
||||
iter = lora_tensors.find(hada_1_mid_name);
|
||||
if (iter != lora_tensors.end()) {
|
||||
hada_1_mid = ggml_ext_cast_f32(ctx, backend, iter->second);
|
||||
if (hada_1_up != nullptr) {
|
||||
hada_1_up = ggml_cont(ctx, ggml_transpose(ctx, hada_1_up));
|
||||
}
|
||||
}
|
||||
|
||||
iter = lora_tensors.find(hada_2_down_name);
|
||||
if (iter != lora_tensors.end()) {
|
||||
@ -360,10 +358,8 @@ struct LoraModel : public GGMLRunner {
|
||||
iter = lora_tensors.find(hada_2_mid_name);
|
||||
if (iter != lora_tensors.end()) {
|
||||
hada_2_mid = ggml_ext_cast_f32(ctx, backend, iter->second);
|
||||
if (hada_2_up != nullptr) {
|
||||
hada_2_up = ggml_cont(ctx, ggml_transpose(ctx, hada_2_up));
|
||||
}
|
||||
}
|
||||
|
||||
if (hada_1_up == nullptr || hada_1_down == nullptr || hada_2_up == nullptr || hada_2_down == nullptr) {
|
||||
break;
|
||||
|
||||
@ -1657,10 +1657,6 @@ namespace LLM {
|
||||
model.get_param_tensors(tensors, prefix);
|
||||
}
|
||||
|
||||
void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) {
|
||||
model.get_param_tensor_ops(tensor_ops);
|
||||
}
|
||||
|
||||
ggml_tensor* forward(GGMLRunnerContext* ctx,
|
||||
ggml_tensor* input_ids,
|
||||
ggml_tensor* input_pos,
|
||||
|
||||
@ -2,7 +2,6 @@
|
||||
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
@ -513,51 +512,8 @@ static bool parse_storage_type(const std::string& global_name, PickleStorageInfo
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool checked_pickle_byte_count(int64_t element_count,
|
||||
uint64_t element_nbytes,
|
||||
uint64_t* byte_count) {
|
||||
if (element_count < 0 || element_nbytes == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint64_t count = static_cast<uint64_t>(element_count);
|
||||
if (count > std::numeric_limits<uint64_t>::max() / element_nbytes) {
|
||||
return false;
|
||||
}
|
||||
|
||||
*byte_count = count * element_nbytes;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool tensor_layout_is_valid(const PickleTensorInfo& tensor, uint64_t raw_element_nbytes) {
|
||||
if (raw_element_nbytes == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool has_zero_dimension = false;
|
||||
uint64_t element_count = 1;
|
||||
for (int i = 0; i < tensor.tensor_storage.n_dims; ++i) {
|
||||
int64_t dimension = tensor.tensor_storage.ne[i];
|
||||
if (dimension < 0) {
|
||||
return false;
|
||||
}
|
||||
if (dimension == 0) {
|
||||
has_zero_dimension = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
uint64_t size = static_cast<uint64_t>(dimension);
|
||||
if (element_count > static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) / size) {
|
||||
return false;
|
||||
}
|
||||
element_count *= size;
|
||||
}
|
||||
|
||||
if (!has_zero_dimension &&
|
||||
element_count > static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) / raw_element_nbytes) {
|
||||
return false;
|
||||
}
|
||||
if (has_zero_dimension) {
|
||||
static bool tensor_is_contiguous(const PickleTensorInfo& tensor) {
|
||||
if (tensor.tensor_storage.nelements() == 0) {
|
||||
return true;
|
||||
}
|
||||
if (tensor.stride_n_dims != tensor.tensor_storage.n_dims) {
|
||||
@ -976,12 +932,7 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
|
||||
if (storage.key.empty() || !parse_storage_type(pid.items[1].str_value, &storage)) {
|
||||
return false;
|
||||
}
|
||||
if (!checked_pickle_byte_count(pid.items[4].int_value,
|
||||
storage.raw_element_nbytes,
|
||||
&storage.nbytes)) {
|
||||
set_error(error, "invalid storage size in torch pickle");
|
||||
return false;
|
||||
}
|
||||
storage.nbytes = (uint64_t)pid.items[4].int_value * storage.raw_element_nbytes;
|
||||
storage_nbytes[storage.key] = storage.nbytes;
|
||||
stack.push_back(make_storage_value(storage));
|
||||
} break;
|
||||
@ -1012,12 +963,7 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
|
||||
tensor.tensor_storage.is_f64 = args.items[0].storage.is_f64;
|
||||
tensor.tensor_storage.is_i64 = args.items[0].storage.is_i64;
|
||||
tensor.tensor_storage.storage_key = args.items[0].storage.key;
|
||||
if (!checked_pickle_byte_count(args.items[1].int_value,
|
||||
args.items[0].storage.raw_element_nbytes,
|
||||
&tensor.tensor_storage.offset)) {
|
||||
set_error(error, "invalid tensor storage offset in torch pickle");
|
||||
return false;
|
||||
}
|
||||
tensor.tensor_storage.offset = (uint64_t)args.items[1].int_value * args.items[0].storage.raw_element_nbytes;
|
||||
|
||||
for (const auto& item : args.items[2].items) {
|
||||
if (item.kind != PickleValue::INT || tensor.tensor_storage.n_dims >= SD_MAX_DIMS) {
|
||||
@ -1033,8 +979,7 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
|
||||
tensor.stride[tensor.stride_n_dims++] = item.int_value;
|
||||
}
|
||||
|
||||
if (!tensor_layout_is_valid(tensor, args.items[0].storage.raw_element_nbytes)) {
|
||||
set_error(error, "invalid tensor shape or stride in torch pickle");
|
||||
if (!tensor_is_contiguous(tensor)) {
|
||||
return false;
|
||||
}
|
||||
stack.push_back(make_tensor_value(tensor));
|
||||
|
||||
@ -139,16 +139,11 @@ bool read_torch_legacy_file(const std::string& file_path,
|
||||
if (it == legacy_storage_map.end()) {
|
||||
return false;
|
||||
}
|
||||
if (current_offset > file_size ||
|
||||
LEGACY_STORAGE_HEADER_SIZE > file_size - current_offset) {
|
||||
if (current_offset + LEGACY_STORAGE_HEADER_SIZE + it->second > file_size) {
|
||||
return false;
|
||||
}
|
||||
uint64_t storage_offset = current_offset + LEGACY_STORAGE_HEADER_SIZE;
|
||||
if (it->second > file_size - storage_offset) {
|
||||
return false;
|
||||
}
|
||||
storage_offsets[storage_key] = storage_offset;
|
||||
current_offset = storage_offset + it->second;
|
||||
storage_offsets[storage_key] = current_offset + LEGACY_STORAGE_HEADER_SIZE;
|
||||
current_offset += LEGACY_STORAGE_HEADER_SIZE + it->second;
|
||||
}
|
||||
|
||||
for (auto& tensor_storage : tensor_storages) {
|
||||
@ -164,10 +159,8 @@ bool read_torch_legacy_file(const std::string& file_path,
|
||||
|
||||
uint64_t base_offset = it_offset->second;
|
||||
uint64_t storage_nbytes = it_size->second;
|
||||
int64_t tensor_nbytes = tensor_storage.nbytes_to_read();
|
||||
if (tensor_nbytes < 0 ||
|
||||
tensor_storage.offset > storage_nbytes ||
|
||||
static_cast<uint64_t>(tensor_nbytes) > storage_nbytes - tensor_storage.offset) {
|
||||
uint64_t tensor_nbytes = tensor_storage.nbytes_to_read();
|
||||
if (tensor_storage.offset + tensor_nbytes > storage_nbytes) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@ -76,10 +76,8 @@ static bool parse_zip_data_pkl(const uint8_t* buffer,
|
||||
return false;
|
||||
}
|
||||
|
||||
int64_t tensor_nbytes = tensor_storage.nbytes_to_read();
|
||||
if (tensor_nbytes < 0 ||
|
||||
tensor_storage.offset > entry_size ||
|
||||
static_cast<uint64_t>(tensor_nbytes) > entry_size - tensor_storage.offset) {
|
||||
uint64_t tensor_nbytes = tensor_storage.nbytes_to_read();
|
||||
if (tensor_storage.offset + tensor_nbytes > entry_size) {
|
||||
set_error(error, "tensor '" + tensor_storage.name + "' exceeds storage entry '" + entry_name + "'");
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -1053,7 +1053,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
|
||||
std::atomic<size_t> tensor_idx(0);
|
||||
std::atomic<bool> failed(false);
|
||||
std::vector<std::thread> workers;
|
||||
std::mutex backend_tensor_set_mutex;
|
||||
std::mutex rpc_backend_mutex;
|
||||
|
||||
for (int i = 0; i < n_threads; ++i) {
|
||||
workers.emplace_back([&, file_path, is_zip]() {
|
||||
@ -1077,7 +1077,6 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
|
||||
|
||||
std::vector<uint8_t> read_buffer;
|
||||
std::vector<uint8_t> convert_buffer;
|
||||
std::vector<uint8_t> zip_entry_buffer;
|
||||
|
||||
while (true) {
|
||||
int64_t t0, t1;
|
||||
@ -1116,60 +1115,34 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
|
||||
|
||||
size_t nbytes_to_read = tensor_storage.nbytes_to_read();
|
||||
|
||||
auto read_data = [&](char* buf, size_t n) -> bool {
|
||||
auto read_data = [&](char* buf, size_t n) {
|
||||
if (zip != nullptr) {
|
||||
if (zip_entry_openbyindex(zip, tensor_storage.index_in_zip) != 0) {
|
||||
LOG_ERROR("failed to open zip entry for tensor '%s'", tensor_storage.name.c_str());
|
||||
return false;
|
||||
}
|
||||
zip_entry_openbyindex(zip, tensor_storage.index_in_zip);
|
||||
size_t entry_size = zip_entry_size(zip);
|
||||
if (tensor_storage.offset > entry_size) {
|
||||
LOG_ERROR("tensor '%s' exceeds its zip storage entry", tensor_storage.name.c_str());
|
||||
zip_entry_close(zip);
|
||||
return false;
|
||||
}
|
||||
size_t tensor_offset = static_cast<size_t>(tensor_storage.offset);
|
||||
if (n > entry_size - tensor_offset) {
|
||||
LOG_ERROR("tensor '%s' exceeds its zip storage entry", tensor_storage.name.c_str());
|
||||
zip_entry_close(zip);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (entry_size != n) {
|
||||
int64_t t_memcpy_start;
|
||||
zip_entry_buffer.resize(entry_size);
|
||||
auto bytes_read = zip_entry_noallocread(zip, (void*)zip_entry_buffer.data(), entry_size);
|
||||
if (bytes_read < 0 || static_cast<size_t>(bytes_read) != entry_size) {
|
||||
LOG_ERROR("failed to read zip entry for tensor '%s'", tensor_storage.name.c_str());
|
||||
zip_entry_close(zip);
|
||||
return false;
|
||||
}
|
||||
read_buffer.resize(entry_size);
|
||||
zip_entry_noallocread(zip, (void*)read_buffer.data(), entry_size);
|
||||
t_memcpy_start = ggml_time_ms();
|
||||
memcpy((void*)buf, (void*)(zip_entry_buffer.data() + tensor_offset), n);
|
||||
memcpy((void*)buf, (void*)(read_buffer.data() + tensor_storage.offset), n);
|
||||
memcpy_time_ms.fetch_add(ggml_time_ms() - t_memcpy_start);
|
||||
} else {
|
||||
auto bytes_read = zip_entry_noallocread(zip, (void*)buf, n);
|
||||
if (bytes_read < 0 || static_cast<size_t>(bytes_read) != n) {
|
||||
LOG_ERROR("failed to read zip entry for tensor '%s'", tensor_storage.name.c_str());
|
||||
zip_entry_close(zip);
|
||||
return false;
|
||||
}
|
||||
zip_entry_noallocread(zip, (void*)buf, n);
|
||||
}
|
||||
zip_entry_close(zip);
|
||||
} else if (mmapped) {
|
||||
if (!mmapped->copy_data(buf, n, tensor_storage.offset)) {
|
||||
LOG_ERROR("read tensor data failed: '%s'", file_path.c_str());
|
||||
return false;
|
||||
failed = true;
|
||||
}
|
||||
} else {
|
||||
file.seekg(tensor_storage.offset);
|
||||
file.read(buf, n);
|
||||
if (!file) {
|
||||
LOG_ERROR("read tensor data failed: '%s'", file_path.c_str());
|
||||
return false;
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
char* read_buf = nullptr;
|
||||
@ -1203,10 +1176,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
|
||||
}
|
||||
|
||||
t0 = ggml_time_ms();
|
||||
if (!read_data(read_buf, nbytes_to_read)) {
|
||||
failed = true;
|
||||
break;
|
||||
}
|
||||
read_data(read_buf, nbytes_to_read);
|
||||
t1 = ggml_time_ms();
|
||||
read_time_ms.fetch_add(t1 - t0);
|
||||
|
||||
@ -1244,8 +1214,17 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
|
||||
if (dst_tensor->buffer != nullptr && !ggml_backend_buffer_is_host(dst_tensor->buffer)) {
|
||||
t0 = ggml_time_ms();
|
||||
|
||||
std::lock_guard<std::mutex> lock(backend_tensor_set_mutex);
|
||||
// RPC backends require serialized access to prevent concurrency issues
|
||||
const char* buffer_type_name = ggml_backend_buft_name(ggml_backend_buffer_get_type(dst_tensor->buffer));
|
||||
bool is_rpc_buffer = buffer_type_name != nullptr &&
|
||||
std::string(buffer_type_name).find("RPC") != std::string::npos;
|
||||
|
||||
if (is_rpc_buffer) {
|
||||
std::lock_guard<std::mutex> lock(rpc_backend_mutex);
|
||||
ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor));
|
||||
} else {
|
||||
ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor));
|
||||
}
|
||||
|
||||
t1 = ggml_time_ms();
|
||||
copy_to_backend_time_ms.fetch_add(t1 - t0);
|
||||
|
||||
@ -53,48 +53,6 @@ static bool backend_supports_host_buffer(ggml_backend_t backend) {
|
||||
return props.caps.buffer_from_host_ptr;
|
||||
}
|
||||
|
||||
static bool device_supports_param_op(ggml_backend_dev_t device,
|
||||
ggml_tensor* weight,
|
||||
enum ggml_op op,
|
||||
ggml_backend_buffer_type_t buft) {
|
||||
if (op == GGML_OP_NONE) {
|
||||
return true;
|
||||
}
|
||||
if (device == nullptr || weight == nullptr || buft == nullptr || weight->buffer != nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ggml_init_params params;
|
||||
params.mem_size = ggml_tensor_overhead() * 2;
|
||||
params.mem_buffer = nullptr;
|
||||
params.no_alloc = true;
|
||||
ggml_context* ctx = ggml_init(params);
|
||||
if (ctx == nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ggml_tensor* op_tensor = nullptr;
|
||||
if (op == GGML_OP_GET_ROWS) {
|
||||
ggml_tensor* indices = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);
|
||||
op_tensor = ggml_get_rows(ctx, weight, indices);
|
||||
}
|
||||
if (op_tensor == nullptr) {
|
||||
ggml_free(ctx);
|
||||
return false;
|
||||
}
|
||||
|
||||
weight->buffer = ggml_backend_buft_alloc_buffer(buft, 0);
|
||||
if (weight->buffer == nullptr) {
|
||||
ggml_free(ctx);
|
||||
return false;
|
||||
}
|
||||
bool supported = ggml_backend_dev_supports_op(device, op_tensor);
|
||||
ggml_backend_buffer_free(weight->buffer);
|
||||
weight->buffer = nullptr;
|
||||
ggml_free(ctx);
|
||||
return supported;
|
||||
}
|
||||
|
||||
ModelManager::~ModelManager() {
|
||||
release_all();
|
||||
}
|
||||
@ -177,8 +135,7 @@ bool ModelManager::register_param_tensors(const std::string& desc,
|
||||
ggml_backend_t params_backend,
|
||||
size_t* registered_tensor_size,
|
||||
bool allow_split_buffer,
|
||||
bool params_follow_compute_backend,
|
||||
const std::map<ggml_tensor*, enum ggml_op>* tensor_ops) {
|
||||
bool params_follow_compute_backend) {
|
||||
if (desc.empty()) {
|
||||
LOG_ERROR("model manager tensor desc is empty");
|
||||
return false;
|
||||
@ -211,12 +168,6 @@ bool ModelManager::register_param_tensors(const std::string& desc,
|
||||
state->params_backend = params_backend;
|
||||
state->allow_split_buffer = allow_split_buffer;
|
||||
state->params_follow_compute_backend = params_follow_compute_backend;
|
||||
if (tensor_ops != nullptr) {
|
||||
auto op_it = tensor_ops->find(tensor);
|
||||
if (op_it != tensor_ops->end()) {
|
||||
state->usage_op = op_it->second;
|
||||
}
|
||||
}
|
||||
new_states.push_back(std::move(state));
|
||||
}
|
||||
|
||||
@ -893,22 +844,6 @@ ggml_backend_buffer_type_t ModelManager::params_buffer_type_for(const TensorStat
|
||||
if (params_buft == nullptr) {
|
||||
params_buft = ggml_backend_get_default_buffer_type(state.params_backend);
|
||||
}
|
||||
if (state.usage_op != GGML_OP_NONE &&
|
||||
state.compute_backend != nullptr) {
|
||||
ggml_backend_dev_t compute_dev = ggml_backend_get_device(state.compute_backend);
|
||||
if (device_supports_param_op(compute_dev, state.tensor, state.usage_op, params_buft)) {
|
||||
return params_buft;
|
||||
}
|
||||
|
||||
ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
|
||||
params_buft = cpu_dev != nullptr ? ggml_backend_dev_buffer_type(cpu_dev) : nullptr;
|
||||
if (!device_supports_param_op(cpu_dev, state.tensor, state.usage_op, params_buft)) {
|
||||
LOG_ERROR("model manager has no compatible buffer for tensor '%s' used by %s",
|
||||
state.name.c_str(),
|
||||
ggml_op_name(state.usage_op));
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
return params_buft;
|
||||
}
|
||||
|
||||
|
||||
@ -39,7 +39,6 @@ private:
|
||||
bool allow_split_buffer = false;
|
||||
bool params_follow_compute_backend = false;
|
||||
bool metadata_validated = false;
|
||||
enum ggml_op usage_op = GGML_OP_NONE;
|
||||
|
||||
int active_prepare_count = 0;
|
||||
|
||||
@ -133,8 +132,7 @@ public:
|
||||
ggml_backend_t params_backend,
|
||||
size_t* registered_tensor_size = nullptr,
|
||||
bool allow_split_buffer = false,
|
||||
bool params_follow_compute_backend = false,
|
||||
const std::map<ggml_tensor*, enum ggml_op>* tensor_ops = nullptr);
|
||||
bool params_follow_compute_backend = false);
|
||||
|
||||
bool unregister_param_tensors(const std::string& desc,
|
||||
size_t* registered_tensor_size = nullptr);
|
||||
|
||||
@ -320,11 +320,7 @@ public:
|
||||
return true;
|
||||
}
|
||||
std::map<std::string, ggml_tensor*> group_tensors;
|
||||
std::map<ggml_tensor*, enum ggml_op> tensor_ops;
|
||||
model->get_param_tensors(group_tensors);
|
||||
if constexpr (std::is_base_of_v<Conditioner, T>) {
|
||||
model->get_param_tensor_ops(tensor_ops);
|
||||
}
|
||||
if (model_manager == nullptr) {
|
||||
return true;
|
||||
}
|
||||
@ -341,7 +337,6 @@ public:
|
||||
module,
|
||||
module_backends,
|
||||
std::move(group_tensors),
|
||||
tensor_ops,
|
||||
residency_mode,
|
||||
params_mem_size);
|
||||
}
|
||||
@ -350,7 +345,6 @@ public:
|
||||
module,
|
||||
module_backends,
|
||||
std::move(group_tensors),
|
||||
tensor_ops,
|
||||
residency_mode,
|
||||
params_mem_size);
|
||||
}
|
||||
@ -364,10 +358,7 @@ public:
|
||||
residency_mode,
|
||||
backend_for(module),
|
||||
params_backend_for(module),
|
||||
params_mem_size,
|
||||
false,
|
||||
false,
|
||||
&tensor_ops);
|
||||
params_mem_size);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -376,7 +367,6 @@ public:
|
||||
SDBackendModule module,
|
||||
const std::vector<ggml_backend_t>& module_backends,
|
||||
std::map<std::string, ggml_tensor*> group_tensors,
|
||||
const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
|
||||
ModelManager::ResidencyMode residency_mode,
|
||||
size_t* params_mem_size) {
|
||||
ggml_backend_t main_backend = module_backends[0];
|
||||
@ -388,7 +378,6 @@ public:
|
||||
module,
|
||||
module_backends,
|
||||
std::move(group_tensors),
|
||||
tensor_ops,
|
||||
residency_mode,
|
||||
params_mem_size);
|
||||
};
|
||||
@ -463,9 +452,7 @@ public:
|
||||
main_backend,
|
||||
params_backend_for(module),
|
||||
params_mem_size,
|
||||
/*allow_split_buffer=*/true,
|
||||
false,
|
||||
&tensor_ops)) {
|
||||
/*allow_split_buffer=*/true)) {
|
||||
return false;
|
||||
}
|
||||
return model_manager->register_param_tensors(desc,
|
||||
@ -473,10 +460,7 @@ public:
|
||||
residency_mode,
|
||||
main_backend,
|
||||
params_backend_for(module),
|
||||
params_mem_size,
|
||||
false,
|
||||
false,
|
||||
&tensor_ops);
|
||||
params_mem_size);
|
||||
}
|
||||
|
||||
// Register graph-cut layer-split tensors on the primary backend first.
|
||||
@ -488,7 +472,6 @@ public:
|
||||
SDBackendModule module,
|
||||
const std::vector<ggml_backend_t>& module_backends,
|
||||
std::map<std::string, ggml_tensor*> group_tensors,
|
||||
const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
|
||||
ModelManager::ResidencyMode residency_mode,
|
||||
size_t* params_mem_size) {
|
||||
bool has_cpu_device = false;
|
||||
@ -510,10 +493,7 @@ public:
|
||||
residency_mode,
|
||||
module_backends[0],
|
||||
params_backend_for(module),
|
||||
params_mem_size,
|
||||
false,
|
||||
false,
|
||||
&tensor_ops);
|
||||
params_mem_size);
|
||||
}
|
||||
|
||||
model->set_runtime_backends(module_backends);
|
||||
@ -538,8 +518,7 @@ public:
|
||||
initial_params_backend,
|
||||
params_mem_size,
|
||||
false,
|
||||
params_follow_runtime,
|
||||
&tensor_ops);
|
||||
params_follow_runtime);
|
||||
}
|
||||
|
||||
bool unload_control_net() {
|
||||
|
||||
@ -205,7 +205,7 @@ std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t
|
||||
ss << "\"" << token << "\", ";
|
||||
}
|
||||
ss << "]";
|
||||
LOG_DEBUG("split prompt \"%s\" to %zu tokens %s", text.c_str(), bpe_tokens.size(), ss.str().c_str());
|
||||
LOG_DEBUG("split prompt \"%s\" to tokens %s", text.c_str(), ss.str().c_str());
|
||||
return bpe_tokens;
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user