DarthAffe/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-09-24 20:20:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: reduce CPU overhead in graph execution and sampling (#1997)

Browse Source
This commit is contained in:
leejet 2026-09-19 18:32:58 +08:00 committed by GitHub
parent d32b4e893b
commit 275ab58e01
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 195 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
src/core/ggml_graph_cut.cpp
View File

@ -426,8 +426,8 @@ namespace sd::ggml_graph_cut {
if (tensor == nullptr || tensor->name[0] == '\0') {
return false;
}
return starts_with(tensor->name, GGML_RUNNER_CUT_PREFIX) &&
ends_with(tensor->name, GGML_RUNNER_CUT_SUFFIX);
return std::strncmp(tensor->name, GGML_RUNNER_CUT_PREFIX, std::strlen(GGML_RUNNER_CUT_PREFIX)) == 0 &&
tensor->name[std::strlen(tensor->name) - 1] == GGML_RUNNER_CUT_SUFFIX[0];
}
std::string make_graph_cut_name(const std::string& group, const std::string& output) {
@ -492,35 +492,88 @@ namespace sd::ggml_graph_cut {
}
}
std::vector<uint64_t> graph_layout(ggml_cgraph* graph, bool include_bindings) {
std::vector<const ggml_tensor*> tensors;
std::unordered_map<const ggml_tensor*, size_t> indices;
auto add = [&](const ggml_tensor* tensor) {
if (tensor != nullptr && indices.emplace(tensor, tensors.size() + 1).second) {
struct GraphLayoutTensors {
struct Entry {
const ggml_tensor* tensor = nullptr;
size_t index = 0;
};
std::vector<ggml_tensor*> tensors;
std::vector<Entry> entries;
explicit GraphLayoutTensors(size_t graph_size) {
tensors.reserve(graph_size);
size_t capacity = 2;
while (capacity < 2 * graph_size) {
capacity *= 2;
}
entries.resize(capacity);
}
size_t find(const ggml_tensor* tensor) const {
size_t hash = reinterpret_cast<uintptr_t>(tensor) >> 4;
hash ^= hash >> 16;
const size_t mask = entries.size() - 1;
size_t slot = hash & mask;
while (entries[slot].tensor != nullptr && entries[slot].tensor != tensor) {
slot = (slot + 1) & mask;
}
return slot;
}
void add(ggml_tensor* tensor) {
if (tensor == nullptr) {
return;
}
size_t slot = find(tensor);
if (entries[slot].tensor != nullptr) {
return;
}
if (2 * (tensors.size() + 1) > entries.size()) {
// Segment graphs can reference tensors outside their node and leaf arrays.
std::vector<Entry> next(2 * entries.size());
entries.swap(next);
for (size_t i = 0; i < tensors.size(); ++i) {
entries[find(tensors[i])] = {tensors[i], i + 1};
}
slot = find(tensor);
}
entries[slot] = {tensor, tensors.size() + 1};
tensors.push_back(tensor);
}
size_t index(const ggml_tensor* tensor) const {
return tensor == nullptr ? 0 : entries[find(tensor)].index;
}
};
std::vector<uint64_t> graph_layout(ggml_cgraph* graph, bool include_bindings) {
const size_t graph_size = static_cast<size_t>(graph->n_leafs) + graph->n_nodes;
GraphLayoutTensors layout_tensors(graph_size);
const auto& tensors = layout_tensors.tensors;
for (int i = 0; i < graph->n_leafs; ++i) {
add(graph->leafs[i]);
layout_tensors.add(graph->leafs[i]);
}
for (int i = 0; i < graph->n_nodes; ++i) {
add(graph->nodes[i]);
layout_tensors.add(graph->nodes[i]);
}
for (size_t i = 0; i < tensors.size(); ++i) {
add(tensors[i]->view_src);
layout_tensors.add(tensors[i]->view_src);
for (auto source : tensors[i]->src) {
add(source);
layout_tensors.add(source);
}
}
std::vector<uint64_t> signature;
signature.reserve(tensors.size() * 24);
const size_t tensor_fields = 5 + 2 * GGML_MAX_DIMS + GGML_MAX_SRC +
GGML_MAX_OP_PARAMS / sizeof(int32_t) + (include_bindings ? 2 : 0);
signature.reserve(2 + graph_size + tensors.size() * tensor_fields);
signature.push_back(graph->n_nodes);
signature.push_back(graph->n_leafs);
for (int i = 0; i < graph->n_leafs; ++i) {
signature.push_back(indices.at(graph->leafs[i]));
signature.push_back(layout_tensors.index(graph->leafs[i]));
}
for (int i = 0; i < graph->n_nodes; ++i) {
signature.push_back(indices.at(graph->nodes[i]));
signature.push_back(layout_tensors.index(graph->nodes[i]));
}
for (auto tensor : tensors) {
signature.push_back(tensor->op);
@ -536,9 +589,9 @@ namespace sd::ggml_graph_cut {
signature.push_back(tensor->ne[d]);
signature.push_back(tensor->nb[d]);
}
signature.push_back(tensor->view_src == nullptr ? 0 : indices.at(tensor->view_src));
signature.push_back(layout_tensors.index(tensor->view_src));
for (auto source : tensor->src) {
signature.push_back(source == nullptr ? 0 : indices.at(source));
signature.push_back(layout_tensors.index(source));
}
if (!can_ignore_op_params(tensor->op)) {
for (int value : tensor->op_params) {
@ -562,14 +615,18 @@ namespace sd::ggml_graph_cut {
return false;
}
}
std::vector<std::pair<int, std::string>> cut_markers;
for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) {
auto node = ggml_graph_node(gf, i);
size_t cut_index = 0;
for (int i = 0; i < gf->n_nodes; ++i) {
auto node = gf->nodes[i];
if (is_graph_cut_tensor(node)) {
cut_markers.emplace_back(i, node->name);
if (cut_index >= plan.cut_markers.size() ||
plan.cut_markers[cut_index].first != i || plan.cut_markers[cut_index].second != node->name) {
return false;
}
++cut_index;
}
}
return cut_markers == plan.cut_markers;
return cut_index == plan.cut_markers.size();
}
bool plan_matches_graph(ggml_cgraph* gf, const Plan& plan) {
@ -948,7 +1005,7 @@ namespace sd::ggml_graph_cut {
return plan;
}
Plan resolve_plan(ggml_backend_t backend,
const Plan& resolve_plan(ggml_backend_t backend,
ggml_cgraph* gf,
PlanCache* cache,
const std::unordered_set<const ggml_tensor*>& params_tensor_set,

3
src/core/ggml_graph_cut.h
View File

@ -94,7 +94,8 @@ namespace sd::ggml_graph_cut {
ggml_cgraph* gf,
const std::unordered_set<const ggml_tensor*>& params_tensor_set,
const char* log_desc);
Plan resolve_plan(ggml_backend_t backend,
// The returned reference is valid until its cache entry is evicted or the cache is destroyed.
const Plan& resolve_plan(ggml_backend_t backend,
ggml_cgraph* gf,
PlanCache* cache,
const std::unordered_set<const ggml_tensor*>& params_tensor_set,

41
src/core/ggml_runner.cpp
View File

@ -342,21 +342,17 @@ void GGMLRunner::copy_data_to_backend_tensor(ggml_cgraph* gf, bool clear_after_c
}
}
bool GGMLRunner::resolve_graph_cut_plan(ggml_cgraph* gf,
GraphCutPlan* plan_out) {
GGML_ASSERT(plan_out != nullptr);
const GGMLRunner::GraphCutPlan& GGMLRunner::resolve_graph_cut_plan(ggml_cgraph* gf) {
GGML_ASSERT(gf != nullptr);
*plan_out = sd::ggml_graph_cut::resolve_plan(runtime_backend,
return sd::ggml_graph_cut::resolve_plan(runtime_backend,
gf,
&graph_cut_plan_cache_,
params_tensor_set_,
get_desc().c_str());
return true;
}
bool GGMLRunner::resolve_graph_cut_layer_split_plan(ggml_cgraph* gf,
GraphCutPlan* plan_out) {
return resolve_graph_cut_plan(gf, plan_out);
const GGMLRunner::GraphCutPlan& GGMLRunner::resolve_graph_cut_layer_split_plan(ggml_cgraph* gf) {
return resolve_graph_cut_plan(gf);
}
bool GGMLRunner::assign_graph_cut_layer_split_backends(ggml_cgraph* gf) {
@ -369,10 +365,7 @@ bool GGMLRunner::assign_graph_cut_layer_split_backends(ggml_cgraph* gf) {
return false;
}
GraphCutPlan plan;
if (!resolve_graph_cut_layer_split_plan(gf, &plan)) {
return false;
}
const auto& plan = resolve_graph_cut_layer_split_plan(gf);
if (!plan.valid || !plan.has_cuts || plan.segments.size() <= 1) {
auto manager = residency_manager.lock();
if (manager == nullptr) {
@ -821,23 +814,24 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
return std::nullopt;
}
const auto params = collect_used_param_tensors(graph);
ggml_graph_cut::Plan plan;
if (!resolve_graph_cut_plan(graph, &plan)) {
return std::nullopt;
}
const auto full_measurement = measure(graph, plan.compute_buffer_size);
const auto& cached_plan = resolve_graph_cut_plan(graph);
const auto full_measurement = measure(graph, cached_plan.compute_buffer_size);
if (full_measurement.buffers.empty()) {
return std::nullopt;
}
auto manager = residency_manager.lock();
const bool segmented = !is_multi_device() && !sd_backend_is_cpu(runtime_backend) &&
manager != nullptr && manager->segmented_compute_enabled() &&
plan.valid && plan.has_cuts && plan.segments.size() > 1 &&
cached_plan.valid && cached_plan.has_cuts && cached_plan.segments.size() > 1 &&
!fits(memory_requests(full_measurement.buffers, cache_.pending_bytes(graph)), params);
ggml_graph_cut::Plan monolithic_plan;
if (!segmented) {
ggml_graph_cut::Segment segment;
monolithic_plan.segments.emplace_back();
auto& segment = monolithic_plan.segments.back();
segment.group_name = "graph";
segment.compute_buffer_size = plan.compute_buffer_size;
segment.compute_buffer_size = cached_plan.compute_buffer_size;
segment.internal_node_indices.reserve(ggml_graph_n_nodes(graph));
segment.input_refs.reserve(ggml_graph_cut::leaf_count(graph));
for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {
segment.internal_node_indices.push_back(i);
}
@ -850,8 +844,8 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
: ggml_graph_cut::Segment::INPUT_EXTERNAL;
segment.input_refs.push_back(input);
}
plan.segments = {std::move(segment)};
}
const auto& plan = segmented ? cached_plan : monolithic_plan;
const bool segments_changed = plan.segments.size() != logged_segment_count_;
if (segments_changed && (segmented || logged_segment_count_ > 1)) {
LOG_VERBOSE("%s using %zu segment%s", get_desc().c_str(),
@ -910,7 +904,10 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
auto ensure_capacity = [&]() {
sync_runtime_residency();
auto requests = memory_requests(measurement.buffers, new_cache_bytes);
if (!fits(requests, weights.params(index)) && workspace_.release_excess(measurement)) {
if (fits(requests, weights.params(index))) {
return true;
}
if (workspace_.release_excess(measurement)) {
sync_runtime_residency();
requests = memory_requests(measurement.buffers, new_cache_bytes);
}

6
src/core/ggml_runner.h
View File

@ -265,11 +265,9 @@ protected:
void copy_data_to_backend_tensor(ggml_cgraph* gf, bool clear_after_copy = true);
bool resolve_graph_cut_plan(ggml_cgraph* gf,
GraphCutPlan* plan_out);
const GraphCutPlan& resolve_graph_cut_plan(ggml_cgraph* gf);
bool resolve_graph_cut_layer_split_plan(ggml_cgraph* gf,
GraphCutPlan* plan_out);
const GraphCutPlan& resolve_graph_cut_layer_split_plan(ggml_cgraph* gf);
bool assign_graph_cut_layer_split_backends(ggml_cgraph* gf);

68
src/core/rng_philox.hpp
View File

@ -1,6 +1,7 @@
#ifndef __SD_CORE_RNG_PHILOX_HPP__
#define __SD_CORE_RNG_PHILOX_HPP__
#include <array>
#include <cmath>
#include <vector>
@ -14,60 +15,35 @@ private:
uint32_t offset;
private:
std::vector<uint32_t> philox_m = {0xD2511F53, 0xCD9E8D57};
std::vector<uint32_t> philox_w = {0x9E3779B9, 0xBB67AE85};
using Counter = std::array<std::vector<uint32_t>, 4>;
static constexpr uint32_t philox_m[2] = {0xD2511F53, 0xCD9E8D57};
static constexpr uint32_t philox_w[2] = {0x9E3779B9, 0xBB67AE85};
float two_pow32_inv = 2.3283064e-10f;
float two_pow32_inv_2pi = 2.3283064e-10f * 6.2831855f;
std::vector<std::vector<uint32_t>> uint32(const std::vector<uint64_t>& x) {
uint32_t N = (uint32_t)x.size();
std::vector<std::vector<uint32_t>> result(2, std::vector<uint32_t>(N));
for (uint32_t i = 0; i < N; ++i) {
result[0][i] = static_cast<uint32_t>(x[i] & 0xFFFFFFFF);
result[1][i] = static_cast<uint32_t>(x[i] >> 32);
}
return result;
}
// A single round of the Philox 4x32 random number generator.
void philox4_round(std::vector<std::vector<uint32_t>>& counter,
const std::vector<std::vector<uint32_t>>& key) {
void philox4_round(Counter& counter, uint32_t key0, uint32_t key1) {
uint32_t N = (uint32_t)counter[0].size();
for (uint32_t i = 0; i < N; i++) {
const uint64_t v1 = static_cast<uint64_t>(counter[0][i]) * static_cast<uint64_t>(philox_m[0]);
const uint64_t v2 = static_cast<uint64_t>(counter[2][i]) * static_cast<uint64_t>(philox_m[1]);
counter[0][i] = static_cast<uint32_t>(v2 >> 32) ^ counter[1][i] ^ key[0][i];
counter[0][i] = static_cast<uint32_t>(v2 >> 32) ^ counter[1][i] ^ key0;
counter[1][i] = static_cast<uint32_t>(v2);
counter[2][i] = static_cast<uint32_t>(v1 >> 32) ^ counter[3][i] ^ key[1][i];
counter[2][i] = static_cast<uint32_t>(v1 >> 32) ^ counter[3][i] ^ key1;
counter[3][i] = static_cast<uint32_t>(v1);
}
}
// Generates 32-bit random numbers using the Philox 4x32 random number generator.
// Parameters:
// counter : A 4xN array of 32-bit integers representing the counter values (offset into generation).
// key : A 2xN array of 32-bit integers representing the key values (seed).
// rounds : The number of rounds to perform.
// Returns:
// std::vector<std::vector<uint32_t>>: A 4xN array of 32-bit integers containing the generated random numbers.
std::vector<std::vector<uint32_t>> philox4_32(std::vector<std::vector<uint32_t>>& counter,
std::vector<std::vector<uint32_t>>& key,
int rounds = 10) {
uint32_t N = (uint32_t)counter[0].size();
void philox4_32(Counter& counter, uint32_t key0, uint32_t key1, int rounds = 10) {
for (int i = 0; i < rounds - 1; ++i) {
philox4_round(counter, key);
for (uint32_t j = 0; j < N; ++j) {
key[0][j] += philox_w[0];
key[1][j] += philox_w[1];
}
philox4_round(counter, key0, key1);
key0 += philox_w[0];
key1 += philox_w[1];
}
philox4_round(counter, key);
return counter;
philox4_round(counter, key0, key1);
}
float box_muller(float x, float y) {
@ -96,24 +72,22 @@ public:
}
std::vector<float> randn(uint32_t n) override {
std::vector<std::vector<uint32_t>> counter(4, std::vector<uint32_t>(n, 0));
for (uint32_t i = 0; i < n; i++) {
counter[0][i] = this->offset;
}
Counter counter;
counter[0].resize(n, this->offset);
counter[1].resize(n);
counter[2].resize(n);
counter[3].resize(n);
for (uint32_t i = 0; i < n; i++) {
counter[2][i] = i;
}
this->offset += 1;
std::vector<uint64_t> key(n, this->seed);
std::vector<std::vector<uint32_t>> key_uint32 = uint32(key);
philox4_32(counter, static_cast<uint32_t>(this->seed), static_cast<uint32_t>(this->seed >> 32));
std::vector<std::vector<uint32_t>> g = philox4_32(counter, key_uint32);
std::vector<float> result;
std::vector<float> result(n);
for (uint32_t i = 0; i < n; ++i) {
result.push_back(box_muller((float)g[0][i], (float)g[1][i]));
result[i] = box_muller((float)counter[0][i], (float)counter[1][i]);
}
return result;
}

50
src/model_manager.cpp
View File

@ -274,6 +274,7 @@ bool ModelManager::register_param_tensors(ModelComponent component,
new_states.push_back(std::move(state));
}
resolved_tensor_states_.clear();
for (auto& state : new_states) {
TensorState* registered_state = state.get();
tensor_states_by_tensor_[registered_state->tensor] = registered_state;
@ -369,6 +370,7 @@ bool ModelManager::unregister_tensor_states(const std::unordered_set<TensorState
}
}
resolved_tensor_states_.clear();
for (auto it = tensor_states_by_tensor_.begin(); it != tensor_states_by_tensor_.end();) {
if (target_states.count(it->second) > 0) {
it = tensor_states_by_tensor_.erase(it);
@ -1199,22 +1201,52 @@ bool ModelManager::resolve_required_tensor_states(const std::vector<ggml_tensor*
std::vector<TensorState*>& required_states,
ggml_backend_t compute_backend) const {
required_states.clear();
required_states.reserve(tensors.size());
auto append_states = [&](const std::vector<TensorState*>& states) {
for (TensorState* state : states) {
if (compute_backend == nullptr || state->compute_backend == nullptr ||
state->compute_backend == compute_backend) {
required_states.push_back(state);
}
}
};
for (auto it = resolved_tensor_states_.begin(); it != resolved_tensor_states_.end(); ++it) {
if (it->tensors == tensors) {
append_states(it->states);
resolved_tensor_states_.splice(resolved_tensor_states_.begin(), resolved_tensor_states_, it);
return true;
}
}
std::vector<TensorState*> states;
states.reserve(tensors.size());
std::unordered_set<TensorState*> seen;
seen.reserve(tensors.size());
bool cacheable = true;
for (ggml_tensor* tensor : tensors) {
if (tensor == nullptr) {
continue;
}
auto param = resolve_param_tensor(tensor);
auto found = tensor_states_by_tensor_.find(param);
auto found = tensor_states_by_tensor_.find(tensor);
// Unregistered views can be rebound without changing the parameter list.
cacheable &= found != tensor_states_by_tensor_.end();
for (auto view = tensor->view_src; found == tensor_states_by_tensor_.end() && view != nullptr; view = view->view_src) {
found = tensor_states_by_tensor_.find(view);
}
if (found == tensor_states_by_tensor_.end()) {
LOG_ERROR("model manager tensor '%s' is not registered", ggml_get_name(tensor));
return false;
}
TensorState* state = found->second;
if ((compute_backend == nullptr || state->compute_backend == nullptr ||
state->compute_backend == compute_backend) &&
seen.insert(state).second) {
required_states.push_back(state);
if (seen.insert(state).second) {
states.push_back(state);
}
}
append_states(states);
if (cacheable && !tensors.empty()) {
static constexpr size_t MAX_RESOLVED_LISTS = 4;
resolved_tensor_states_.push_front({tensors, std::move(states)});
if (resolved_tensor_states_.size() > MAX_RESOLVED_LISTS) {
resolved_tensor_states_.pop_back();
}
}
return true;
@ -1274,8 +1306,7 @@ size_t ModelManager::compute_backend_alloc_size(const std::vector<TensorState*>&
size_t total_size = 0;
std::unordered_set<TensorState*> seen;
for (TensorState* state : states) {
if (state == nullptr || state->tensor == nullptr || !seen.insert(state).second ||
should_ignore(*state) || is_optional_missing_tensor(state->name)) {
if (state == nullptr || state->tensor == nullptr) {
continue;
}
const bool compute_resident =
@ -1285,6 +1316,9 @@ size_t ModelManager::compute_backend_alloc_size(const std::vector<TensorState*>&
if (missing_only && compute_resident) {
continue;
}
if (!seen.insert(state).second || should_ignore(*state) || is_optional_missing_tensor(state->name)) {
continue;
}
ggml_backend_buffer_type_t buffer_type = nullptr;
if (state->compute_backend == state->params_backend) {

7
src/model_manager.h
View File

@ -2,6 +2,7 @@
#define __MODEL_MANAGER_H__
#include <cstdint>
#include <list>
#include <map>
#include <memory>
#include <set>
@ -84,9 +85,15 @@ private:
size_t resident_bytes = 0;
};
struct ResolvedTensorStates {
std::vector<ggml_tensor*> tensors;
std::vector<TensorState*> states;
};
ModelLoader model_loader_;
std::vector<std::unique_ptr<TensorState>> tensor_states_;
std::map<const ggml_tensor*, TensorState*> tensor_states_by_tensor_;
mutable std::list<ResolvedTensorStates> resolved_tensor_states_;
std::vector<std::unique_ptr<ParamsStorageBlock>> params_storage_blocks_;
std::vector<std::unique_ptr<ComputeStagingBlock>> compute_staging_blocks_;
std::map<ggml_backend_t, ggml_backend_buffer_type_t> split_buffer_types_;

10
src/pipeline/diffusion_engine.cpp
View File

@ -2158,6 +2158,10 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
RunnerEndOnExit sample_control_runner_end{!control_image.empty() && control_net != nullptr ? control_net.get() : nullptr};
const bool apply_denoise_mask = !denoise_mask.empty() &&
std::any_of(denoise_mask.values().begin(), denoise_mask.values().end(),
[](float value) { return value != 1.f; });
std::vector<int> skip_layers(guidance.slg.layers, guidance.slg.layers + guidance.slg.layer_count);
float cfg_scale = guidance.txt_cfg;
float img_cfg_scale = guidance.img_cfg;
@ -2287,13 +2291,13 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
hunyuan_timestep_r_tensor = sd::Tensor<float>::from_vector({sigmas[step + 1]});
}
sd::Tensor<float> noised_input = x * c_in;
if (!denoise_mask.empty() && (version == VERSION_WAN2_2_TI2V || sd_version_is_ltxav(version) || sd_version_is_lingbot_video(version))) {
if (apply_denoise_mask && (version == VERSION_WAN2_2_TI2V || sd_version_is_ltxav(version) || sd_version_is_lingbot_video(version))) {
noised_input = noised_input * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
}
if (cache_runtime.spectrum_enabled && cache_runtime.spectrum.should_predict()) {
cache_runtime.spectrum.predict(&denoised);
if (!denoise_mask.empty()) {
if (apply_denoise_mask) {
denoised = denoised * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
}
if (preview_needed && sd_should_preview_denoised()) {
@ -2516,7 +2520,7 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
if (cache_runtime.spectrum_enabled) {
cache_runtime.spectrum.update(denoised);
}
if (!denoise_mask.empty()) {
if (apply_denoise_mask) {
denoised = denoised * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
}
if (preview_needed && sd_should_preview_denoised()) {