diff --git a/src/core/ggml_graph_cut.cpp b/src/core/ggml_graph_cut.cpp index a6e2d99e..78ebb2d5 100644 --- a/src/core/ggml_graph_cut.cpp +++ b/src/core/ggml_graph_cut.cpp @@ -426,8 +426,8 @@ namespace sd::ggml_graph_cut { if (tensor == nullptr || tensor->name[0] == '\0') { return false; } - return starts_with(tensor->name, GGML_RUNNER_CUT_PREFIX) && - ends_with(tensor->name, GGML_RUNNER_CUT_SUFFIX); + return std::strncmp(tensor->name, GGML_RUNNER_CUT_PREFIX, std::strlen(GGML_RUNNER_CUT_PREFIX)) == 0 && + tensor->name[std::strlen(tensor->name) - 1] == GGML_RUNNER_CUT_SUFFIX[0]; } std::string make_graph_cut_name(const std::string& group, const std::string& output) { @@ -492,35 +492,88 @@ namespace sd::ggml_graph_cut { } } - std::vector graph_layout(ggml_cgraph* graph, bool include_bindings) { - std::vector tensors; - std::unordered_map indices; - auto add = [&](const ggml_tensor* tensor) { - if (tensor != nullptr && indices.emplace(tensor, tensors.size() + 1).second) { - tensors.push_back(tensor); - } + struct GraphLayoutTensors { + struct Entry { + const ggml_tensor* tensor = nullptr; + size_t index = 0; }; + + std::vector tensors; + std::vector entries; + + explicit GraphLayoutTensors(size_t graph_size) { + tensors.reserve(graph_size); + size_t capacity = 2; + while (capacity < 2 * graph_size) { + capacity *= 2; + } + entries.resize(capacity); + } + + size_t find(const ggml_tensor* tensor) const { + size_t hash = reinterpret_cast(tensor) >> 4; + hash ^= hash >> 16; + const size_t mask = entries.size() - 1; + size_t slot = hash & mask; + while (entries[slot].tensor != nullptr && entries[slot].tensor != tensor) { + slot = (slot + 1) & mask; + } + return slot; + } + + void add(ggml_tensor* tensor) { + if (tensor == nullptr) { + return; + } + size_t slot = find(tensor); + if (entries[slot].tensor != nullptr) { + return; + } + if (2 * (tensors.size() + 1) > entries.size()) { + // Segment graphs can reference tensors outside their node and leaf arrays. + std::vector next(2 * entries.size()); + entries.swap(next); + for (size_t i = 0; i < tensors.size(); ++i) { + entries[find(tensors[i])] = {tensors[i], i + 1}; + } + slot = find(tensor); + } + entries[slot] = {tensor, tensors.size() + 1}; + tensors.push_back(tensor); + } + + size_t index(const ggml_tensor* tensor) const { + return tensor == nullptr ? 0 : entries[find(tensor)].index; + } + }; + + std::vector graph_layout(ggml_cgraph* graph, bool include_bindings) { + const size_t graph_size = static_cast(graph->n_leafs) + graph->n_nodes; + GraphLayoutTensors layout_tensors(graph_size); + const auto& tensors = layout_tensors.tensors; for (int i = 0; i < graph->n_leafs; ++i) { - add(graph->leafs[i]); + layout_tensors.add(graph->leafs[i]); } for (int i = 0; i < graph->n_nodes; ++i) { - add(graph->nodes[i]); + layout_tensors.add(graph->nodes[i]); } for (size_t i = 0; i < tensors.size(); ++i) { - add(tensors[i]->view_src); + layout_tensors.add(tensors[i]->view_src); for (auto source : tensors[i]->src) { - add(source); + layout_tensors.add(source); } } std::vector signature; - signature.reserve(tensors.size() * 24); + const size_t tensor_fields = 5 + 2 * GGML_MAX_DIMS + GGML_MAX_SRC + + GGML_MAX_OP_PARAMS / sizeof(int32_t) + (include_bindings ? 2 : 0); + signature.reserve(2 + graph_size + tensors.size() * tensor_fields); signature.push_back(graph->n_nodes); signature.push_back(graph->n_leafs); for (int i = 0; i < graph->n_leafs; ++i) { - signature.push_back(indices.at(graph->leafs[i])); + signature.push_back(layout_tensors.index(graph->leafs[i])); } for (int i = 0; i < graph->n_nodes; ++i) { - signature.push_back(indices.at(graph->nodes[i])); + signature.push_back(layout_tensors.index(graph->nodes[i])); } for (auto tensor : tensors) { signature.push_back(tensor->op); @@ -536,9 +589,9 @@ namespace sd::ggml_graph_cut { signature.push_back(tensor->ne[d]); signature.push_back(tensor->nb[d]); } - signature.push_back(tensor->view_src == nullptr ? 0 : indices.at(tensor->view_src)); + signature.push_back(layout_tensors.index(tensor->view_src)); for (auto source : tensor->src) { - signature.push_back(source == nullptr ? 0 : indices.at(source)); + signature.push_back(layout_tensors.index(source)); } if (!can_ignore_op_params(tensor->op)) { for (int value : tensor->op_params) { @@ -562,14 +615,18 @@ namespace sd::ggml_graph_cut { return false; } } - std::vector> cut_markers; - for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) { - auto node = ggml_graph_node(gf, i); + size_t cut_index = 0; + for (int i = 0; i < gf->n_nodes; ++i) { + auto node = gf->nodes[i]; if (is_graph_cut_tensor(node)) { - cut_markers.emplace_back(i, node->name); + if (cut_index >= plan.cut_markers.size() || + plan.cut_markers[cut_index].first != i || plan.cut_markers[cut_index].second != node->name) { + return false; + } + ++cut_index; } } - return cut_markers == plan.cut_markers; + return cut_index == plan.cut_markers.size(); } bool plan_matches_graph(ggml_cgraph* gf, const Plan& plan) { @@ -948,11 +1005,11 @@ namespace sd::ggml_graph_cut { return plan; } - Plan resolve_plan(ggml_backend_t backend, - ggml_cgraph* gf, - PlanCache* cache, - const std::unordered_set& params_tensor_set, - const char* log_desc) { + const Plan& resolve_plan(ggml_backend_t backend, + ggml_cgraph* gf, + PlanCache* cache, + const std::unordered_set& params_tensor_set, + const char* log_desc) { GGML_ASSERT(backend != nullptr); GGML_ASSERT(gf != nullptr); GGML_ASSERT(cache != nullptr); diff --git a/src/core/ggml_graph_cut.h b/src/core/ggml_graph_cut.h index c93f368c..5a30f36f 100644 --- a/src/core/ggml_graph_cut.h +++ b/src/core/ggml_graph_cut.h @@ -94,11 +94,12 @@ namespace sd::ggml_graph_cut { ggml_cgraph* gf, const std::unordered_set& params_tensor_set, const char* log_desc); - Plan resolve_plan(ggml_backend_t backend, - ggml_cgraph* gf, - PlanCache* cache, - const std::unordered_set& params_tensor_set, - const char* log_desc); + // The returned reference is valid until its cache entry is evicted or the cache is destroyed. + const Plan& resolve_plan(ggml_backend_t backend, + ggml_cgraph* gf, + PlanCache* cache, + const std::unordered_set& params_tensor_set, + const char* log_desc); } // namespace sd::ggml_graph_cut diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp index bb08bb14..50173aab 100644 --- a/src/core/ggml_runner.cpp +++ b/src/core/ggml_runner.cpp @@ -342,21 +342,17 @@ void GGMLRunner::copy_data_to_backend_tensor(ggml_cgraph* gf, bool clear_after_c } } -bool GGMLRunner::resolve_graph_cut_plan(ggml_cgraph* gf, - GraphCutPlan* plan_out) { - GGML_ASSERT(plan_out != nullptr); +const GGMLRunner::GraphCutPlan& GGMLRunner::resolve_graph_cut_plan(ggml_cgraph* gf) { GGML_ASSERT(gf != nullptr); - *plan_out = sd::ggml_graph_cut::resolve_plan(runtime_backend, - gf, - &graph_cut_plan_cache_, - params_tensor_set_, - get_desc().c_str()); - return true; + return sd::ggml_graph_cut::resolve_plan(runtime_backend, + gf, + &graph_cut_plan_cache_, + params_tensor_set_, + get_desc().c_str()); } -bool GGMLRunner::resolve_graph_cut_layer_split_plan(ggml_cgraph* gf, - GraphCutPlan* plan_out) { - return resolve_graph_cut_plan(gf, plan_out); +const GGMLRunner::GraphCutPlan& GGMLRunner::resolve_graph_cut_layer_split_plan(ggml_cgraph* gf) { + return resolve_graph_cut_plan(gf); } bool GGMLRunner::assign_graph_cut_layer_split_backends(ggml_cgraph* gf) { @@ -369,10 +365,7 @@ bool GGMLRunner::assign_graph_cut_layer_split_backends(ggml_cgraph* gf) { return false; } - GraphCutPlan plan; - if (!resolve_graph_cut_layer_split_plan(gf, &plan)) { - return false; - } + const auto& plan = resolve_graph_cut_layer_split_plan(gf); if (!plan.valid || !plan.has_cuts || plan.segments.size() <= 1) { auto manager = residency_manager.lock(); if (manager == nullptr) { @@ -820,24 +813,25 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n if (!assign_graph_cut_layer_split_backends(graph)) { return std::nullopt; } - const auto params = collect_used_param_tensors(graph); - ggml_graph_cut::Plan plan; - if (!resolve_graph_cut_plan(graph, &plan)) { - return std::nullopt; - } - const auto full_measurement = measure(graph, plan.compute_buffer_size); + const auto params = collect_used_param_tensors(graph); + const auto& cached_plan = resolve_graph_cut_plan(graph); + const auto full_measurement = measure(graph, cached_plan.compute_buffer_size); if (full_measurement.buffers.empty()) { return std::nullopt; } auto manager = residency_manager.lock(); const bool segmented = !is_multi_device() && !sd_backend_is_cpu(runtime_backend) && manager != nullptr && manager->segmented_compute_enabled() && - plan.valid && plan.has_cuts && plan.segments.size() > 1 && + cached_plan.valid && cached_plan.has_cuts && cached_plan.segments.size() > 1 && !fits(memory_requests(full_measurement.buffers, cache_.pending_bytes(graph)), params); + ggml_graph_cut::Plan monolithic_plan; if (!segmented) { - ggml_graph_cut::Segment segment; + monolithic_plan.segments.emplace_back(); + auto& segment = monolithic_plan.segments.back(); segment.group_name = "graph"; - segment.compute_buffer_size = plan.compute_buffer_size; + segment.compute_buffer_size = cached_plan.compute_buffer_size; + segment.internal_node_indices.reserve(ggml_graph_n_nodes(graph)); + segment.input_refs.reserve(ggml_graph_cut::leaf_count(graph)); for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) { segment.internal_node_indices.push_back(i); } @@ -850,8 +844,8 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n : ggml_graph_cut::Segment::INPUT_EXTERNAL; segment.input_refs.push_back(input); } - plan.segments = {std::move(segment)}; } + const auto& plan = segmented ? cached_plan : monolithic_plan; const bool segments_changed = plan.segments.size() != logged_segment_count_; if (segments_changed && (segmented || logged_segment_count_ > 1)) { LOG_VERBOSE("%s using %zu segment%s", get_desc().c_str(), @@ -910,7 +904,10 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n auto ensure_capacity = [&]() { sync_runtime_residency(); auto requests = memory_requests(measurement.buffers, new_cache_bytes); - if (!fits(requests, weights.params(index)) && workspace_.release_excess(measurement)) { + if (fits(requests, weights.params(index))) { + return true; + } + if (workspace_.release_excess(measurement)) { sync_runtime_residency(); requests = memory_requests(measurement.buffers, new_cache_bytes); } diff --git a/src/core/ggml_runner.h b/src/core/ggml_runner.h index 273098e8..2152ec8a 100644 --- a/src/core/ggml_runner.h +++ b/src/core/ggml_runner.h @@ -265,11 +265,9 @@ protected: void copy_data_to_backend_tensor(ggml_cgraph* gf, bool clear_after_copy = true); - bool resolve_graph_cut_plan(ggml_cgraph* gf, - GraphCutPlan* plan_out); + const GraphCutPlan& resolve_graph_cut_plan(ggml_cgraph* gf); - bool resolve_graph_cut_layer_split_plan(ggml_cgraph* gf, - GraphCutPlan* plan_out); + const GraphCutPlan& resolve_graph_cut_layer_split_plan(ggml_cgraph* gf); bool assign_graph_cut_layer_split_backends(ggml_cgraph* gf); diff --git a/src/core/rng_philox.hpp b/src/core/rng_philox.hpp index 56760d37..9799a875 100644 --- a/src/core/rng_philox.hpp +++ b/src/core/rng_philox.hpp @@ -1,6 +1,7 @@ #ifndef __SD_CORE_RNG_PHILOX_HPP__ #define __SD_CORE_RNG_PHILOX_HPP__ +#include #include #include @@ -14,60 +15,35 @@ private: uint32_t offset; private: - std::vector philox_m = {0xD2511F53, 0xCD9E8D57}; - std::vector philox_w = {0x9E3779B9, 0xBB67AE85}; - float two_pow32_inv = 2.3283064e-10f; - float two_pow32_inv_2pi = 2.3283064e-10f * 6.2831855f; + using Counter = std::array, 4>; - std::vector> uint32(const std::vector& x) { - uint32_t N = (uint32_t)x.size(); - std::vector> result(2, std::vector(N)); - - for (uint32_t i = 0; i < N; ++i) { - result[0][i] = static_cast(x[i] & 0xFFFFFFFF); - result[1][i] = static_cast(x[i] >> 32); - } - - return result; - } + static constexpr uint32_t philox_m[2] = {0xD2511F53, 0xCD9E8D57}; + static constexpr uint32_t philox_w[2] = {0x9E3779B9, 0xBB67AE85}; + float two_pow32_inv = 2.3283064e-10f; + float two_pow32_inv_2pi = 2.3283064e-10f * 6.2831855f; // A single round of the Philox 4x32 random number generator. - void philox4_round(std::vector>& counter, - const std::vector>& key) { + void philox4_round(Counter& counter, uint32_t key0, uint32_t key1) { uint32_t N = (uint32_t)counter[0].size(); for (uint32_t i = 0; i < N; i++) { const uint64_t v1 = static_cast(counter[0][i]) * static_cast(philox_m[0]); const uint64_t v2 = static_cast(counter[2][i]) * static_cast(philox_m[1]); - counter[0][i] = static_cast(v2 >> 32) ^ counter[1][i] ^ key[0][i]; + counter[0][i] = static_cast(v2 >> 32) ^ counter[1][i] ^ key0; counter[1][i] = static_cast(v2); - counter[2][i] = static_cast(v1 >> 32) ^ counter[3][i] ^ key[1][i]; + counter[2][i] = static_cast(v1 >> 32) ^ counter[3][i] ^ key1; counter[3][i] = static_cast(v1); } } - // Generates 32-bit random numbers using the Philox 4x32 random number generator. - // Parameters: - // counter : A 4xN array of 32-bit integers representing the counter values (offset into generation). - // key : A 2xN array of 32-bit integers representing the key values (seed). - // rounds : The number of rounds to perform. - // Returns: - // std::vector>: A 4xN array of 32-bit integers containing the generated random numbers. - std::vector> philox4_32(std::vector>& counter, - std::vector>& key, - int rounds = 10) { - uint32_t N = (uint32_t)counter[0].size(); + void philox4_32(Counter& counter, uint32_t key0, uint32_t key1, int rounds = 10) { for (int i = 0; i < rounds - 1; ++i) { - philox4_round(counter, key); - - for (uint32_t j = 0; j < N; ++j) { - key[0][j] += philox_w[0]; - key[1][j] += philox_w[1]; - } + philox4_round(counter, key0, key1); + key0 += philox_w[0]; + key1 += philox_w[1]; } - philox4_round(counter, key); - return counter; + philox4_round(counter, key0, key1); } float box_muller(float x, float y) { @@ -96,24 +72,22 @@ public: } std::vector randn(uint32_t n) override { - std::vector> counter(4, std::vector(n, 0)); - for (uint32_t i = 0; i < n; i++) { - counter[0][i] = this->offset; - } + Counter counter; + counter[0].resize(n, this->offset); + counter[1].resize(n); + counter[2].resize(n); + counter[3].resize(n); for (uint32_t i = 0; i < n; i++) { counter[2][i] = i; } this->offset += 1; - std::vector key(n, this->seed); - std::vector> key_uint32 = uint32(key); + philox4_32(counter, static_cast(this->seed), static_cast(this->seed >> 32)); - std::vector> g = philox4_32(counter, key_uint32); - - std::vector result; + std::vector result(n); for (uint32_t i = 0; i < n; ++i) { - result.push_back(box_muller((float)g[0][i], (float)g[1][i])); + result[i] = box_muller((float)counter[0][i], (float)counter[1][i]); } return result; } diff --git a/src/model_manager.cpp b/src/model_manager.cpp index 734eb053..9a9fe062 100644 --- a/src/model_manager.cpp +++ b/src/model_manager.cpp @@ -274,6 +274,7 @@ bool ModelManager::register_param_tensors(ModelComponent component, new_states.push_back(std::move(state)); } + resolved_tensor_states_.clear(); for (auto& state : new_states) { TensorState* registered_state = state.get(); tensor_states_by_tensor_[registered_state->tensor] = registered_state; @@ -369,6 +370,7 @@ bool ModelManager::unregister_tensor_states(const std::unordered_setsecond) > 0) { it = tensor_states_by_tensor_.erase(it); @@ -1199,22 +1201,52 @@ bool ModelManager::resolve_required_tensor_states(const std::vector& required_states, ggml_backend_t compute_backend) const { required_states.clear(); + required_states.reserve(tensors.size()); + auto append_states = [&](const std::vector& states) { + for (TensorState* state : states) { + if (compute_backend == nullptr || state->compute_backend == nullptr || + state->compute_backend == compute_backend) { + required_states.push_back(state); + } + } + }; + for (auto it = resolved_tensor_states_.begin(); it != resolved_tensor_states_.end(); ++it) { + if (it->tensors == tensors) { + append_states(it->states); + resolved_tensor_states_.splice(resolved_tensor_states_.begin(), resolved_tensor_states_, it); + return true; + } + } + std::vector states; + states.reserve(tensors.size()); std::unordered_set seen; + seen.reserve(tensors.size()); + bool cacheable = true; for (ggml_tensor* tensor : tensors) { if (tensor == nullptr) { continue; } - auto param = resolve_param_tensor(tensor); - auto found = tensor_states_by_tensor_.find(param); + auto found = tensor_states_by_tensor_.find(tensor); + // Unregistered views can be rebound without changing the parameter list. + cacheable &= found != tensor_states_by_tensor_.end(); + for (auto view = tensor->view_src; found == tensor_states_by_tensor_.end() && view != nullptr; view = view->view_src) { + found = tensor_states_by_tensor_.find(view); + } if (found == tensor_states_by_tensor_.end()) { LOG_ERROR("model manager tensor '%s' is not registered", ggml_get_name(tensor)); return false; } TensorState* state = found->second; - if ((compute_backend == nullptr || state->compute_backend == nullptr || - state->compute_backend == compute_backend) && - seen.insert(state).second) { - required_states.push_back(state); + if (seen.insert(state).second) { + states.push_back(state); + } + } + append_states(states); + if (cacheable && !tensors.empty()) { + static constexpr size_t MAX_RESOLVED_LISTS = 4; + resolved_tensor_states_.push_front({tensors, std::move(states)}); + if (resolved_tensor_states_.size() > MAX_RESOLVED_LISTS) { + resolved_tensor_states_.pop_back(); } } return true; @@ -1274,8 +1306,7 @@ size_t ModelManager::compute_backend_alloc_size(const std::vector& size_t total_size = 0; std::unordered_set seen; for (TensorState* state : states) { - if (state == nullptr || state->tensor == nullptr || !seen.insert(state).second || - should_ignore(*state) || is_optional_missing_tensor(state->name)) { + if (state == nullptr || state->tensor == nullptr) { continue; } const bool compute_resident = @@ -1285,6 +1316,9 @@ size_t ModelManager::compute_backend_alloc_size(const std::vector& if (missing_only && compute_resident) { continue; } + if (!seen.insert(state).second || should_ignore(*state) || is_optional_missing_tensor(state->name)) { + continue; + } ggml_backend_buffer_type_t buffer_type = nullptr; if (state->compute_backend == state->params_backend) { diff --git a/src/model_manager.h b/src/model_manager.h index 95b1fe90..7f8df59c 100644 --- a/src/model_manager.h +++ b/src/model_manager.h @@ -2,6 +2,7 @@ #define __MODEL_MANAGER_H__ #include +#include #include #include #include @@ -84,9 +85,15 @@ private: size_t resident_bytes = 0; }; + struct ResolvedTensorStates { + std::vector tensors; + std::vector states; + }; + ModelLoader model_loader_; std::vector> tensor_states_; std::map tensor_states_by_tensor_; + mutable std::list resolved_tensor_states_; std::vector> params_storage_blocks_; std::vector> compute_staging_blocks_; std::map split_buffer_types_; diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index ecad31a5..4fb90807 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2158,6 +2158,10 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr skip_layers(guidance.slg.layers, guidance.slg.layers + guidance.slg.layer_count); float cfg_scale = guidance.txt_cfg; float img_cfg_scale = guidance.img_cfg; @@ -2287,13 +2291,13 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr::from_vector({sigmas[step + 1]}); } sd::Tensor noised_input = x * c_in; - if (!denoise_mask.empty() && (version == VERSION_WAN2_2_TI2V || sd_version_is_ltxav(version) || sd_version_is_lingbot_video(version))) { + if (apply_denoise_mask && (version == VERSION_WAN2_2_TI2V || sd_version_is_ltxav(version) || sd_version_is_lingbot_video(version))) { noised_input = noised_input * denoise_mask + sampling_init_latent * (1.0f - denoise_mask); } if (cache_runtime.spectrum_enabled && cache_runtime.spectrum.should_predict()) { cache_runtime.spectrum.predict(&denoised); - if (!denoise_mask.empty()) { + if (apply_denoise_mask) { denoised = denoised * denoise_mask + sampling_init_latent * (1.0f - denoise_mask); } if (preview_needed && sd_should_preview_denoised()) { @@ -2516,7 +2520,7 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr