DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 19:36:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: avoid CPU fallback based on backend allocation chunk size (#2294)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-11-08 16:24:53 +08:00
1 parent de00bafc3d
commit 0b3b98d7b8
2 files changed
+17 -13
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/core/ggml_runner.cpp
+5 -2
View File
@@ -212,6 +212,10 @@ ggml_backend_t GGMLRunner::backend_for_weight(const ggml_tensor* tensor) const {
}
static bool backend_can_allocate_op(ggml_backend_t backend, const ggml_tensor* node) {
// Vulkan reports a preferred allocation chunk size, not a hard tensor limit.
if (sd_backend_is(backend, "Vulkan")) {
return true;
}
auto buft = ggml_backend_get_default_buffer_type(backend);
size_t alignment = ggml_backend_buft_get_alignment(buft);
size_t max_size = ggml_backend_buft_get_max_size(buft);
@@ -219,6 +223,7 @@ static bool backend_can_allocate_op(ggml_backend_t backend, const ggml_tensor* n
if (ggml_backend_buft_get_alloc_size(buft, node) > max_size) {
return false;
}
// Cross-backend copies must also fit the destination's allocation limit.
for (auto src : node->src) {
if (src != nullptr && ggml_backend_buft_get_alloc_size(buft, src) > max_size) {
return false;
@@ -251,8 +256,6 @@ void GGMLRunner::pin_multi_device_nodes(ggml_backend_sched_t sched, ggml_cgraph*
node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE) {
continue;
}
// Operation support can allow tensors larger than the backend's allocation limit.
// Check inputs too, since cross-backend copies must also fit a single allocation.
if (!backend_can_allocate_op(current, node)) {
auto cpu = workspace_.cpu_backend();
if (cpu != nullptr && ggml_backend_supports_op(cpu, node)) {
src/model/diffusion/minimax_h3.hpp
+12 -11
View File
@@ -174,14 +174,14 @@ namespace MiniMaxH3 {
int64_t chunk_size = x->ne[1];
if (ctx->backend != nullptr) {
auto buft = ggml_backend_get_default_buffer_type(ctx->backend);
size_t alignment = ggml_backend_buft_get_alignment(buft);
size_t max_size = ggml_backend_buft_get_max_size(buft);
max_size -= max_size % alignment;
// Matmul outputs are F32 even for quantized weights. Split before fc1
// so its expanded activation never needs an oversized backend buffer.
auto buft = ggml_backend_get_default_buffer_type(ctx->backend);
size_t alignment = ggml_backend_buft_get_alignment(buft);
size_t chunk_budget = ggml_backend_buft_get_max_size(buft);
chunk_budget -= chunk_budget % alignment;
// Use the backend's preferred allocation size as a conservative chunk budget.
// Matmul outputs are F32 even for quantized weights.
size_t token_bytes = ggml_row_size(GGML_TYPE_F32, std::max(x->ne[0], 2 * ffn_hidden_size)) * x->ne[2] * x->ne[3];
chunk_size = static_cast<int64_t>(std::min<size_t>(chunk_size, std::max<size_t>(1, max_size / token_bytes)));
chunk_size = static_cast<int64_t>(std::min<size_t>(chunk_size, std::max<size_t>(1, chunk_budget / token_bytes)));
}
if (chunk_size == x->ne[1]) {
return forward_chunk(x);
@@ -269,10 +269,11 @@ namespace MiniMaxH3 {
if (ctx->backend != nullptr) {
auto buft = ggml_backend_get_default_buffer_type(ctx->backend);
size_t alignment = ggml_backend_buft_get_alignment(buft);
size_t max_size = ggml_backend_buft_get_max_size(buft);
max_size -= max_size % alignment;
// This is a conservative chunk budget, not an operation support limit.
size_t chunk_budget = ggml_backend_buft_get_max_size(buft);
chunk_budget -= chunk_budget % alignment;
size_t token_bytes = ggml_row_size(GGML_TYPE_F32, 3 * heads * head_dim) * inputs[0]->ne[2] * inputs[0]->ne[3];
chunk_size = static_cast<int64_t>(std::min<size_t>(chunk_size, std::max<size_t>(1, max_size / token_bytes)));
chunk_size = static_cast<int64_t>(std::min<size_t>(chunk_size, std::max<size_t>(1, chunk_budget / token_bytes)));
}
if (chunk_size == sequence) {
ggml_tensor* projected = nullptr;
@@ -284,7 +285,7 @@ namespace MiniMaxH3 {
return split_qkv(ctx, projected);
}
// Split Q/K/V before concatenating tokens so the full packed QKV never exceeds the buffer limit.
// Split Q/K/V before concatenating tokens to avoid materializing the full packed QKV.
std::vector<ggml_tensor*> chunks[3];
for (size_t i = 0; i < inputs.size(); ++i) {
auto input = inputs[i];