From 0b3b98d7b83ec433925252e1fbc00118629a3c5f Mon Sep 17 00:00:00 2001
From: leejet <leejet714@gmail.com>
Date: Sun, 8 Nov 2026 16:24:53 +0800
Subject: [PATCH] fix: avoid CPU fallback based on backend allocation chunk
 size (#2294)

---
 src/core/ggml_runner.cpp           |  7 +++++--
 src/model/diffusion/minimax_h3.hpp | 23 ++++++++++++-----------
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp
index 2fa14b15..770f46b2 100644
--- a/src/core/ggml_runner.cpp
+++ b/src/core/ggml_runner.cpp
@@ -212,6 +212,10 @@ ggml_backend_t GGMLRunner::backend_for_weight(const ggml_tensor* tensor) const {
 }
 
 static bool backend_can_allocate_op(ggml_backend_t backend, const ggml_tensor* node) {
+    // Vulkan reports a preferred allocation chunk size, not a hard tensor limit.
+    if (sd_backend_is(backend, "Vulkan")) {
+        return true;
+    }
     auto buft        = ggml_backend_get_default_buffer_type(backend);
     size_t alignment = ggml_backend_buft_get_alignment(buft);
     size_t max_size  = ggml_backend_buft_get_max_size(buft);
@@ -219,6 +223,7 @@ static bool backend_can_allocate_op(ggml_backend_t backend, const ggml_tensor* n
     if (ggml_backend_buft_get_alloc_size(buft, node) > max_size) {
         return false;
     }
+    // Cross-backend copies must also fit the destination's allocation limit.
     for (auto src : node->src) {
         if (src != nullptr && ggml_backend_buft_get_alloc_size(buft, src) > max_size) {
             return false;
@@ -251,8 +256,6 @@ void GGMLRunner::pin_multi_device_nodes(ggml_backend_sched_t sched, ggml_cgraph*
             node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE) {
             continue;
         }
-        // Operation support can allow tensors larger than the backend's allocation limit.
-        // Check inputs too, since cross-backend copies must also fit a single allocation.
         if (!backend_can_allocate_op(current, node)) {
             auto cpu = workspace_.cpu_backend();
             if (cpu != nullptr && ggml_backend_supports_op(cpu, node)) {
diff --git a/src/model/diffusion/minimax_h3.hpp b/src/model/diffusion/minimax_h3.hpp
index 97fd230e..5f388d84 100644
--- a/src/model/diffusion/minimax_h3.hpp
+++ b/src/model/diffusion/minimax_h3.hpp
@@ -174,14 +174,14 @@ namespace MiniMaxH3 {
 
             int64_t chunk_size = x->ne[1];
             if (ctx->backend != nullptr) {
-                auto buft        = ggml_backend_get_default_buffer_type(ctx->backend);
-                size_t alignment = ggml_backend_buft_get_alignment(buft);
-                size_t max_size  = ggml_backend_buft_get_max_size(buft);
-                max_size -= max_size % alignment;
-                // Matmul outputs are F32 even for quantized weights. Split before fc1
-                // so its expanded activation never needs an oversized backend buffer.
+                auto buft           = ggml_backend_get_default_buffer_type(ctx->backend);
+                size_t alignment    = ggml_backend_buft_get_alignment(buft);
+                size_t chunk_budget = ggml_backend_buft_get_max_size(buft);
+                chunk_budget -= chunk_budget % alignment;
+                // Use the backend's preferred allocation size as a conservative chunk budget.
+                // Matmul outputs are F32 even for quantized weights.
                 size_t token_bytes = ggml_row_size(GGML_TYPE_F32, std::max(x->ne[0], 2 * ffn_hidden_size)) * x->ne[2] * x->ne[3];
-                chunk_size         = static_cast<int64_t>(std::min<size_t>(chunk_size, std::max<size_t>(1, max_size / token_bytes)));
+                chunk_size         = static_cast<int64_t>(std::min<size_t>(chunk_size, std::max<size_t>(1, chunk_budget / token_bytes)));
             }
             if (chunk_size == x->ne[1]) {
                 return forward_chunk(x);
@@ -269,10 +269,11 @@ namespace MiniMaxH3 {
             if (ctx->backend != nullptr) {
                 auto buft        = ggml_backend_get_default_buffer_type(ctx->backend);
                 size_t alignment = ggml_backend_buft_get_alignment(buft);
-                size_t max_size  = ggml_backend_buft_get_max_size(buft);
-                max_size -= max_size % alignment;
+                // This is a conservative chunk budget, not an operation support limit.
+                size_t chunk_budget = ggml_backend_buft_get_max_size(buft);
+                chunk_budget -= chunk_budget % alignment;
                 size_t token_bytes = ggml_row_size(GGML_TYPE_F32, 3 * heads * head_dim) * inputs[0]->ne[2] * inputs[0]->ne[3];
-                chunk_size         = static_cast<int64_t>(std::min<size_t>(chunk_size, std::max<size_t>(1, max_size / token_bytes)));
+                chunk_size         = static_cast<int64_t>(std::min<size_t>(chunk_size, std::max<size_t>(1, chunk_budget / token_bytes)));
             }
             if (chunk_size == sequence) {
                 ggml_tensor* projected = nullptr;
@@ -284,7 +285,7 @@ namespace MiniMaxH3 {
                 return split_qkv(ctx, projected);
             }
 
-            // Split Q/K/V before concatenating tokens so the full packed QKV never exceeds the buffer limit.
+            // Split Q/K/V before concatenating tokens to avoid materializing the full packed QKV.
             std::vector<ggml_tensor*> chunks[3];
             for (size_t i = 0; i < inputs.size(); ++i) {
                 auto input = inputs[i];