CMakeLists.txt

@@ -343,11 +343,6 @@ add_subdirectory(thirdparty)
 target_sources(${SD_LIB} PRIVATE $<TARGET_OBJECTS:zip>)
 target_link_libraries(${SD_LIB} PUBLIC ggml)
 target_link_libraries(${SD_LIB} PRIVATE onig sd-utf8proc)
-if (SD_CUDA)
-    find_package(CUDAToolkit REQUIRED)
-    target_link_libraries(${SD_LIB} PRIVATE CUDA::cuda_driver)
-    set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_USE_CUDA)
-endif()
 target_include_directories(${SD_LIB} PUBLIC . src include)
 target_include_directories(${SD_LIB} PRIVATE src/core)
 target_include_directories(${SD_LIB} PUBLIC . thirdparty)

cmake/stable-diffusion-config.cmake.in

@@ -10,9 +10,6 @@ set(SD_BIN_DIR "@PACKAGE_SD_BIN_INSTALL_DIR@")
 
 include(CMakeFindDependencyMacro)
 find_dependency(ggml REQUIRED HINTS "${SD_LIB_DIR}/cmake")
-if(@SD_CUDA@ AND NOT SD_SHARED_LIB)
-    find_dependency(CUDAToolkit REQUIRED)
-endif()
 
 if(NOT TARGET stable-diffusion)
     find_library(stable-diffusion_LIBRARY stable-diffusion
@@ -31,10 +28,6 @@ if(NOT TARGET stable-diffusion)
             INTERFACE_COMPILE_FEATURES "c_std_11;cxx_std_17"
             POSITION_INDEPENDENT_CODE ON)
 
-    if(@SD_CUDA@ AND NOT SD_SHARED_LIB)
-        set_property(TARGET stable-diffusion APPEND PROPERTY INTERFACE_LINK_LIBRARIES CUDA::cuda_driver)
-    endif()
-
     if(SD_SHARED_LIB)
         target_compile_definitions(stable-diffusion
             INTERFACE SD_BUILD_SHARED_LIB)

ggml

@@ -1 +1 @@
-Subproject commit 1e22ec0d04b43afa69963e4b4ea6683535d54d79
+Subproject commit e20c3a14aa70ee84ca58499814206dd08d8026bc

src/core/backend_fit.cpp

@@ -102,7 +102,7 @@ namespace sd::backend_fit {
         for (const auto& [name, stored_tensor] : loader.get_tensor_storage_map()) {
             TensorStorage ts = stored_tensor;
             ComponentKind kind;
-            if (!classify_tensor(ts.name, kind)) {
+            if (is_unused_tensor(ts.name) || !classify_tensor(ts.name, kind)) {
                 continue;
             }
             if (ts.expected_type != GGML_TYPE_COUNT) {

src/core/ggml_extend.cpp

@@ -643,14 +643,6 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
     ggml_tensor* kqv = nullptr;
 
     auto build_kqv = [&](ggml_tensor* q_in, ggml_tensor* k_in, ggml_tensor* v_in, ggml_tensor* mask_in) -> ggml_tensor* {
-        const bool pad_head = d_head > 0 && d_head < 64 && q_in->ne[0] == d_head && k_in->ne[0] == d_head &&
-                              q_in->type == GGML_TYPE_F32 && k_in->type == GGML_TYPE_F32 &&
-                              v_in->type == GGML_TYPE_F32 && sd_backend_supports_cuda_mma(backend);
-        if (pad_head) {
-            // CUDA FA MMA starts at 64 channels; keep the original head's attention scale.
-            q_in = ggml_pad(ctx, q_in, 64 - d_head, 0, 0, 0);
-            k_in = ggml_pad(ctx, k_in, 64 - d_head, 0, 0, 0);
-        }
         if (kv_scale != 1.0f) {
             k_in = ggml_ext_scale(ctx, k_in, kv_scale);
         }
@@ -658,9 +650,6 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
 
         v_in = ggml_ext_cont(ctx, ggml_permute(ctx, v_in, 0, 2, 1, 3));
         v_in = ggml_reshape_3d(ctx, v_in, d_head, L_k, n_kv_head * N);
-        if (pad_head) {
-            v_in = ggml_pad(ctx, v_in, 64 - d_head, 0, 0, 0);
-        }
         if (kv_scale != 1.0f) {
             v_in = ggml_ext_scale(ctx, v_in, kv_scale);
         }
@@ -690,9 +679,6 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
         if (kv_scale != 1.0f) {
             out = ggml_ext_scale(ctx, out, 1.0f / kv_scale);
         }
-        if (pad_head) {
-            out = ggml_ext_slice(ctx, out, 0, 0, d_head);
-        }
         return out;
     };
 

src/core/ggml_extend_backend.cpp

@@ -8,10 +8,6 @@
 #include <stdexcept>
 #include <vector>
 
-#ifdef SD_USE_CUDA
-#include <cuda.h>
-#endif
-
 #include "core/util.h"
 #include "ggml/src/ggml-impl.h"
 #include "stable-diffusion.h"
@@ -433,70 +429,6 @@ bool sd_backend_is_cpu(ggml_backend_t backend) {
     return dev != nullptr && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU;
 }
 
-bool sd_backend_supports_cuda_mma(ggml_backend_t backend) {
-#ifdef SD_USE_CUDA
-    if (!sd_backend_is(backend, "CUDA")) {
-        return false;
-    }
-    auto dev = ggml_backend_get_device(backend);
-    if (dev == nullptr) {
-        return false;
-    }
-    static std::mutex mutex;
-    static std::unordered_map<ggml_backend_dev_t, bool> cache;
-    std::lock_guard<std::mutex> lock(mutex);
-    auto it = cache.find(dev);
-    if (it != cache.end()) {
-        return it->second;
-    }
-    const bool supported = [&]() {
-        ggml_backend_dev_props props{};
-        ggml_backend_dev_get_props(dev, &props);
-        CUdevice device;
-        int major = 0, minor = 0;
-        if (props.device_id == nullptr || cuInit(0) != CUDA_SUCCESS ||
-            cuDeviceGetByPCIBusId(&device, props.device_id) != CUDA_SUCCESS ||
-            cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device) != CUDA_SUCCESS ||
-            cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device) != CUDA_SUCCESS) {
-            return false;
-        }
-        auto reg          = ggml_backend_dev_backend_reg(dev);
-        auto get_features = reinterpret_cast<ggml_backend_get_features_t>(
-            ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features"));
-        if (get_features == nullptr) {
-            return false;
-        }
-        // Match ggml's highest compiled architecture for this device, including PTX fallback.
-        const int cc      = 100 * major + 10 * minor;
-        int compiled_arch = 0;
-        for (auto feature = get_features(reg); feature != nullptr && feature->name != nullptr; ++feature) {
-            if (std::strcmp(feature->name, "ARCHS") != 0 || feature->value == nullptr) {
-                continue;
-            }
-            const char* arch = feature->value;
-            while (*arch != '\0') {
-                char* end        = nullptr;
-                const long value = std::strtol(arch, &end, 10);
-                if (end == arch) {
-                    ++arch;
-                    continue;
-                }
-                if (value <= cc && value > compiled_arch) {
-                    compiled_arch = static_cast<int>(value);
-                }
-                arch = end;
-            }
-        }
-        return compiled_arch == 700 || compiled_arch >= 750;
-    }();
-    cache.emplace(dev, supported);
-    return supported;
-#else
-    (void)backend;
-    return false;
-#endif
-}
-
 ggml_backend_t sd_backend_cpu_init() {
     ggml_backend_load_all_once();
     return ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);

src/core/ggml_extend_backend.h

@@ -87,7 +87,6 @@ private:
 
 bool sd_backend_is(ggml_backend_t backend, const std::string& name);
 bool sd_backend_is_cpu(ggml_backend_t backend);
-bool sd_backend_supports_cuda_mma(ggml_backend_t backend);
 ggml_backend_t sd_backend_cpu_init();
 bool sd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads);
 ggml_status sd_backend_graph_compute_with_eval_callback(ggml_backend_t backend,

src/model/adapter/lora.hpp

@@ -67,7 +67,7 @@ struct LoraModel : public GGMLRunner {
         std::map<std::string, ggml_tensor*> scalars;
         std::set<std::string> scalar_names;
         for (const auto& [name, source] : sources) {
-            if (filter && !filter(name))
+            if (is_unused_tensor(name) || (filter && !filter(name)))
                 continue;
             const bool scalar  = source.nelements() == 1 && (ends_with(name, ".alpha") || ends_with(name, ".scale"));
             auto* tensor       = ggml_new_tensor(params_ctx, scalar ? GGML_TYPE_F32 : source.type, source.n_dims, source.ne);

src/model/te/llm.hpp

@@ -1377,7 +1377,7 @@ namespace LLM {
                 x        = ggml_ext_cont(ctx->ggml_ctx, kqv);
                 x        = ggml_reshape_3d(ctx->ggml_ctx, x, head_dim * num_heads, n_token, N);
             } else {
-                x = ggml_ext_attention_ext(ctx, q, k, v, num_heads, attention_mask, true, ctx->flash_attn_enabled);  // [N, n_token, hidden_size]
+                x = ggml_ext_attention_ext(ctx, q, k, v, num_heads, attention_mask, true, false);  // [N, n_token, hidden_size]
             }
 
             x = out_proj->forward(ctx, x);  // [N, n_token, hidden_size]

src/model_loader.cpp

@@ -35,6 +35,52 @@
 
 /*================================================= Preprocess ==================================================*/
 
+const char* unused_tensors[] = {
+    "betas",
+    "alphas_cumprod_prev",
+    "sqrt_alphas_cumprod",
+    "sqrt_one_minus_alphas_cumprod",
+    "log_one_minus_alphas_cumprod",
+    "sqrt_recip_alphas_cumprod",
+    "sqrt_recipm1_alphas_cumprod",
+    "posterior_variance",
+    "posterior_log_variance_clipped",
+    "posterior_mean_coef1",
+    "posterior_mean_coef2",
+    "cond_stage_model.transformer.text_model.embeddings.position_ids",
+    "cond_stage_model.1.model.text_model.embeddings.position_ids",
+    "cond_stage_model.transformer.vision_model.embeddings.position_ids",
+    "cond_stage_model.model.logit_scale",
+    "conditioner.embedders.0.transformer.text_model.embeddings.position_ids",
+    "conditioner.embedders.0.model.logit_scale",
+    "conditioner.embedders.1.model.logit_scale",
+    "model.diffusion_model.time_embedding.cond_proj.weight",
+    "unet.time_embedding.cond_proj.weight",
+    "model_ema.decay",
+    "model_ema.num_updates",
+    "model_ema.diffusion_model",
+    "embedding_manager",
+    "denoiser.sigmas",
+    "text_encoders.t5xxl.transformer.encoder.embed_tokens.weight",  // only used during training
+    "ztsnr",                                                        // Found in some SDXL vpred models
+    "edm_vpred.sigma_min",                                          // Found in CosXL
+    // TODO: find another way to avoid the "unknown tensor" for these two
+    // "edm_vpred.sigma_max", // Used to detect CosXL
+    // "v_pred", // Used to detect SDXL vpred models
+    "text_encoders.llm.output.weight",
+    "text_encoders.llm.lm_head.",
+    "language_model.lm_head.",
+};
+
+bool is_unused_tensor(const std::string& name) {
+    for (size_t i = 0; i < sizeof(unused_tensors) / sizeof(const char*); i++) {
+        if (starts_with(name, unused_tensors[i])) {
+            return true;
+        }
+    }
+    return false;
+}
+
 void f64_to_f32_vec(double* src, float* dst, int64_t n) {
     // support inplace op
     for (int64_t i = 0; i < n; i++) {
@@ -238,6 +284,10 @@ bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const
     size_t file_index = add_file_path(file_path);
 
     for (auto& tensor_storage : tensor_storages) {
+        if (is_unused_tensor(tensor_storage.name)) {
+            continue;
+        }
+
         if (!starts_with(tensor_storage.name, prefix)) {
             tensor_storage.name = prefix + tensor_storage.name;
         }
@@ -306,6 +356,10 @@ bool ModelLoader::init_from_torch_legacy_file(const std::string& file_path, cons
     size_t file_index = add_file_path(file_path);
 
     for (auto& tensor_storage : tensor_storages) {
+        if (is_unused_tensor(tensor_storage.name)) {
+            continue;
+        }
+
         if (!starts_with(tensor_storage.name, prefix)) {
             tensor_storage.name = prefix + tensor_storage.name;
         }
@@ -620,6 +674,10 @@ SDVersion ModelLoader::get_sd_version() const {
 std::map<ggml_type, uint32_t> ModelLoader::get_wtype_stat() const {
     std::map<ggml_type, uint32_t> wtype_stat;
     for (auto& [name, tensor_storage] : tensor_storage_map) {
+        if (is_unused_tensor(tensor_storage.name)) {
+            continue;
+        }
+
         auto iter = wtype_stat.find(tensor_storage.type);
         if (iter != wtype_stat.end()) {
             iter->second++;
@@ -633,6 +691,10 @@ std::map<ggml_type, uint32_t> ModelLoader::get_wtype_stat() const {
 std::map<ggml_type, uint32_t> ModelLoader::get_conditioner_wtype_stat() const {
     std::map<ggml_type, uint32_t> wtype_stat;
     for (auto& [name, tensor_storage] : tensor_storage_map) {
+        if (is_unused_tensor(tensor_storage.name)) {
+            continue;
+        }
+
         if ((tensor_storage.name.find("text_encoders") == std::string::npos &&
              tensor_storage.name.find("cond_stage_model") == std::string::npos &&
              tensor_storage.name.find("te.text_model.") == std::string::npos &&
@@ -653,6 +715,10 @@ std::map<ggml_type, uint32_t> ModelLoader::get_conditioner_wtype_stat() const {
 std::map<ggml_type, uint32_t> ModelLoader::get_diffusion_model_wtype_stat() const {
     std::map<ggml_type, uint32_t> wtype_stat;
     for (auto& [name, tensor_storage] : tensor_storage_map) {
+        if (is_unused_tensor(tensor_storage.name)) {
+            continue;
+        }
+
         if (tensor_storage.name.find("model.diffusion_model.") == std::string::npos && tensor_storage.name.find("unet.") == std::string::npos) {
             continue;
         }
@@ -670,6 +736,10 @@ std::map<ggml_type, uint32_t> ModelLoader::get_diffusion_model_wtype_stat() cons
 std::map<ggml_type, uint32_t> ModelLoader::get_vae_wtype_stat() const {
     std::map<ggml_type, uint32_t> wtype_stat;
     for (auto& [name, tensor_storage] : tensor_storage_map) {
+        if (is_unused_tensor(tensor_storage.name)) {
+            continue;
+        }
+
         if (tensor_storage.name.find("vae.") == std::string::npos &&
             tensor_storage.name.find("first_stage_model") == std::string::npos) {
             continue;
@@ -753,6 +823,9 @@ void ModelLoader::process_model_files(bool enable_mmap, bool writable_mmap) {
 
     std::vector<TensorStorage> processed_tensor_storages;
     for (const auto& [name, tensor_storage] : tensor_storage_map) {
+        if (is_unused_tensor(tensor_storage.name)) {
+            continue;
+        }
         processed_tensor_storages.push_back(tensor_storage);
     }
 
@@ -1478,6 +1551,9 @@ int64_t ModelLoader::get_params_mem_size(ggml_backend_t backend, ggml_type type)
     int64_t mem_size = 0;
     std::vector<TensorStorage> processed_tensor_storages;
     for (auto [name, tensor_storage] : tensor_storage_map) {
+        if (is_unused_tensor(tensor_storage.name)) {
+            continue;
+        }
         if (tensor_should_be_converted(tensor_storage, type)) {
             tensor_storage.type = type;
         }

src/model_loader.h

@@ -28,6 +28,8 @@ struct MmapTensorStore {
     std::shared_ptr<struct ggml_backend_buffer> mmbuffer;
 };
 
+bool is_unused_tensor(const std::string& name);
+
 class ModelLoader {
 public:
     using FileId       = uint64_t;