diff --git a/.gitmodules b/.gitmodules index a26210d9..9e1837e1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "thirdparty/libwebm"] path = thirdparty/libwebm url = https://github.com/webmproject/libwebm.git +[submodule "thirdparty/cutlass"] + path = thirdparty/cutlass + url = https://github.com/NVIDIA/cutlass.git diff --git a/CMakeLists.txt b/CMakeLists.txt index db8ebde6..e6681317 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -346,12 +346,13 @@ if (SD_CUDA) target_link_libraries(${SD_LIB} PUBLIC CUDA::cuda_driver) set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_USE_CUDA) if (NOT SD_USE_UPSTREAM_GGML) + include(cmake/cutlass.cmake) enable_language(CUDA) set_target_properties(${SD_LIB} PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON) - if (GGML_CUDA_CUTLASS_PATH) + if (SD_CUDA_CUTLASS_PATH) target_sources(${SD_LIB} PRIVATE src/core/cuda/cute_attention.cu src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu) set_source_files_properties(src/core/cuda/cute_attention.cu src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu PROPERTIES - INCLUDE_DIRECTORIES "${GGML_CUDA_CUTLASS_PATH}/include" + INCLUDE_DIRECTORIES "${SD_CUDA_CUTLASS_PATH}/include" COMPILE_OPTIONS "--use_fast_math;--extended-lambda;--expt-relaxed-constexpr") if (MSVC) set_property(SOURCE src/core/cuda/cute_attention.cu src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu APPEND PROPERTY COMPILE_OPTIONS -Xcompiler=/Zc:preprocessor,/Zc:__cplusplus) diff --git a/cmake/cutlass.cmake b/cmake/cutlass.cmake new file mode 100644 index 00000000..0ec6f32f --- /dev/null +++ b/cmake/cutlass.cmake @@ -0,0 +1,20 @@ +set(sd_cutlass_default "${PROJECT_SOURCE_DIR}/thirdparty/cutlass") +if(NOT DEFINED SD_CUDA_CUTLASS_PATH AND DEFINED GGML_CUDA_CUTLASS_PATH) + get_filename_component(sd_old_cutlass_default "${SD_GGML_SOURCE_DIR}/src/ggml-cuda/vendors/cutlass" ABSOLUTE) + if(NOT GGML_CUDA_CUTLASS_PATH STREQUAL sd_old_cutlass_default) + set(sd_cutlass_default "${GGML_CUDA_CUTLASS_PATH}") + endif() + message(STATUS "Migrating GGML_CUDA_CUTLASS_PATH to SD_CUDA_CUTLASS_PATH") +endif() +set(SD_CUDA_CUTLASS_PATH "${sd_cutlass_default}" CACHE PATH "sd: CUTLASS source directory for INT8 GEMM and CuTe attention, empty to disable") +unset(GGML_CUDA_CUTLASS_PATH CACHE) +if(SD_CUDA_CUTLASS_PATH) + if(NOT EXISTS "${SD_CUDA_CUTLASS_PATH}/include/cutlass/gemm/kernel/default_gemm_universal_with_visitor.h" OR + NOT EXISTS "${SD_CUDA_CUTLASS_PATH}/include/cute/tensor.hpp") + message(FATAL_ERROR + "CUTLASS headers not found at ${SD_CUDA_CUTLASS_PATH}. " + "Run git submodule update --init --recursive from the sd.cpp repository root, " + "set SD_CUDA_CUTLASS_PATH to a CUTLASS source tree (tested with 3.9.2), " + "or set SD_CUDA_CUTLASS_PATH to an empty string to disable CUTLASS.") + endif() +endif() diff --git a/docs/build.md b/docs/build.md index ef79c263..62966ec6 100644 --- a/docs/build.md +++ b/docs/build.md @@ -111,7 +111,7 @@ cmake .. -DSD_CUDA=ON cmake --build . --config Release ``` -CUDA builds with the bundled patched GGML use the CUTLASS submodule by default +CUDA extensions use sd.cpp's `thirdparty/cutlass` submodule by default for eligible INT8 GEMM and CuTe attention operations. If its headers are missing, CMake stops with an initialization hint; it does not download them automatically. Run `git submodule update --init --recursive` from the repository root before @@ -121,14 +121,17 @@ To build with the existing CUDA implementations without CUTLASS, configure from the repository root with: ```shell -cmake -S . -B build -DSD_CUDA=ON -DGGML_CUDA_CUTLASS_PATH:PATH= +cmake -S . -B build -DSD_CUDA=ON -DSD_CUDA_CUTLASS_PATH:PATH= cmake --build build --config Release ``` -Alternatively, set `GGML_CUDA_CUTLASS_PATH` to another local CUTLASS source +Alternatively, set `SD_CUDA_CUTLASS_PATH` to another local CUTLASS source directory (tested with 3.9.2). Existing CMake caches retain their configured path; -pass `-UGGML_CUDA_CUTLASS_PATH` when reconfiguring to restore the bundled default. -Non-CUDA builds do not require CUTLASS. +pass `-USD_CUDA_CUTLASS_PATH` when reconfiguring to restore the bundled default. +Existing `GGML_CUDA_CUTLASS_PATH` caches are migrated on first configuration: +the old bundled path becomes `thirdparty/cutlass`, while custom paths and an +empty disabled value are preserved. An explicitly set `SD_CUDA_CUTLASS_PATH` +takes precedence. Non-CUDA builds do not require CUTLASS. ## Build with HipBLAS diff --git a/ggml b/ggml index 50ebf58e..a57ef572 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit 50ebf58e14e0974ba043183c8c2b2c8bf6657a2c +Subproject commit a57ef57267625ff1ea3858718c167ee0aa426807 diff --git a/src/core/cuda/CMakeLists.txt b/src/core/cuda/CMakeLists.txt index e6834c66..a1258d94 100644 --- a/src/core/cuda/CMakeLists.txt +++ b/src/core/cuda/CMakeLists.txt @@ -26,7 +26,7 @@ if(SD_CUDA) target_sources(${SD_CUDA_TARGET} PRIVATE sage_attention.cu sol_attention.cu) set_source_files_properties(sage_attention.cu sol_attention.cu PROPERTIES COMPILE_OPTIONS "--use_fast_math") set_source_files_properties(${SD_CUDA_SOURCES} PROPERTIES COMPILE_OPTIONS "--use_fast_math") - if(GGML_CUDA_CUTLASS_PATH) + if(SD_CUDA_CUTLASS_PATH) set_property(SOURCE "${CMAKE_CURRENT_BINARY_DIR}/int8_cuda.cu" APPEND PROPERTY COMPILE_DEFINITIONS SD_CUDA_I8_CUTLASS) set_property(SOURCE "${CMAKE_CURRENT_BINARY_DIR}/registration_cuda.cu" APPEND PROPERTY COMPILE_DEFINITIONS SD_CUDA_CUTE_ATTENTION) set_property(SOURCE "${CMAKE_CURRENT_BINARY_DIR}/int8_cuda.cu" APPEND PROPERTY INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/src/core/cuda") diff --git a/thirdparty/cutlass b/thirdparty/cutlass new file mode 160000 index 00000000..ad7b2f5e --- /dev/null +++ b/thirdparty/cutlass @@ -0,0 +1 @@ +Subproject commit ad7b2f5e84fcfa124cb02b91d5bd26d238c0459e