ci: cache CUDA compilation for Windows and Docker builds

.github/workflows/build.yml

@@ -16,6 +16,7 @@ on:
         ".github/workflows/**",
         ".dockerignore",
         "docker/Dockerfile*",
+        "scripts/ci-cuda-cache.sh",
         "**/CMakeLists.txt",
         "**/Makefile",
         "**/*.h",
@@ -38,6 +39,7 @@ on:
         ".github/workflows/**",
         ".dockerignore",
         "docker/Dockerfile*",
+        "scripts/ci-cuda-cache.sh",
         "**/CMakeLists.txt",
         "**/Makefile",
         "**/*.h",
@@ -253,6 +255,7 @@ jobs:
         run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
 
       - name: Set up Docker Buildx
+        id: buildx
         uses: docker/setup-buildx-action@v4
 
       - name: Log in to the container registry
@@ -275,6 +278,21 @@ jobs:
           # if set to "true" but frees about 6 GB
           tool-cache: false
 
+      - name: Cache CUDA compilation
+        if: matrix.variant == 'cuda'
+        uses: actions/cache@v5
+        with:
+          path: ${{ runner.temp }}/cuda-ccache
+          key: cuda-ccache-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('docker/Dockerfile.cuda') }}-${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: |
+            cuda-ccache-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('docker/Dockerfile.cuda') }}-
+
+      - name: Restore CUDA compiler cache mount
+        if: matrix.variant == 'cuda'
+        env:
+          CUDA_CACHE_BUILDER: ${{ steps.buildx.outputs.name }}
+        run: bash scripts/ci-cuda-cache.sh restore
+
       - name: Build and push Docker image
         id: build-push
         uses: docker/build-push-action@v7
@@ -288,6 +306,12 @@ jobs:
           annotations: ${{ steps.meta.outputs.annotations }}
           build-args: ${{ matrix.build-args }}
 
+      - name: Export CUDA compiler cache mount
+        if: matrix.variant == 'cuda'
+        env:
+          CUDA_CACHE_BUILDER: ${{ steps.buildx.outputs.name }}
+        run: bash scripts/ci-cuda-cache.sh save
+
   macOS-latest-cmake:
     runs-on: macos-latest
 
@@ -365,7 +389,7 @@ jobs:
           - build: "cpu"
             defines: "-DGGML_NATIVE=OFF -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
           - build: "cuda12"
-            defines: "-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_ARCHITECTURES='61;70;75;80;86;89;90;100;120' -DCMAKE_CUDA_FLAGS='-Xcudafe \"--diag_suppress=177\" -Xcudafe \"--diag_suppress=550\"' -DGGML_NATIVE=OFF -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
+            defines: "-DSD_CUDA=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_CUDA_COMPILER_LAUNCHER=sccache -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_ARCHITECTURES='61;70;75;80;86;89;90;100;120' -DCMAKE_CUDA_FLAGS='-Xcudafe \"--diag_suppress=177\" -Xcudafe \"--diag_suppress=550\"' -DGGML_NATIVE=OFF -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
           - build: "vulkan"
             defines: "-DSD_VULKAN=ON -DSD_BUILD_SHARED_LIBS=ON -DGGML_NATIVE=OFF -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
     steps:
@@ -394,6 +418,14 @@ jobs:
           method: "network"
           sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
 
+      - name: Cache CUDA compilation
+        if: matrix.build == 'cuda12'
+        uses: ggml-org/ccache-action@v1.2.21
+        with:
+          variant: sccache
+          key: windows-cuda-12.8.1-x64
+          max-size: "2G"
+
       - name: Install Vulkan SDK
         id: get_vulkan
         if: ${{ matrix.build == 'vulkan' }}

docker/Dockerfile.cuda

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1
+
 ARG CUDA_VERSION=12.6.3
 ARG UBUNTU_VERSION=24.04
 
@@ -25,8 +27,15 @@ ARG CUDA_ARCHITECTURES=""
 ARG GGML_CUDA_FA_ALL_QUANTS=""
 ARG GGML_CUDA_ENABLE_DYNAMIC_CPU_BACKENDS=ON
 
+ENV CCACHE_DIR=/root/.cache/ccache \
+    CCACHE_MAXSIZE=2G \
+    CCACHE_COMPILERCHECK=content
+
 RUN set -- \
-        -DSD_CUDA=ON; \
+        -DSD_CUDA=ON \
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache; \
     if [ "${GGML_CUDA_ENABLE_DYNAMIC_CPU_BACKENDS}" = "ON" ]; then \
         set -- "$@" \
             -DSD_BUILD_SHARED_LIBS=ON \
@@ -44,7 +53,11 @@ RUN set -- \
         set -- "$@" "-DGGML_CUDA_FA_ALL_QUANTS=${GGML_CUDA_FA_ALL_QUANTS}"; \
     fi; \
     cmake . -B ./build "$@"
-RUN cmake --build ./build --config Release -j$(nproc)
+RUN --mount=type=cache,target=/root/.cache/ccache \
+    ccache --zero-stats && \
+    cmake --build ./build --config Release -j$(nproc) && \
+    ccache --cleanup && \
+    ccache --show-stats
 
 FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION} AS runtime
 

scripts/ci-cuda-cache.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+mode="${1:-}"
+if [[ "$mode" != restore && "$mode" != save ]]; then
+    echo "Usage: $0 restore|save" >&2
+    exit 1
+fi
+
+cache_dir="${RUNNER_TEMP:?}/cuda-ccache"
+builder="${CUDA_CACHE_BUILDER:?}"
+work_dir="$(mktemp -d "${RUNNER_TEMP}/cuda-cache.XXXXXX")"
+trap 'rm -rf -- "$work_dir"' EXIT
+mkdir -p "$cache_dir"
+
+# Cache mounts are outside image layers; always execute the transfer RUN.
+if [[ "$mode" == restore ]]; then
+    cat > "$work_dir/Dockerfile" <<'DOCKERFILE'
+FROM ubuntu:24.04
+RUN --mount=type=cache,target=/root/.cache/ccache \
+    --mount=type=bind,source=.,target=/cache-input \
+    cp -a /cache-input/. /root/.cache/ccache/
+DOCKERFILE
+    docker buildx build --builder "$builder" --no-cache \
+        --file "$work_dir/Dockerfile" --output type=cacheonly "$cache_dir"
+else
+    cat > "$work_dir/Dockerfile" <<'DOCKERFILE'
+FROM ubuntu:24.04 AS transfer
+RUN --mount=type=cache,target=/root/.cache/ccache \
+    mkdir /cache-output && cp -a /root/.cache/ccache/. /cache-output/
+FROM scratch
+COPY --from=transfer /cache-output/ /
+DOCKERFILE
+    docker buildx build --builder "$builder" --no-cache \
+        --file "$work_dir/Dockerfile" \
+        --output "type=local,dest=$work_dir/export" "$work_dir"
+    # Replace only after export succeeds, including removal of evicted entries.
+    mv "$cache_dir" "$work_dir/previous-cache"
+    mv "$work_dir/export" "$cache_dir"
+fi