.github/pull_request_template.md

@@ -1,15 +0,0 @@
-## Summary
-
-<!-- Describe what changed and why. Keep the PR focused on one clear change. -->
-
-## Related Issue / Discussion
-
-<!-- Link related issues, discussions, or previous PRs if applicable. -->
-
-## Additional Information
-
-<!-- Add verification notes, screenshots, sample output, or other context when applicable. -->
-
-## Checklist
-
-- [ ] I have read and confirmed this PR follows the [contribution guidelines](https://github.com/leejet/stable-diffusion.cpp/blob/master/CONTRIBUTING.md).

.github/workflows/build.yml

@@ -14,8 +14,6 @@ on:
     paths:
       [
         ".github/workflows/**",
-        ".dockerignore",
-        "Dockerfile*",
         "**/CMakeLists.txt",
         "**/Makefile",
         "**/*.h",
@@ -23,7 +21,6 @@ on:
         "**/*.c",
         "**/*.cpp",
         "**/*.cu",
-        "examples/server/frontend",
         "examples/server/frontend/**",
       ]
   pull_request:
@@ -31,8 +28,6 @@ on:
     paths:
       [
         ".github/workflows/**",
-        ".dockerignore",
-        "Dockerfile*",
         "**/CMakeLists.txt",
         "**/Makefile",
         "**/*.h",
@@ -40,7 +35,6 @@ on:
         "**/*.c",
         "**/*.cpp",
         "**/*.cu",
-        "examples/server/frontend",
         "examples/server/frontend/**",
       ]
 
@@ -70,7 +64,7 @@ jobs:
       - name: Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 10.15.1
+          version: 9
 
       - name: Dependencies
         id: depends
@@ -133,13 +127,13 @@ jobs:
       - name: Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 10.15.1
+          version: 9
 
       - name: Dependencies
         id: depends
         run: |
           sudo apt-get update
-          sudo apt-get install build-essential libvulkan-dev glslc spirv-headers
+          sudo apt-get install build-essential libvulkan-dev glslc
 
       - name: Build
         id: cmake_build
@@ -180,8 +174,7 @@ jobs:
 
   build-and-push-docker-images:
     name: Build and push container images
-    if: ${{ github.event_name != 'pull_request' }}
+    runs-on: ubuntu-latest
-    runs-on: ${{ matrix.runner }}
 
     permissions:
       contents: read
@@ -193,20 +186,6 @@ jobs:
     strategy:
       matrix:
         variant: [musa, sycl, vulkan, cuda]
-        platform: [linux/amd64]
-        runner: [ubuntu-latest]
-        build-args: [""]
-        tag-suffix: [""]
-        include:
-          - variant: cuda
-            platform: linux/arm64
-            runner: ubuntu-24.04-arm
-            tag-suffix: "-spark"
-            build-args: |
-              CUDA_VERSION=13.0.0
-              UBUNTU_VERSION=24.04
-              CUDA_ARCHITECTURES=121
-              GGML_CUDA_FA_ALL_QUANTS=ON
 
     env:
       REGISTRY: ghcr.io
@@ -226,7 +205,7 @@ jobs:
       - name: Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 10.15.1
+          version: 9
 
       - name: Get commit hash
         id: commit
@@ -260,14 +239,12 @@ jobs:
         id: build-push
         uses: docker/build-push-action@v6
         with:
-          context: .
+          platforms: linux/amd64
-          platforms: ${{ matrix.platform }}
           push: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
           file: Dockerfile.${{ matrix.variant }}
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH_NAME }}-${{ matrix.variant }}${{ matrix.tag-suffix }}
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH_NAME }}-${{ matrix.variant }}
           labels: ${{ steps.meta.outputs.labels }}
           annotations: ${{ steps.meta.outputs.annotations }}
-          build-args: ${{ matrix.build-args }}
 
   macOS-latest-cmake:
     runs-on: macos-latest
@@ -287,7 +264,7 @@ jobs:
       - name: Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 10.15.1
+          version: 9
 
       - name: Dependencies
         id: depends
@@ -368,7 +345,7 @@ jobs:
       - name: Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 10.15.1
+          version: 9
 
       - name: Install cuda-toolkit
         id: cuda-toolkit
@@ -463,129 +440,12 @@ jobs:
           path: |
             sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip
 
-  windows-latest-rocm:
-    runs-on: windows-2022
-
-    env:
-      ROCM_VERSION: "7.13.0"
-      GPU_TARGETS: "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: recursive
-
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 10.15.1
-
-      - name: Cache ROCm Installation
-        id: cache-rocm
-        uses: actions/cache@v4
-        with:
-          path: C:\TheRock\build
-          key: rocm-${{ env.ROCM_VERSION }}-gfx1151-${{ runner.os }}
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.16
-        with:
-          key: windows-latest-rocm-${{ env.ROCM_VERSION }}-x64
-          evict-old-files: 1d
-
-      - name: Install ROCm
-        if: steps.cache-rocm.outputs.cache-hit != 'true'
-        run: |
-          $ErrorActionPreference = "Stop"
-          write-host "Downloading AMD ROCm ${{ env.ROCM_VERSION }} tarball"
-          Invoke-WebRequest -Uri "https://repo.amd.com/rocm/tarball/therock-dist-windows-gfx1151-${{ env.ROCM_VERSION }}.tar.gz" -OutFile "${env:RUNNER_TEMP}\rocm.tar.gz"
-          write-host "Extracting ROCm tarball"
-          mkdir C:\TheRock\build -Force
-          tar -xzf "${env:RUNNER_TEMP}\rocm.tar.gz" -C C:\TheRock\build --strip-components=1
-          write-host "Completed ROCm extraction"
-
-      - name: Setup ROCm Environment
-        run: |
-          $rocmPath = "C:\TheRock\build"
-          echo "HIP_PATH=$rocmPath" >> $env:GITHUB_ENV
-          echo "HIP_DEVICE_LIB_PATH=$rocmPath\lib\llvm\amdgcn\bitcode" >> $env:GITHUB_ENV
-          echo "HIP_PLATFORM=amd" >> $env:GITHUB_ENV
-          echo "LLVM_PATH=$rocmPath\lib\llvm" >> $env:GITHUB_ENV
-          echo "$rocmPath\bin" >> $env:GITHUB_PATH
-          echo "$rocmPath\lib\llvm\bin" >> $env:GITHUB_PATH
-
-      - name: Build
-        run: |
-          mkdir build
-          cd build
-          cmake .. `
-            -G "Unix Makefiles" `
-            -DCMAKE_PREFIX_PATH="${env:HIP_PATH}" `
-            -DSD_HIPBLAS=ON `
-            -DSD_BUILD_SHARED_LIBS=ON `
-            -DGGML_NATIVE=OFF `
-            -DCMAKE_C_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
-            -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang++.exe" `
-            -DCMAKE_HIP_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
-            -DHIP_PATH="${env:HIP_PATH}" `
-            -DCMAKE_BUILD_TYPE=Release `
-            -DGPU_TARGETS="${{ env.GPU_TARGETS }}"
-          cmake --build . --config Release --parallel ${env:NUMBER_OF_PROCESSORS}
-
-      - name: Get commit hash
-        id: commit
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: pr-mpt/actions-commit-hash@v2
-
-      - name: Pack artifacts
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        run: |
-          $ErrorActionPreference = "Stop"
-          $dst = "build\bin"
-          $rocmBin = Join-Path "${env:HIP_PATH}" "bin"
-          $requiredRocmPaths = @(
-            (Join-Path $rocmBin "rocblas.dll"),
-            (Join-Path $rocmBin "rocblas\library")
-          )
-          foreach ($path in $requiredRocmPaths) {
-            if (!(Test-Path $path)) {
-              throw "Missing ROCm runtime dependency: $path"
-            }
-          }
-
-          foreach ($pattern in @("rocblas*.dll", "hipblas*.dll", "libhipblas*.dll")) {
-            Copy-Item -Path (Join-Path $rocmBin $pattern) -Destination $dst -Force -ErrorAction SilentlyContinue
-          }
-
-          foreach ($dir in @("rocblas", "hipblaslt")) {
-            $src = Join-Path $rocmBin $dir
-            if (Test-Path $src) {
-              Copy-Item -Path $src -Destination $dst -Recurse -Force
-            }
-          }
-
-          7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip .\build\bin\*
-
-      - name: Upload artifacts
-        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip
-          path: |
-            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip
-
   windows-latest-cmake-hip:
     runs-on: windows-2022
 
     env:
-      HIPSDK_INSTALLER_VERSION: "26.Q1"
+      HIPSDK_INSTALLER_VERSION: "25.Q3"
-      ROCM_VERSION: "7.1.1"
+      GPU_TARGETS: "gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
-      GPU_TARGETS: "gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
 
     steps:
       - uses: actions/checkout@v3
@@ -600,7 +460,7 @@ jobs:
       - name: Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 10.15.1
+          version: 9
 
       - name: Cache ROCm Installation
         id: cache-rocm
@@ -620,7 +480,7 @@ jobs:
         run: |
           $ErrorActionPreference = "Stop"
           write-host "Downloading AMD HIP SDK Installer"
-          Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-Win11-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe"
+          Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-WinSvr2022-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe"
           write-host "Installing AMD HIP SDK"
           $proc = Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -PassThru
           $completed = $proc.WaitForExit(600000)
@@ -673,75 +533,47 @@ jobs:
         run: |
           md "build\bin\rocblas\library\"
           md "build\bin\hipblaslt\library"
-          cp "${env:HIP_PATH}\bin\libhipblas.dll" "build\bin\"
+          cp "${env:HIP_PATH}\bin\hipblas.dll" "build\bin\"
-          cp "${env:HIP_PATH}\bin\libhipblaslt.dll" "build\bin\"
+          cp "${env:HIP_PATH}\bin\hipblaslt.dll" "build\bin\"
           cp "${env:HIP_PATH}\bin\rocblas.dll" "build\bin\"
           cp "${env:HIP_PATH}\bin\rocblas\library\*" "build\bin\rocblas\library\"
           cp "${env:HIP_PATH}\bin\hipblaslt\library\*" "build\bin\hipblaslt\library\"
-          7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip .\build\bin\*
+          7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-x64.zip .\build\bin\*
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
         uses: actions/upload-artifact@v4
         with:
-          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip
+          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-x64.zip
           path: |
-            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip
+            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-x64.zip
 
   ubuntu-latest-rocm:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
+    container: rocm/dev-ubuntu-24.04:7.2
 
     env:
+      ROCM_VERSION: "7.2"
       UBUNTU_VERSION: "24.04"
-
+      GPU_TARGETS: "gfx1151;gfx1150;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201"
-    strategy:
-      matrix:
-        include:
-          - ROCM_VERSION: "7.2.1"
-            gpu_targets: "gfx908;gfx90a;gfx942;gfx1030;gfx1031;gfx1032;gfx1100;gfx1101;gfx1102;gfx1151;gfx1150;gfx1200;gfx1201"
-            build: 'x64'
-          - ROCM_VERSION: "7.13.0"
-            gpu_targets: "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
-            build: x64
 
     steps:
+      - run: apt-get update && apt-get install -y git
       - name: Clone
         id: checkout
         uses: actions/checkout@v6
         with:
           submodules: recursive
 
-      - name: ccache
+      - name: Setup Node
-        uses: ggml-org/ccache-action@v1.2.16
+        uses: actions/setup-node@v4
         with:
-          key: ubuntu-rocm-cmake-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
+          node-version: 20
-          evict-old-files: 1d
 
-      - name: Dependencies
+      - name: Setup pnpm
-        id: depends
+        uses: pnpm/action-setup@v4
-        run: |
+        with:
-          sudo apt install -y build-essential cmake wget zip ninja-build
+          version: 9
-
-      - name: Setup Legacy ROCm
-        if: matrix.ROCM_VERSION == '7.2.1'
-        id: legacy_env
-        run: |
-          sudo mkdir --parents --mode=0755 /etc/apt/keyrings
-          wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \
-            gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
-
-          sudo tee /etc/apt/sources.list.d/rocm.list << EOF
-          deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${{ matrix.ROCM_VERSION }} noble main
-          EOF
-
-          sudo tee /etc/apt/preferences.d/rocm-pin-600 << EOF
-          Package: *
-          Pin: release o=repo.radeon.com
-          Pin-Priority: 600
-          EOF
-
-          sudo apt update
-          sudo apt-get install -y libssl-dev rocm-hip-sdk
 
       - name: Free disk space
         run: |
@@ -756,29 +588,51 @@ jobs:
           sudo rm -rf /var/lib/apt/lists/* || true
           sudo apt clean
 
-      - name: Setup TheRock
+      - name: Dependencies
-        if: matrix.ROCM_VERSION != '7.2.1'
+        id: depends
-        id: therock_env
         run: |
-          wget https://repo.amd.com/rocm/tarball/therock-dist-linux-gfx1151-${{ matrix.ROCM_VERSION }}.tar.gz
+          sudo apt-get update
-          mkdir install
+          sudo apt install -y \
-          tar -xf *.tar.gz -C install
+            cmake \
-          export ROCM_PATH=$(pwd)/install
+            hip-dev \
-          echo ROCM_PATH=$ROCM_PATH >> $GITHUB_ENV
+            hipblas-dev \
-          echo PATH=$PATH:$ROCM_PATH/bin >> $GITHUB_ENV
+            ninja-build \
-          echo LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/llvm/lib:$ROCM_PATH/lib/rocprofiler-systems >> $GITHUB_ENV
+            rocm-dev \
+            zip
+          # Clean apt caches to recover disk space
+          sudo apt clean
+          sudo rm -rf /var/lib/apt/lists/* || true
 
-      # setup-node installs into /opt/hostedtoolcache, which is removed above.
+      - name: Setup ROCm Environment
-      # Keep Node/pnpm setup after disk cleanup so the server frontend can be embedded.
+        run: |
-      - name: Setup Node
+          # Add ROCm to PATH for current session
-        uses: actions/setup-node@v4
+          echo "/opt/rocm/bin" >> $GITHUB_PATH
-        with:
-          node-version: 20
 
-      - name: Setup pnpm
+          # Build regex pattern from ${{ env.GPU_TARGETS }} (match target as substring)
-        uses: pnpm/action-setup@v4
+          TARGET_REGEX="($(printf '%s' "${{ env.GPU_TARGETS }}" | sed 's/;/|/g'))"
-        with:
+
-          version: 10.15.1
+          # Remove library files for architectures we're not building for to save disk space
+          echo "Cleaning up unneeded architecture files..."
+          cd /opt/rocm/lib/rocblas/library
+          # Keep only our target architectures
+          for file in *; do
+            if printf '%s' "$file" | grep -q 'gfx'; then
+              if ! printf '%s' "$file" | grep -Eq "$TARGET_REGEX"; then
+                echo "Removing $file" &&
+                sudo rm -f "$file";
+              fi
+            fi
+          done
+
+          cd /opt/rocm/lib/hipblaslt/library
+          for file in *; do
+            if printf '%s' "$file" | grep -q 'gfx'; then
+              if ! printf '%s' "$file" | grep -Eq "$TARGET_REGEX"; then
+                echo "Removing $file" &&
+                sudo rm -f "$file";
+              fi
+            fi
+          done
 
       - name: Build
         id: cmake_build
@@ -786,12 +640,12 @@ jobs:
           mkdir build
           cd build
           cmake .. -G Ninja \
-            -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
+            -DCMAKE_CXX_COMPILER=amdclang++ \
-            -DCMAKE_HIP_FLAGS="-mllvm --amdgpu-unroll-threshold-local=600" \
+            -DCMAKE_C_COMPILER=amdclang \
             -DCMAKE_BUILD_TYPE=Release \
             -DSD_HIPBLAS=ON \
-            -DHIP_PLATFORM=amd \
+            -DGPU_TARGETS="${{ env.GPU_TARGETS }}" \
-            -DGPU_TARGETS="${{ matrix.gpu_targets }}" \
+            -DAMDGPU_TARGETS="${{ env.GPU_TARGETS }}" \
             -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
             -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
             -DSD_BUILD_SHARED_LIBS=ON
@@ -810,6 +664,16 @@ jobs:
           cp ggml/LICENSE ./build/bin/ggml.txt
           cp LICENSE ./build/bin/stable-diffusion.cpp.txt
 
+          # Move ROCm runtime libraries (to avoid double space consumption)
+          sudo mv /opt/rocm/lib/librocsparse.so* ./build/bin/
+          sudo mv /opt/rocm/lib/libhsa-runtime64.so* ./build/bin/
+          sudo mv /opt/rocm/lib/libamdhip64.so* ./build/bin/
+          sudo mv /opt/rocm/lib/libhipblas.so* ./build/bin/
+          sudo mv /opt/rocm/lib/libhipblaslt.so* ./build/bin/
+          sudo mv /opt/rocm/lib/librocblas.so* ./build/bin/
+          sudo mv /opt/rocm/lib/rocblas/ ./build/bin/
+          sudo mv /opt/rocm/lib/hipblaslt/ ./build/bin/
+
       - name: Fetch system info
         id: system-info
         run: |
@@ -824,15 +688,15 @@ jobs:
         run: |
           cp ggml/LICENSE ./build/bin/ggml.txt
           cp LICENSE ./build/bin/stable-diffusion.cpp.txt
-          zip -y -r sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip ./build/bin
+          zip -y -r sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm.zip ./build/bin
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
         uses: actions/upload-artifact@v4
         with:
-          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip
+          name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm.zip
           path: |
-            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip
+            sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm.zip
 
   release:
     if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
@@ -847,7 +711,6 @@ jobs:
       - macOS-latest-cmake
       - windows-latest-cmake
       - windows-latest-cmake-hip
-      - windows-latest-rocm
 
     steps:
       - name: Clone

.github/workflows/stale-prs.yml

@@ -1,55 +0,0 @@
-name: Close inactive PRs
-
-on:
-  schedule:
-    # Run daily. GitHub cron schedules use UTC.
-    - cron: "30 1 * * *"
-  workflow_dispatch:
-    inputs:
-      debug_only:
-        description: "Dry run: log intended actions without changing PRs"
-        required: false
-        default: false
-        type: boolean
-
-permissions:
-  issues: write
-  pull-requests: write
-
-concurrency:
-  group: ${{ github.workflow }}
-  cancel-in-progress: false
-
-jobs:
-  stale-prs:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Mark and close inactive PRs
-        uses: actions/stale@v10
-        with:
-          days-before-issue-stale: -1
-          days-before-issue-close: -1
-
-          days-before-pr-stale: 365
-          days-before-pr-close: 7
-
-          stale-pr-label: pr:inactive
-          close-pr-label: pr:auto-closed
-          exempt-pr-labels: pr:keep-open
-
-          stale-pr-message: >
-            This PR has been inactive for 365 days. If there is no new activity
-            within 7 days, it will be closed automatically. Comment, push new
-            commits, or remove the pr:inactive label to keep it open. Add
-            pr:keep-open to exempt it from future inactive PR cleanup.
-
-          close-pr-message: >
-            Closing this PR because it has had no activity for 7 days after
-            being marked inactive. If this is still useful or ready to move
-            forward, feel free to reopen it with fresh context or updated
-            details. Sorry for any inconvenience.
-
-          remove-pr-stale-when-updated: true
-          delete-branch: false
-          operations-per-run: 100
-          debug-only: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_only || false }}

.gitmodules

@@ -1,12 +1,6 @@
 [submodule "ggml"]
     path = ggml
-	url = https://github.com/leejet/ggml.git
+	url = https://github.com/ggml-org/ggml.git
 [submodule "examples/server/frontend"]
 	path = examples/server/frontend
-	url = https://github.com/leejet/sdcpp-webui.git
+	url = https://github.com/leejet/stable-ui.git
-[submodule "thirdparty/libwebp"]
-	path = thirdparty/libwebp
-	url = https://github.com/webmproject/libwebp.git
-[submodule "thirdparty/libwebm"]
-	path = thirdparty/libwebm
-	url = https://github.com/webmproject/libwebm.git

CMakeLists.txt

@@ -11,68 +11,17 @@ endif()
 if (MSVC)
     add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
     add_compile_definitions(_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING)
-    add_compile_options(
-        $<$<COMPILE_LANGUAGE:C>:/MP>
-        $<$<COMPILE_LANGUAGE:C>:/utf-8>
-        $<$<COMPILE_LANGUAGE:CXX>:/MP>
-        $<$<COMPILE_LANGUAGE:CXX>:/utf-8>
-    )
 endif()
 
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
-if(APPLE)
-    function(sd_set_macos_rpaths target)
-        get_target_property(target_type ${target} TYPE)
-        if(target_type STREQUAL "EXECUTABLE")
-            set(runtime_paths "@executable_path" "@executable_path/../lib")
-        elseif(target_type STREQUAL "SHARED_LIBRARY" OR target_type STREQUAL "MODULE_LIBRARY")
-            set(runtime_paths "@loader_path" "@loader_path/../lib")
-            set_target_properties(${target} PROPERTIES
-                MACOSX_RPATH ON
-                INSTALL_NAME_DIR "@rpath"
-                BUILD_WITH_INSTALL_NAME_DIR ON
-            )
-        else()
-            return()
-        endif()
-
-        # Release artifacts zip the build output directly, so keep macOS rpaths relocatable.
-        set_target_properties(${target} PROPERTIES
-            BUILD_RPATH "${runtime_paths}"
-            INSTALL_RPATH "${runtime_paths}"
-            BUILD_WITH_INSTALL_RPATH ON
-        )
-    endfunction()
-endif()
-
 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     set(SD_STANDALONE ON)
 else()
     set(SD_STANDALONE OFF)
 endif()
 
-set(SD_SUBMODULE_WEBP FALSE)
-if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/libwebp/CMakeLists.txt")
-    set(SD_SUBMODULE_WEBP TRUE)
-endif()
-if(SD_SUBMODULE_WEBP)
-    set(SD_WEBP_DEFAULT ON)
-else()
-    set(SD_WEBP_DEFAULT ${SD_USE_SYSTEM_WEBP})
-endif()
-
-set(SD_SUBMODULE_WEBM FALSE)
-if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/libwebm/CMakeLists.txt")
-    set(SD_SUBMODULE_WEBM TRUE)
-endif()
-if(SD_SUBMODULE_WEBM)
-    set(SD_WEBM_DEFAULT ON)
-else()
-    set(SD_WEBM_DEFAULT ${SD_USE_SYSTEM_WEBM})
-endif()
-
 #
 # Option list
 #
@@ -80,10 +29,6 @@ endif()
 # general
 #option(SD_BUILD_TESTS                "sd: build tests"    ${SD_STANDALONE})
 option(SD_BUILD_EXAMPLES             "sd: build examples" ${SD_STANDALONE})
-option(SD_WEBP                       "sd: enable WebP image I/O support" ${SD_WEBP_DEFAULT})
-option(SD_USE_SYSTEM_WEBP            "sd: link against system libwebp" OFF)
-option(SD_WEBM                       "sd: enable WebM video output support" ${SD_WEBM_DEFAULT})
-option(SD_USE_SYSTEM_WEBM            "sd: link against system libwebm" OFF)
 option(SD_CUDA                       "sd: cuda backend" OFF)
 option(SD_HIPBLAS                    "sd: rocm backend" OFF)
 option(SD_METAL                      "sd: metal backend" OFF)
@@ -96,141 +41,50 @@ option(SD_BUILD_SHARED_GGML_LIB      "sd: build ggml as a separate shared lib" O
 option(SD_USE_SYSTEM_GGML            "sd: use system-installed GGML library" OFF)
 #option(SD_BUILD_SERVER               "sd: build server example"                           ON)
 
-set(CMAKE_C_STANDARD 11)
-set(CMAKE_C_STANDARD_REQUIRED true)
-
-set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_STANDARD_REQUIRED true)
-
 if(SD_CUDA)
     message("-- Use CUDA as backend stable-diffusion")
     set(GGML_CUDA ON)
+    add_definitions(-DSD_USE_CUDA)
 endif()
 
 if(SD_METAL)
     message("-- Use Metal as backend stable-diffusion")
     set(GGML_METAL ON)
+    add_definitions(-DSD_USE_METAL)
 endif()
 
 if (SD_VULKAN)
     message("-- Use Vulkan as backend stable-diffusion")
     set(GGML_VULKAN ON)
+    add_definitions(-DSD_USE_VULKAN)
 endif ()
 
 if (SD_OPENCL)
     message("-- Use OpenCL as backend stable-diffusion")
     set(GGML_OPENCL ON)
+    add_definitions(-DSD_USE_OPENCL)
 endif ()
 
 if (SD_HIPBLAS)
     message("-- Use HIPBLAS as backend stable-diffusion")
     set(GGML_HIP ON)
-    # ggml-hip's device-stub objects must be position-independent, or the
+    add_definitions(-DSD_USE_CUDA)
-    # default-PIE sd-cli link fails with `relocation R_X86_64_32 ... cannot be
-    # used when making a PIE object` on distros that default to PIE
-    # (Ubuntu 24.04, Fedora 40+, Debian 12+). The shared-library branch below
-    # already sets this; the static build (the HIP default) did not.
-    set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 endif ()
 
 if(SD_MUSA)
     message("-- Use MUSA as backend stable-diffusion")
     set(GGML_MUSA ON)
-endif()
+    add_definitions(-DSD_USE_CUDA)
-
-if(SD_WEBP)
-    if(NOT SD_SUBMODULE_WEBP AND NOT SD_USE_SYSTEM_WEBP)
-        message(FATAL_ERROR "WebP support enabled but no source found.
-          Either initialize the submodule:\n  git submodule update --init thirdparty/libwebp\n\n"
-          "Or link against system library:\n  cmake (...) -DSD_USE_SYSTEM_WEBP=ON")
-    endif()
-    if(SD_USE_SYSTEM_WEBP)
-        find_package(WebP)
-        if(WebP_FOUND)
-            add_library(webp ALIAS WebP::webp)
-            # libwebp CMake target naming is not consistent across versions/distros.
-            # Some export WebP::libwebpmux, others export WebP::webpmux.
-            if(TARGET WebP::libwebpmux)
-                add_library(libwebpmux ALIAS WebP::libwebpmux)
-            elseif(TARGET WebP::webpmux)
-                add_library(libwebpmux ALIAS WebP::webpmux)
-            else()
-                message(FATAL_ERROR
-                    "Could not find a compatible webpmux target in system WebP package. "
-                    "Expected WebP::libwebpmux or WebP::webpmux."
-                )
-            endif()
-        else()
-            find_package(PkgConfig REQUIRED)
-            pkg_check_modules(WebP REQUIRED IMPORTED_TARGET GLOBAL libwebp)
-            pkg_check_modules(WebPMux REQUIRED IMPORTED_TARGET GLOBAL libwebpmux)
-            link_libraries(PkgConfig::WebP)
-            link_libraries(PkgConfig::WebPMux)
-            add_library(libwebpmux ALIAS PkgConfig::WebPMux)
-        endif()
-    endif()
-endif()
-
-if(SD_WEBM)
-    if(NOT SD_WEBP)
-        message(FATAL_ERROR "SD_WEBM requires SD_WEBP because WebM output reuses libwebp VP8 encoding.")
-    endif()
-    if(NOT SD_SUBMODULE_WEBM AND NOT SD_USE_SYSTEM_WEBM)
-        message(FATAL_ERROR "WebM support enabled but no source found.
-          Either initialize the submodule:\n  git submodule update --init thirdparty/libwebm\n\n"
-          "Or link against system library:\n  cmake (...) -DSD_USE_SYSTEM_WEBM=ON")
-    endif()
-    if(SD_USE_SYSTEM_WEBM)
-        find_package(PkgConfig)
-        if(PkgConfig_FOUND)
-            pkg_check_modules(WebM REQUIRED IMPORTED_TARGET GLOBAL libwebm)
-        endif()
-        if(PkgConfig_FOUND AND WebM_FOUND)
-            link_libraries(PkgConfig::WebM)
-        else()
-            find_path(WEBM_INCLUDE_DIR
-                NAMES mkvmuxer/mkvmuxer.h mkvparser/mkvparser.h common/webmids.h
-                PATH_SUFFIXES webm
-                REQUIRED)
-            find_library(WEBM_LIBRARY
-                NAMES webm libwebm
-                REQUIRED)
-
-            add_library(webm UNKNOWN IMPORTED)
-            set_target_properties(webm PROPERTIES
-                IMPORTED_LOCATION "${WEBM_LIBRARY}"
-                INTERFACE_INCLUDE_DIRECTORIES "${WEBM_INCLUDE_DIR}")
-        endif()
-    endif()
 endif()
 
 set(SD_LIB stable-diffusion)
 
-file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
+file(GLOB SD_LIB_SOURCES
     "src/*.h"
     "src/*.cpp"
     "src/*.hpp"
-    "src/conditioning/*.h"
+    "src/vocab/*.h"
-    "src/conditioning/*.cpp"
+    "src/vocab/*.cpp"
-    "src/conditioning/*.hpp"
-    "src/core/*.h"
-    "src/core/*.cpp"
-    "src/core/*.hpp"
-    "src/extensions/*.h"
-    "src/extensions/*.cpp"
-    "src/extensions/*.hpp"
-    "src/model/*/*.h"
-    "src/model/*/*.cpp"
-    "src/model/*/*.hpp"
-    "src/runtime/*.h"
-    "src/runtime/*.cpp"
-    "src/runtime/*.hpp"
-    "src/model_io/*.h"
-    "src/model_io/*.cpp"
-    "src/tokenizers/*.h"
-    "src/tokenizers/*.cpp"
-    "src/tokenizers/vocab/*.h"
-    "src/tokenizers/vocab/*.cpp"
 )
 
 find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH)
@@ -283,14 +137,11 @@ else()
     add_library(${SD_LIB} STATIC ${SD_LIB_SOURCES})
 endif()
 
-if(APPLE)
-    sd_set_macos_rpaths(${SD_LIB})
-endif()
-
 if(SD_SYCL)
     message("-- Use SYCL as backend stable-diffusion")
     set(GGML_SYCL ON)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing -fsycl")
+    add_definitions(-DSD_USE_SYCL)
     # disable fast-math on host, see:
     # https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/2021-10/fp-model-fp.html
     if (WIN32)
@@ -326,8 +177,7 @@ endif()
 add_subdirectory(thirdparty)
 
 target_link_libraries(${SD_LIB} PUBLIC ggml zip)
-target_include_directories(${SD_LIB} PUBLIC . src include)
+target_include_directories(${SD_LIB} PUBLIC . include)
-target_include_directories(${SD_LIB} PRIVATE src/core)
 target_include_directories(${SD_LIB} PUBLIC . thirdparty)
 target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17)
 

CONTRIBUTING.md

@@ -1,67 +0,0 @@
-# Contributing
-
-This document collects general contribution conventions for this repository.
-
-## Before You Start
-
-Before opening a PR, please search existing PRs to avoid duplicating ongoing work.
-
-For large-scale refactors or changes with broad impact, please open an issue first to discuss the approach before submitting a PR.
-
-If you want to update a third-party dependency, please open an issue first instead of submitting a direct PR. See [Dependency Updates](#dependency-updates) for details.
-
-## Pull Requests
-
-Keep each PR focused on one clear change. Large or overly complex PRs are harder to review and may not be merged.
-
-Follow Conventional Commit-style subjects seen in history: `feat:`, `fix:`, `refactor:`, `ci:`, `docs:`, `chore:`. Keep subjects imperative and scoped.
-
-PRs should include:
-
-- What changed and why (short problem/solution summary).
-- Verification evidence when applicable (commands and key outputs).
-- Linked issue/PR context when applicable.
-- Screenshots or sample outputs for UI/visual behavior changes.
-
-## Code Style
-
-Format code according to the repository style before submitting changes.
-
-Formatting follows `.clang-format` (Chromium base, 4-space indent, no tabs). Run `format-code.sh` before opening a PR. Keep C++ standard at C++17-compatible patterns used in this repo.
-
-Naming conventions:
-
-- Use `PascalCase` for class/struct/type names.
-- In `PascalCase` names, preserve common abbreviations in uppercase, for example `SD`, `API`, `HTTP`, `JSON`, `RGB`, `VAE`, `TAE`, `LoRA`, and `WebP`.
-- Use `snake_case` for functions, methods, variables, and file names unless an existing API requires a different style.
-- Use a trailing underscore for private data member names, for example `hidden_size_` or `tokenizer_`.
-- Use `.h` for C and C++ header files. Do not introduce new `.hpp` headers.
-- Use macro-based header include guards instead of `#pragma once`.
-- Format header include guards as `__SD_{PATH}__`, where `{PATH}` is the header path in uppercase snake case without the file extension. For example, `src/sample.h` should use `__SD_SAMPLE_H__`.
-- Do not introduce anonymous namespaces in new or modified code; prefer `static` file-local functions/variables or an explicit named namespace when scoping is needed.
-- In `class`/`struct` definitions, place data members before member functions unless an existing type already clearly follows a different pattern.
-- Keep `test_*.cpp` / `test_*.py` naming for tests.
-
-Some older code in the project may not fully follow the current conventions. Please do not submit PRs that only rewrite existing code to match style rules.
-
-When adding or modifying model implementations, follow the model config and weight detection conventions in [docs/model_config.md](docs/model_config.md).
-
-## AI-Assisted Contributions
-
-AI tools may be used to assist development, but contributors are responsible for the quality and correctness of the submitted code.
-
-If any part of a contribution was generated with AI assistance, the contributor must perform a thorough human review before submitting the PR and understand every changed line.
-
-Do not list AI tools as co-authors. The human contributor is the sole responsible author of the submitted code.
-
-Please do not submit AI-generated code that you do not understand, and do not include meaningless experiments, temporary test code, or unrelated generated output in a PR.
-
-## Dependency Updates
-
-Do not submit PRs that update `ggml`. `ggml` updates are performed only after local validation by the maintainer.
-
-Other third-party dependencies are not updated unless necessary. If you want to update a dependency, please open an issue first instead of submitting a direct PR.
-
-## Security & Configuration
-
-Do not commit model weights, secrets, or local absolute paths. Keep large binaries out of git unless intentionally tracked release assets.

Dockerfile

@@ -2,18 +2,7 @@ ARG UBUNTU_VERSION=24.04
 
 FROM ubuntu:$UBUNTU_VERSION AS build
 
-# sd-server embeds the web UI at build time, so the build image needs Node/pnpm.
+RUN apt-get update && apt-get install -y --no-install-recommends build-essential git cmake
-RUN apt-get update && apt-get install -y --no-install-recommends build-essential git cmake ca-certificates curl gnupg && \
-    mkdir -p /etc/apt/keyrings && \
-    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
-    gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
-    rm /tmp/nodesource-repo.gpg.key && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends nodejs && \
-    npm install -g pnpm@10.15.1 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /sd.cpp
 

Dockerfile.cuda

@@ -3,31 +3,14 @@ ARG UBUNTU_VERSION=24.04
 
 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu${UBUNTU_VERSION} AS build
 
-# sd-server embeds the web UI at build time, so the build image needs Node/pnpm.
+RUN apt-get update && apt-get install -y --no-install-recommends build-essential git ccache cmake
-RUN apt-get update && apt-get install -y --no-install-recommends build-essential git ccache cmake ca-certificates curl gnupg && \
-    mkdir -p /etc/apt/keyrings && \
-    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
-    gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
-    rm /tmp/nodesource-repo.gpg.key && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends nodejs && \
-    npm install -g pnpm@10.15.1 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /sd.cpp
 
 COPY . .
 
 ARG CUDACXX=/usr/local/cuda/bin/nvcc
-ARG CUDA_ARCHITECTURES=""
+RUN cmake . -B ./build -DSD_CUDA=ON
-ARG GGML_CUDA_FA_ALL_QUANTS=""
-
-RUN cmake . -B ./build \
-    -DSD_CUDA=ON \
-    ${CUDA_ARCHITECTURES:+-DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHITECTURES}"} \
-    ${GGML_CUDA_FA_ALL_QUANTS:+-DGGML_CUDA_FA_ALL_QUANTS=${GGML_CUDA_FA_ALL_QUANTS}}
 RUN cmake --build ./build --config Release -j$(nproc)
 
 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-runtime-ubuntu${UBUNTU_VERSION} AS runtime

Dockerfile.musa

@@ -3,18 +3,7 @@ ARG UBUNTU_VERSION=22.04
 
 FROM mthreads/musa:${MUSA_VERSION}-devel-ubuntu${UBUNTU_VERSION}-amd64 as build
 
-# sd-server embeds the web UI at build time, so the build image needs Node/pnpm.
+RUN apt-get update && apt-get install -y ccache cmake git
-RUN apt-get update && apt-get install -y --no-install-recommends ccache cmake git ca-certificates curl gnupg && \
-    mkdir -p /etc/apt/keyrings && \
-    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
-    gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
-    rm /tmp/nodesource-repo.gpg.key && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends nodejs && \
-    npm install -g pnpm@10.15.1 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /sd.cpp
 

Dockerfile.sycl

@@ -1,20 +1,8 @@
-# ggml SYCL hardware detection uses BMG G31/WCL architecture enums added in oneAPI 2025.3.
+ARG SYCL_VERSION=2025.1.0-0
-ARG SYCL_VERSION=2025.3.2-0
 
 FROM intel/oneapi-basekit:${SYCL_VERSION}-devel-ubuntu24.04 AS build
 
-# sd-server embeds the web UI at build time, so the build image needs Node/pnpm.
+RUN apt-get update && apt-get install -y cmake
-RUN apt-get update && apt-get install -y --no-install-recommends cmake ca-certificates curl gnupg && \
-    mkdir -p /etc/apt/keyrings && \
-    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
-    gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
-    rm /tmp/nodesource-repo.gpg.key && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends nodejs && \
-    npm install -g pnpm@10.15.1 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /sd.cpp
 

Dockerfile.vulkan

@@ -2,18 +2,7 @@ ARG UBUNTU_VERSION=24.04
 
 FROM ubuntu:$UBUNTU_VERSION AS build
 
-# sd-server embeds the web UI at build time, so the build image needs Node/pnpm.
+RUN apt-get update && apt-get install -y --no-install-recommends build-essential git cmake libvulkan-dev glslc
-RUN apt-get update && apt-get install -y --no-install-recommends build-essential git cmake libvulkan-dev glslc spirv-headers ca-certificates curl gnupg && \
-    mkdir -p /etc/apt/keyrings && \
-    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
-    gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
-    rm /tmp/nodesource-repo.gpg.key && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends nodejs && \
-    npm install -g pnpm@10.15.1 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /sd.cpp
 

README.md

@@ -15,18 +15,26 @@ API and command-line option may change frequently.***
 
 ## 🔥Important News
 
-* **2026/06/04** 🚀 stable-diffusion.cpp now supports **Ideogram4**
-* **2026/05/31** 🚀 stable-diffusion.cpp now supports **PiD**
-* **2026/05/27** 🚀 stable-diffusion.cpp now supports **Lens**
-* **2026/05/17** 🚀 stable-diffusion.cpp now supports **LTX-2.3**
-* **2026/04/11** 🚀 stable-diffusion.cpp now uses a brand-new embedded web UI.  
 * **2026/01/18** 🚀 stable-diffusion.cpp now supports **FLUX.2-klein**  
+  👉 Details: [PR #1193](https://github.com/leejet/stable-diffusion.cpp/pull/1193)
+
 * **2025/12/01** 🚀 stable-diffusion.cpp now supports **Z-Image**  
+  👉 Details: [PR #1020](https://github.com/leejet/stable-diffusion.cpp/pull/1020)
+
 * **2025/11/30** 🚀 stable-diffusion.cpp now supports **FLUX.2-dev**  
+  👉 Details: [PR #1016](https://github.com/leejet/stable-diffusion.cpp/pull/1016)
+
 * **2025/10/13** 🚀 stable-diffusion.cpp now supports **Qwen-Image-Edit / Qwen-Image-Edit 2509**  
+  👉 Details: [PR #877](https://github.com/leejet/stable-diffusion.cpp/pull/877)
+
 * **2025/10/12** 🚀 stable-diffusion.cpp now supports **Qwen-Image**  
+  👉 Details: [PR #851](https://github.com/leejet/stable-diffusion.cpp/pull/851)
+
 * **2025/09/14** 🚀 stable-diffusion.cpp now supports **Wan2.1 Vace**  
+  👉 Details: [PR #819](https://github.com/leejet/stable-diffusion.cpp/pull/819)
+
 * **2025/09/06** 🚀 stable-diffusion.cpp now supports **Wan2.1 / Wan2.2**  
+  👉 Details: [PR #778](https://github.com/leejet/stable-diffusion.cpp/pull/778)
 
 ## Features
 
@@ -40,25 +48,17 @@ API and command-line option may change frequently.***
     - [SD3/SD3.5](./docs/sd3.md)
     - [FLUX.1-dev/FLUX.1-schnell](./docs/flux.md)
     - [FLUX.2-dev/FLUX.2-klein](./docs/flux2.md)
-    - [Lens](./docs/lens.md)
     - [Chroma](./docs/chroma.md)
     - [Chroma1-Radiance](./docs/chroma_radiance.md)
     - [Qwen Image](./docs/qwen_image.md)
-    - [PiD](./docs/pid.md)
-    - [LongCat Image](./docs/longcat_image.md)
     - [Z-Image](./docs/z_image.md)
     - [Ovis-Image](./docs/ovis_image.md)
     - [Anima](./docs/anima.md)
-    - [ERNIE-Image](./docs/ernie_image.md)
-    - [HiDream-O1-Image](./docs/hidream_o1_image.md)
-    - [Ideogram4](./docs/ideogram4.md)
   - Image Edit Models
     - [FLUX.1-Kontext-dev](./docs/kontext.md)
     - [Qwen Image Edit series](./docs/qwen_image_edit.md)
-    - [LongCat Image Edit](./docs/longcat_image.md)
   - Video Models
     - [Wan2.1/Wan2.2](./docs/wan.md)
-    - [LTX-2.3](./docs/ltx2.md)
   - [PhotoMaker](https://github.com/TencentARC/PhotoMaker) support.
   - Control Net support with SD 1.5
   - LoRA support, same as [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora)
@@ -73,10 +73,9 @@ API and command-line option may change frequently.***
   - OpenCL
   - SYCL
 - Supported weight formats
-  - Pytorch checkpoint (`.ckpt` or `.pth` or `.pt`)
+  - Pytorch checkpoint (`.ckpt` or `.pth`)
   - Safetensors (`.safetensors`)
   - GGUF (`.gguf`)
-- Convert mode supports converting model weights to `.gguf` or `.safetensors`
 - Supported platforms
     - Linux
     - Mac OS
@@ -94,7 +93,6 @@ API and command-line option may change frequently.***
     - `DPM++ 2M`
     - [`DPM++ 2M v2`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/8457)
     - `DPM++ 2S a`
-    - `ER-SDE`
     - [`LCM`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/13952)
 - Cross-platform reproducibility
     - `--rng cuda`, default, consistent with the `stable-diffusion-webui GPU RNG`
@@ -128,11 +126,9 @@ API and command-line option may change frequently.***
 ## Performance
 
 If you want to improve performance or reduce VRAM/RAM usage, please refer to [performance guide](./docs/performance.md).
-For runtime and parameter backend placement, see the [backend selection guide](./docs/backend.md).
 
 ## More Guides
 
-- [Backend selection](./docs/backend.md)
 - [SD1.x/SD2.x/SDXL](./docs/sd.md)
 - [SD3/SD3.5](./docs/sd3.md)
 - [FLUX.1-dev/FLUX.1-schnell](./docs/flux.md)
@@ -142,14 +138,9 @@ For runtime and parameter backend placement, see the [backend selection guide](.
 - [🔥Qwen Image](./docs/qwen_image.md)
 - [🔥Qwen Image Edit series](./docs/qwen_image_edit.md)
 - [🔥Wan2.1/Wan2.2](./docs/wan.md)
-- [🔥LTX-2.3](./docs/ltx2.md)
 - [🔥Z-Image](./docs/z_image.md)
 - [Ovis-Image](./docs/ovis_image.md)
 - [Anima](./docs/anima.md)
-- [ERNIE-Image](./docs/ernie_image.md)
-- [HiDream-O1-Image](./docs/hidream_o1_image.md)
-- [Lens](./docs/lens.md)
-- [LongCat Image / LongCat Image Edit](./docs/longcat_image.md)
 - [LoRA](./docs/lora.md)
 - [LCM/LCM-LoRA](./docs/lcm.md)
 - [Using PhotoMaker to personalize image generation](./docs/photo_maker.md)
@@ -165,7 +156,6 @@ These projects wrap `stable-diffusion.cpp` for easier use in other languages/fra
 
 * Golang (non-cgo): [seasonjs/stable-diffusion](https://github.com/seasonjs/stable-diffusion)
 * Golang (cgo): [Binozo/GoStableDiffusion](https://github.com/Binozo/GoStableDiffusion)
-* Golang (non-cgo): [l8bloom/gosd](https://github.com/l8bloom/gosd)
 * C#: [DarthAffe/StableDiffusion.NET](https://github.com/DarthAffe/StableDiffusion.NET)
 * Python: [william-murray1204/stable-diffusion-cpp-python](https://github.com/william-murray1204/stable-diffusion-cpp-python)
 * Rust: [newfla/diffusion-rs](https://github.com/newfla/diffusion-rs)

docs/backend.md

@@ -1,122 +0,0 @@
-# Backend selection
-
-`stable-diffusion.cpp` has two backend assignments:
-
-- `--backend` selects the runtime backend used to execute model graphs.
-- `--params-backend` selects the backend used to allocate model parameters.
-
-If `--params-backend` is not set, parameters use the same backend as their module runtime backend.
-
-## Syntax
-
-A backend assignment can be a single backend name:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend cpu
-```
-
-This applies to every module that does not have a more specific assignment.
-
-Assignments can also target individual modules:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend te=cpu,vae=cuda0,diffusion=vulkan0
-```
-
-The same syntax is used for parameter placement:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend cuda0 --params-backend te=cpu,vae=cpu
-```
-
-Module names are case-insensitive. Hyphens and underscores in module names are ignored, so `clip_vision`, `clip-vision`, and `clipvision` are equivalent.
-
-`all=`, `default=`, and `*=` can be used to set the default backend inside a mixed assignment:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend all=cuda0,te=cpu
-```
-
-## Modules
-
-| Module | Purpose | Accepted names |
-| --- | --- | --- |
-| `diffusion` | UNet, DiT, MMDiT, Flux, Wan, Qwen Image, and other diffusion models | `diffusion`, `model`, `unet`, `dit` |
-| `te` | Text encoders and conditioners | `te`, `clip`, `text`, `textencoder`, `textencoders`, `conditioner`, `cond`, `llm`, `t5`, `t5xxl` |
-| `clip_vision` | CLIP vision encoder | `clip_vision`, `clipvision`, `clip-vision`, `vision` |
-| `vae` | VAE and TAE | `vae`, `firststage`, `autoencoder`, `tae` |
-| `controlnet` | ControlNet | `controlnet`, `control` |
-| `photomaker` | PhotoMaker ID encoder and PhotoMaker LoRA | `photomaker`, `photomakerid`, `pmid`, `photo` |
-| `upscaler` | ESRGAN upscaler | `upscaler`, `esrgan`, `hires` |
-
-`te` is the preferred module name for text encoders. `clip` is kept as an accepted alias because many existing commands and model names use CLIP terminology.
-
-## Backend names
-
-Backend names are resolved against the GGML backend device list. Matching is case-insensitive and accepts exact names or unique prefixes, so common values include names such as:
-
-- `cpu`
-- `cuda0`
-- `vulkan0`
-- `metal`
-
-The special values `auto`, `default`, and an empty backend name select the default backend. The default preference is GPU, then integrated GPU, then CPU.
-
-The special value `gpu` selects the first GPU backend, falling back to the first integrated GPU backend.
-
-## Runtime backend vs. parameter backend
-
-The runtime backend controls where graph execution runs. The parameter backend controls where model weights are allocated.
-
-For example:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend cuda0 --params-backend cpu
-```
-
-This runs all modules on `cuda0`, but stores parameters in CPU RAM. During execution, parameters are moved to the runtime backend as needed.
-
-Per-module assignments can be mixed:
-
-```shell
-sd-cli -m model.safetensors -p "a cat" --backend diffusion=cuda0,te=cpu,vae=cpu --params-backend diffusion=cuda0,te=cpu,vae=cpu
-```
-
-This keeps text encoding and VAE execution on CPU while the diffusion model runs on GPU.
-
-## Backend sharing and lifetime
-
-Backends are managed by `SDBackendManager`.
-
-Within one manager, backend instances are cached by resolved backend device name. If multiple modules request the same backend, they share the same `ggml_backend_t`.
-
-For example:
-
-```shell
---backend te=cpu,vae=cpu
-```
-
-uses one shared CPU backend for both `te` and `vae` runtime execution.
-
-Runtime and parameter assignments also share the same backend cache. If `--backend diffusion=cuda0` and `--params-backend diffusion=cuda0` resolve to the same device, both use the same backend instance.
-
-`SDBackendManager` owns the backend instances and frees them when the context or upscaler is destroyed. Model runners receive non-owning runtime and parameter backend pointers and do not free them.
-
-## Compatibility flags
-
-The older CPU placement flags are still supported:
-
-- `--clip-on-cpu`
-- `--vae-on-cpu`
-- `--control-net-cpu`
-- `--offload-to-cpu`
-
-`--clip-on-cpu`, `--vae-on-cpu`, and `--control-net-cpu` affect runtime backend assignment only when `--backend` is not set. They map to `te=cpu`, `vae=cpu`, and `controlnet=cpu`.
-
-`--offload-to-cpu` affects parameter backend assignment only when `--params-backend` is not set. It is equivalent to:
-
-```shell
---params-backend cpu
-```
-
-Explicit `--backend` and `--params-backend` assignments are preferred for new commands.

docs/build.md

@@ -16,26 +16,6 @@ git submodule init
 git submodule update
 ```
 
-## WebP and WebM Support in Examples
-
-The example applications (`examples/cli` and `examples/server`) use `libwebp` to support WebP image I/O, and `examples/cli` can also use `libwebm` for `.webm` video output. Both are enabled by default. WebM output currently reuses `libwebp` to encode each frame as VP8 before muxing with `libwebm`.
-
-If you do not want WebP/WebM support, you can disable them at configure time:
-
-```shell
-mkdir build && cd build
-cmake .. -DSD_WEBP=OFF -DSD_WEBM=OFF
-cmake --build . --config Release
-```
-
-If the submodules are not available, you can also link against system packages instead:
-
-```shell
-mkdir build && cd build
-cmake .. -DSD_USE_SYSTEM_WEBP=ON -DSD_USE_SYSTEM_WEBM=ON
-cmake --build . --config Release
-```
-
 ## Build (CPU only)
 
 If you don't have a GPU or CUDA installed, you can build a CPU-only version.
@@ -102,11 +82,6 @@ cmake --build . --config Release
 ## Build with Vulkan
 
 Install Vulkan SDK from https://www.lunarg.com/vulkan-sdk/.
-On Ubuntu, install the Vulkan development packages and SPIR-V headers:
-
-```shell
-sudo apt-get install build-essential libvulkan-dev glslc spirv-headers
-```
 
 ```shell
 mkdir build && cd build

docs/caching.md

@@ -131,6 +131,8 @@ sd-cli -m model.safetensors -p "a cat" --cache-mode spectrum
 | `warmup` | Steps to always compute before caching starts | 4 |
 | `stop` | Stop caching at this fraction of total steps | 0.9 |
 
+```
+
 ### Performance Tips
 
 - Start with default thresholds and adjust based on output quality

docs/distilled_sd.md

@@ -87,32 +87,51 @@ pipe.save_pretrained("segmindtiny-sd", safe_serialization=True)
 ```bash
 python convert_diffusers_to_original_stable_diffusion.py \
       --model_path  ./segmindtiny-sd \
-      --checkpoint_path ./segmind_tiny-sd.safetensors  --half --use_safetensors
+      --checkpoint_path ./segmind_tiny-sd.ckpt --half
 ```
 
-The file segmind_tiny-sd.safetensors will be generated and is now ready for use with sd.cpp. You can follow a similar process for the other models mentioned above.
+The file segmind_tiny-sd.ckpt will be generated and is now ready for use with sd.cpp. You can follow a similar process for the other models mentioned above.
 
 
-### SDXS-512-DreamShaper
+##### Another available .ckpt file:
+
+ * https://huggingface.co/ClashSAN/small-sd/resolve/main/tinySDdistilled.ckpt
+
+To use this file, you must first adjust its non-contiguous tensors:
+
+```python
+import torch
+ckpt = torch.load("tinySDdistilled.ckpt", map_location=torch.device('cpu'))
+for key, value in ckpt['state_dict'].items():
+    if isinstance(value, torch.Tensor):
+        ckpt['state_dict'][key] = value.contiguous()
+torch.save(ckpt, "tinySDdistilled_fixed.ckpt")
+```
+
+
+### SDXS-512
 
 Another very tiny and **incredibly fast**  model is SDXS by IDKiro et al.  The authors refer to it as *"Real-Time One-Step Latent Diffusion Models with Image Conditions"*. For details read the paper: https://arxiv.org/pdf/2403.16627 . Once again the authors removed some more blocks of U-Net part and unlike other SD1 models they use an adjusted _AutoEncoderTiny_ instead of default _AutoEncoderKL_ for the VAE part.
-##### Some ready-to-run SDXS-512 model files are available online, such as:
 
-* https://huggingface.co/akleine/sdxs-512
+##### 1. Download the diffusers model from  Hugging Face using Python:
-* https://huggingface.co/concedo/sdxs-512-tinySDdistilled-GGUF
+
+```python
+from diffusers import StableDiffusionPipeline
+pipe = StableDiffusionPipeline.from_pretrained("IDKiro/sdxs-512-dreamshaper")
+pipe.save_pretrained(save_directory="sdxs")
+```
+##### 2. Create a safetensors file
+
+```bash
+python convert_diffusers_to_original_stable_diffusion.py \
+    --model_path  sdxs  --checkpoint_path sdxs.safetensors --half --use_safetensors
+```
+
+##### 3. Run the model as follows:
 
-##### Run the model as follows:
 ```bash
 ~/stable-diffusion.cpp/build/bin/sd-cli -m sdxs.safetensors -p "portrait of a lovely cat" \
   --cfg-scale 1 --steps 1
 ```
+
 Both options: ``` --cfg-scale 1 ``` and  ``` --steps 1 ``` are mandatory here.                                                 
-
-### SDXS-512-0.9
-
-Even though the name "SDXS-512-0.9" is similar to "SDXS-512-DreamShaper", it is *completely different* but also **incredibly fast**. Sometimes it is preferred, so try it yourself.
-##### Download a ready-to-run file from here:
-
-* https://huggingface.co/akleine/sdxs-09
-
-For the use of this model, both options ``` --cfg-scale 1 ``` and ``` --steps 1 ``` are again absolutely necessary.

docs/ernie_image.md

@@ -1,35 +0,0 @@
-# How to Use
-
-You can run ERNIE-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or even less.
-
-## Download weights
-
-- Download ERNIE-Image-Turbo
-    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/diffusion_models
-    - gguf: https://huggingface.co/unsloth/ERNIE-Image-Turbo-GGUF/tree/main
-- Download ERNIE-Image
-    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/diffusion_models
-    - gguf: https://huggingface.co/unsloth/ERNIE-Image-GGUF/tree/main
-- Download vae
-    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/vae
-- Download ministral 3b
-    - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/text_encoders
-    - gguf: https://huggingface.co/unsloth/Ministral-3-3B-Instruct-2512-GGUF/tree/main
-
-## Examples
-
-### ERNIE-Image-Turbo
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\ernie-image-turbo.safetensors --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 8 -v --offload-to-cpu --diffusion-fa
-```
-
-<img width="256" alt="ERNIE-Image Turbo example" src="../assets/ernie_image/turbo_example.png" />
-
-### ERNIE-Image
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\ernie-image-UD-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors  --llm ..\..\ComfyUI\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa
-```
-
-<img width="256" alt="ERNIE-Image example" src="../assets/ernie_image/example.png" />

docs/flux2.md

@@ -8,8 +8,6 @@
     - gguf: https://huggingface.co/city96/FLUX.2-dev-gguf/tree/main
 - Download vae
     - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main
-- Download FLUX.2-small-decoder (full_encoder_small_decoder.safetensors) as an alternative VAE option
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-small-decoder/tree/main
 - Download Mistral-Small-3.2-24B-Instruct-2506-GGUF
     - gguf: https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/tree/main
 
@@ -33,8 +31,6 @@
     - gguf: https://huggingface.co/leejet/FLUX.2-klein-base-4B-GGUF/tree/main
 - Download vae
     - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main
-- Download FLUX.2-small-decoder (full_encoder_small_decoder.safetensors) as an alternative VAE option
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-small-decoder/tree/main
 - Download Qwen3 4b
     - safetensors: https://huggingface.co/Comfy-Org/flux2-klein-4B/tree/main/split_files/text_encoders
     - gguf: https://huggingface.co/unsloth/Qwen3-4B-GGUF/tree/main

docs/hidream_o1_image.md

@@ -1,20 +0,0 @@
-# How to Use
-
-## Download weights
-
-- Download HiDream-O1-Image-Dev
-    - safetensors: https://huggingface.co/Comfy-Org/HiDream-O1-Image/tree/main/checkpoints
-- Download HiDream-O1-Image
-    - safetensors: https://huggingface.co/Comfy-Org/HiDream-O1-Image/tree/main/checkpoints
-
-## Examples
-
-### HiDream-O1-Image-Dev
-
-```
-.\bin\Release\sd-cli.exe -m  ..\..\ComfyUI\models\diffusion_models\hidream_o1_image_dev_bf16.safetensors -p "a lovely cat holding a sign says 
-'hidream o1 cpp'" --cfg-scale 1.0  -v -H 1024 -W 1024
-```
-
-<img width="256" alt="HiDream-O1-Image-Dev example" src="../assets/hidream-o1/dev_example.png" />
-

docs/hipBLAS_on_Windows.md

@@ -26,12 +26,12 @@ Fortunately, `AMD` provides complete help documentation, you can use the help do
 
 Then we must set `ROCM` as environment variables before running cmake.
 
-Usually if you install according to the official tutorial and do not modify the ROCM path, then there is a high probability that it is here `C:\Program Files\AMD\ROCm\7.1.1\bin`
+Usually if you install according to the official tutorial and do not modify the ROCM path, then there is a high probability that it is here `C:\Program Files\AMD\ROCm\5.5\bin`
 
 This is what I use to set the clang:
 ```Commandline
-set CC=C:\Program Files\AMD\ROCm\7.1.1\bin\clang.exe
+set CC=C:\Program Files\AMD\ROCm\5.5\bin\clang.exe
-set CXX=C:\Program Files\AMD\ROCm\7.1.1\bin\clang++.exe
+set CXX=C:\Program Files\AMD\ROCm\5.5\bin\clang++.exe
 ```
 
 ## Ninja
@@ -46,7 +46,7 @@ set ninja=C:\Program Files\ninja\ninja.exe
 ## Building stable-diffusion.cpp
 
 The thing different from the regular CPU build is `-DSD_HIPBLAS=ON` ,
-`-G "Ninja"`, `-DCMAKE_C_COMPILER=clang`, `-DCMAKE_CXX_COMPILER=clang++`, `-DAMDGPU_TARGETS=gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032`
+`-G "Ninja"`, `-DCMAKE_C_COMPILER=clang`, `-DCMAKE_CXX_COMPILER=clang++`, `-DAMDGPU_TARGETS=gfx1100`
 
 >**Notice**: check the `clang` and `clang++` information:
 ```Commandline
@@ -59,29 +59,26 @@ If you see like this, we can continue:
 clang version 17.0.0 (git@github.amd.com:Compute-Mirrors/llvm-project e3201662d21c48894f2156d302276eb1cf47c7be)
 Target: x86_64-pc-windows-msvc
 Thread model: posix
-InstalledDir: C:\Program Files\AMD\ROCm\7.1.1\bin
+InstalledDir: C:\Program Files\AMD\ROCm\5.5\bin
 ```
 
 ```
 clang version 17.0.0 (git@github.amd.com:Compute-Mirrors/llvm-project e3201662d21c48894f2156d302276eb1cf47c7be)
 Target: x86_64-pc-windows-msvc
 Thread model: posix
-InstalledDir: C:\Program Files\AMD\ROCm\7.1.1\bin
+InstalledDir: C:\Program Files\AMD\ROCm\5.5\bin
 ```
 
->**Notice** that the GPU targets are now compatible with multiple GPU architectures (ROCm 7.1.1 targets). You can change them to match your GPU architecture. Click here to see your architecture [LLVM Target](https://rocm.docs.amd.com/en/latest/release/windows_support.html#windows-supported-gpus)
+>**Notice** that the `gfx1100` is the GPU architecture of my GPU, you can change it to your GPU architecture. Click here to see your architecture [LLVM Target](https://rocm.docs.amd.com/en/latest/release/windows_support.html#windows-supported-gpus)
 
-Examples:
+My GPU is AMD Radeon™ RX 7900 XTX Graphics, so I set it to `gfx1100`.
-- AMD Radeon™ RX 7900 XTX Graphics: `gfx1100`
-- AMD Radeon™ RX 7900 XT Graphics: `gfx1101`
-- AMD Radeon™ RX 7900 GRE Graphics: `gfx1102`
 
 option:
 
 ```commandline
 mkdir build
 cd build
-cmake .. -G "Ninja" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS="gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
+cmake .. -G "Ninja" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS=gfx1100
 cmake --build . --config Release
 ```
 

docs/ideogram4.md

@@ -1,40 +0,0 @@
-# How to Use
-
-## Download weights
-
-- Download Ideogram4
-    - safetensors: https://huggingface.co/ideogram-ai/ideogram-4-fp8/tree/main/transformer
-- Download Ideogram4 uncond
-    - safetensors: https://huggingface.co/ideogram-ai/ideogram-4-fp8/tree/main/unconditional_transformer
-- Download vae
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main
-- Download Qwen3-VL-8B-Instruct
-    - gguf: https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/tree/main
-
-## Convert weights
-
-fp8 scale -> bf16
-
-```
-python .\convert_fp8_scale_to_bf16.py --input .\ideogram4_fp8.safetensors --output ideogram4_bf16.safetensors
-python .\convert_fp8_scale_to_bf16.py --input .\ideogram4_uncond_fp8.safetensors --output ideogram4_uncond_bf16.safetensors
-```
-
-bf16 -> q8
-
-```
-.\bin\Release\sd-cli.exe -M convert -m ideogram4_bf16.safetensors -o ideogram4-Q8_0.gguf --tensor-type-rules "^layers.*adaln_modulation.*weight=q8_0,layers.*attention.o.*weight=q8_0,layers.*attention.qkv.*weight=q8_0,layers.*feed_forward.*weight=q8_0" -v
-
-.\bin\Release\sd-cli.exe -M convert -m ideogram4_uncond_bf16.safetensors -o ideogram4_uncond-Q8_0.gguf --tensor-type-rules "^layers.*adaln_modulation.*weight=q8_0,layers.*attention.o.*weight=q8_0,layers.*attention.qkv.*weight=q8_0,layers.*feed_forward.*weight=q8_0" -v
-```
-
-If you want lower VRAM usage, you can change the quantization from q8_0 to a lower-level quantization, such as q4_0.
-
-
-## Examples
-
-```sh
-.\bin\Release\sd-cli.exe --diffusion-model ideogram4-Q8_0.gguf --uncond-diffusion-model ideogram4_uncond-Q8_0.gguf --llm ..\..\llm\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors -p '{"high_level_description":"A square 1024 x 1024 luxury fashion magazine cover featuring exactly one short chubby fluffy cat as the main model. The cat sits on a soft ivory studio floor, facing the viewer with a stylish calm expression, wearing tiny black sunglasses, a red silk scarf, and a small gold collar charm. In front of the cat on the floor is a wide horizontal luxury nameplate that clearly reads ideogram4.cpp. The whole design feels premium, fashionable, clean, and editorial.","style_description":{"aesthetics":"luxury fashion magazine cover, high-end pet couture campaign, minimalist editorial design, elegant studio photography, soft paper texture, refined typography, fashionable and polished","lighting":"Soft diffused studio lighting, gentle spotlight on the cat, subtle floor shadow, warm ivory highlights, clean separation between subject and background","photo":"high-resolution fashion editorial photography look, front-facing cat portrait, crisp fur details, glossy sunglasses, clear readable nameplate text, shallow depth of field","medium":"mixed media fashion photography and premium editorial graphic design","color_palette":["#F4EFE7","#111111","#D8B56D","#B73A3A","#FFFFFF","#8A7A6A"]},"compositional_deconstruction":{"canvas":"Square 1024 x 1024 canvas with a normal upright orientation. Do not rotate the poster or any text. Use a clean fashion magazine cover layout.","background":"Warm ivory studio backdrop with subtle paper grain, a soft spotlight gradient, faint floor shadow, and a few minimal gold editorial lines. The background is spacious, premium, and uncluttered.","layout":"Top center has a small elegant headline. Center area features one cat as the main fashion model. Lower foreground has a wide horizontal luxury nameplate placed on the floor in front of the cat. Bottom center has a small footer. All text is horizontal, upright, and readable left to right.","elements":[{"type":"text","desc":"Top center headline reading LOOK WHAT I FOUND in a refined high-fashion serif font. The headline is horizontal, centered, elegant, and secondary to the nameplate text."},{"type":"obj","desc":"Exactly one short chubby fluffy cat sitting in the center like a luxury fashion model. The cat has a large round head, compact body, short legs, soft detailed fur, expressive eyes, and a calm confident pose. The cat is cute and rounded, not tall, not stretched, not duplicated."},{"type":"obj","desc":"Tiny glossy black sunglasses worn naturally by the cat, slightly oversized but still showing the cat face clearly. The sunglasses add a chic fashion-editorial attitude."},{"type":"obj","desc":"A red silk scarf tied neatly around the cat neck, with soft folds and a couture feeling. The scarf must not cover the cat face or the nameplate."},{"type":"obj","desc":"A small gold collar charm or fashion accessory under the scarf, subtle and premium, adding a luxury campaign detail."},{"type":"obj","desc":"In the lower foreground, place a wide horizontal luxury nameplate on the floor in front of the cat. The nameplate is low, flat, landscape-oriented, much wider than tall, like a fashion show seat card or premium display plaque. It is centered, front-facing, level, and fully visible. It must not become vertical, tall, standing, rotated, or side-facing."},{"type":"text","desc":"Print the exact text ideogram4.cpp only on the wide horizontal nameplate. Use clean bold black lettering, perfectly spelled, lowercase, with the number 4 and .cpp extension. The text must fit completely inside the nameplate, stay horizontal, and be readable from left to right."},{"type":"obj","desc":"Add sparse premium editorial accents around the edges: thin gold lines, small code brackets, tiny cursor marks, subtle dots, and minimal geometric details. No extra cats, no stickers, no animal faces, no busy decorations."},{"type":"text","desc":"Bottom center footer reading tiny paws, big compile energy in a small refined monospace or editorial font. The footer is horizontal, centered, understated, and much smaller than the nameplate text."}]}}'  --diffusion-fa -v --offload-to-cpu -H 1024 -W 1024
-```
-
-<img alt="ideogram4 image example" src="../assets/ideogram4/example.png" />

docs/lens.md

@@ -1,32 +0,0 @@
-# How to Use
-
-Lens uses a Lens diffusion transformer, the FLUX.2 VAE, and GPT-OSS-20B as the LLM text encoder.
-
-## Download weights
-
-- Download Lens
-    - safetensors: https://huggingface.co/Comfy-Org/Lens/tree/main/diffusion_models
-- Download Lens Turbo
-    - safetensors: https://huggingface.co/Comfy-Org/Lens/tree/main/diffusion_models
-- Download vae
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main
-- Download GPT-OSS-20B
-    - gguf: https://huggingface.co/unsloth/gpt-oss-20b-GGUF/tree/main
-
-## Examples
-
-### Lens
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model ..\..\ComfyUI\models\diffusion_models\lens_bf16.safetensors --llm "..\..\llm\gpt-oss-20b-UD-Q8_K_XL.gguf" --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors --cfg-scale 5.0  -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" --diffusion-fa -v
-```
-
-<img width="256" alt="Lens example" src="../assets/lens/example.png" />
-
-### Lens Turbo
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model ..\..\ComfyUI\models\diffusion_models\lens_turbo_bf16.safetensors --llm "..\..\llm\gpt-oss-20b-UD-Q8_K_XL.gguf" --vae ..\..\ComfyUI\models\vae\flux2_ae.safetensors --cfg-scale 1.0  -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" --diffusion-fa -v --steps 4
-```
-
-<img width="256" alt="Lens Turbo example" src="../assets/lens/turbo_example.png" />

docs/longcat_image.md

@@ -1,30 +0,0 @@
-# How to Use
-
-LongCat-Image uses a LongCat diffusion transformer, the FLUX VAE, and Qwen2.5-VL as the LLM text encoder.
-
-## Download weights
-
-- Download LongCat Image
-    - safetensors: https://huggingface.co/Comfy-Org/LongCat-Image/tree/main/split_files/diffusion_models
-    - gguf: https://huggingface.co/vantagewithai/LongCat-Image-GGUF/tree/main/comfy
-- Download LongCat Image Edit
-    - LongCat Image Edit Turbo: https://huggingface.co/meituan-longcat/LongCat-Image-Edit-Turbo
-    - gguf: https://huggingface.co/vantagewithai/LongCat-Image-Edit-GGUF/tree/main
-- Download vae
-    - safetensors: https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/ae.safetensors
-- Download qwen_2.5_vl 7b
-    - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders
-    - gguf: https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-GGUF/tree/main
-    - For image editing with GGUF text encoders, also download the matching mmproj file and pass it with `--llm_vision`.
-
-## Run
-
-LongCat uses quoted text for character-level text rendering. Put target text inside single quotes, double quotes, or Chinese quotes.
-
-### LongCat Image
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\..\ComfyUI\models\diffusion_models\LongCat-Image-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\ae.sft --llm ..\..\ComfyUI\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf -p "a lovely cat holding a sign says 'longcat.cpp'" --cfg-scale 5.0 --sampling-method euler --flow-shift 3 -v --offload-to-cpu --diffusion-fa
-```
-
-<img alt="longcat example" src="../assets/longcat/example.png" />

docs/ltx2.md

@@ -1,77 +0,0 @@
-# How to Use
-
-## Download weights
-
-- Download LTX-2.3
-    - safetensors: https://huggingface.co/Kijai/LTX2.3_comfy/tree/main/diffusion_models
-    - gguf: https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main
-- Download gemma-3-12b-it
-    - gguf: https://huggingface.co/unsloth/gemma-3-12b-it-GGUF/tree/main
-- Download embeddings connectors
-    - safetensors: https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main/text_encoders
-- Download vae
-    - safetensors: https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main/vae
-- Download audio vae
-    - safetensors: https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main/vae
-- Download LTX spatial latent upscaler
-    - safetensors: https://huggingface.co/Lightricks/LTX-2.3/resolve/main/ltx-2.3-spatial-upscaler-x2-1.1.safetensors
-
-## Examples
-
-### LTX-2.3 dev T2V
-
-```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\..\ComfyUI\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\..\ComfyUI\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\..\ComfyUI\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 33 --fps 24 -o t2v.webm
-```
-
-<video
-  src="../assets/ltx2/t2v.webm"
-  controls
-  muted
-  style="max-width: 100%; height: auto;"></video>
-
-### LTX-2.3 dev I2V
-
-```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\..\ComfyUI\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\..\ComfyUI\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\..\ComfyUI\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v  -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 33 -i ..\assets\ernie_image\turbo_example.png -o i2v.webm
-```
-
-<video
-  src="../assets/ltx2/i2v.webm"
-  controls
-  muted
-  style="max-width: 100%; height: auto;"></video>
-
-### LTX-2.3 dev FLF2V
-
-```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\..\ComfyUI\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\..\ComfyUI\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\..\ComfyUI\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors  -p "glass flower blossom" --cfg-scale 6.0 --sampling-method euler -v  -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 33 --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png -o flf2v.webm
-```
-
-<video
-  src="../assets/ltx2/flf2v.webm"
-  controls
-  muted
-  style="max-width: 100%; height: auto;"></video>
-
-### LTX-2.3 spatial latent upscale
-
-LTX spatial latent upscale runs a model-backed x2 latent upsampler between the low-resolution video pass and the high-resolution refine pass. `-W` and `-H` are the pre-upscale generation size; the spatial upsampler produces x2 latent dimensions.
-
-Put `ltx-2.3-spatial-upscaler-x2-1.1.safetensors` under the directory passed to `--hires-upscalers-dir`, then use the model name without path or extension in `--hires-upscaler`.
-
-```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\..\ComfyUI\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\..\ComfyUI\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\..\ComfyUI\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\..\ComfyUI\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors --hires-upscalers-dir ..\..\ComfyUI\models\latent_upscale_models --hires-upscaler ltx-2.3-spatial-upscaler-x2-1.1 --hires --hires-steps 4 -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v  -W 640 -H 360 --diffusion-fa --offload-to-cpu --video-frames 33 -i ..\assets\ernie_image\turbo_example.png -o hires_i2v.webm
-```
-
-By default, the hires refine pass uses the main sampler and scheduler, then trims the second-pass sigma schedule by `--hires-denoising-strength` (`0.7` by default). To reproduce a ComfyUI-style explicit refine schedule, pass custom hires sigmas:
-
-```
---hires-sigmas "0.85,0.725,0.421875,0.0"
-```
-
-<video
-  src="../assets/ltx2/hires_i2v.webm"
-  controls
-  muted
-  style="max-width: 100%; height: auto;"></video>

docs/model_config.md

@@ -1,118 +0,0 @@
-# Model Configuration Conventions
-
-This document describes the conventions for model configuration structs and
-weight-based configuration detection.
-
-## Config Types
-
-Model configuration should live in a model-specific `*Config` struct.
-
-Examples:
-
-- `ZImageConfig`
-- `UNetConfig`
-- `MMDiTConfig`
-- `LLMConfig`
-
-Preserve established acronym casing in type names, such as `UNet`, `MMDiT`,
-`LLM`, `VAE`, and `T5`.
-
-Place the config struct near the top of the model header, before the main model
-blocks and runner types that consume it.
-
-## Config Variables
-
-Variables and members that hold a config should be named `config`.
-
-Examples:
-
-```cpp
-UNetConfig config;
-UnetModelBlock unet;
-
-MMDiTRunner(...)
-    : DiffusionModelRunner(backend, params_backend, prefix),
-      config(MMDiTConfig::detect_from_weights(tensor_storage_map, prefix)),
-      mmdit(config) {
-}
-```
-
-Avoid alternate names such as `params`, `params_cfg`, `model_params`, or
-model-specific aliases unless an existing public API requires them.
-
-## Weight Detection
-
-If a model can derive configuration from loaded weight metadata, expose that
-logic as a static method on the config type:
-
-```cpp
-static XxxConfig detect_from_weights(const String2TensorStorage& tensor_storage_map,
-                                     const std::string& prefix);
-```
-
-Additional selector arguments are allowed when required by an existing model
-family, for example `SDVersion version` or an architecture enum:
-
-```cpp
-static UNetConfig detect_from_weights(const String2TensorStorage& tensor_storage_map,
-                                      const std::string& prefix,
-                                      SDVersion version = VERSION_SD1);
-```
-
-Use `TensorStorage` metadata, especially `n_dims` and `ne`, to infer shapes.
-Do not load or parse tensor data for config detection.
-
-Detection should respect `prefix`. For nested weights, construct full names from
-`prefix + "." + suffix` or filter entries with `starts_with(name, prefix)`.
-
-Do not add persistent config fields such as `inferred_from_weights` only to
-record whether detection happened. If the function needs to decide whether to
-print a debug line, keep that as local control flow inside `detect_from_weights`.
-
-## Logging
-
-When config values are inferred from weights, print one `LOG_DEBUG` line at the
-end of `detect_from_weights`.
-
-Example:
-
-```cpp
-LOG_DEBUG("llm: num_layers = %" PRId64 ", vocab_size = %" PRId64 ", hidden_size = %" PRId64 ", intermediate_size = %" PRId64,
-          config.num_layers,
-          config.vocab_size,
-          config.hidden_size,
-          config.intermediate_size);
-```
-
-Only print the config detection log when the function actually inferred values
-from weights. Do not duplicate the same config summary in runner constructors or
-model loading code.
-
-Use the correct format specifiers for field types, such as `%" PRId64 "` for
-`int64_t` and `%d` for `int`.
-
-## Runner And Model Responsibilities
-
-Runners should detect the config once and pass it into the model block:
-
-```cpp
-struct XxxRunner : public DiffusionModelRunner {
-    XxxConfig config;
-    XxxModel model;
-
-    XxxRunner(..., const String2TensorStorage& tensor_storage_map, const std::string prefix)
-        : DiffusionModelRunner(backend, params_backend, prefix),
-          config(XxxConfig::detect_from_weights(tensor_storage_map, prefix)),
-          model(config) {
-        model.init(params_ctx, tensor_storage_map, prefix);
-    }
-};
-```
-
-Model blocks should consume `config` directly instead of re-scanning weights in
-their constructors. Keep config-derived behavior centralized in the config
-struct.
-
-If a model has no weight-derived config today, it may still provide
-`detect_from_weights` for API consistency, but it should not print a config
-detection log unless it actually derives values from weights.

docs/pid.md

@@ -1,39 +0,0 @@
-# How to Use
-
-PiD is NVIDIA's Pixel Diffusion Decoder. It replaces the usual VAE decode or decode-then-upscale path with a pixel-space diffusion decoder conditioned on a
-source latent and text prompt.
-
-In stable-diffusion.cpp, PiD currently runs as an image edit pipeline: provide a reference image with `-r`/`--ref-image`, encode that image with a matching VAE, then let the PiD diffusion model decode/upscale directly to RGB.
-
-## Download weights
-
-- Download PiD
-    - safetensors: https://huggingface.co/Comfy-Org/PixelDiT/tree/main/diffusion_models
-- Download Gemma 2 2B
-    - safetensors: https://huggingface.co/Comfy-Org/PixelDiT/tree/main/text_encoders
-- Download the VAE that matches the PiD checkpoint backbone
-    - safetensors: https://huggingface.co/nvidia/PiD/tree/main/checkpoints
-    - Flux / Z-Image PiD: use the Flux VAE and pass `--vae-format flux`
-    - SD3 PiD: use the SD3 VAE and pass `--vae-format sd3`
-    - Flux.2 PiD: use the Flux.2 VAE and pass `--vae-format flux2`
-
-The official PiD model card should be checked before use. At the time of the initial PiD release, the official weights are under the NSCLv1 non-commercial license.
-
-## Examples
-
-```
-.\bin\Release\sd-cli.exe --diffusion-model ..\..\ComfyUI\models\diffusion_models\pid_flux1_512_to_2048_4step_bf16.safetensors --llm "..\..\ComfyUI\models\text_encoders\gemma_2_2b_it_elm_bf16.safetensors" --vae ..\..\ComfyUI\models\vae\ae.sft --vae-format flux --cfg-scale 1.0  -p "a lovely cat" -r ..\assets\ernie_image\turbo_example.png --diffusion-fa -v --steps 4 -H 2048 -W 2048 --rng cpu
-```
-
-Before:
-
-<img width="256" alt="ERNIE-Image Turbo example" src="../assets/ernie_image/turbo_example.png" />
-
-After:
-<img width="1024" alt="PiD example" src="../assets/pid/example.png" />
-
-## Notes
-
-- `-r`/`--ref-image` is required. PiD uses the first reference image as the source latent condition.
-- `--vae-format` should match the VAE latent layout used by the PiD checkpoint. This is important when using standalone VAE files because the PiD diffusion
-  checkpoint alone does not identify the VAE format.

docs/z_image.md

@@ -21,7 +21,7 @@ You can run Z-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or ev
 ### Z-Image-Turbo
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  z_image_turbo-Q3_K.gguf --vae ..\..\ComfyUI\models\vae\ae.sft  --llm ..\..\ComfyUI\models\text_encoders\Qwen3-4B-Instruct-2507-Q4_K_M.gguf -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 1.0 -v --offload-to-cpu --diffusion-fa -H 1024 -W 512 --steps 8
+.\bin\Release\sd-cli.exe --diffusion-model  z_image_turbo-Q3_K.gguf --vae ..\..\ComfyUI\models\vae\ae.sft  --llm ..\..\ComfyUI\models\text_encoders\Qwen3-4B-Instruct-2507-Q4_K_M.gguf -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 1.0 -v --offload-to-cpu --diffusion-fa -H 1024 -W 512
 ```
 
 <img width="256" alt="z-image example" src="../assets/z_image/q3_K.png" />

examples/cli/CMakeLists.txt

@@ -1,27 +1,6 @@
 set(TARGET sd-cli)
 
-add_executable(${TARGET}
+add_executable(${TARGET} main.cpp)
-    ../common/common.cpp
-    ../common/log.cpp
-    ../common/media_io.cpp
-    image_metadata.cpp
-    main.cpp
-)
-if(APPLE)
-    sd_set_macos_rpaths(${TARGET})
-endif()
-target_include_directories(${TARGET} PRIVATE
-    "${CMAKE_CURRENT_SOURCE_DIR}/.."
-    "${PROJECT_SOURCE_DIR}/src"
-)
 install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE stable-diffusion zip ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE stable-diffusion ${CMAKE_THREAD_LIBS_INIT})
-if(SD_WEBP)
-    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBP)
-    target_link_libraries(${TARGET} PRIVATE webp libwebpmux)
-endif()
-if(SD_WEBM)
-    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBM)
-    target_link_libraries(${TARGET} PRIVATE webm)
-endif()
 target_compile_features(${TARGET} PUBLIC c_std_11 cxx_std_17)

examples/cli/README.md

@@ -4,29 +4,21 @@
 usage: ./bin/sd-cli  [options]
 
 CLI Options:
-  -o, --output <string>         path to write result image to. you can use printf-style %d format specifiers for image
+  -o, --output <string>       path to write result image to. you can use printf-style %d format specifiers for image sequences (default:
-                                sequences (default: ./output.png) (eg. output_%03d.png). Single-file video outputs
+                              ./output.png) (eg. output_%03d.png)
-                                support .avi, .webm, and animated .webp
+  --preview-path <string>     path to write preview image to (default: ./preview.png)
-  --image <string>              path to the image to inspect (for metadata mode)
+  --preview-interval <int>    interval in denoising steps between consecutive updates of the image preview file (default is 1, meaning updating at
-  --metadata-format <string>    metadata output format, one of [text, json] (default: text)
+                              every step)
-  --preview-path <string>       path to write preview image to (default: ./preview.png). Multi-frame previews support
+  --output-begin-idx <int>    starting index for output image sequence, must be non-negative (default 0 if specified %d in output path, 1 otherwise)
-                                .avi, .webm, and animated .webp
+  --canny                     apply canny preprocessor (edge detection)
-  --preview-interval <int>      interval in denoising steps between consecutive updates of the image preview file
+  --convert-name              convert tensor name (for convert mode)
-                                (default is 1, meaning updating at every step)
+  -v, --verbose               print extra info
-  --output-begin-idx <int>      starting index for output image sequence, must be non-negative (default 0 if specified
+  --color                     colors the logging tags according to level
-                                %d in output path, 1 otherwise)
+  --taesd-preview-only        prevents usage of taesd for decoding the final image. (for use with --preview tae)
-  --canny                       apply canny preprocessor (edge detection)
+  --preview-noisy             enables previewing noisy inputs of the models rather than the denoised outputs
-  --convert-name                convert tensor name (for convert mode)
+  -M, --mode                  run mode, one of [img_gen, vid_gen, upscale, convert], default: img_gen
-  -v, --verbose                 print extra info
+  --preview                   preview method. must be one of the following [none, proj, tae, vae] (default is none)
-  --color                       colors the logging tags according to level
+  -h, --help                  show this help message and exit
-  --taesd-preview-only          prevents usage of taesd for decoding the final image. (for use with --preview tae)
-  --preview-noisy               enables previewing noisy inputs of the models rather than the denoised outputs
-  --metadata-raw                include raw hex previews for unparsed metadata payloads
-  --metadata-brief              truncate long metadata text values in text output
-  --metadata-all                include structural/container entries such as IHDR, IDAT, and non-metadata JPEG segments
-  -M, --mode                    run mode, one of [img_gen, vid_gen, upscale, convert, metadata], default: img_gen
-  --preview                     preview method. must be one of the following [none, proj, tae, vae] (default is none)
-  -h, --help                    show this help message and exit
 
 Context Options:
   -m, --model <string>                     path to full model
@@ -34,34 +26,28 @@ Context Options:
   --clip_g <string>                        path to the clip-g text encoder
   --clip_vision <string>                   path to the clip-vision encoder
   --t5xxl <string>                         path to the t5xxl text encoder
-  --llm <string>                           path to the llm text encoder. For example: (qwenvl2.5 for qwen-image,
+  --llm <string>                           path to the llm text encoder. For example: (qwenvl2.5 for qwen-image, mistral-small3.2 for flux2, ...)
-                                           mistral-small3.2 for flux2, ...)
   --llm_vision <string>                    path to the llm vit
   --qwen2vl <string>                       alias of --llm. Deprecated.
   --qwen2vl_vision <string>                alias of --llm_vision. Deprecated.
   --diffusion-model <string>               path to the standalone diffusion model
   --high-noise-diffusion-model <string>    path to the standalone high noise diffusion model
-  --uncond-diffusion-model <string>        path to the standalone unconditional diffusion model, currently used by
-                                           Ideogram4 CFG
   --vae <string>                           path to standalone vae model
   --taesd <string>                         path to taesd. Using Tiny AutoEncoder for fast decoding (low quality)
   --tae <string>                           alias of --taesd
   --control-net <string>                   path to control net model
   --embd-dir <string>                      embeddings directory
   --lora-model-dir <string>                lora model directory
-  --hires-upscalers-dir <string>           highres fix upscaler model directory
   --tensor-type-rules <string>             weight type per tensor pattern (example: "^vae\.=f16,model\.=q8_0")
   --photo-maker <string>                   path to PHOTOMAKER model
   --upscale-model <string>                 path to esrgan model.
-  -t, --threads <int>                      number of threads to use during computation (default: -1). If threads <= 0,
+  -t, --threads <int>                      number of threads to use during computation (default: -1). If threads <= 0, then threads will be set to the number of
-                                           then threads will be set to the number of CPU physical cores
+                                           CPU physical cores
   --chroma-t5-mask-pad <int>               t5 mask pad size of chroma
-  --max-vram <float>                       maximum VRAM budget in GiB for graph-cut segmented execution. 0 disables
+  --vae-tile-overlap <float>               tile overlap for vae tiling, in fraction of tile size (default: 0.5)
-                                           graph splitting; a negative value auto-detects free VRAM, sparing the
+  --vae-tiling                             process vae in tiles to reduce memory usage
-                                           specified value (e.g. -0.5 will keep at least 0.5 GiB free)
   --force-sdxl-vae-conv-scale              force use of conv scale on sdxl vae
-  --offload-to-cpu                         place the weights in RAM to save VRAM, and automatically load them into VRAM
+  --offload-to-cpu                         place the weights in RAM to save VRAM, and automatically load them into VRAM when needed
-                                           when needed
   --mmap                                   whether to memory-map model
   --control-net-cpu                        keep controlnet in cpu (for low vram)
   --clip-on-cpu                            keep clip in cpu (for low vram)
@@ -76,19 +62,20 @@ Context Options:
   --chroma-disable-dit-mask                disable dit mask for chroma
   --qwen-image-zero-cond-t                 enable zero_cond_t for qwen image
   --chroma-enable-t5-mask                  enable t5 mask for chroma
-  --type                                   weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K,
+  --type                                   weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K, q4_K). If not specified, the default is the
-                                           q4_K). If not specified, the default is the type of the weight file
+                                           type of the weight file
   --rng                                    RNG, one of [std_default, cuda, cpu], default: cuda(sd-webui), cpu(comfyui)
   --sampler-rng                            sampler RNG, one of [std_default, cuda, cpu]. If not specified, use --rng
-  --prediction                             prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow,
+  --prediction                             prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow, flux2_flow]
-                                           flux2_flow]
+  --lora-apply-mode                        the way to apply LoRA, one of [auto, immediately, at_runtime], default is auto. In auto mode, if the model weights
-  --lora-apply-mode                        the way to apply LoRA, one of [auto, immediately, at_runtime], default is
+                                           contain any quantized parameters, the at_runtime mode will be used; otherwise,
-                                           auto. In auto mode, if the model weights contain any quantized parameters,
+                                           immediately will be used.The immediately mode may have precision and
-                                           the at_runtime mode will be used; otherwise, immediately will be used.The
+                                           compatibility issues with quantized parameters, but it usually offers faster inference
-                                           immediately mode may have precision and compatibility issues with quantized
+                                           speed and, in some cases, lower memory usage. The at_runtime mode, on the
-                                           parameters, but it usually offers faster inference speed and, in some cases,
+                                           other hand, is exactly the opposite.
-                                           lower memory usage. The at_runtime mode, on the other hand, is exactly the
+  --vae-tile-size                          tile size for vae tiling, format [X]x[Y] (default: 32x32)
-                                           opposite.
+  --vae-relative-tile-size                 relative tile size for vae tiling, format [X]x[Y], in fraction of image size if < 1, in number of tiles per dim if >=1
+                                           (overrides --vae-tile-size)
 
 Generation Options:
   -p, --prompt <string>                    the prompt to render
@@ -97,115 +84,67 @@ Generation Options:
   --end-img <string>                       path to the end image, required by flf2v
   --mask <string>                          path to the mask image
   --control-image <string>                 path to control image, control net
-  --control-video <string>                 path to control video frames, It must be a directory path. The video frames
+  --control-video <string>                 path to control video frames, It must be a directory path. The video frames inside should be stored as images in
-                                           inside should be stored as images in lexicographical (character) order. For
+                                           lexicographical (character) order. For example, if the control video path is
-                                           example, if the control video path is `frames`, the directory contain images
+                                           `frames`, the directory contain images such as 00.png, 01.png, ... etc.
-                                           such as 00.png, 01.png, ... etc.
   --pm-id-images-dir <string>              path to PHOTOMAKER input id images dir
   --pm-id-embed-path <string>              path to PHOTOMAKER v2 id embed
-  --hires-upscaler <string>                highres fix upscaler, Lanczos, Nearest, Latent, Latent (nearest), Latent
-                                           (nearest-exact), Latent (antialiased), Latent (bicubic), Latent (bicubic
-                                           antialiased), or a model name under --hires-upscalers-dir (default: Latent)
-  --extra-sample-args <string>             extra sampler/scheduler/guidance args, key=value list. APG supports apg_eta,
-                                           apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports
-                                           slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end;
-                                           ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma
-  --extra-tiling-args <string>             extra VAE tiling args, key=value list. LTX video VAE supports
-                                           temporal_tile_frames (default: 4), temporal_tile_overlap (default: 1)
   -H, --height <int>                       image height, in pixel space (default: 512)
   -W, --width <int>                        image width, in pixel space (default: 512)
   --steps <int>                            number of sample steps (default: 20)
   --high-noise-steps <int>                 (high noise) number of sample steps (default: -1 = auto)
-  --clip-skip <int>                        ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer
+  --clip-skip <int>                        ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1). <= 0 represents unspecified,
-                                           (default: -1). <= 0 represents unspecified, will be 1 for SD1.x, 2 for SD2.x
+                                           will be 1 for SD1.x, 2 for SD2.x
   -b, --batch-count <int>                  batch count
   --video-frames <int>                     video frames (default: 1)
   --fps <int>                              fps (default: 24)
-  --timestep-shift <int>                   shift timestep for NitroFusion models (default: 0). recommended N for
+  --timestep-shift <int>                   shift timestep for NitroFusion models (default: 0). recommended N for NitroSD-Realism around 250 and 500 for
-                                           NitroSD-Realism around 250 and 500 for NitroSD-Vibrant
+                                           NitroSD-Vibrant
   --upscale-repeats <int>                  Run the ESRGAN upscaler this many times (default: 1)
   --upscale-tile-size <int>                tile size for ESRGAN upscaling (default: 128)
-  --hires-width <int>                      highres fix target width, 0 to use --hires-scale (default: 0)
-  --hires-height <int>                     highres fix target height, 0 to use --hires-scale (default: 0)
-  --hires-steps <int>                      highres fix second pass sample steps, 0 to reuse --steps (default: 0)
-  --hires-upscale-tile-size <int>          highres fix upscaler tile size, reserved for model-backed upscalers (default:
-                                           128)
   --cfg-scale <float>                      unconditional guidance scale: (default: 7.0)
-  --img-cfg-scale <float>                  image guidance scale for inpaint or image edit models: (default: same as
+  --img-cfg-scale <float>                  image guidance scale for inpaint or instruct-pix2pix models: (default: same as --cfg-scale)
-                                           --cfg-scale)
   --guidance <float>                       distilled guidance scale for models with guidance input (default: 3.5)
-  --slg-scale <float>                      skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means
+  --slg-scale <float>                      skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means disabled, a value of 2.5 is nice for sd3.5
-                                           disabled, a value of 2.5 is nice for sd3.5 medium
+                                           medium
   --skip-layer-start <float>               SLG enabling point (default: 0.01)
   --skip-layer-end <float>                 SLG disabling point (default: 0.2)
-  --eta <float>                            noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and
+  --eta <float>                            eta in DDIM, only for DDIM and TCD (default: 0)
-                                           res_2s; 1 for euler_a, er_sde and dpm++2s_a)
   --flow-shift <float>                     shift value for Flow models like SD3.x or WAN (default: auto)
   --high-noise-cfg-scale <float>           (high noise) unconditional guidance scale: (default: 7.0)
-  --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or image edit models (default:
+  --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or instruct-pix2pix models (default: same as --cfg-scale)
-                                           same as --cfg-scale)
+  --high-noise-guidance <float>            (high noise) distilled guidance scale for models with guidance input (default: 3.5)
-  --high-noise-guidance <float>            (high noise) distilled guidance scale for models with guidance input
+  --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default: 0)
-                                           (default: 3.5)
-  --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default:
-                                           0)
   --high-noise-skip-layer-start <float>    (high noise) SLG enabling point (default: 0.01)
   --high-noise-skip-layer-end <float>      (high noise) SLG disabling point (default: 0.2)
-  --high-noise-eta <float>                 (high noise) noise multiplier (default: 0 for ddim_trailing, tcd,
+  --high-noise-eta <float>                 (high noise) eta in DDIM, only for DDIM and TCD (default: 0)
-                                           res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)
   --strength <float>                       strength for noising/unnoising (default: 0.75)
   --pm-style-strength <float>
-  --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full
+  --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image
-                                           destruction of information in init image
+  --moe-boundary <float>                   timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if `--high-noise-steps` is set to -1
-  --moe-boundary <float>                   timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if
-                                           `--high-noise-steps` is set to -1
   --vace-strength <float>                  wan vace strength
-  --vae-tile-overlap <float>               tile overlap for vae tiling, in fraction of tile size (default: 0.5)
+  --increase-ref-index                     automatically increase the indices of references images based on the order they are listed (starting with 1).
-  --hires-scale <float>                    highres fix scale when target size is not set (default: 2.0)
-  --hires-denoising-strength <float>       highres fix second pass denoising strength (default: 0.7)
-  --increase-ref-index                     automatically increase the indices of references images based on the order
-                                           they are listed (starting with 1).
   --disable-auto-resize-ref-image          disable auto resize of ref images
   --disable-image-metadata                 do not embed generation metadata on image files
-  --vae-tiling                             process vae in tiles to reduce memory usage
-  --temporal-tiling                        enable temporal tiling for LTX video VAE decode
-  --hires                                  enable highres fix
   -s, --seed                               RNG seed (default: 42, use random seed for < 0)
-  --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m,
+  --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing,
-                                           dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s,
+                                           tcd, res_multistep, res_2s] (default: euler for Flux/SD3/Wan, euler_a
-                                           er_sde, euler_cfg_pp, euler_a_cfg_pp] (default: euler for Flux/SD3/Wan, euler_a otherwise)
+                                           otherwise)
-  --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a,
+  --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm,
-                                           dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep,
+                                           ddim_trailing, tcd, res_multistep, res_2s] default: euler for Flux/SD3/Wan,
-                                           res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for Flux/SD3/Wan, euler_a otherwise
+                                           euler_a otherwise
-  --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits,
+  --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, smoothstep, sgm_uniform, simple,
-                                           smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent, ltx2], default:
+                                           kl_optimal, lcm, bong_tangent], default: discrete
-                                           model-specific
+  --sigmas                                 custom sigma values for the sampler, comma-separated (e.g., "14.61,7.8,3.5,0.0").
-  --sigmas                                 custom sigma values for the sampler, comma-separated (e.g.,
-                                           "14.61,7.8,3.5,0.0").
-  --hires-sigmas                           custom sigma values for the highres fix second pass, comma-separated (e.g.,
-                                           "0.85,0.725,0.421875,0.0").
   --skip-layers                            layers to skip for SLG steps (default: [7,8,9])
   --high-noise-skip-layers                 (high noise) layers to skip for SLG steps (default: [7,8,9])
   -r, --ref-image                          reference image for Flux Kontext models (can be used multiple times)
-  --cache-mode                             caching method: 'easycache' (DiT), 'ucache' (UNET),
+  --cache-mode                             caching method: 'easycache' (DiT), 'ucache' (UNET), 'dbcache'/'taylorseer'/'cache-dit' (DiT block-level),
-                                           'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT
+                                           'spectrum' (UNET/DiT Chebyshev+Taylor forecasting)
-                                           Chebyshev+Taylor forecasting)
   --cache-option                           named cache params (key=value format, comma-separated). easycache/ucache:
-                                           threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit:
+                                           threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit: Fn=,Bn=,threshold=,warmup=;
-                                           Fn=,Bn=,threshold=,warmup=; spectrum: w=,m=,lam=,window=,flex=,warmup=,stop=.
+                                           spectrum: w=,m=,lam=,window=,flex=,warmup=,stop=. Examples:
-                                           Examples: "threshold=0.25" or "threshold=1.5,reset=0"
+                                           "threshold=0.25" or "threshold=1.5,reset=0" or "w=0.4,window=2"
-  --scm-mask                               SCM steps mask for cache-dit: comma-separated 0/1 (e.g.,
+  --scm-mask                               SCM steps mask for cache-dit: comma-separated 0/1 (e.g., "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache
-                                           "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache
   --scm-policy                             SCM policy: 'dynamic' (default) or 'static'
-  --vae-tile-size                          tile size for vae tiling, format [X]x[Y] (default: 32x32)
-  --vae-relative-tile-size                 relative tile size for vae tiling, format [X]x[Y], in fraction of image size
-                                           if < 1, in number of tiles per dim if >=1 (overrides --vae-tile-size)
-```
-
-Metadata mode inspects PNG/JPEG container metadata without loading any model:
-
-```bash
-./bin/sd-cli -M metadata --image ./output.png
-./bin/sd-cli -M metadata --image ./output.jpg --metadata-format json
-./bin/sd-cli -M metadata --image ./output.png --metadata-raw
-./bin/sd-cli -M metadata --image ./output.png --metadata-all
 ```

examples/cli/avi_writer.h

@@ -0,0 +1,217 @@
+#ifndef __AVI_WRITER_H__
+#define __AVI_WRITER_H__
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+#include "stable-diffusion.h"
+
+#ifndef INCLUDE_STB_IMAGE_WRITE_H
+#include "stb_image_write.h"
+#endif
+
+typedef struct {
+    uint32_t offset;
+    uint32_t size;
+} avi_index_entry;
+
+// Write 32-bit little-endian integer
+void write_u32_le(FILE* f, uint32_t val) {
+    fwrite(&val, 4, 1, f);
+}
+
+// Write 16-bit little-endian integer
+void write_u16_le(FILE* f, uint16_t val) {
+    fwrite(&val, 2, 1, f);
+}
+
+/**
+ * Create an MJPG AVI file from an array of sd_image_t images.
+ * Images are encoded to JPEG using stb_image_write.
+ *
+ * @param filename Output AVI file name.
+ * @param images Array of input images.
+ * @param num_images Number of images in the array.
+ * @param fps Frames per second for the video.
+ * @param quality JPEG quality (0-100).
+ * @return 0 on success, -1 on failure.
+ */
+int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality = 90) {
+    if (num_images == 0) {
+        fprintf(stderr, "Error: Image array is empty.\n");
+        return -1;
+    }
+
+    FILE* f = fopen(filename, "wb");
+    if (!f) {
+        perror("Error opening file for writing");
+        return -1;
+    }
+
+    uint32_t width    = images[0].width;
+    uint32_t height   = images[0].height;
+    uint32_t channels = images[0].channel;
+    if (channels != 3 && channels != 4) {
+        fprintf(stderr, "Error: Unsupported channel count: %u\n", channels);
+        fclose(f);
+        return -1;
+    }
+
+    // --- RIFF AVI Header ---
+    fwrite("RIFF", 4, 1, f);
+    long riff_size_pos = ftell(f);
+    write_u32_le(f, 0);  // Placeholder for file size
+    fwrite("AVI ", 4, 1, f);
+
+    // 'hdrl' LIST (header list)
+    fwrite("LIST", 4, 1, f);
+    write_u32_le(f, 4 + 8 + 56 + 8 + 4 + 8 + 56 + 8 + 40);
+    fwrite("hdrl", 4, 1, f);
+
+    // 'avih' chunk (AVI main header)
+    fwrite("avih", 4, 1, f);
+    write_u32_le(f, 56);
+    write_u32_le(f, 1000000 / fps);       // Microseconds per frame
+    write_u32_le(f, 0);                   // Max bytes per second
+    write_u32_le(f, 0);                   // Padding granularity
+    write_u32_le(f, 0x110);               // Flags (HASINDEX | ISINTERLEAVED)
+    write_u32_le(f, num_images);          // Total frames
+    write_u32_le(f, 0);                   // Initial frames
+    write_u32_le(f, 1);                   // Number of streams
+    write_u32_le(f, width * height * 3);  // Suggested buffer size
+    write_u32_le(f, width);
+    write_u32_le(f, height);
+    write_u32_le(f, 0);  // Reserved
+    write_u32_le(f, 0);  // Reserved
+    write_u32_le(f, 0);  // Reserved
+    write_u32_le(f, 0);  // Reserved
+
+    // 'strl' LIST (stream list)
+    fwrite("LIST", 4, 1, f);
+    write_u32_le(f, 4 + 8 + 56 + 8 + 40);
+    fwrite("strl", 4, 1, f);
+
+    // 'strh' chunk (stream header)
+    fwrite("strh", 4, 1, f);
+    write_u32_le(f, 56);
+    fwrite("vids", 4, 1, f);              // Stream type: video
+    fwrite("MJPG", 4, 1, f);              // Codec: Motion JPEG
+    write_u32_le(f, 0);                   // Flags
+    write_u16_le(f, 0);                   // Priority
+    write_u16_le(f, 0);                   // Language
+    write_u32_le(f, 0);                   // Initial frames
+    write_u32_le(f, 1);                   // Scale
+    write_u32_le(f, fps);                 // Rate
+    write_u32_le(f, 0);                   // Start
+    write_u32_le(f, num_images);          // Length
+    write_u32_le(f, width * height * 3);  // Suggested buffer size
+    write_u32_le(f, (uint32_t)-1);        // Quality
+    write_u32_le(f, 0);                   // Sample size
+    write_u16_le(f, 0);                   // rcFrame.left
+    write_u16_le(f, 0);                   // rcFrame.top
+    write_u16_le(f, 0);                   // rcFrame.right
+    write_u16_le(f, 0);                   // rcFrame.bottom
+
+    // 'strf' chunk (stream format: BITMAPINFOHEADER)
+    fwrite("strf", 4, 1, f);
+    write_u32_le(f, 40);
+    write_u32_le(f, 40);  // biSize
+    write_u32_le(f, width);
+    write_u32_le(f, height);
+    write_u16_le(f, 1);                   // biPlanes
+    write_u16_le(f, 24);                  // biBitCount
+    fwrite("MJPG", 4, 1, f);              // biCompression (FOURCC)
+    write_u32_le(f, width * height * 3);  // biSizeImage
+    write_u32_le(f, 0);                   // XPelsPerMeter
+    write_u32_le(f, 0);                   // YPelsPerMeter
+    write_u32_le(f, 0);                   // Colors used
+    write_u32_le(f, 0);                   // Colors important
+
+    // 'movi' LIST (video frames)
+    // long movi_list_pos = ftell(f);
+    fwrite("LIST", 4, 1, f);
+    long movi_size_pos = ftell(f);
+    write_u32_le(f, 0);  // Placeholder for movi size
+    fwrite("movi", 4, 1, f);
+
+    avi_index_entry* index = (avi_index_entry*)malloc(sizeof(avi_index_entry) * num_images);
+    if (!index) {
+        fclose(f);
+        return -1;
+    }
+
+    // Encode and write each frame as JPEG
+    struct {
+        uint8_t* buf;
+        size_t size;
+    } jpeg_data;
+
+    for (int i = 0; i < num_images; i++) {
+        jpeg_data.buf  = nullptr;
+        jpeg_data.size = 0;
+
+        // Callback function to collect JPEG data into memory
+        auto write_to_buf = [](void* context, void* data, int size) {
+            auto jd = (decltype(jpeg_data)*)context;
+            jd->buf = (uint8_t*)realloc(jd->buf, jd->size + size);
+            memcpy(jd->buf + jd->size, data, size);
+            jd->size += size;
+        };
+
+        // Encode to JPEG in memory
+        stbi_write_jpg_to_func(
+            write_to_buf,
+            &jpeg_data,
+            images[i].width,
+            images[i].height,
+            channels,
+            images[i].data,
+            quality);
+
+        // Write '00dc' chunk (video frame)
+        fwrite("00dc", 4, 1, f);
+        write_u32_le(f, (uint32_t)jpeg_data.size);
+        index[i].offset = ftell(f) - 8;
+        index[i].size   = (uint32_t)jpeg_data.size;
+        fwrite(jpeg_data.buf, 1, jpeg_data.size, f);
+
+        // Align to even byte size
+        if (jpeg_data.size % 2)
+            fputc(0, f);
+
+        free(jpeg_data.buf);
+    }
+
+    // Finalize 'movi' size
+    long cur_pos   = ftell(f);
+    long movi_size = cur_pos - movi_size_pos - 4;
+    fseek(f, movi_size_pos, SEEK_SET);
+    write_u32_le(f, movi_size);
+    fseek(f, cur_pos, SEEK_SET);
+
+    // Write 'idx1' index
+    fwrite("idx1", 4, 1, f);
+    write_u32_le(f, num_images * 16);
+    for (int i = 0; i < num_images; i++) {
+        fwrite("00dc", 4, 1, f);
+        write_u32_le(f, 0x10);
+        write_u32_le(f, index[i].offset);
+        write_u32_le(f, index[i].size);
+    }
+
+    // Finalize RIFF size
+    cur_pos        = ftell(f);
+    long file_size = cur_pos - riff_size_pos - 4;
+    fseek(f, riff_size_pos, SEEK_SET);
+    write_u32_le(f, file_size);
+    fseek(f, cur_pos, SEEK_SET);
+
+    fclose(f);
+    free(index);
+
+    return 0;
+}
+
+#endif  // __AVI_WRITER_H__

examples/cli/image_metadata.h

@@ -1,21 +0,0 @@
-#pragma once
-
-#include <iosfwd>
-#include <string>
-
-enum class MetadataOutputFormat {
-    TEXT,
-    JSON,
-};
-
-struct MetadataReadOptions {
-    MetadataOutputFormat output_format = MetadataOutputFormat::TEXT;
-    bool include_raw                   = false;
-    bool brief                         = false;
-    bool include_structural            = false;
-};
-
-bool print_image_metadata(const std::string& image_path,
-                          const MetadataReadOptions& options,
-                          std::ostream& out,
-                          std::string& error);

examples/cli/main.cpp

@@ -15,12 +15,9 @@
 // #include "preprocessing.hpp"
 #include "stable-diffusion.h"
 
-#include "common/common.h"
+#include "common/common.hpp"
-#include "common/media_io.h"
-#include "common/resource_owners.hpp"
-#include "image_metadata.h"
 
-namespace fs = std::filesystem;
+#include "avi_writer.h"
 
 const char* previews_str[] = {
     "none",
@@ -35,8 +32,6 @@ struct SDCliParams {
     SDMode mode             = IMG_GEN;
     std::string output_path = "output.png";
     int output_begin_idx    = -1;
-    std::string image_path;
-    std::string metadata_format = "text";
 
     bool verbose          = false;
     bool canny_preprocess = false;
@@ -49,9 +44,6 @@ struct SDCliParams {
     bool taesd_preview       = false;
     bool preview_noisy       = false;
     bool color               = false;
-    bool metadata_raw        = false;
-    bool metadata_brief      = false;
-    bool metadata_all        = false;
 
     bool normal_exit = false;
 
@@ -61,19 +53,11 @@ struct SDCliParams {
         options.string_options = {
             {"-o",
              "--output",
-             "path to write result image to. you can use printf-style %d format specifiers for image sequences (default: ./output.png) (eg. output_%03d.png). Single-file video outputs support .avi, .webm, and animated .webp",
+             "path to write result image to. you can use printf-style %d format specifiers for image sequences (default: ./output.png) (eg. output_%03d.png)",
              &output_path},
-            {"",
-             "--image",
-             "path to the image to inspect (for metadata mode)",
-             &image_path},
-            {"",
-             "--metadata-format",
-             "metadata output format, one of [text, json] (default: text)",
-             &metadata_format},
             {"",
              "--preview-path",
-             "path to write preview image to (default: ./preview.png). Multi-frame previews support .avi, .webm, and animated .webp",
+             "path to write preview image to (default: ./preview.png)",
              &preview_path},
         };
 
@@ -113,18 +97,6 @@ struct SDCliParams {
              "--preview-noisy",
              "enables previewing noisy inputs of the models rather than the denoised outputs",
              true, &preview_noisy},
-            {"",
-             "--metadata-raw",
-             "include raw hex previews for unparsed metadata payloads",
-             true, &metadata_raw},
-            {"",
-             "--metadata-brief",
-             "truncate long metadata text values in text output",
-             true, &metadata_brief},
-            {"",
-             "--metadata-all",
-             "include structural/container entries such as IHDR, IDAT, and non-metadata JPEG segments",
-             true, &metadata_all},
 
         };
 
@@ -169,16 +141,15 @@ struct SDCliParams {
             return 1;
         };
 
-        auto on_help_arg = [&](int argc, const char** argv, int index, bool& valid) {
+        auto on_help_arg = [&](int argc, const char** argv, int index) {
             normal_exit = true;
-            valid       = true;
             return -1;
         };
 
         options.manual_options = {
             {"-M",
              "--mode",
-             "run mode, one of [img_gen, vid_gen, upscale, convert, metadata], default: img_gen",
+             "run mode, one of [img_gen, vid_gen, upscale, convert], default: img_gen",
              on_mode_arg},
             {"",
              "--preview",
@@ -193,7 +164,12 @@ struct SDCliParams {
         return options;
     };
 
-    bool resolve() {
+    bool process_and_check() {
+        if (output_path.length() == 0) {
+            LOG_ERROR("error: the following arguments are required: output_path");
+            return false;
+        }
+
         if (mode == CONVERT) {
             if (output_path == "output.png") {
                 output_path = "output.gguf";
@@ -202,43 +178,11 @@ struct SDCliParams {
         return true;
     }
 
-    bool validate() {
-        if (mode != METADATA) {
-            if (output_path.length() == 0) {
-                LOG_ERROR("error: the following arguments are required: output_path");
-                return false;
-            }
-        } else {
-            if (image_path.empty()) {
-                LOG_ERROR("error: metadata mode needs an image path (--image)");
-                return false;
-            }
-            if (metadata_format != "text" && metadata_format != "json") {
-                LOG_ERROR("error: invalid metadata format %s, must be one of [text, json]",
-                          metadata_format.c_str());
-                return false;
-            }
-        }
-        return true;
-    }
-
-    bool resolve_and_validate() {
-        if (!resolve()) {
-            return false;
-        }
-        if (!validate()) {
-            return false;
-        }
-        return true;
-    }
-
     std::string to_string() const {
         std::ostringstream oss;
         oss << "SDCliParams {\n"
             << "  mode: " << modes_str[mode] << ",\n"
             << "  output_path: \"" << output_path << "\",\n"
-            << "  image_path: \"" << image_path << "\",\n"
-            << "  metadata_format: \"" << metadata_format << "\",\n"
             << "  verbose: " << (verbose ? "true" : "false") << ",\n"
             << "  color: " << (color ? "true" : "false") << ",\n"
             << "  canny_preprocess: " << (canny_preprocess ? "true" : "false") << ",\n"
@@ -248,10 +192,7 @@ struct SDCliParams {
             << "  preview_path: \"" << preview_path << "\",\n"
             << "  preview_fps: " << preview_fps << ",\n"
             << "  taesd_preview: " << (taesd_preview ? "true" : "false") << ",\n"
-            << "  preview_noisy: " << (preview_noisy ? "true" : "false") << ",\n"
+            << "  preview_noisy: " << (preview_noisy ? "true" : "false") << "\n"
-            << "  metadata_raw: " << (metadata_raw ? "true" : "false") << ",\n"
-            << "  metadata_brief: " << (metadata_brief ? "true" : "false") << ",\n"
-            << "  metadata_all: " << (metadata_all ? "true" : "false") << "\n"
             << "}";
         return oss.str();
     }
@@ -276,15 +217,9 @@ void parse_args(int argc, const char** argv, SDCliParams& cli_params, SDContextP
         exit(cli_params.normal_exit ? 0 : 1);
     }
 
-    bool valid = cli_params.resolve_and_validate();
+    if (!cli_params.process_and_check() ||
-    if (valid && cli_params.mode != METADATA) {
+        !ctx_params.process_and_check(cli_params.mode) ||
-        valid = ctx_params.resolve_and_validate(cli_params.mode) &&
+        !gen_params.process_and_check(cli_params.mode, ctx_params.lora_model_dir)) {
-                gen_params.resolve_and_validate(cli_params.mode,
-                                                ctx_params.lora_model_dir,
-                                                ctx_params.hires_upscalers_dir);
-    }
-
-    if (!valid) {
         print_usage(argc, argv, options_vec);
         exit(1);
     }
@@ -296,7 +231,7 @@ void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
 }
 
 bool load_images_from_dir(const std::string dir,
-                          std::vector<SDImageOwner>& images,
+                          std::vector<sd_image_t>& images,
                           int expected_width  = 0,
                           int expected_height = 0,
                           int max_image_num   = 0,
@@ -323,7 +258,7 @@ bool load_images_from_dir(const std::string dir,
         std::string ext  = entry.path().extension().string();
         std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
 
-        if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".bmp" || ext == ".webp") {
+        if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".bmp") {
             LOG_DEBUG("load image %zu from '%s'", images.size(), path.c_str());
             int width             = 0;
             int height            = 0;
@@ -333,12 +268,12 @@ bool load_images_from_dir(const std::string dir,
                 return false;
             }
 
-            images.emplace_back(sd_image_t{(uint32_t)width,
+            images.push_back({(uint32_t)width,
-                                           (uint32_t)height,
+                              (uint32_t)height,
-                                           3,
+                              3,
-                                           image_buffer});
+                              image_buffer});
 
-            if (max_image_num > 0 && static_cast<int>(images.size()) >= max_image_num) {
+            if (max_image_num > 0 && images.size() >= max_image_num) {
                 break;
             }
         }
@@ -353,17 +288,9 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy,
     // is_noisy is set to true if the preview corresponds to noisy latents, false if it's denoised latents
     // unused in this app, it will either be always noisy or always denoised here
     if (frame_count == 1) {
-        if (!write_image_to_file(cli_params->preview_path,
+        stbi_write_png(cli_params->preview_path.c_str(), image->width, image->height, image->channel, image->data, 0);
-                                 image->data,
-                                 image->width,
-                                 image->height,
-                                 image->channel)) {
-            LOG_ERROR("save preview image to '%s' failed", cli_params->preview_path.c_str());
-        }
     } else {
-        if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps) != 0) {
+        create_mjpg_avi_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps);
-            LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str());
-        }
     }
 }
 
@@ -386,32 +313,11 @@ std::string format_frame_idx(std::string pattern, int frame_idx) {
     return result;
 }
 
-static fs::path get_video_audio_sidecar_path(const SDCliParams& cli_params) {
-    fs::path out_path     = cli_params.output_path;
-    fs::path base_path    = out_path;
-    fs::path ext          = out_path.has_extension() ? out_path.extension() : fs::path{};
-    std::string ext_lower = ext.string();
-    std::transform(ext_lower.begin(), ext_lower.end(), ext_lower.begin(), ::tolower);
-    const EncodedImageFormat output_format = encoded_image_format_from_path(out_path.string());
-    if (!ext.empty()) {
-        if (output_format == EncodedImageFormat::JPEG ||
-            output_format == EncodedImageFormat::PNG ||
-            output_format == EncodedImageFormat::WEBP ||
-            ext_lower == ".avi" ||
-            ext_lower == ".webm") {
-            base_path.replace_extension();
-        }
-    }
-    base_path += ".wav";
-    return base_path;
-}
-
 bool save_results(const SDCliParams& cli_params,
                   const SDContextParams& ctx_params,
                   const SDGenerationParams& gen_params,
                   sd_image_t* results,
-                  int num_results,
+                  int num_results) {
-                  const sd_audio_t* generated_audio = nullptr) {
     if (results == nullptr || num_results <= 0) {
         return false;
     }
@@ -434,13 +340,9 @@ bool save_results(const SDCliParams& cli_params,
 
     std::string ext_lower = ext.string();
     std::transform(ext_lower.begin(), ext_lower.end(), ext_lower.begin(), ::tolower);
-    const EncodedImageFormat output_format = encoded_image_format_from_path(out_path.string());
+    bool is_jpg = (ext_lower == ".jpg" || ext_lower == ".jpeg" || ext_lower == ".jpe");
     if (!ext.empty()) {
-        if (output_format == EncodedImageFormat::JPEG ||
+        if (is_jpg || ext_lower == ".png") {
-            output_format == EncodedImageFormat::PNG ||
-            output_format == EncodedImageFormat::WEBP ||
-            ext_lower == ".avi" ||
-            ext_lower == ".webm") {
             base_path.replace_extension();
         }
     }
@@ -455,34 +357,23 @@ bool save_results(const SDCliParams& cli_params,
         if (!img.data)
             return false;
 
-        const int64_t metadata_seed = cli_params.mode == VID_GEN ? gen_params.seed : gen_params.seed + idx;
+        std::string params = gen_params.embed_image_metadata
-        std::string params          = gen_params.embed_image_metadata
+                                 ? get_image_params(ctx_params, gen_params, gen_params.seed + idx)
-                                          ? get_image_params(ctx_params, gen_params, metadata_seed, cli_params.mode)
+                                 : "";
-                                          : "";
+        int ok             = 0;
-        const bool ok               = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, 90);
+        if (is_jpg) {
-        LOG_INFO("save result image %d to '%s' (%s)", idx, path.string().c_str(), ok ? "success" : "failure");
+            ok = stbi_write_jpg(path.string().c_str(), img.width, img.height, img.channel, img.data, 90, params.size() > 0 ? params.c_str() : nullptr);
-        return ok;
-    };
-
-    auto write_audio_sidecar = [&](const fs::path& wav_path) {
-        if (generated_audio == nullptr) {
-            return;
-        }
-        if (write_wav_to_file(wav_path.string(),
-                              generated_audio->data,
-                              generated_audio->sample_count,
-                              generated_audio->channels,
-                              generated_audio->sample_rate)) {
-            LOG_INFO("save result audio to '%s'", wav_path.string().c_str());
         } else {
-            LOG_WARN("failed to save result audio to '%s'", wav_path.string().c_str());
+            ok = stbi_write_png(path.string().c_str(), img.width, img.height, img.channel, img.data, 0, params.size() > 0 ? params.c_str() : nullptr);
         }
+        LOG_INFO("save result image %d to '%s' (%s)", idx, path.string().c_str(), ok ? "success" : "failure");
+        return ok != 0;
     };
 
     int sucessful_reults = 0;
 
     if (std::regex_search(cli_params.output_path, format_specifier_regex)) {
-        if (output_format == EncodedImageFormat::UNKNOWN)
+        if (!is_jpg && ext_lower != ".png")
             ext = ".png";
         fs::path pattern = base_path;
         pattern += ext;
@@ -498,28 +389,20 @@ bool save_results(const SDCliParams& cli_params,
     }
 
     if (cli_params.mode == VID_GEN && num_results > 1) {
-        if (ext_lower != ".avi" && ext_lower != ".webp" && ext_lower != ".webm")
+        if (ext_lower != ".avi")
             ext = ".avi";
         fs::path video_path = base_path;
         video_path += ext;
-        std::string final_ext_lower = ext.string();
+        if (create_mjpg_avi_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps) == 0) {
-        std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower);
+            LOG_INFO("save result MJPG AVI video to '%s'", video_path.string().c_str());
-        const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm");
-        if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, 90, mux_audio ? generated_audio : nullptr) == 0) {
-            LOG_INFO("save result video to '%s'", video_path.string().c_str());
-            if (generated_audio != nullptr && !mux_audio) {
-                fs::path wav_path = video_path;
-                wav_path.replace_extension(".wav");
-                write_audio_sidecar(wav_path);
-            }
             return true;
         } else {
-            LOG_ERROR("Failed to save result video to '%s'", video_path.string().c_str());
+            LOG_ERROR("Failed to save result MPG AVI video to '%s'", video_path.string().c_str());
             return false;
         }
     }
 
-    if (output_format == EncodedImageFormat::UNKNOWN)
+    if (!is_jpg && ext_lower != ".png")
         ext = ".png";
 
     for (int i = 0; i < num_results; ++i) {
@@ -533,9 +416,6 @@ bool save_results(const SDCliParams& cli_params,
         }
     }
     LOG_INFO("%d/%d images saved", sucessful_reults, num_results);
-    if (generated_audio != nullptr) {
-        write_audio_sidecar(get_video_audio_sidecar_path(cli_params));
-    }
     return sucessful_reults != 0;
 }
 
@@ -550,27 +430,6 @@ int main(int argc, const char* argv[]) {
     SDGenerationParams gen_params;
 
     parse_args(argc, argv, cli_params, ctx_params, gen_params);
-    sd_set_log_callback(sd_log_cb, (void*)&cli_params);
-    log_verbose = cli_params.verbose;
-    log_color   = cli_params.color;
-
-    if (cli_params.mode == METADATA) {
-        MetadataReadOptions options;
-        options.output_format      = cli_params.metadata_format == "json"
-                                         ? MetadataOutputFormat::JSON
-                                         : MetadataOutputFormat::TEXT;
-        options.include_raw        = cli_params.metadata_raw;
-        options.brief              = cli_params.metadata_brief;
-        options.include_structural = cli_params.metadata_all;
-
-        std::string error;
-        if (!print_image_metadata(cli_params.image_path, options, std::cout, error)) {
-            LOG_ERROR("%s", error.c_str());
-            return 1;
-        }
-        return 0;
-    }
-
     if (gen_params.video_frames > 4) {
         size_t last_dot_pos   = cli_params.preview_path.find_last_of(".");
         std::string base_path = cli_params.preview_path;
@@ -588,6 +447,9 @@ int main(int argc, const char* argv[]) {
     if (cli_params.preview_method == PREVIEW_PROJ)
         cli_params.preview_fps /= 4;
 
+    sd_set_log_callback(sd_log_cb, (void*)&cli_params);
+    log_verbose = cli_params.verbose;
+    log_color   = cli_params.color;
     sd_set_preview_callback(step_callback,
                             cli_params.preview_method,
                             cli_params.preview_interval,
@@ -623,10 +485,39 @@ int main(int argc, const char* argv[]) {
         }
     }
 
-    bool vae_decode_only = true;
+    bool vae_decode_only     = true;
+    sd_image_t init_image    = {0, 0, 3, nullptr};
+    sd_image_t end_image     = {0, 0, 3, nullptr};
+    sd_image_t control_image = {0, 0, 3, nullptr};
+    sd_image_t mask_image    = {0, 0, 1, nullptr};
+    std::vector<sd_image_t> ref_images;
+    std::vector<sd_image_t> pmid_images;
+    std::vector<sd_image_t> control_frames;
+
+    auto release_all_resources = [&]() {
+        free(init_image.data);
+        free(end_image.data);
+        free(control_image.data);
+        free(mask_image.data);
+        for (auto image : ref_images) {
+            free(image.data);
+            image.data = nullptr;
+        }
+        ref_images.clear();
+        for (auto image : pmid_images) {
+            free(image.data);
+            image.data = nullptr;
+        }
+        pmid_images.clear();
+        for (auto image : control_frames) {
+            free(image.data);
+            image.data = nullptr;
+        }
+        control_frames.clear();
+    };
 
     auto load_image_and_update_size = [&](const std::string& path,
-                                          SDImageOwner& image,
+                                          sd_image_t& image,
                                           bool resize_image    = true,
                                           int expected_channel = 3) -> bool {
         int expected_width  = 0;
@@ -636,73 +527,74 @@ int main(int argc, const char* argv[]) {
             expected_height = gen_params.height;
         }
 
-        if (!load_sd_image_from_file(image.put(), path.c_str(), expected_width, expected_height, expected_channel)) {
+        if (!load_sd_image_from_file(&image, path.c_str(), expected_width, expected_height, expected_channel)) {
             LOG_ERROR("load image from '%s' failed", path.c_str());
+            release_all_resources();
             return false;
         }
 
-        gen_params.set_width_and_height_if_unset(image.get().width, image.get().height);
+        gen_params.set_width_and_height_if_unset(image.width, image.height);
         return true;
     };
 
     if (gen_params.init_image_path.size() > 0) {
         vae_decode_only = false;
-        if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image)) {
+        if (!load_image_and_update_size(gen_params.init_image_path, init_image)) {
             return 1;
         }
     }
 
     if (gen_params.end_image_path.size() > 0) {
         vae_decode_only = false;
-        if (!load_image_and_update_size(gen_params.end_image_path, gen_params.end_image)) {
+        if (!load_image_and_update_size(gen_params.end_image_path, end_image)) {
             return 1;
         }
     }
 
     if (gen_params.ref_image_paths.size() > 0) {
         vae_decode_only = false;
-        gen_params.ref_images.clear();
         for (auto& path : gen_params.ref_image_paths) {
-            SDImageOwner ref_image({0, 0, 3, nullptr});
+            sd_image_t ref_image = {0, 0, 3, nullptr};
             if (!load_image_and_update_size(path, ref_image, false)) {
                 return 1;
             }
-            gen_params.ref_images.push_back(std::move(ref_image));
+            ref_images.push_back(ref_image);
         }
     }
 
     if (gen_params.mask_image_path.size() > 0) {
-        if (!load_sd_image_from_file(gen_params.mask_image.put(),
+        if (!load_sd_image_from_file(&mask_image,
                                      gen_params.mask_image_path.c_str(),
                                      gen_params.get_resolved_width(),
                                      gen_params.get_resolved_height(),
                                      1)) {
             LOG_ERROR("load image from '%s' failed", gen_params.mask_image_path.c_str());
+            release_all_resources();
             return 1;
         }
     } else {
-        sd_image_t generated_mask = {0, 0, 1, nullptr};
+        mask_image.data = (uint8_t*)malloc(gen_params.get_resolved_width() * gen_params.get_resolved_height());
-        generated_mask.data       = (uint8_t*)malloc(gen_params.get_resolved_width() * gen_params.get_resolved_height());
+        if (mask_image.data == nullptr) {
-        if (generated_mask.data == nullptr) {
             LOG_ERROR("malloc mask image failed");
+            release_all_resources();
             return 1;
         }
-        generated_mask.width  = gen_params.get_resolved_width();
+        mask_image.width  = gen_params.get_resolved_width();
-        generated_mask.height = gen_params.get_resolved_height();
+        mask_image.height = gen_params.get_resolved_height();
-        memset(generated_mask.data, 255, gen_params.get_resolved_width() * gen_params.get_resolved_height());
+        memset(mask_image.data, 255, gen_params.get_resolved_width() * gen_params.get_resolved_height());
-        gen_params.mask_image.reset(generated_mask);
     }
 
     if (gen_params.control_image_path.size() > 0) {
-        if (!load_sd_image_from_file(gen_params.control_image.put(),
+        if (!load_sd_image_from_file(&control_image,
                                      gen_params.control_image_path.c_str(),
                                      gen_params.get_resolved_width(),
                                      gen_params.get_resolved_height())) {
             LOG_ERROR("load image from '%s' failed", gen_params.control_image_path.c_str());
+            release_all_resources();
             return 1;
         }
         if (cli_params.canny_preprocess) {  // apply preprocessor
-            preprocess_canny(gen_params.control_image.get(),
+            preprocess_canny(control_image,
                              0.08f,
                              0.08f,
                              0.8f,
@@ -712,25 +604,25 @@ int main(int argc, const char* argv[]) {
     }
 
     if (!gen_params.control_video_path.empty()) {
-        gen_params.control_frames.clear();
         if (!load_images_from_dir(gen_params.control_video_path,
-                                  gen_params.control_frames,
+                                  control_frames,
                                   gen_params.get_resolved_width(),
                                   gen_params.get_resolved_height(),
                                   gen_params.video_frames,
                                   cli_params.verbose)) {
+            release_all_resources();
             return 1;
         }
     }
 
     if (!gen_params.pm_id_images_dir.empty()) {
-        gen_params.pm_id_images.clear();
         if (!load_images_from_dir(gen_params.pm_id_images_dir,
-                                  gen_params.pm_id_images,
+                                  pmid_images,
                                   0,
                                   0,
                                   0,
                                   cli_params.verbose)) {
+            release_all_resources();
             return 1;
         }
     }
@@ -739,71 +631,119 @@ int main(int argc, const char* argv[]) {
         vae_decode_only = false;
     }
 
-    if (gen_params.hires_enabled &&
-        (gen_params.resolved_hires_upscaler == SD_HIRES_UPSCALER_MODEL ||
-         gen_params.resolved_hires_upscaler == SD_HIRES_UPSCALER_LANCZOS ||
-         gen_params.resolved_hires_upscaler == SD_HIRES_UPSCALER_NEAREST)) {
-        vae_decode_only = false;
-    }
-
     sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(vae_decode_only, true, cli_params.taesd_preview);
 
-    SDImageVec results;
+    sd_image_t* results = nullptr;
-    int num_results             = 0;
+    int num_results     = 0;
-    sd_audio_t* generated_audio = nullptr;
 
     if (cli_params.mode == UPSCALE) {
         num_results = 1;
-        results.push_back(gen_params.init_image.release());
+        results     = (sd_image_t*)calloc(num_results, sizeof(sd_image_t));
+        if (results == nullptr) {
+            LOG_INFO("failed to allocate results array");
+            release_all_resources();
+            return 1;
+        }
+
+        results[0]      = init_image;
+        init_image.data = nullptr;
     } else {
-        SDCtxPtr sd_ctx(new_sd_ctx(&sd_ctx_params));
+        sd_ctx_t* sd_ctx = new_sd_ctx(&sd_ctx_params);
 
         if (sd_ctx == nullptr) {
             LOG_INFO("new_sd_ctx_t failed");
+            release_all_resources();
             return 1;
         }
 
         if (gen_params.sample_params.sample_method == SAMPLE_METHOD_COUNT) {
-            gen_params.sample_params.sample_method = sd_get_default_sample_method(sd_ctx.get());
+            gen_params.sample_params.sample_method = sd_get_default_sample_method(sd_ctx);
         }
 
         if (gen_params.high_noise_sample_params.sample_method == SAMPLE_METHOD_COUNT) {
-            gen_params.high_noise_sample_params.sample_method = sd_get_default_sample_method(sd_ctx.get());
+            gen_params.high_noise_sample_params.sample_method = sd_get_default_sample_method(sd_ctx);
         }
 
         if (gen_params.sample_params.scheduler == SCHEDULER_COUNT) {
-            gen_params.sample_params.scheduler = sd_get_default_scheduler(sd_ctx.get(), gen_params.sample_params.sample_method);
+            gen_params.sample_params.scheduler = sd_get_default_scheduler(sd_ctx, gen_params.sample_params.sample_method);
         }
 
         if (cli_params.mode == IMG_GEN) {
-            sd_img_gen_params_t img_gen_params = gen_params.to_sd_img_gen_params_t();
+            sd_img_gen_params_t img_gen_params = {
+                gen_params.lora_vec.data(),
+                static_cast<uint32_t>(gen_params.lora_vec.size()),
+                gen_params.prompt.c_str(),
+                gen_params.negative_prompt.c_str(),
+                gen_params.clip_skip,
+                init_image,
+                ref_images.data(),
+                (int)ref_images.size(),
+                gen_params.auto_resize_ref_image,
+                gen_params.increase_ref_index,
+                mask_image,
+                gen_params.get_resolved_width(),
+                gen_params.get_resolved_height(),
+                gen_params.sample_params,
+                gen_params.strength,
+                gen_params.seed,
+                gen_params.batch_count,
+                control_image,
+                gen_params.control_strength,
+                {
+                    pmid_images.data(),
+                    (int)pmid_images.size(),
+                    gen_params.pm_id_embed_path.c_str(),
+                    gen_params.pm_style_strength,
+                },  // pm_params
+                gen_params.vae_tiling_params,
+                gen_params.cache_params,
+            };
 
+            results     = generate_image(sd_ctx, &img_gen_params);
             num_results = gen_params.batch_count;
-            results.adopt(generate_image(sd_ctx.get(), &img_gen_params), num_results);
         } else if (cli_params.mode == VID_GEN) {
-            sd_vid_gen_params_t vid_gen_params = gen_params.to_sd_vid_gen_params_t();
+            sd_vid_gen_params_t vid_gen_params = {
-            sd_image_t* generated_video        = nullptr;
+                gen_params.lora_vec.data(),
-            if (!generate_video(sd_ctx.get(), &vid_gen_params, &generated_video, &num_results, &generated_audio)) {
+                static_cast<uint32_t>(gen_params.lora_vec.size()),
-                generated_video = nullptr;
+                gen_params.prompt.c_str(),
-            }
+                gen_params.negative_prompt.c_str(),
-            results.adopt(generated_video, num_results);
+                gen_params.clip_skip,
+                init_image,
+                end_image,
+                control_frames.data(),
+                (int)control_frames.size(),
+                gen_params.get_resolved_width(),
+                gen_params.get_resolved_height(),
+                gen_params.sample_params,
+                gen_params.high_noise_sample_params,
+                gen_params.moe_boundary,
+                gen_params.strength,
+                gen_params.seed,
+                gen_params.video_frames,
+                gen_params.vace_strength,
+                gen_params.vae_tiling_params,
+                gen_params.cache_params,
+            };
+
+            results = generate_video(sd_ctx, &vid_gen_params, &num_results);
         }
 
-        if (!results) {
+        if (results == nullptr) {
             LOG_ERROR("generate failed");
+            free_sd_ctx(sd_ctx);
             return 1;
         }
+
+        free_sd_ctx(sd_ctx);
     }
 
     int upscale_factor = 4;  // unused for RealESRGAN_x4plus_anime_6B.pth
     if (ctx_params.esrgan_path.size() > 0 && gen_params.upscale_repeats > 0) {
-        UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(ctx_params.esrgan_path.c_str(),
+        upscaler_ctx_t* upscaler_ctx = new_upscaler_ctx(ctx_params.esrgan_path.c_str(),
-                                                     ctx_params.offload_params_to_cpu,
+                                                        ctx_params.offload_params_to_cpu,
-                                                     ctx_params.diffusion_conv_direct,
+                                                        ctx_params.diffusion_conv_direct,
-                                                     ctx_params.n_threads,
+                                                        ctx_params.n_threads,
-                                                     gen_params.upscale_tile_size,
+                                                        gen_params.upscale_tile_size);
-                                                     ctx_params.backend.c_str(),
-                                                     ctx_params.params_backend.c_str()));
 
         if (upscaler_ctx == nullptr) {
             LOG_ERROR("new_upscaler_ctx failed");
@@ -812,27 +752,32 @@ int main(int argc, const char* argv[]) {
                 if (results[i].data == nullptr) {
                     continue;
                 }
-                SDImageOwner current_image(results[i]);
+                sd_image_t current_image = results[i];
-                results[i] = {0, 0, 0, nullptr};
                 for (int u = 0; u < gen_params.upscale_repeats; ++u) {
-                    SDImageOwner upscaled_image(upscale(upscaler_ctx.get(), current_image.get(), upscale_factor));
+                    sd_image_t upscaled_image = upscale(upscaler_ctx, current_image, upscale_factor);
-                    if (upscaled_image.get().data == nullptr) {
+                    if (upscaled_image.data == nullptr) {
                         LOG_ERROR("upscale failed");
                         break;
                     }
-                    current_image = std::move(upscaled_image);
+                    free(current_image.data);
+                    current_image = upscaled_image;
                 }
-                results[i] = current_image.release();  // Set the final upscaled image as the result
+                results[i] = current_image;  // Set the final upscaled image as the result
             }
         }
     }
 
-    if (!save_results(cli_params, ctx_params, gen_params, results.data(), num_results, generated_audio)) {
+    if (!save_results(cli_params, ctx_params, gen_params, results, num_results)) {
-        free_sd_audio(generated_audio);
         return 1;
     }
 
-    free_sd_audio(generated_audio);
+    for (int i = 0; i < num_results; i++) {
+        free(results[i].data);
+        results[i].data = nullptr;
+    }
+    free(results);
+
+    release_all_resources();
 
     return 0;
 }

examples/common/common.h

@@ -1,304 +0,0 @@
-#ifndef __EXAMPLES_COMMON_COMMON_H__
-#define __EXAMPLES_COMMON_COMMON_H__
-
-#include <cmath>
-#include <cstdint>
-#include <functional>
-#include <map>
-#include <string>
-#include <vector>
-
-#include "log.h"
-#include "resource_owners.hpp"
-#include "stable-diffusion.h"
-
-#define SAFE_STR(s) ((s) ? (s) : "")
-#define BOOL_STR(b) ((b) ? "true" : "false")
-
-extern const char* const modes_str[];
-#define SD_ALL_MODES_STR "img_gen, vid_gen, convert, upscale, metadata"
-
-enum SDMode {
-    IMG_GEN,
-    VID_GEN,
-    CONVERT,
-    UPSCALE,
-    METADATA,
-    MODE_COUNT
-};
-
-struct StringOption {
-    std::string short_name;
-    std::string long_name;
-    std::string desc;
-    std::string* target;
-};
-
-struct IntOption {
-    std::string short_name;
-    std::string long_name;
-    std::string desc;
-    int* target;
-};
-
-struct FloatOption {
-    std::string short_name;
-    std::string long_name;
-    std::string desc;
-    float* target;
-};
-
-struct BoolOption {
-    std::string short_name;
-    std::string long_name;
-    std::string desc;
-    bool keep_true;
-    bool* target;
-};
-
-struct ManualFunction {
-    std::function<int(int, const char**, int, bool&)> _func;
-
-    ManualFunction() = default;
-
-    ManualFunction(std::function<int(int argc, const char** argv, int index, bool& valid)> func)
-        : _func(std::move(func)) {
-    }
-
-    template <typename F>
-    ManualFunction(F func)
-        : _func(make_function(func)) {
-    }
-
-    int operator()(int argc, const char** argv, int index, bool& valid) const {
-        return _func(argc, argv, index, valid);
-    }
-
-private:
-    template <typename F>
-    static std::function<int(int, const char**, int, bool&)> make_function(F func) {
-        if constexpr (std::is_invocable_v<F, int, const char**, int, bool&>) {
-            return func;
-        } else {
-            return [func](int argc, const char** argv, int index, bool&) {
-                return func(argc, argv, index);
-            };
-        }
-    }
-};
-
-struct ManualOption {
-    std::string short_name;
-    std::string long_name;
-    std::string desc;
-    ManualFunction cb;
-};
-
-struct ArgOptions {
-    std::vector<StringOption> string_options;
-    std::vector<IntOption> int_options;
-    std::vector<FloatOption> float_options;
-    std::vector<BoolOption> bool_options;
-    std::vector<ManualOption> manual_options;
-
-    static std::string wrap_text(const std::string& text, size_t width, size_t indent);
-    void print() const;
-};
-
-bool parse_options(int argc, const char** argv, const std::vector<ArgOptions>& options_list);
-bool decode_base64_image(const std::string& encoded_input,
-                         int target_channels,
-                         int expected_width,
-                         int expected_height,
-                         SDImageOwner& out_image);
-
-struct SDContextParams {
-    int n_threads = -1;
-    std::string model_path;
-    std::string clip_l_path;
-    std::string clip_g_path;
-    std::string clip_vision_path;
-    std::string t5xxl_path;
-    std::string llm_path;
-    std::string llm_vision_path;
-    std::string diffusion_model_path;
-    std::string high_noise_diffusion_model_path;
-    std::string uncond_diffusion_model_path;
-    std::string embeddings_connectors_path;
-    std::string vae_path;
-    std::string vae_format = "auto";
-    std::string audio_vae_path;
-    std::string taesd_path;
-    std::string esrgan_path;
-    std::string control_net_path;
-    std::string embedding_dir;
-    std::string photo_maker_path;
-    sd_type_t wtype = SD_TYPE_COUNT;
-    std::string tensor_type_rules;
-    std::string lora_model_dir = ".";
-    std::string hires_upscalers_dir;
-
-    std::map<std::string, std::string> embedding_map;
-    std::vector<sd_embedding_t> embedding_vec;
-
-    rng_type_t rng_type         = CUDA_RNG;
-    rng_type_t sampler_rng_type = RNG_TYPE_COUNT;
-    bool offload_params_to_cpu  = false;
-    float max_vram              = 0.f;
-    bool stream_layers          = false;
-    std::string backend;
-    std::string params_backend;
-    bool enable_mmap           = false;
-    bool control_net_cpu       = false;
-    bool clip_on_cpu           = false;
-    bool vae_on_cpu            = false;
-    bool flash_attn            = false;
-    bool diffusion_flash_attn  = false;
-    bool diffusion_conv_direct = false;
-    bool vae_conv_direct       = false;
-
-    bool circular   = false;
-    bool circular_x = false;
-    bool circular_y = false;
-
-    bool chroma_use_dit_mask = true;
-    bool chroma_use_t5_mask  = false;
-    int chroma_t5_mask_pad   = 1;
-
-    bool qwen_image_zero_cond_t = false;
-
-    prediction_t prediction           = PREDICTION_COUNT;
-    lora_apply_mode_t lora_apply_mode = LORA_APPLY_AUTO;
-
-    bool force_sdxl_vae_conv_scale = false;
-
-    float flow_shift = INFINITY;
-    ArgOptions get_options();
-    void build_embedding_map();
-    bool resolve(SDMode mode);
-    bool validate(SDMode mode);
-    bool resolve_and_validate(SDMode mode);
-    std::string to_string() const;
-    sd_ctx_params_t to_sd_ctx_params_t(bool vae_decode_only, bool free_params_immediately, bool taesd_preview);
-};
-
-struct SDGenerationParams {
-    // User-facing input fields.
-    std::string prompt;
-    std::string negative_prompt;
-    int clip_skip              = -1;  // <= 0 represents unspecified
-    int width                  = -1;
-    int height                 = -1;
-    int batch_count            = 1;
-    int64_t seed               = 42;
-    float strength             = 0.75f;
-    float control_strength     = 0.9f;
-    bool auto_resize_ref_image = true;
-    bool increase_ref_index    = false;
-    bool embed_image_metadata  = true;
-
-    std::string init_image_path;
-    std::string end_image_path;
-    std::string mask_image_path;
-    std::string control_image_path;
-    std::vector<std::string> ref_image_paths;
-    std::string control_video_path;
-
-    sd_sample_params_t sample_params;
-    sd_sample_params_t high_noise_sample_params;
-    std::string extra_sample_args;
-    std::string high_noise_extra_sample_args;
-    std::vector<int> skip_layers            = {7, 8, 9};
-    std::vector<int> high_noise_skip_layers = {7, 8, 9};
-
-    std::vector<float> custom_sigmas;
-
-    std::string cache_mode;
-    std::string cache_option;
-    std::string scm_mask;
-    bool scm_policy_dynamic = true;
-    sd_cache_params_t cache_params{};
-
-    float moe_boundary                   = 0.875f;
-    int video_frames                     = 1;
-    int fps                              = 16;
-    float vace_strength                  = 1.f;
-    sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
-    std::string extra_tiling_args;
-
-    std::string pm_id_images_dir;
-    std::string pm_id_embed_path;
-    float pm_style_strength = 20.f;
-
-    int upscale_repeats   = 1;
-    int upscale_tile_size = 128;
-
-    bool hires_enabled         = false;
-    std::string hires_upscaler = "Latent";
-    std::string hires_upscaler_model_path;
-    float hires_scale              = 2.f;
-    int hires_width                = 0;
-    int hires_height               = 0;
-    int hires_steps                = 0;
-    float hires_denoising_strength = 0.7f;
-    int hires_upscale_tile_size    = 128;
-    std::vector<float> hires_custom_sigmas;
-
-    std::map<std::string, float> lora_map;
-    std::map<std::string, float> high_noise_lora_map;
-
-    // Derived and normalized fields.
-    std::string prompt_with_lora;  // for metadata record only
-    std::vector<sd_lora_t> lora_vec;
-    sd_hires_upscaler_t resolved_hires_upscaler;
-
-    // Owned execution payload.
-    SDImageOwner init_image;
-    SDImageOwner end_image;
-    std::vector<SDImageOwner> ref_images;
-    SDImageOwner mask_image;
-    SDImageOwner control_image;
-    std::vector<SDImageOwner> pm_id_images;
-    std::vector<SDImageOwner> control_frames;
-
-    // Backing storage for sd_img_gen_params_t view fields.
-    std::vector<sd_image_t> ref_image_views;
-    std::vector<sd_image_t> pm_id_image_views;
-    std::vector<sd_image_t> control_frame_views;
-
-    SDGenerationParams();
-    SDGenerationParams(const SDGenerationParams& other)                = default;
-    SDGenerationParams& operator=(const SDGenerationParams& other)     = default;
-    SDGenerationParams(SDGenerationParams&& other) noexcept            = default;
-    SDGenerationParams& operator=(SDGenerationParams&& other) noexcept = default;
-    ArgOptions get_options();
-    bool from_json_str(const std::string& json_str,
-                       const std::function<std::string(const std::string&)>& lora_path_resolver = {});
-    bool initialize_cache_params();
-    void extract_and_remove_lora(const std::string& lora_model_dir);
-    bool width_and_height_are_set() const;
-    void set_width_and_height_if_unset(int w, int h);
-    int get_resolved_width() const;
-    int get_resolved_height() const;
-    bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false);
-    bool validate(SDMode mode);
-    bool resolve_and_validate(SDMode mode,
-                              const std::string& lora_model_dir,
-                              const std::string& hires_upscalers_dir,
-                              bool strict = false);
-    sd_img_gen_params_t to_sd_img_gen_params_t();
-    sd_vid_gen_params_t to_sd_vid_gen_params_t();
-    std::string to_string() const;
-};
-
-std::string version_string();
-std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
-                                            const SDGenerationParams& gen_params,
-                                            int64_t seed,
-                                            SDMode mode = IMG_GEN);
-std::string get_image_params(const SDContextParams& ctx_params,
-                             const SDGenerationParams& gen_params,
-                             int64_t seed,
-                             SDMode mode = IMG_GEN);
-
-#endif  // __EXAMPLES_COMMON_COMMON_H__

examples/common/log.cpp

@@ -1,115 +0,0 @@
-#include "log.h"
-
-#include <vector>
-
-bool log_verbose = false;
-bool log_color   = false;
-
-std::string sd_basename(const std::string& path) {
-    size_t pos = path.find_last_of('/');
-    if (pos != std::string::npos) {
-        return path.substr(pos + 1);
-    }
-    pos = path.find_last_of('\\');
-    if (pos != std::string::npos) {
-        return path.substr(pos + 1);
-    }
-    return path;
-}
-
-void print_utf8(FILE* stream, const char* utf8) {
-    if (!utf8) {
-        return;
-    }
-
-#ifdef _WIN32
-    HANDLE h = (stream == stderr)
-                   ? GetStdHandle(STD_ERROR_HANDLE)
-                   : GetStdHandle(STD_OUTPUT_HANDLE);
-
-    DWORD mode;
-    BOOL is_console = GetConsoleMode(h, &mode);
-
-    if (is_console) {
-        int wlen = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0);
-        if (wlen <= 0) {
-            return;
-        }
-
-        std::vector<wchar_t> wbuf(static_cast<size_t>(wlen));
-
-        MultiByteToWideChar(CP_UTF8, 0, utf8, -1, wbuf.data(), wlen);
-
-        DWORD written;
-        WriteConsoleW(h, wbuf.data(), wlen - 1, &written, NULL);
-    } else {
-        DWORD written;
-        WriteFile(h, utf8, (DWORD)strlen(utf8), &written, NULL);
-    }
-#else
-    fputs(utf8, stream);
-#endif
-}
-
-void log_print(enum sd_log_level_t level, const char* log, bool verbose, bool color) {
-    int tag_color;
-    const char* level_str;
-    FILE* out_stream = (level == SD_LOG_ERROR) ? stderr : stdout;
-
-    if (!log || (!verbose && level <= SD_LOG_DEBUG)) {
-        return;
-    }
-
-    switch (level) {
-        case SD_LOG_DEBUG:
-            tag_color = 37;
-            level_str = "DEBUG";
-            break;
-        case SD_LOG_INFO:
-            tag_color = 34;
-            level_str = "INFO";
-            break;
-        case SD_LOG_WARN:
-            tag_color = 35;
-            level_str = "WARN";
-            break;
-        case SD_LOG_ERROR:
-            tag_color = 31;
-            level_str = "ERROR";
-            break;
-        default:
-            tag_color = 33;
-            level_str = "?????";
-            break;
-    }
-
-    if (color) {
-        fprintf(out_stream, "\033[%d;1m[%-5s]\033[0m ", tag_color, level_str);
-    } else {
-        fprintf(out_stream, "[%-5s] ", level_str);
-    }
-    print_utf8(out_stream, log);
-    fflush(out_stream);
-}
-
-void example_log_printf(sd_log_level_t level, const char* file, int line, const char* format, ...) {
-    constexpr size_t LOG_BUFFER_SIZE = 4096;
-
-    va_list args;
-    va_start(args, format);
-
-    static char log_buffer[LOG_BUFFER_SIZE + 1];
-    int written = snprintf(log_buffer, LOG_BUFFER_SIZE, "%s:%-4d - ", sd_basename(file).c_str(), line);
-
-    if (written >= 0 && written < static_cast<int>(LOG_BUFFER_SIZE)) {
-        vsnprintf(log_buffer + written, LOG_BUFFER_SIZE - written, format, args);
-    }
-    size_t len = strlen(log_buffer);
-    if (len == 0 || log_buffer[len - 1] != '\n') {
-        strncat(log_buffer, "\n", LOG_BUFFER_SIZE - len);
-    }
-
-    log_print(level, log_buffer, log_verbose, log_color);
-
-    va_end(args);
-}

examples/common/log.h

@@ -1,32 +0,0 @@
-#ifndef __EXAMPLE_LOG_H__
-#define __EXAMPLE_LOG_H__
-
-#include <cstdarg>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <string>
-
-#if defined(_WIN32)
-#ifndef NOMINMAX
-#define NOMINMAX
-#endif
-#include <windows.h>
-#endif  // _WIN32
-
-#include "stable-diffusion.h"
-
-extern bool log_verbose;
-extern bool log_color;
-
-std::string sd_basename(const std::string& path);
-void print_utf8(FILE* stream, const char* utf8);
-void log_print(sd_log_level_t level, const char* log, bool verbose, bool color);
-void example_log_printf(sd_log_level_t level, const char* file, int line, const char* format, ...);
-
-#define LOG_DEBUG(format, ...) example_log_printf(SD_LOG_DEBUG, __FILE__, __LINE__, format, ##__VA_ARGS__)
-#define LOG_INFO(format, ...) example_log_printf(SD_LOG_INFO, __FILE__, __LINE__, format, ##__VA_ARGS__)
-#define LOG_WARN(format, ...) example_log_printf(SD_LOG_WARN, __FILE__, __LINE__, format, ##__VA_ARGS__)
-#define LOG_ERROR(format, ...) example_log_printf(SD_LOG_ERROR, __FILE__, __LINE__, format, ##__VA_ARGS__)
-
-#endif  // __EXAMPLE_LOG_H__

examples/common/media_io.h

@@ -1,113 +0,0 @@
-#ifndef __MEDIA_IO_H__
-#define __MEDIA_IO_H__
-
-#include <cstdint>
-#include <string>
-#include <vector>
-
-#include "stable-diffusion.h"
-
-enum class EncodedImageFormat {
-    JPEG,
-    PNG,
-    WEBP,
-    UNKNOWN,
-};
-
-EncodedImageFormat encoded_image_format_from_path(const std::string& path);
-
-std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
-                                            const uint8_t* image,
-                                            int width,
-                                            int height,
-                                            int channels,
-                                            const std::string& parameters = "",
-                                            int quality                   = 90);
-
-bool write_image_to_file(const std::string& path,
-                         const uint8_t* image,
-                         int width,
-                         int height,
-                         int channels,
-                         const std::string& parameters = "",
-                         int quality                   = 90);
-
-uint8_t* load_image_from_file(const char* image_path,
-                              int& width,
-                              int& height,
-                              int expected_width   = 0,
-                              int expected_height  = 0,
-                              int expected_channel = 3);
-
-bool load_sd_image_from_file(sd_image_t* image,
-                             const char* image_path,
-                             int expected_width   = 0,
-                             int expected_height  = 0,
-                             int expected_channel = 3);
-
-uint8_t* load_image_from_memory(const char* image_bytes,
-                                int len,
-                                int& width,
-                                int& height,
-                                int expected_width   = 0,
-                                int expected_height  = 0,
-                                int expected_channel = 3);
-
-int create_mjpg_avi_from_sd_images(const char* filename,
-                                   sd_image_t* images,
-                                   int num_images,
-                                   int fps,
-                                   int quality             = 90,
-                                   const sd_audio_t* audio = nullptr);
-std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images,
-                                                              int num_images,
-                                                              int fps,
-                                                              int quality             = 90,
-                                                              const sd_audio_t* audio = nullptr);
-
-#ifdef SD_USE_WEBP
-int create_animated_webp_from_sd_images(const char* filename,
-                                        sd_image_t* images,
-                                        int num_images,
-                                        int fps,
-                                        int quality = 90);
-std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(sd_image_t* images,
-                                                                   int num_images,
-                                                                   int fps,
-                                                                   int quality = 90);
-#endif
-
-#ifdef SD_USE_WEBM
-int create_webm_from_sd_images(const char* filename,
-                               sd_image_t* images,
-                               int num_images,
-                               int fps,
-                               int quality             = 90,
-                               const sd_audio_t* audio = nullptr);
-std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images,
-                                                          int num_images,
-                                                          int fps,
-                                                          int quality             = 90,
-                                                          const sd_audio_t* audio = nullptr);
-#endif
-
-int create_video_from_sd_images(const char* filename,
-                                sd_image_t* images,
-                                int num_images,
-                                int fps,
-                                int quality             = 90,
-                                const sd_audio_t* audio = nullptr);
-std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& output_format,
-                                                           sd_image_t* images,
-                                                           int num_images,
-                                                           int fps,
-                                                           int quality             = 90,
-                                                           const sd_audio_t* audio = nullptr);
-
-bool write_wav_to_file(const std::string& path,
-                       const float* interleaved_samples,
-                       uint64_t sample_count,
-                       uint32_t channels,
-                       uint32_t sample_rate);
-
-#endif  // __MEDIA_IO_H__

examples/common/resource_owners.hpp

@@ -1,236 +0,0 @@
-#ifndef __EXAMPLE_RESOURCE_OWNERS_H__
-#define __EXAMPLE_RESOURCE_OWNERS_H__
-
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <memory>
-#include <utility>
-#include <vector>
-
-#include "stable-diffusion.h"
-
-struct FreeDeleter {
-    void operator()(void* ptr) const {
-        free(ptr);
-    }
-};
-
-struct FileCloser {
-    void operator()(FILE* file) const {
-        if (file != nullptr) {
-            fclose(file);
-        }
-    }
-};
-
-struct SDCtxDeleter {
-    void operator()(sd_ctx_t* ctx) const {
-        if (ctx != nullptr) {
-            free_sd_ctx(ctx);
-        }
-    }
-};
-
-struct UpscalerCtxDeleter {
-    void operator()(upscaler_ctx_t* ctx) const {
-        if (ctx != nullptr) {
-            free_upscaler_ctx(ctx);
-        }
-    }
-};
-
-template <typename T>
-using FreeUniquePtr = std::unique_ptr<T, FreeDeleter>;
-
-using FilePtr        = std::unique_ptr<FILE, FileCloser>;
-using SDCtxPtr       = std::unique_ptr<sd_ctx_t, SDCtxDeleter>;
-using UpscalerCtxPtr = std::unique_ptr<upscaler_ctx_t, UpscalerCtxDeleter>;
-
-class SDImageOwner {
-private:
-    static sd_image_t copy_image(const sd_image_t& image) {
-        if (image.data == nullptr) {
-            return {image.width, image.height, image.channel, nullptr};
-        }
-
-        const size_t byte_count = static_cast<size_t>(image.width) * image.height * image.channel;
-        uint8_t* raw_copy       = static_cast<uint8_t*>(malloc(byte_count));
-        if (raw_copy == nullptr) {
-            return {0, 0, 0, nullptr};
-        }
-
-        std::memcpy(raw_copy, image.data, byte_count);
-        return {image.width, image.height, image.channel, raw_copy};
-    }
-
-    sd_image_t image_ = {0, 0, 0, nullptr};
-
-public:
-    SDImageOwner() = default;
-    explicit SDImageOwner(sd_image_t image)
-        : image_(image) {
-    }
-
-    SDImageOwner(const SDImageOwner& other)
-        : image_(copy_image(other.image_)) {
-    }
-
-    SDImageOwner& operator=(const SDImageOwner& other) {
-        if (this != &other) {
-            reset(copy_image(other.image_));
-        }
-        return *this;
-    }
-
-    SDImageOwner(SDImageOwner&& other) noexcept
-        : image_(other.release()) {
-    }
-
-    SDImageOwner& operator=(SDImageOwner&& other) noexcept {
-        if (this != &other) {
-            reset();
-            image_ = other.release();
-        }
-        return *this;
-    }
-
-    ~SDImageOwner() {
-        reset();
-    }
-
-    sd_image_t* put() {
-        if (image_.data != nullptr) {
-            free(image_.data);
-            image_.data = nullptr;
-        }
-        image_.width   = 0;
-        image_.height  = 0;
-        image_.channel = 0;
-        return &image_;
-    }
-
-    sd_image_t& get() {
-        return image_;
-    }
-
-    const sd_image_t& get() const {
-        return image_;
-    }
-
-    sd_image_t release() {
-        sd_image_t image = image_;
-        image_           = {0, 0, 0, nullptr};
-        return image;
-    }
-
-    void reset(sd_image_t image = {0, 0, 0, nullptr}) {
-        if (image_.data != nullptr) {
-            free(image_.data);
-        }
-        image_ = image;
-    }
-};
-
-class SDImageVec {
-private:
-    std::vector<sd_image_t> images_;
-
-public:
-    SDImageVec() = default;
-
-    SDImageVec(const SDImageVec&)            = delete;
-    SDImageVec& operator=(const SDImageVec&) = delete;
-
-    SDImageVec(SDImageVec&& other) noexcept
-        : images_(std::move(other.images_)) {
-    }
-
-    SDImageVec& operator=(SDImageVec&& other) noexcept {
-        if (this != &other) {
-            clear();
-            images_ = std::move(other.images_);
-        }
-        return *this;
-    }
-
-    ~SDImageVec() {
-        clear();
-    }
-
-    void push_back(sd_image_t image) {
-        images_.push_back(image);
-    }
-
-    void push_back(SDImageOwner&& image) {
-        images_.push_back(image.release());
-    }
-
-    void reserve(size_t count) {
-        images_.reserve(count);
-    }
-
-    void adopt(sd_image_t* images, int count) {
-        clear();
-        if (images == nullptr || count <= 0) {
-            free(images);
-            return;
-        }
-
-        images_.reserve(static_cast<size_t>(count));
-        for (int i = 0; i < count; ++i) {
-            images_.push_back(images[i]);
-        }
-        free(images);
-    }
-
-    size_t size() const {
-        return images_.size();
-    }
-
-    bool empty() const {
-        return images_.empty();
-    }
-
-    int count() const {
-        return static_cast<int>(images_.size());
-    }
-
-    explicit operator bool() const {
-        return !images_.empty();
-    }
-
-    sd_image_t* data() {
-        return images_.data();
-    }
-
-    const sd_image_t* data() const {
-        return images_.data();
-    }
-
-    sd_image_t& operator[](size_t index) {
-        return images_[index];
-    }
-
-    const sd_image_t& operator[](size_t index) const {
-        return images_[index];
-    }
-
-    std::vector<sd_image_t>& raw() {
-        return images_;
-    }
-
-    const std::vector<sd_image_t>& raw() const {
-        return images_;
-    }
-
-    void clear() {
-        for (sd_image_t& image : images_) {
-            free(image.data);
-            image.data = nullptr;
-        }
-        images_.clear();
-    }
-};
-
-#endif  // __EXAMPLE_RESOURCE_OWNERS_H__

examples/server/CMakeLists.txt

@@ -50,33 +50,13 @@ if(SD_SERVER_BUILD_FRONTEND AND EXISTS "${FRONTEND_DIR}")
 
         set_source_files_properties("${GENERATED_HTML_HEADER}" PROPERTIES GENERATED TRUE)
     else()
-        if(EXISTS "${GENERATED_HTML_HEADER}")
+        message(WARNING "pnpm not found, frontend build disabled")
-            message(STATUS "pnpm not found; using pre-built frontend header detected at ${GENERATED_HTML_HEADER}")
-            set(HAVE_FRONTEND_BUILD ON)
-            add_custom_target(${TARGET}_frontend)
-        else()
-            message(WARNING "pnpm not found; frontend build disabled.")
-        endif()
     endif()
 else()
     message(STATUS "Frontend disabled or directory not found: ${FRONTEND_DIR}")
 endif()
 
-add_executable(${TARGET}
+add_executable(${TARGET} main.cpp)
-    ../common/common.cpp
-    ../common/log.cpp
-    ../common/media_io.cpp
-    main.cpp
-    runtime.cpp
-    async_jobs.cpp
-    routes_index.cpp
-    routes_openai.cpp
-    routes_sdapi.cpp
-    routes_sdcpp.cpp
-)
-if(APPLE)
-    sd_set_macos_rpaths(${TARGET})
-endif()
 
 if(HAVE_FRONTEND_BUILD)
     add_dependencies(${TARGET} ${TARGET}_frontend)
@@ -90,14 +70,6 @@ endif()
 
 install(TARGETS ${TARGET} RUNTIME)
 target_link_libraries(${TARGET} PRIVATE stable-diffusion ${CMAKE_THREAD_LIBS_INIT})
-if(SD_WEBP)
-    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBP)
-    target_link_libraries(${TARGET} PRIVATE webp libwebpmux)
-endif()
-if(SD_WEBM)
-    target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBM)
-    target_link_libraries(${TARGET} PRIVATE webm)
-endif()
 
 # due to httplib; it contains a pragma for MSVC, but other things need explicit flags
 if(WIN32 AND NOT MSVC)

examples/server/README.md

@@ -1,33 +1,3 @@
-# Example
-
-The following example starts `sd-server` with a standalone diffusion model, VAE, and LLM text encoder:
-
-```
-.\bin\Release\sd-server.exe --diffusion-model  ..\models\diffusion_models\z_image_turbo_bf16.safetensors --vae ..\models\vae\ae.sft  --llm ..\models\text_encoders\qwen_3_4b.safetensors --diffusion-fa --offload-to-cpu -v --cfg-scale 1.0
-```
-
-What this example does:
-
-* `--diffusion-model` selects the standalone diffusion model
-* `--vae` selects the VAE decoder
-* `--llm` selects the text encoder / language model used by this pipeline
-* `--diffusion-fa` enables flash attention in the diffusion model
-* `--offload-to-cpu` reduces VRAM pressure by keeping weights in RAM when possible
-* `-v` enables verbose logging
-* `--cfg-scale 1.0` sets the default CFG scale for generation
-
-After the server starts successfully:
-
-* the web UI is available at `http://127.0.0.1:1234/`
-* the native async API is available under `/sdcpp/v1/...`
-* the compatibility APIs are available under `/v1/...` and `/sdapi/v1/...`
-
-If you want to use a different host or port, pass:
-
-```bash
---listen-ip <ip> --listen-port <port>
-```
-
 # Frontend
 
 ## Build with Frontend
@@ -38,7 +8,7 @@ The server can optionally build the web frontend and embed it into the binary as
 
 Install the following tools:
 
-* **Node.js** ≥ 20
+* **Node.js** ≥ 22.18
   https://nodejs.org/
 
 * **pnpm** ≥ 10
@@ -84,7 +54,7 @@ and embed the generated frontend into the server binary.
 
 ## Frontend Repository
 
-The web frontend is maintained in a **separate repository**, https://github.com/leejet/sdcpp-webui.
+The web frontend is maintained in a **separate repository**, https://github.com/leejet/stable-ui.
 
 If you want to modify the UI or frontend logic, please submit pull requests to the **frontend repository**.
 
@@ -136,34 +106,28 @@ Context Options:
   --clip_g <string>                        path to the clip-g text encoder
   --clip_vision <string>                   path to the clip-vision encoder
   --t5xxl <string>                         path to the t5xxl text encoder
-  --llm <string>                           path to the llm text encoder. For example: (qwenvl2.5 for qwen-image,
+  --llm <string>                           path to the llm text encoder. For example: (qwenvl2.5 for qwen-image, mistral-small3.2 for flux2, ...)
-                                           mistral-small3.2 for flux2, ...)
   --llm_vision <string>                    path to the llm vit
   --qwen2vl <string>                       alias of --llm. Deprecated.
   --qwen2vl_vision <string>                alias of --llm_vision. Deprecated.
   --diffusion-model <string>               path to the standalone diffusion model
   --high-noise-diffusion-model <string>    path to the standalone high noise diffusion model
-  --uncond-diffusion-model <string>        path to the standalone unconditional diffusion model, currently used by
-                                           Ideogram4 CFG
   --vae <string>                           path to standalone vae model
   --taesd <string>                         path to taesd. Using Tiny AutoEncoder for fast decoding (low quality)
   --tae <string>                           alias of --taesd
   --control-net <string>                   path to control net model
   --embd-dir <string>                      embeddings directory
   --lora-model-dir <string>                lora model directory
-  --hires-upscalers-dir <string>           highres fix upscaler model directory
   --tensor-type-rules <string>             weight type per tensor pattern (example: "^vae\.=f16,model\.=q8_0")
   --photo-maker <string>                   path to PHOTOMAKER model
   --upscale-model <string>                 path to esrgan model.
-  -t, --threads <int>                      number of threads to use during computation (default: -1). If threads <= 0,
+  -t, --threads <int>                      number of threads to use during computation (default: -1). If threads <= 0, then threads will be set to the number of
-                                           then threads will be set to the number of CPU physical cores
+                                           CPU physical cores
   --chroma-t5-mask-pad <int>               t5 mask pad size of chroma
-  --max-vram <float>                       maximum VRAM budget in GiB for graph-cut segmented execution. 0 disables
+  --vae-tile-overlap <float>               tile overlap for vae tiling, in fraction of tile size (default: 0.5)
-                                           graph splitting; a negative value auto-detects free VRAM, sparing the
+  --vae-tiling                             process vae in tiles to reduce memory usage
-                                           specified value (e.g. -0.5 will keep at least 0.5 GiB free)
   --force-sdxl-vae-conv-scale              force use of conv scale on sdxl vae
-  --offload-to-cpu                         place the weights in RAM to save VRAM, and automatically load them into VRAM
+  --offload-to-cpu                         place the weights in RAM to save VRAM, and automatically load them into VRAM when needed
-                                           when needed
   --mmap                                   whether to memory-map model
   --control-net-cpu                        keep controlnet in cpu (for low vram)
   --clip-on-cpu                            keep clip in cpu (for low vram)
@@ -178,19 +142,20 @@ Context Options:
   --chroma-disable-dit-mask                disable dit mask for chroma
   --qwen-image-zero-cond-t                 enable zero_cond_t for qwen image
   --chroma-enable-t5-mask                  enable t5 mask for chroma
-  --type                                   weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K,
+  --type                                   weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K, q4_K). If not specified, the default is the
-                                           q4_K). If not specified, the default is the type of the weight file
+                                           type of the weight file
   --rng                                    RNG, one of [std_default, cuda, cpu], default: cuda(sd-webui), cpu(comfyui)
   --sampler-rng                            sampler RNG, one of [std_default, cuda, cpu]. If not specified, use --rng
-  --prediction                             prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow,
+  --prediction                             prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow, flux2_flow]
-                                           flux2_flow]
+  --lora-apply-mode                        the way to apply LoRA, one of [auto, immediately, at_runtime], default is auto. In auto mode, if the model weights
-  --lora-apply-mode                        the way to apply LoRA, one of [auto, immediately, at_runtime], default is
+                                           contain any quantized parameters, the at_runtime mode will be used; otherwise,
-                                           auto. In auto mode, if the model weights contain any quantized parameters,
+                                           immediately will be used.The immediately mode may have precision and
-                                           the at_runtime mode will be used; otherwise, immediately will be used.The
+                                           compatibility issues with quantized parameters, but it usually offers faster inference
-                                           immediately mode may have precision and compatibility issues with quantized
+                                           speed and, in some cases, lower memory usage. The at_runtime mode, on the
-                                           parameters, but it usually offers faster inference speed and, in some cases,
+                                           other hand, is exactly the opposite.
-                                           lower memory usage. The at_runtime mode, on the other hand, is exactly the
+  --vae-tile-size                          tile size for vae tiling, format [X]x[Y] (default: 32x32)
-                                           opposite.
+  --vae-relative-tile-size                 relative tile size for vae tiling, format [X]x[Y], in fraction of image size if < 1, in number of tiles per dim if >=1
+                                           (overrides --vae-tile-size)
 
 Default Generation Options:
   -p, --prompt <string>                    the prompt to render
@@ -199,106 +164,65 @@ Default Generation Options:
   --end-img <string>                       path to the end image, required by flf2v
   --mask <string>                          path to the mask image
   --control-image <string>                 path to control image, control net
-  --control-video <string>                 path to control video frames, It must be a directory path. The video frames
+  --control-video <string>                 path to control video frames, It must be a directory path. The video frames inside should be stored as images in
-                                           inside should be stored as images in lexicographical (character) order. For
+                                           lexicographical (character) order. For example, if the control video path is
-                                           example, if the control video path is `frames`, the directory contain images
+                                           `frames`, the directory contain images such as 00.png, 01.png, ... etc.
-                                           such as 00.png, 01.png, ... etc.
   --pm-id-images-dir <string>              path to PHOTOMAKER input id images dir
   --pm-id-embed-path <string>              path to PHOTOMAKER v2 id embed
-  --hires-upscaler <string>                highres fix upscaler, Lanczos, Nearest, Latent, Latent (nearest), Latent
-                                           (nearest-exact), Latent (antialiased), Latent (bicubic), Latent (bicubic
-                                           antialiased), or a model name under --hires-upscalers-dir (default: Latent)
-  --extra-sample-args <string>             extra sampler/scheduler/guidance args, key=value list. APG supports apg_eta,
-                                           apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports
-                                           slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end;
-                                           ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma
-  --extra-tiling-args <string>             extra VAE tiling args, key=value list. LTX video VAE supports
-                                           temporal_tile_frames (default: 4), temporal_tile_overlap (default: 1)
   -H, --height <int>                       image height, in pixel space (default: 512)
   -W, --width <int>                        image width, in pixel space (default: 512)
   --steps <int>                            number of sample steps (default: 20)
   --high-noise-steps <int>                 (high noise) number of sample steps (default: -1 = auto)
-  --clip-skip <int>                        ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer
+  --clip-skip <int>                        ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1). <= 0 represents unspecified,
-                                           (default: -1). <= 0 represents unspecified, will be 1 for SD1.x, 2 for SD2.x
+                                           will be 1 for SD1.x, 2 for SD2.x
   -b, --batch-count <int>                  batch count
   --video-frames <int>                     video frames (default: 1)
   --fps <int>                              fps (default: 24)
-  --timestep-shift <int>                   shift timestep for NitroFusion models (default: 0). recommended N for
+  --timestep-shift <int>                   shift timestep for NitroFusion models (default: 0). recommended N for NitroSD-Realism around 250 and 500 for
-                                           NitroSD-Realism around 250 and 500 for NitroSD-Vibrant
+                                           NitroSD-Vibrant
   --upscale-repeats <int>                  Run the ESRGAN upscaler this many times (default: 1)
   --upscale-tile-size <int>                tile size for ESRGAN upscaling (default: 128)
-  --hires-width <int>                      highres fix target width, 0 to use --hires-scale (default: 0)
-  --hires-height <int>                     highres fix target height, 0 to use --hires-scale (default: 0)
-  --hires-steps <int>                      highres fix second pass sample steps, 0 to reuse --steps (default: 0)
-  --hires-upscale-tile-size <int>          highres fix upscaler tile size, reserved for model-backed upscalers (default:
-                                           128)
   --cfg-scale <float>                      unconditional guidance scale: (default: 7.0)
-  --img-cfg-scale <float>                  image guidance scale for inpaint or image edit models: (default: same as
+  --img-cfg-scale <float>                  image guidance scale for inpaint or instruct-pix2pix models: (default: same as --cfg-scale)
-                                           --cfg-scale)
   --guidance <float>                       distilled guidance scale for models with guidance input (default: 3.5)
-  --slg-scale <float>                      skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means
+  --slg-scale <float>                      skip layer guidance (SLG) scale, only for DiT models: (default: 0). 0 means disabled, a value of 2.5 is nice for sd3.5
-                                           disabled, a value of 2.5 is nice for sd3.5 medium
+                                           medium
   --skip-layer-start <float>               SLG enabling point (default: 0.01)
   --skip-layer-end <float>                 SLG disabling point (default: 0.2)
-  --eta <float>                            noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and
+  --eta <float>                            eta in DDIM, only for DDIM and TCD (default: 0)
-                                           res_2s; 1 for euler_a, er_sde and dpm++2s_a)
   --flow-shift <float>                     shift value for Flow models like SD3.x or WAN (default: auto)
   --high-noise-cfg-scale <float>           (high noise) unconditional guidance scale: (default: 7.0)
-  --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or image edit models (default:
+  --high-noise-img-cfg-scale <float>       (high noise) image guidance scale for inpaint or instruct-pix2pix models (default: same as --cfg-scale)
-                                           same as --cfg-scale)
+  --high-noise-guidance <float>            (high noise) distilled guidance scale for models with guidance input (default: 3.5)
-  --high-noise-guidance <float>            (high noise) distilled guidance scale for models with guidance input
+  --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default: 0)
-                                           (default: 3.5)
-  --high-noise-slg-scale <float>           (high noise) skip layer guidance (SLG) scale, only for DiT models: (default:
-                                           0)
   --high-noise-skip-layer-start <float>    (high noise) SLG enabling point (default: 0.01)
   --high-noise-skip-layer-end <float>      (high noise) SLG disabling point (default: 0.2)
-  --high-noise-eta <float>                 (high noise) noise multiplier (default: 0 for ddim_trailing, tcd,
+  --high-noise-eta <float>                 (high noise) eta in DDIM, only for DDIM and TCD (default: 0)
-                                           res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a)
   --strength <float>                       strength for noising/unnoising (default: 0.75)
   --pm-style-strength <float>
-  --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full
+  --control-strength <float>               strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image
-                                           destruction of information in init image
+  --moe-boundary <float>                   timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if `--high-noise-steps` is set to -1
-  --moe-boundary <float>                   timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if
-                                           `--high-noise-steps` is set to -1
   --vace-strength <float>                  wan vace strength
-  --vae-tile-overlap <float>               tile overlap for vae tiling, in fraction of tile size (default: 0.5)
+  --increase-ref-index                     automatically increase the indices of references images based on the order they are listed (starting with 1).
-  --hires-scale <float>                    highres fix scale when target size is not set (default: 2.0)
-  --hires-denoising-strength <float>       highres fix second pass denoising strength (default: 0.7)
-  --increase-ref-index                     automatically increase the indices of references images based on the order
-                                           they are listed (starting with 1).
   --disable-auto-resize-ref-image          disable auto resize of ref images
   --disable-image-metadata                 do not embed generation metadata on image files
-  --vae-tiling                             process vae in tiles to reduce memory usage
-  --temporal-tiling                        enable temporal tiling for LTX video VAE decode
-  --hires                                  enable highres fix
   -s, --seed                               RNG seed (default: 42, use random seed for < 0)
-  --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m,
+  --sampling-method                        sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing,
-                                           dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s,
+                                           tcd, res_multistep, res_2s] (default: euler for Flux/SD3/Wan, euler_a
-                                           er_sde, euler_cfg_pp, euler_a_cfg_pp] (default: euler for Flux/SD3/Wan, euler_a otherwise)
+                                           otherwise)
-  --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a,
+  --high-noise-sampling-method             (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm,
-                                           dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep,
+                                           ddim_trailing, tcd, res_multistep, res_2s] default: euler for Flux/SD3/Wan,
-                                           res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for Flux/SD3/Wan, euler_a otherwise
+                                           euler_a otherwise
-  --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits,
+  --scheduler                              denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, smoothstep, sgm_uniform, simple,
-                                           smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent, ltx2], default:
+                                           kl_optimal, lcm, bong_tangent], default: discrete
-                                           model-specific
+  --sigmas                                 custom sigma values for the sampler, comma-separated (e.g., "14.61,7.8,3.5,0.0").
-  --sigmas                                 custom sigma values for the sampler, comma-separated (e.g.,
-                                           "14.61,7.8,3.5,0.0").
-  --hires-sigmas                           custom sigma values for the highres fix second pass, comma-separated (e.g.,
-                                           "0.85,0.725,0.421875,0.0").
   --skip-layers                            layers to skip for SLG steps (default: [7,8,9])
   --high-noise-skip-layers                 (high noise) layers to skip for SLG steps (default: [7,8,9])
   -r, --ref-image                          reference image for Flux Kontext models (can be used multiple times)
-  --cache-mode                             caching method: 'easycache' (DiT), 'ucache' (UNET),
+  --cache-mode                             caching method: 'easycache' (DiT), 'ucache' (UNET), 'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT Chebyshev+Taylor forecasting)
-                                           'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT
-                                           Chebyshev+Taylor forecasting)
   --cache-option                           named cache params (key=value format, comma-separated). easycache/ucache:
-                                           threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit:
+                                           threshold=,start=,end=,decay=,relative=,reset=; dbcache/taylorseer/cache-dit: Fn=,Bn=,threshold=,warmup=. Examples:
-                                           Fn=,Bn=,threshold=,warmup=; spectrum: w=,m=,lam=,window=,flex=,warmup=,stop=.
+                                           "threshold=0.25" or "threshold=1.5,reset=0"
-                                           Examples: "threshold=0.25" or "threshold=1.5,reset=0"
+  --scm-mask                               SCM steps mask for cache-dit: comma-separated 0/1 (e.g., "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache
-  --scm-mask                               SCM steps mask for cache-dit: comma-separated 0/1 (e.g.,
-                                           "1,1,1,0,0,1,0,0,1,0") - 1=compute, 0=can cache
   --scm-policy                             SCM policy: 'dynamic' (default) or 'static'
-  --vae-tile-size                          tile size for vae tiling, format [X]x[Y] (default: 32x32)
-  --vae-relative-tile-size                 relative tile size for vae tiling, format [X]x[Y], in fraction of image size
-                                           if < 1, in number of tiles per dim if >=1 (overrides --vae-tile-size)
 ```

examples/server/async_jobs.cpp

@@ -1,356 +0,0 @@
-// Extracted from main.cpp during server refactor.
-
-#include "async_jobs.h"
-
-#include <iomanip>
-#include <sstream>
-
-#include "common/log.h"
-#include "common/media_io.h"
-#include "common/resource_owners.hpp"
-
-const char* async_job_kind_name(AsyncJobKind kind) {
-    switch (kind) {
-        case AsyncJobKind::ImgGen:
-            return "img_gen";
-        case AsyncJobKind::VidGen:
-            return "vid_gen";
-        default:
-            return "img_gen";
-    }
-}
-
-const char* async_job_status_name(AsyncJobStatus status) {
-    switch (status) {
-        case AsyncJobStatus::Queued:
-            return "queued";
-        case AsyncJobStatus::Generating:
-            return "generating";
-        case AsyncJobStatus::Completed:
-            return "completed";
-        case AsyncJobStatus::Failed:
-            return "failed";
-        case AsyncJobStatus::Cancelled:
-            return "cancelled";
-        default:
-            return "failed";
-    }
-}
-
-void purge_expired_jobs(AsyncJobManager& manager) {
-    const int64_t now = unix_timestamp_now();
-
-    for (auto it = manager.expired_jobs.begin(); it != manager.expired_jobs.end();) {
-        if (it->second <= now) {
-            it = manager.expired_jobs.erase(it);
-        } else {
-            ++it;
-        }
-    }
-
-    for (auto it = manager.jobs.begin(); it != manager.jobs.end();) {
-        const auto& job = it->second;
-        if (job->completed_at == 0) {
-            ++it;
-            continue;
-        }
-
-        int64_t ttl_seconds = job->status == AsyncJobStatus::Completed
-                                  ? manager.completed_ttl_seconds
-                                  : manager.failed_ttl_seconds;
-        if (now - job->completed_at >= ttl_seconds) {
-            manager.expired_jobs[job->id] = now + std::max<int64_t>(ttl_seconds, 60);
-            it                            = manager.jobs.erase(it);
-        } else {
-            ++it;
-        }
-    }
-}
-
-size_t count_pending_jobs(const AsyncJobManager& manager) {
-    size_t pending = 0;
-    for (const auto& entry : manager.jobs) {
-        if (entry.second->status == AsyncJobStatus::Queued ||
-            entry.second->status == AsyncJobStatus::Generating) {
-            ++pending;
-        }
-    }
-    return pending;
-}
-
-std::string make_async_job_id(AsyncJobManager& manager) {
-    std::ostringstream oss;
-    oss << "job_" << std::hex << unix_timestamp_now() << "_" << std::setw(8)
-        << std::setfill('0') << manager.next_id++;
-    return oss.str();
-}
-
-bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job) {
-    auto new_end = std::remove(manager.queue.begin(), manager.queue.end(), job.id);
-    if (new_end == manager.queue.end()) {
-        return false;
-    }
-
-    manager.queue.erase(new_end, manager.queue.end());
-    job.status       = AsyncJobStatus::Cancelled;
-    job.completed_at = unix_timestamp_now();
-    job.result_images_b64.clear();
-    job.result_media_b64.clear();
-    job.result_media_mime_type.clear();
-    job.result_frame_count = 0;
-    job.result_fps         = 0;
-    job.error_code         = "cancelled";
-    job.error_message      = "job cancelled by client";
-    return true;
-}
-
-json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJob& job) {
-    json result;
-    result["id"]             = job.id;
-    result["kind"]           = async_job_kind_name(job.kind);
-    result["status"]         = async_job_status_name(job.status);
-    result["created"]        = job.created_at;
-    result["started"]        = job.started_at == 0 ? json(nullptr) : json(job.started_at);
-    result["completed"]      = job.completed_at == 0 ? json(nullptr) : json(job.completed_at);
-    result["queue_position"] = 0;
-
-    if (job.status == AsyncJobStatus::Queued) {
-        size_t position = 1;
-        for (const auto& queued_id : manager.queue) {
-            if (queued_id == job.id) {
-                result["queue_position"] = position;
-                break;
-            }
-            ++position;
-        }
-    }
-
-    if (job.status == AsyncJobStatus::Completed) {
-        if (job.kind == AsyncJobKind::VidGen) {
-            result["result"] = {
-                {"output_format", job.vid_gen.output_format},
-                {"mime_type", job.result_media_mime_type},
-                {"fps", job.result_fps},
-                {"frame_count", job.result_frame_count},
-                {"b64_json", job.result_media_b64},
-            };
-        } else {
-            json images = json::array();
-            for (size_t i = 0; i < job.result_images_b64.size(); ++i) {
-                images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}});
-            }
-            result["result"] = {
-                {"output_format", job.img_gen.output_format},
-                {"images", images},
-            };
-        }
-        result["error"] = nullptr;
-    } else if (job.status == AsyncJobStatus::Failed ||
-               job.status == AsyncJobStatus::Cancelled) {
-        result["result"] = nullptr;
-        result["error"]  = {
-             {"code",
-             job.error_code.empty()
-                  ? (job.status == AsyncJobStatus::Cancelled ? "cancelled" : "generation_failed")
-                  : job.error_code},
-             {"message", job.error_message},
-        };
-    } else {
-        result["result"] = nullptr;
-        result["error"]  = nullptr;
-    }
-
-    return result;
-}
-
-bool execute_img_gen_job(ServerRuntime& runtime,
-                         AsyncGenerationJob& job,
-                         std::vector<std::string>& output_images,
-                         std::string& error_message) {
-    sd_img_gen_params_t params = job.img_gen.to_sd_img_gen_params_t();
-
-    SDImageVec results;
-
-    {
-        std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
-        sd_image_t* raw_results = generate_image(runtime.sd_ctx, &params);
-        results.adopt(raw_results, params.batch_count);
-    }
-
-    const int num_results = results.count();
-    if (num_results <= 0) {
-        error_message = "generate_image returned no results";
-        return false;
-    }
-
-    EncodedImageFormat encoded_format = EncodedImageFormat::PNG;
-    if (job.img_gen.output_format == "jpeg") {
-        encoded_format = EncodedImageFormat::JPEG;
-    } else if (job.img_gen.output_format == "webp") {
-        encoded_format = EncodedImageFormat::WEBP;
-    }
-
-    for (int i = 0; i < num_results; ++i) {
-        if (results[i].data == nullptr) {
-            continue;
-        }
-
-        const std::string metadata = job.img_gen.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime.ctx_params,
-                                                            job.img_gen.gen_params,
-                                                            job.img_gen.gen_params.seed + i)
-                                         : "";
-        auto image_bytes           = encode_image_to_vector(encoded_format,
-                                                            results[i].data,
-                                                            results[i].width,
-                                                            results[i].height,
-                                                            results[i].channel,
-                                                            metadata,
-                                                            job.img_gen.output_compression);
-        if (image_bytes.empty()) {
-            continue;
-        }
-        output_images.push_back(base64_encode(image_bytes));
-    }
-
-    if (output_images.empty()) {
-        error_message = "generate_image returned empty encoded outputs";
-        return false;
-    }
-
-    return true;
-}
-
-bool execute_vid_gen_job(ServerRuntime& runtime,
-                         AsyncGenerationJob& job,
-                         std::string& output_media_b64,
-                         std::string& output_media_mime_type,
-                         int& output_frame_count,
-                         int& output_fps,
-                         std::string& error_message) {
-    sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t();
-
-    SDImageVec results;
-    int num_results             = 0;
-    sd_audio_t* generated_audio = nullptr;
-
-    {
-        std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
-        sd_image_t* raw_results = nullptr;
-        if (!generate_video(runtime.sd_ctx, &params, &raw_results, &num_results, &generated_audio)) {
-            raw_results = nullptr;
-        }
-        results.adopt(raw_results, num_results);
-    }
-
-    num_results = results.count();
-    if (num_results <= 0) {
-        free_sd_audio(generated_audio);
-        error_message = "generate_video returned no results";
-        return false;
-    }
-
-    std::vector<uint8_t> video_bytes = create_video_from_sd_images_to_vector(job.vid_gen.output_format,
-                                                                             results.data(),
-                                                                             num_results,
-                                                                             job.vid_gen.gen_params.fps,
-                                                                             job.vid_gen.output_compression,
-                                                                             generated_audio);
-    free_sd_audio(generated_audio);
-    if (video_bytes.empty()) {
-        error_message = "failed to encode generated video container";
-        return false;
-    }
-
-    output_media_b64       = base64_encode(video_bytes);
-    output_media_mime_type = video_mime_type(job.vid_gen.output_format);
-    output_frame_count     = num_results;
-    output_fps             = job.vid_gen.gen_params.fps;
-    return true;
-}
-
-void async_job_worker(ServerRuntime& runtime) {
-    AsyncJobManager& manager = *runtime.async_job_manager;
-
-    while (true) {
-        std::shared_ptr<AsyncGenerationJob> job;
-        {
-            std::unique_lock<std::mutex> lock(manager.mutex);
-            manager.cv.wait(lock, [&]() { return manager.stop || !manager.queue.empty(); });
-
-            if (manager.stop && manager.queue.empty()) {
-                break;
-            }
-
-            purge_expired_jobs(manager);
-            if (manager.queue.empty()) {
-                continue;
-            }
-
-            const std::string job_id = manager.queue.front();
-            manager.queue.pop_front();
-
-            auto it = manager.jobs.find(job_id);
-            if (it == manager.jobs.end()) {
-                continue;
-            }
-
-            job             = it->second;
-            job->status     = AsyncJobStatus::Generating;
-            job->started_at = unix_timestamp_now();
-        }
-
-        std::vector<std::string> output_images;
-        std::string output_media_b64;
-        std::string output_media_mime_type;
-        int output_frame_count = 0;
-        int output_fps         = 0;
-        std::string error_message;
-        bool ok = false;
-
-        if (job->kind == AsyncJobKind::ImgGen) {
-            ok = execute_img_gen_job(runtime, *job, output_images, error_message);
-        } else if (job->kind == AsyncJobKind::VidGen) {
-            ok = execute_vid_gen_job(runtime,
-                                     *job,
-                                     output_media_b64,
-                                     output_media_mime_type,
-                                     output_frame_count,
-                                     output_fps,
-                                     error_message);
-        } else {
-            error_message = "unsupported job kind";
-        }
-
-        {
-            std::lock_guard<std::mutex> lock(manager.mutex);
-            auto it = manager.jobs.find(job->id);
-            if (it == manager.jobs.end()) {
-                continue;
-            }
-
-            job->completed_at = unix_timestamp_now();
-            if (ok) {
-                job->status                 = AsyncJobStatus::Completed;
-                job->result_images_b64      = std::move(output_images);
-                job->result_media_b64       = std::move(output_media_b64);
-                job->result_media_mime_type = std::move(output_media_mime_type);
-                job->result_frame_count     = output_frame_count;
-                job->result_fps             = output_fps;
-                job->error_code.clear();
-                job->error_message.clear();
-            } else {
-                job->status        = AsyncJobStatus::Failed;
-                job->error_code    = "generation_failed";
-                job->error_message = error_message.empty() ? "unknown generation error" : error_message;
-                job->result_images_b64.clear();
-                job->result_media_b64.clear();
-                job->result_media_mime_type.clear();
-                job->result_frame_count = 0;
-                job->result_fps         = 0;
-            }
-
-            purge_expired_jobs(manager);
-        }
-    }
-}

examples/server/async_jobs.h

@@ -1,78 +0,0 @@
-#pragma once
-
-#include <condition_variable>
-#include <cstdint>
-#include <deque>
-#include <memory>
-#include <mutex>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-
-#include "runtime.h"
-
-enum class AsyncJobKind {
-    ImgGen,
-    VidGen,
-};
-
-enum class AsyncJobStatus {
-    Queued,
-    Generating,
-    Completed,
-    Failed,
-    Cancelled,
-};
-
-const char* async_job_kind_name(AsyncJobKind kind);
-const char* async_job_status_name(AsyncJobStatus status);
-
-struct AsyncGenerationJob {
-    std::string id;
-    AsyncJobKind kind     = AsyncJobKind::ImgGen;
-    AsyncJobStatus status = AsyncJobStatus::Queued;
-    int64_t created_at    = unix_timestamp_now();
-    int64_t started_at    = 0;
-    int64_t completed_at  = 0;
-    ImgGenJobRequest img_gen;
-    VidGenJobRequest vid_gen;
-    std::vector<std::string> result_images_b64;
-    std::string result_media_b64;
-    std::string result_media_mime_type;
-    int result_frame_count = 0;
-    int result_fps         = 0;
-    std::string error_code;
-    std::string error_message;
-};
-
-struct AsyncJobManager {
-    std::mutex mutex;
-    std::condition_variable cv;
-    std::unordered_map<std::string, std::shared_ptr<AsyncGenerationJob>> jobs;
-    std::unordered_map<std::string, int64_t> expired_jobs;
-    std::deque<std::string> queue;
-    uint64_t next_id              = 0;
-    bool stop                     = false;
-    size_t max_pending_jobs       = 64;
-    int64_t completed_ttl_seconds = 600;
-    int64_t failed_ttl_seconds    = 600;
-};
-
-void purge_expired_jobs(AsyncJobManager& manager);
-size_t count_pending_jobs(const AsyncJobManager& manager);
-std::string make_async_job_id(AsyncJobManager& manager);
-bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job);
-json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJob& job);
-bool execute_img_gen_job(ServerRuntime& runtime,
-                         AsyncGenerationJob& job,
-                         std::vector<std::string>& output_images,
-                         std::string& error_message);
-bool execute_vid_gen_job(ServerRuntime& runtime,
-                         AsyncGenerationJob& job,
-                         std::string& output_media_b64,
-                         std::string& output_media_mime_type,
-                         int& output_frame_count,
-                         int& output_fps,
-                         std::string& error_message);
-void async_job_worker(ServerRuntime& runtime);

examples/server/frontend

@@ -1 +1 @@
-Subproject commit 797ccf80825cc035508ba9b599b2a21953e7f835
+Subproject commit 1a34176cd6d39ad3a226b2b69047e71f6797f6bc

examples/server/routes.h

@@ -1,11 +0,0 @@
-#pragma once
-
-#include <string>
-
-#include "httplib.h"
-#include "runtime.h"
-
-void register_index_endpoints(httplib::Server& svr, const SDSvrParams& svr_params, const std::string& index_html);
-void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt);
-void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt);
-void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt);

examples/server/routes_index.cpp

@@ -1,22 +0,0 @@
-#include "routes.h"
-
-#include <fstream>
-#include <iterator>
-
-void register_index_endpoints(httplib::Server& svr, const SDSvrParams& svr_params, const std::string& index_html) {
-    const std::string serve_html_path = svr_params.serve_html_path;
-    svr.Get("/", [serve_html_path, index_html](const httplib::Request&, httplib::Response& res) {
-        if (!serve_html_path.empty()) {
-            std::ifstream file(serve_html_path);
-            if (file) {
-                std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
-                res.set_content(content, "text/html");
-            } else {
-                res.status = 500;
-                res.set_content("Error: Unable to read HTML file", "text/plain");
-            }
-        } else {
-            res.set_content(index_html, "text/html");
-        }
-    });
-}

examples/server/routes_openai.cpp

@@ -1,388 +0,0 @@
-#include "routes.h"
-
-#include <algorithm>
-#include <ctime>
-#include <regex>
-
-#include "common/common.h"
-#include "common/media_io.h"
-#include "common/resource_owners.hpp"
-
-static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) {
-    std::regex re("<sd_cpp_extra_args>(.*?)</sd_cpp_extra_args>");
-    std::smatch match;
-
-    std::string extracted;
-    if (std::regex_search(text, match, re)) {
-        extracted = match[1].str();
-        text      = std::regex_replace(text, re, "");
-    }
-    return extracted;
-}
-
-static bool build_openai_generation_request(const httplib::Request& req,
-                                            ServerRuntime& runtime,
-                                            ImgGenJobRequest& request,
-                                            std::string& error_message) {
-    if (req.body.empty()) {
-        error_message = "empty body";
-        return false;
-    }
-
-    json j                    = json::parse(req.body);
-    std::string prompt        = j.value("prompt", "");
-    int n                     = std::max(1, j.value("n", 1));
-    std::string size          = j.value("size", "");
-    std::string output_format = j.value("output_format", "png");
-    int output_compression    = j.value("output_compression", 100);
-    int width                 = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->width : 512;
-    int height                = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->height : 512;
-    if (!size.empty()) {
-        auto pos = size.find('x');
-        if (pos != std::string::npos) {
-            try {
-                width  = std::stoi(size.substr(0, pos));
-                height = std::stoi(size.substr(pos + 1));
-            } catch (...) {
-            }
-        }
-    }
-
-    if (prompt.empty()) {
-        error_message = "prompt required";
-        return false;
-    }
-
-    request.gen_params = *runtime.default_gen_params;
-    if (!assign_output_options(request, output_format, output_compression, true, error_message)) {
-        return false;
-    }
-
-    request.gen_params.prompt      = prompt;
-    request.gen_params.width       = width;
-    request.gen_params.height      = height;
-    request.gen_params.batch_count = n;
-
-    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
-    if (!sd_cpp_extra_args_str.empty() && !request.gen_params.from_json_str(sd_cpp_extra_args_str)) {
-        error_message = "invalid sd_cpp_extra_args";
-        return false;
-    }
-
-    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
-    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
-        error_message = "invalid params";
-        return false;
-    }
-    return true;
-}
-
-static bool build_openai_edit_request(const httplib::Request& req,
-                                      ServerRuntime& runtime,
-                                      ImgGenJobRequest& request,
-                                      std::string& error_message) {
-    if (!req.is_multipart_form_data()) {
-        error_message = "Content-Type must be multipart/form-data";
-        return false;
-    }
-
-    std::string prompt = req.form.get_field("prompt");
-    if (prompt.empty()) {
-        error_message = "prompt required";
-        return false;
-    }
-
-    size_t image_count    = req.form.get_file_count("image[]");
-    bool has_legacy_image = req.form.has_file("image");
-    if (image_count == 0 && !has_legacy_image) {
-        error_message = "at least one image[] required";
-        return false;
-    }
-
-    std::vector<std::vector<uint8_t>> images_bytes;
-    for (size_t i = 0; i < image_count; ++i) {
-        auto file = req.form.get_file("image[]", i);
-        images_bytes.emplace_back(file.content.begin(), file.content.end());
-    }
-    if (image_count == 0 && has_legacy_image) {
-        auto file = req.form.get_file("image");
-        images_bytes.emplace_back(file.content.begin(), file.content.end());
-    }
-
-    std::vector<uint8_t> mask_bytes;
-    if (req.form.has_file("mask")) {
-        auto file = req.form.get_file("mask");
-        mask_bytes.assign(file.content.begin(), file.content.end());
-    }
-
-    int n = 1;
-    if (req.form.has_field("n")) {
-        try {
-            n = std::stoi(req.form.get_field("n"));
-        } catch (...) {
-        }
-    }
-
-    std::string size = req.form.get_field("size");
-    int width        = -1;
-    int height       = -1;
-    if (!size.empty()) {
-        auto pos = size.find('x');
-        if (pos != std::string::npos) {
-            try {
-                width  = std::stoi(size.substr(0, pos));
-                height = std::stoi(size.substr(pos + 1));
-            } catch (...) {
-            }
-        }
-    }
-
-    std::string output_format = req.form.has_field("output_format")
-                                    ? req.form.get_field("output_format")
-                                    : "png";
-
-    int output_compression = 100;
-    try {
-        output_compression = std::stoi(req.form.get_field("output_compression"));
-    } catch (...) {
-    }
-
-    request.gen_params = *runtime.default_gen_params;
-    if (!assign_output_options(request, output_format, output_compression, false, error_message)) {
-        return false;
-    }
-
-    request.gen_params.prompt      = prompt;
-    request.gen_params.width       = width;
-    request.gen_params.height      = height;
-    request.gen_params.batch_count = n;
-
-    for (auto& bytes : images_bytes) {
-        int img_w           = 0;
-        int img_h           = 0;
-        uint8_t* raw_pixels = load_image_from_memory(
-            reinterpret_cast<const char*>(bytes.data()),
-            static_cast<int>(bytes.size()),
-            img_w, img_h,
-            width, height, 3);
-        if (raw_pixels == nullptr) {
-            continue;
-        }
-
-        SDImageOwner image_owner({(uint32_t)img_w, (uint32_t)img_h, 3, raw_pixels});
-        request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height);
-        request.gen_params.ref_images.push_back(std::move(image_owner));
-    }
-
-    if (!request.gen_params.ref_images.empty()) {
-        request.gen_params.init_image = request.gen_params.ref_images.front();
-    }
-
-    if (!mask_bytes.empty()) {
-        int expected_width  = 0;
-        int expected_height = 0;
-        if (request.gen_params.width_and_height_are_set()) {
-            expected_width  = request.gen_params.width;
-            expected_height = request.gen_params.height;
-        }
-        int mask_w = 0;
-        int mask_h = 0;
-
-        uint8_t* mask_raw = load_image_from_memory(
-            reinterpret_cast<const char*>(mask_bytes.data()),
-            static_cast<int>(mask_bytes.size()),
-            mask_w, mask_h,
-            expected_width, expected_height, 1);
-        request.gen_params.mask_image.reset({(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw});
-        const sd_image_t& mask_image = request.gen_params.mask_image.get();
-        request.gen_params.set_width_and_height_if_unset(mask_image.width, mask_image.height);
-    } else {
-        request.gen_params.mask_image.reset({
-            (uint32_t)request.gen_params.get_resolved_width(),
-            (uint32_t)request.gen_params.get_resolved_height(),
-            1,
-            nullptr,
-        });
-    }
-
-    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
-    if (!sd_cpp_extra_args_str.empty() && !request.gen_params.from_json_str(sd_cpp_extra_args_str)) {
-        error_message = "invalid sd_cpp_extra_args";
-        return false;
-    }
-
-    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
-    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
-        error_message = "invalid params";
-        return false;
-    }
-
-    return true;
-}
-
-static bool execute_sync_img_gen_request(ServerRuntime& runtime,
-                                         ImgGenJobRequest& request,
-                                         SDImageVec& results,
-                                         std::string& error_message) {
-    sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t();
-    int num_results                    = 0;
-
-    {
-        std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
-        sd_image_t* raw_results = generate_image(runtime.sd_ctx, &img_gen_params);
-        num_results             = request.gen_params.batch_count;
-        results.adopt(raw_results, num_results);
-    }
-
-    if (results.empty()) {
-        error_message = "generate_image returned no results";
-        return false;
-    }
-    return true;
-}
-
-void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
-    ServerRuntime* runtime = &rt;
-
-    svr.Get("/v1/models", [runtime](const httplib::Request&, httplib::Response& res) {
-        json r;
-        r["data"] = json::array();
-        r["data"].push_back({{"id", "sd-cpp-local"}, {"object", "model"}, {"owned_by", "local"}});
-        res.set_content(r.dump(), "application/json");
-    });
-
-    svr.Post("/v1/images/generations", [runtime](const httplib::Request& req, httplib::Response& res) {
-        try {
-            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
-                res.status = 400;
-                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
-                return;
-            }
-
-            ImgGenJobRequest request;
-            std::string error_message;
-            if (!build_openai_generation_request(req, *runtime, request, error_message)) {
-                res.status = 400;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
-
-            SDImageVec results;
-            if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) {
-                res.status = 500;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            json out;
-            out["created"]       = static_cast<long long>(std::time(nullptr));
-            out["data"]          = json::array();
-            out["output_format"] = request.output_format;
-
-            for (int i = 0; i < request.gen_params.batch_count; ++i) {
-                if (results[i].data == nullptr) {
-                    continue;
-                }
-                std::string params = request.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime->ctx_params,
-                                                            request.gen_params,
-                                                            request.gen_params.seed + i)
-                                         : "";
-                auto image_bytes   = encode_image_to_vector(request.output_format == "jpeg"
-                                                                ? EncodedImageFormat::JPEG
-                                                            : request.output_format == "webp"
-                                                                ? EncodedImageFormat::WEBP
-                                                                : EncodedImageFormat::PNG,
-                                                          results[i].data,
-                                                          results[i].width,
-                                                          results[i].height,
-                                                          results[i].channel,
-                                                          params,
-                                                          request.output_compression);
-                if (image_bytes.empty()) {
-                    LOG_ERROR("write image to mem failed");
-                    continue;
-                }
-
-                json item;
-                item["b64_json"] = base64_encode(image_bytes);
-                out["data"].push_back(item);
-            }
-
-            res.set_content(out.dump(), "application/json");
-            res.status = 200;
-
-        } catch (const std::exception& e) {
-            res.status = 500;
-            json err;
-            err["error"]   = "server_error";
-            err["message"] = e.what();
-            res.set_content(err.dump(), "application/json");
-        }
-    });
-
-    svr.Post("/v1/images/edits", [runtime](const httplib::Request& req, httplib::Response& res) {
-        try {
-            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
-                res.status = 400;
-                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
-                return;
-            }
-
-            ImgGenJobRequest request;
-            std::string error_message;
-            if (!build_openai_edit_request(req, *runtime, request, error_message)) {
-                res.status = 400;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
-
-            SDImageVec results;
-            if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) {
-                res.status = 500;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            json out;
-            out["created"]       = static_cast<long long>(std::time(nullptr));
-            out["data"]          = json::array();
-            out["output_format"] = request.output_format;
-
-            for (int i = 0; i < request.gen_params.batch_count; ++i) {
-                if (results[i].data == nullptr) {
-                    continue;
-                }
-                std::string params = request.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime->ctx_params,
-                                                            request.gen_params,
-                                                            request.gen_params.seed + i)
-                                         : "";
-                auto image_bytes   = encode_image_to_vector(request.output_format == "jpeg" ? EncodedImageFormat::JPEG : EncodedImageFormat::PNG,
-                                                          results[i].data,
-                                                          results[i].width,
-                                                          results[i].height,
-                                                          results[i].channel,
-                                                          params,
-                                                          request.output_compression);
-                json item;
-                item["b64_json"] = base64_encode(image_bytes);
-                out["data"].push_back(item);
-            }
-
-            res.set_content(out.dump(), "application/json");
-            res.status = 200;
-
-        } catch (const std::exception& e) {
-            res.status = 500;
-            json err;
-            err["error"]   = "server_error";
-            err["message"] = e.what();
-            res.set_content(err.dump(), "application/json");
-        }
-    });
-}

examples/server/routes_sdapi.cpp

@@ -1,473 +0,0 @@
-#include "routes.h"
-
-#include <algorithm>
-#include <cctype>
-#include <cstring>
-#include <regex>
-#include <string_view>
-#include <unordered_map>
-
-#include "common/common.h"
-#include "common/media_io.h"
-#include "common/resource_owners.hpp"
-
-namespace fs = std::filesystem;
-
-static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) {
-    std::regex re("<sd_cpp_extra_args>(.*?)</sd_cpp_extra_args>");
-    std::smatch match;
-
-    std::string extracted;
-    if (std::regex_search(text, match, re)) {
-        extracted = match[1].str();
-        text      = std::regex_replace(text, re, "");
-    }
-    return extracted;
-}
-
-static fs::path resolve_display_model_path(const ServerRuntime& runtime) {
-    const auto& ctx = *runtime.ctx_params;
-    if (!ctx.model_path.empty()) {
-        return fs::path(ctx.model_path);
-    }
-    if (!ctx.diffusion_model_path.empty()) {
-        return fs::path(ctx.diffusion_model_path);
-    }
-    return {};
-}
-
-static std::string lower_ascii(std::string value) {
-    std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
-        return static_cast<char>(std::tolower(c));
-    });
-    return value;
-}
-
-static enum sample_method_t get_sdapi_sample_method(std::string name) {
-    enum sample_method_t result = str_to_sample_method(name.c_str());
-    if (result != SAMPLE_METHOD_COUNT) {
-        return result;
-    }
-
-    name = lower_ascii(name);
-    static const std::unordered_map<std::string_view, sample_method_t> hardcoded{
-        {"euler a", EULER_A_SAMPLE_METHOD},
-        {"k_euler_a", EULER_A_SAMPLE_METHOD},
-        {"euler", EULER_SAMPLE_METHOD},
-        {"k_euler", EULER_SAMPLE_METHOD},
-        {"heun", HEUN_SAMPLE_METHOD},
-        {"k_heun", HEUN_SAMPLE_METHOD},
-        {"dpm2", DPM2_SAMPLE_METHOD},
-        {"k_dpm_2", DPM2_SAMPLE_METHOD},
-        {"lcm", LCM_SAMPLE_METHOD},
-        {"ddim", DDIM_TRAILING_SAMPLE_METHOD},
-        {"dpm++ 2m", DPMPP2M_SAMPLE_METHOD},
-        {"k_dpmpp_2m", DPMPP2M_SAMPLE_METHOD},
-        {"res multistep", RES_MULTISTEP_SAMPLE_METHOD},
-        {"k_res_multistep", RES_MULTISTEP_SAMPLE_METHOD},
-        {"res 2s", RES_2S_SAMPLE_METHOD},
-        {"k_res_2s", RES_2S_SAMPLE_METHOD},
-        {"euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
-        {"k_euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
-        {"euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
-        {"k_euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
-    };
-    auto it = hardcoded.find(name);
-    return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT;
-}
-
-static void assign_solid_mask(SDImageOwner& mask_owner, int width, int height) {
-    const size_t pixel_count = static_cast<size_t>(width) * static_cast<size_t>(height);
-    uint8_t* raw_mask        = static_cast<uint8_t*>(malloc(pixel_count));
-    if (raw_mask == nullptr) {
-        mask_owner.reset({0, 0, 1, nullptr});
-        return;
-    }
-    std::memset(raw_mask, 255, pixel_count);
-    mask_owner.reset({(uint32_t)width, (uint32_t)height, 1, raw_mask});
-}
-
-static bool build_sdapi_img_gen_request(const json& j,
-                                        ServerRuntime& runtime,
-                                        bool img2img,
-                                        ImgGenJobRequest& request,
-                                        std::string& error_message) {
-    std::string prompt          = j.value("prompt", "");
-    std::string negative_prompt = j.value("negative_prompt", "");
-    int width                   = j.value("width", 512);
-    int height                  = j.value("height", 512);
-    int steps                   = j.value("steps", runtime.default_gen_params->sample_params.sample_steps);
-    float cfg_scale             = j.value("cfg_scale", runtime.default_gen_params->sample_params.guidance.txt_cfg);
-    int64_t seed                = j.value("seed", -1);
-    int batch_size              = j.value("batch_size", 1);
-    int clip_skip               = j.value("clip_skip", -1);
-    std::string sampler_name    = j.value("sampler_name", "");
-    std::string scheduler_name  = j.value("scheduler", "");
-
-    if (width <= 0 || height <= 0) {
-        error_message = "width and height must be positive";
-        return false;
-    }
-
-    if (prompt.empty()) {
-        error_message = "prompt required";
-        return false;
-    }
-
-    request.gen_params = *runtime.default_gen_params;
-
-    request.gen_params.prompt                         = prompt;
-    request.gen_params.negative_prompt                = negative_prompt;
-    request.gen_params.seed                           = seed;
-    request.gen_params.sample_params.sample_steps     = steps;
-    request.gen_params.batch_count                    = batch_size;
-    request.gen_params.sample_params.guidance.txt_cfg = cfg_scale;
-    request.gen_params.width                          = j.value("width", -1);
-    request.gen_params.height                         = j.value("height", -1);
-
-    if (!img2img && j.value("enable_hr", false)) {
-        request.gen_params.hires_enabled = true;
-        request.gen_params.hires_scale   = j.value("hr_scale", request.gen_params.hires_scale);
-        request.gen_params.hires_width   = j.value("hr_resize_x", request.gen_params.hires_width);
-        request.gen_params.hires_height  = j.value("hr_resize_y", request.gen_params.hires_height);
-        request.gen_params.hires_steps   = j.value("hr_steps", request.gen_params.hires_steps);
-        request.gen_params.hires_denoising_strength =
-            j.value("denoising_strength", request.gen_params.hires_denoising_strength);
-
-        request.gen_params.hires_upscaler = j.value("hr_upscaler", request.gen_params.hires_upscaler);
-    }
-
-    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
-    if (!sd_cpp_extra_args_str.empty() && !request.gen_params.from_json_str(sd_cpp_extra_args_str)) {
-        error_message = "invalid sd_cpp_extra_args";
-        return false;
-    }
-
-    if (clip_skip > 0) {
-        request.gen_params.clip_skip = clip_skip;
-    }
-
-    enum sample_method_t sample_method = get_sdapi_sample_method(sampler_name);
-    if (sample_method != SAMPLE_METHOD_COUNT) {
-        request.gen_params.sample_params.sample_method = sample_method;
-    }
-
-    enum scheduler_t scheduler = str_to_scheduler(scheduler_name.c_str());
-    if (scheduler != SCHEDULER_COUNT) {
-        request.gen_params.sample_params.scheduler = scheduler;
-    }
-
-    if (j.contains("lora") && j["lora"].is_array()) {
-        request.gen_params.lora_map.clear();
-        request.gen_params.high_noise_lora_map.clear();
-
-        for (const auto& item : j["lora"]) {
-            if (!item.is_object()) {
-                continue;
-            }
-
-            std::string path   = item.value("path", "");
-            float multiplier   = item.value("multiplier", 1.0f);
-            bool is_high_noise = item.value("is_high_noise", false);
-
-            if (path.empty()) {
-                error_message = "lora.path required";
-                return false;
-            }
-
-            std::string fullpath = get_lora_full_path(runtime, path);
-            if (fullpath.empty()) {
-                error_message = "invalid lora path: " + path;
-                return false;
-            }
-
-            if (is_high_noise) {
-                request.gen_params.high_noise_lora_map[fullpath] += multiplier;
-            } else {
-                request.gen_params.lora_map[fullpath] += multiplier;
-            }
-        }
-    }
-
-    if (img2img) {
-        const int expected_width  = request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0;
-        const int expected_height = request.gen_params.width_and_height_are_set() ? request.gen_params.height : 0;
-
-        if (j.contains("init_images") && j["init_images"].is_array() && !j["init_images"].empty()) {
-            if (decode_base64_image(j["init_images"][0].get<std::string>(),
-                                    3,
-                                    expected_width,
-                                    expected_height,
-                                    request.gen_params.init_image)) {
-                const sd_image_t& image = request.gen_params.init_image.get();
-                request.gen_params.set_width_and_height_if_unset(image.width, image.height);
-            }
-        }
-
-        if (j.contains("mask") && j["mask"].is_string()) {
-            if (decode_base64_image(j["mask"].get<std::string>(),
-                                    1,
-                                    expected_width,
-                                    expected_height,
-                                    request.gen_params.mask_image)) {
-                const sd_image_t& image = request.gen_params.mask_image.get();
-                request.gen_params.set_width_and_height_if_unset(image.width, image.height);
-            }
-            sd_image_t& mask_image      = request.gen_params.mask_image.get();
-            bool inpainting_mask_invert = j.value("inpainting_mask_invert", 0) != 0;
-            if (inpainting_mask_invert && mask_image.data != nullptr) {
-                for (uint32_t i = 0; i < mask_image.width * mask_image.height; ++i) {
-                    mask_image.data[i] = 255 - mask_image.data[i];
-                }
-            }
-        } else {
-            const int resolved_width  = request.gen_params.get_resolved_width();
-            const int resolved_height = request.gen_params.get_resolved_height();
-            assign_solid_mask(request.gen_params.mask_image, resolved_width, resolved_height);
-        }
-
-        float denoising_strength = j.value("denoising_strength", -1.f);
-        if (denoising_strength >= 0.f) {
-            request.gen_params.strength = std::min(denoising_strength, 1.0f);
-        }
-    }
-
-    if (j.contains("extra_images") && j["extra_images"].is_array()) {
-        for (const auto& extra_image : j["extra_images"]) {
-            if (!extra_image.is_string()) {
-                continue;
-            }
-            SDImageOwner image_owner;
-            if (decode_base64_image(extra_image.get<std::string>(),
-                                    3,
-                                    request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0,
-                                    request.gen_params.width_and_height_are_set() ? request.gen_params.height : 0,
-                                    image_owner)) {
-                const sd_image_t& image = image_owner.get();
-                request.gen_params.set_width_and_height_if_unset(image.width, image.height);
-                request.gen_params.ref_images.push_back(std::move(image_owner));
-            }
-        }
-    }
-
-    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
-    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
-        error_message = "invalid params";
-        return false;
-    }
-
-    return true;
-}
-
-void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
-    ServerRuntime* runtime = &rt;
-
-    auto sdapi_any2img = [runtime](const httplib::Request& req, httplib::Response& res, bool img2img) {
-        try {
-            if (req.body.empty()) {
-                res.status = 400;
-                res.set_content(R"({"error":"empty body"})", "application/json");
-                return;
-            }
-            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
-                res.status = 400;
-                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
-                return;
-            }
-
-            json j = json::parse(req.body);
-            ImgGenJobRequest request;
-            std::string error_message;
-            if (!build_sdapi_img_gen_request(j, *runtime, img2img, request, error_message)) {
-                res.status = 400;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
-
-            sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t();
-            SDImageVec results;
-            int num_results = 0;
-
-            {
-                std::lock_guard<std::mutex> lock(*runtime->sd_ctx_mutex);
-                sd_image_t* raw_results = generate_image(runtime->sd_ctx, &img_gen_params);
-                num_results             = request.gen_params.batch_count;
-                results.adopt(raw_results, num_results);
-            }
-
-            if (results.empty()) {
-                res.status = 500;
-                res.set_content(R"({"error":"generate_image returned no results"})", "application/json");
-                return;
-            }
-
-            json out;
-            out["images"]     = json::array();
-            out["parameters"] = j;
-            out["info"]       = "";
-
-            for (int i = 0; i < num_results; ++i) {
-                if (results[i].data == nullptr) {
-                    continue;
-                }
-
-                std::string params = request.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime->ctx_params,
-                                                            request.gen_params,
-                                                            request.gen_params.seed + i)
-                                         : "";
-                auto image_bytes   = encode_image_to_vector(EncodedImageFormat::PNG,
-                                                            results[i].data,
-                                                            results[i].width,
-                                                            results[i].height,
-                                                            results[i].channel,
-                                                            params);
-
-                if (image_bytes.empty()) {
-                    LOG_ERROR("write image to mem failed");
-                    continue;
-                }
-
-                out["images"].push_back(base64_encode(image_bytes));
-            }
-
-            res.set_content(out.dump(), "application/json");
-            res.status = 200;
-
-        } catch (const std::exception& e) {
-            res.status = 500;
-            json err;
-            err["error"]   = "server_error";
-            err["message"] = e.what();
-            res.set_content(err.dump(), "application/json");
-        }
-    };
-
-    svr.Post("/sdapi/v1/txt2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) {
-        sdapi_any2img(req, res, false);
-    });
-
-    svr.Post("/sdapi/v1/img2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) {
-        sdapi_any2img(req, res, true);
-    });
-
-    svr.Get("/sdapi/v1/loras", [runtime](const httplib::Request&, httplib::Response& res) {
-        refresh_lora_cache(*runtime);
-
-        json result = json::array();
-        {
-            std::lock_guard<std::mutex> lock(*runtime->lora_mutex);
-            for (const auto& e : *runtime->lora_cache) {
-                json item;
-                item["name"] = e.name;
-                item["path"] = e.path;
-                result.push_back(item);
-            }
-        }
-
-        res.set_content(result.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/upscalers", [runtime](const httplib::Request&, httplib::Response& res) {
-        refresh_upscaler_cache(*runtime);
-
-        auto make_builtin = [](const char* name) {
-            json item;
-            item["name"]       = name;
-            item["model_name"] = nullptr;
-            item["model_path"] = nullptr;
-            item["model_url"]  = nullptr;
-            item["scale"]      = 4;
-            return item;
-        };
-
-        json result = json::array();
-        result.push_back(make_builtin("None"));
-        result.push_back(make_builtin("Lanczos"));
-        result.push_back(make_builtin("Nearest"));
-
-        {
-            std::lock_guard<std::mutex> lock(*runtime->upscaler_mutex);
-            for (const auto& e : *runtime->upscaler_cache) {
-                json item;
-                item["name"]       = e.name;
-                item["model_name"] = e.model_name;
-                item["model_path"] = e.fullpath;
-                item["model_url"]  = nullptr;
-                item["scale"]      = e.scale;
-                result.push_back(item);
-            }
-        }
-
-        res.set_content(result.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/latent-upscale-modes", [](const httplib::Request&, httplib::Response& res) {
-        json result = json::array({
-            {{"name", "Latent"}},
-            {{"name", "Latent (nearest)"}},
-            {{"name", "Latent (nearest-exact)"}},
-            {{"name", "Latent (antialiased)"}},
-            {{"name", "Latent (bicubic)"}},
-            {{"name", "Latent (bicubic antialiased)"}},
-        });
-        res.set_content(result.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/samplers", [runtime](const httplib::Request&, httplib::Response& res) {
-        std::vector<std::string> sampler_names;
-        sampler_names.push_back("default");
-        for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) {
-            sampler_names.push_back(sd_sample_method_name((sample_method_t)i));
-        }
-        json r = json::array();
-        for (auto name : sampler_names) {
-            json entry;
-            entry["name"]    = name;
-            entry["aliases"] = json::array({name});
-            entry["options"] = json::object();
-            r.push_back(entry);
-        }
-        res.set_content(r.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/schedulers", [runtime](const httplib::Request&, httplib::Response& res) {
-        std::vector<std::string> scheduler_names;
-        scheduler_names.push_back("default");
-        for (int i = 0; i < SCHEDULER_COUNT; i++) {
-            scheduler_names.push_back(sd_scheduler_name((scheduler_t)i));
-        }
-        json r = json::array();
-        for (auto name : scheduler_names) {
-            json entry;
-            entry["name"]  = name;
-            entry["label"] = name;
-            r.push_back(entry);
-        }
-        res.set_content(r.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/sd-models", [runtime](const httplib::Request&, httplib::Response& res) {
-        fs::path model_path = resolve_display_model_path(*runtime);
-        json entry;
-        entry["title"]      = model_path.stem();
-        entry["model_name"] = model_path.stem();
-        entry["filename"]   = model_path.filename();
-        entry["hash"]       = "8888888888";
-        entry["sha256"]     = "8888888888888888888888888888888888888888888888888888888888888888";
-        entry["config"]     = nullptr;
-        json r              = json::array();
-        r.push_back(entry);
-        res.set_content(r.dump(), "application/json");
-    });
-
-    svr.Get("/sdapi/v1/options", [runtime](const httplib::Request&, httplib::Response& res) {
-        fs::path model_path = resolve_display_model_path(*runtime);
-        json r;
-        r["samples_format"]      = "png";
-        r["sd_model_checkpoint"] = model_path.stem();
-        res.set_content(r.dump(), "application/json");
-    });
-}

examples/server/routes_sdcpp.cpp

@@ -1,595 +0,0 @@
-#include "routes.h"
-
-#include <algorithm>
-#include <cmath>
-#include <filesystem>
-
-#include "async_jobs.h"
-#include "common/common.h"
-
-namespace fs = std::filesystem;
-
-static bool parse_cache_mode(const std::string& mode_str, sd_cache_mode_t& mode_out) {
-    if (mode_str == "disabled") {
-        mode_out = SD_CACHE_DISABLED;
-        return true;
-    }
-    if (mode_str == "easycache") {
-        mode_out = SD_CACHE_EASYCACHE;
-        return true;
-    }
-    if (mode_str == "ucache") {
-        mode_out = SD_CACHE_UCACHE;
-        return true;
-    }
-    if (mode_str == "dbcache") {
-        mode_out = SD_CACHE_DBCACHE;
-        return true;
-    }
-    if (mode_str == "taylorseer") {
-        mode_out = SD_CACHE_TAYLORSEER;
-        return true;
-    }
-    if (mode_str == "cache-dit") {
-        mode_out = SD_CACHE_CACHE_DIT;
-        return true;
-    }
-    if (mode_str == "spectrum") {
-        mode_out = SD_CACHE_SPECTRUM;
-        return true;
-    }
-    return false;
-}
-
-static json finite_number_or_null(float value) {
-    return std::isfinite(value) ? json(value) : json(nullptr);
-}
-
-static const char* capability_scheduler_name(enum scheduler_t scheduler) {
-    return scheduler < SCHEDULER_COUNT ? sd_scheduler_name(scheduler) : "default";
-}
-
-static const char* capability_sample_method_name(enum sample_method_t sample_method) {
-    return sample_method < SAMPLE_METHOD_COUNT ? sd_sample_method_name(sample_method) : "default";
-}
-
-static json make_vae_tiling_json(const sd_tiling_params_t& params) {
-    return {
-        {"enabled", params.enabled},
-        {"temporal_tiling", params.temporal_tiling},
-        {"tile_size_x", params.tile_size_x},
-        {"tile_size_y", params.tile_size_y},
-        {"target_overlap", params.target_overlap},
-        {"rel_size_x", params.rel_size_x},
-        {"rel_size_y", params.rel_size_y},
-        {"extra_tiling_args", params.extra_tiling_args ? params.extra_tiling_args : ""},
-    };
-}
-
-static fs::path resolve_display_model_path(const ServerRuntime& runtime) {
-    const auto& ctx = *runtime.ctx_params;
-    if (!ctx.model_path.empty()) {
-        return fs::path(ctx.model_path);
-    }
-    if (!ctx.diffusion_model_path.empty()) {
-        return fs::path(ctx.diffusion_model_path);
-    }
-    return {};
-}
-
-static json make_sample_params_json(const sd_sample_params_t& sample_params, const std::vector<int>& skip_layers) {
-    const auto& guidance = sample_params.guidance;
-    return {
-        {"scheduler", capability_scheduler_name(sample_params.scheduler)},
-        {"sample_method", capability_sample_method_name(sample_params.sample_method)},
-        {"sample_steps", sample_params.sample_steps},
-        {"eta", finite_number_or_null(sample_params.eta)},
-        {"shifted_timestep", sample_params.shifted_timestep},
-        {"flow_shift", finite_number_or_null(sample_params.flow_shift)},
-        {"guidance",
-         {
-             {"txt_cfg", guidance.txt_cfg},
-             {"img_cfg", finite_number_or_null(guidance.img_cfg)},
-             {"distilled_guidance", guidance.distilled_guidance},
-             {"slg",
-              {
-                  {"layers", skip_layers},
-                  {"layer_start", guidance.slg.layer_start},
-                  {"layer_end", guidance.slg.layer_end},
-                  {"scale", guidance.slg.scale},
-              }},
-         }},
-    };
-}
-
-static json make_hires_json(const SDGenerationParams& defaults) {
-    return {
-        {"enabled", defaults.hires_enabled},
-        {"upscaler", defaults.hires_upscaler},
-        {"scale", defaults.hires_scale},
-        {"target_width", defaults.hires_width},
-        {"target_height", defaults.hires_height},
-        {"steps", defaults.hires_steps},
-        {"denoising_strength", defaults.hires_denoising_strength},
-        {"custom_sigmas", defaults.hires_custom_sigmas},
-        {"upscale_tile_size", defaults.hires_upscale_tile_size},
-    };
-}
-
-static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const std::string& output_format) {
-    return {
-        {"prompt", defaults.prompt},
-        {"negative_prompt", defaults.negative_prompt},
-        {"clip_skip", defaults.clip_skip},
-        {"width", defaults.width > 0 ? defaults.width : 512},
-        {"height", defaults.height > 0 ? defaults.height : 512},
-        {"strength", defaults.strength},
-        {"seed", defaults.seed},
-        {"batch_count", defaults.batch_count},
-        {"auto_resize_ref_image", defaults.auto_resize_ref_image},
-        {"increase_ref_index", defaults.increase_ref_index},
-        {"control_strength", defaults.control_strength},
-        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)},
-        {"hires", make_hires_json(defaults)},
-        {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)},
-        {"cache_mode", defaults.cache_mode},
-        {"cache_option", defaults.cache_option},
-        {"scm_mask", defaults.scm_mask},
-        {"scm_policy_dynamic", defaults.scm_policy_dynamic},
-        {"output_format", output_format},
-        {"output_compression", 100},
-    };
-}
-
-static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const std::string& output_format) {
-    return {
-        {"prompt", defaults.prompt},
-        {"negative_prompt", defaults.negative_prompt},
-        {"clip_skip", defaults.clip_skip},
-        {"width", defaults.width > 0 ? defaults.width : 512},
-        {"height", defaults.height > 0 ? defaults.height : 512},
-        {"strength", defaults.strength},
-        {"seed", defaults.seed},
-        {"video_frames", defaults.video_frames},
-        {"fps", defaults.fps},
-        {"moe_boundary", defaults.moe_boundary},
-        {"vace_strength", defaults.vace_strength},
-        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)},
-        {"high_noise_sample_params", make_sample_params_json(defaults.high_noise_sample_params, defaults.high_noise_skip_layers)},
-        {"hires", make_hires_json(defaults)},
-        {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)},
-        {"cache_mode", defaults.cache_mode},
-        {"cache_option", defaults.cache_option},
-        {"scm_mask", defaults.scm_mask},
-        {"scm_policy_dynamic", defaults.scm_policy_dynamic},
-        {"output_format", output_format},
-        {"output_compression", 100},
-    };
-}
-
-static json make_img_gen_features_json() {
-    return {
-        {"init_image", true},
-        {"mask_image", true},
-        {"control_image", true},
-        {"ref_images", true},
-        {"lora", true},
-        {"vae_tiling", true},
-        {"hires", true},
-        {"cache", true},
-        {"cancel_queued", true},
-        {"cancel_generating", false},
-    };
-}
-
-static json make_vid_gen_features_json() {
-    return {
-        {"init_image", true},
-        {"end_image", true},
-        {"control_frames", true},
-        {"high_noise_sample_params", true},
-        {"lora", true},
-        {"vae_tiling", true},
-        {"cache", true},
-        {"cancel_queued", true},
-        {"cancel_generating", false},
-    };
-}
-
-static json make_capabilities_json(ServerRuntime& runtime) {
-    refresh_lora_cache(runtime);
-    refresh_upscaler_cache(runtime);
-
-    AsyncJobManager& manager  = *runtime.async_job_manager;
-    const auto& defaults      = *runtime.default_gen_params;
-    const fs::path model_path = resolve_display_model_path(runtime);
-    const bool supports_img   = runtime_supports_generation_mode(runtime, IMG_GEN);
-    const bool supports_vid   = runtime_supports_generation_mode(runtime, VID_GEN);
-    json samplers             = json::array();
-    json schedulers           = json::array();
-    json image_output_formats = supported_img_output_formats();
-    json video_output_formats = supported_vid_output_formats();
-    json available_loras      = json::array();
-    json available_upscalers  = json::array();
-    json supported_modes      = json::array();
-
-    for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) {
-        samplers.push_back(sd_sample_method_name((sample_method_t)i));
-    }
-
-    for (int i = 0; i < SCHEDULER_COUNT; ++i) {
-        schedulers.push_back(sd_scheduler_name((scheduler_t)i));
-    }
-
-    {
-        std::lock_guard<std::mutex> lock(*runtime.lora_mutex);
-        for (const auto& entry : *runtime.lora_cache) {
-            available_loras.push_back({
-                {"name", entry.name},
-                {"path", entry.path},
-            });
-        }
-    }
-
-    available_upscalers.push_back({
-        {"name", "None"},
-    });
-    available_upscalers.push_back({
-        {"name", "Lanczos"},
-    });
-    available_upscalers.push_back({
-        {"name", "Nearest"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent (nearest)"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent (nearest-exact)"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent (antialiased)"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent (bicubic)"},
-    });
-    available_upscalers.push_back({
-        {"name", "Latent (bicubic antialiased)"},
-    });
-    {
-        std::lock_guard<std::mutex> lock(*runtime.upscaler_mutex);
-        for (const auto& entry : *runtime.upscaler_cache) {
-            available_upscalers.push_back({
-                {"name", entry.name},
-            });
-        }
-    }
-
-    if (supports_img) {
-        supported_modes.push_back("img_gen");
-    }
-    if (supports_vid) {
-        supported_modes.push_back("vid_gen");
-    }
-
-    std::string default_img_output_format = "png";
-    std::string default_vid_output_format = "avi";
-    if (!image_output_formats.empty()) {
-        default_img_output_format = image_output_formats[0].get<std::string>();
-    }
-    if (!video_output_formats.empty()) {
-        default_vid_output_format = video_output_formats[0].get<std::string>();
-    }
-
-    json defaults_by_mode       = json::object();
-    json output_formats_by_mode = json::object();
-    json features_by_mode       = json::object();
-    if (supports_img) {
-        defaults_by_mode["img_gen"]       = make_img_gen_defaults_json(defaults, default_img_output_format);
-        output_formats_by_mode["img_gen"] = image_output_formats;
-        features_by_mode["img_gen"]       = make_img_gen_features_json();
-    }
-    if (supports_vid) {
-        defaults_by_mode["vid_gen"]       = make_vid_gen_defaults_json(defaults, default_vid_output_format);
-        output_formats_by_mode["vid_gen"] = video_output_formats;
-        features_by_mode["vid_gen"]       = make_vid_gen_features_json();
-    }
-
-    json top_level_defaults       = json::object();
-    json top_level_output_formats = json::array();
-    json top_level_features       = {
-              {"cancel_queued", true},
-              {"cancel_generating", false},
-    };
-    std::string current_mode = "";
-    if (supports_img) {
-        current_mode             = "img_gen";
-        top_level_defaults       = defaults_by_mode["img_gen"];
-        top_level_output_formats = output_formats_by_mode["img_gen"];
-        top_level_features       = features_by_mode["img_gen"];
-    } else if (supports_vid) {
-        current_mode             = "vid_gen";
-        top_level_defaults       = defaults_by_mode["vid_gen"];
-        top_level_output_formats = output_formats_by_mode["vid_gen"];
-        top_level_features       = features_by_mode["vid_gen"];
-    }
-
-    json result;
-    result["model"] = {
-        {"name", model_path.filename().u8string()},
-        {"stem", model_path.stem().u8string()},
-        {"path", model_path.u8string()},
-    };
-    result["current_mode"]     = current_mode;
-    result["supported_modes"]  = supported_modes;
-    result["defaults"]         = top_level_defaults;
-    result["defaults_by_mode"] = defaults_by_mode;
-    result["limits"]           = {
-                  {"min_width", 64},
-                  {"max_width", 4096},
-                  {"min_height", 64},
-                  {"max_height", 4096},
-                  {"max_batch_count", 8},
-                  {"max_queue_size", manager.max_pending_jobs},
-    };
-    result["samplers"]               = samplers;
-    result["schedulers"]             = schedulers;
-    result["output_formats"]         = top_level_output_formats;
-    result["output_formats_by_mode"] = output_formats_by_mode;
-    result["features"]               = top_level_features;
-    result["features_by_mode"]       = features_by_mode;
-    result["loras"]                  = available_loras;
-    result["upscalers"]              = available_upscalers;
-    return result;
-}
-
-static bool parse_img_gen_request(const json& body,
-                                  ServerRuntime& runtime,
-                                  ImgGenJobRequest& request,
-                                  std::string& error_message) {
-    request.gen_params = *runtime.default_gen_params;
-
-    refresh_lora_cache(runtime);
-    if (!request.gen_params.from_json_str(body.dump(), [&](const std::string& path) {
-            return get_lora_full_path(runtime, path);
-        })) {
-        error_message = "invalid generation parameters";
-        return false;
-    }
-
-    std::string output_format = body.value("output_format", "png");
-    int output_compression    = body.value("output_compression", 100);
-    if (!assign_output_options(request, output_format, output_compression, true, error_message)) {
-        return false;
-    }
-    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
-    if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
-        error_message = "invalid generation parameters";
-        return false;
-    }
-    return true;
-}
-
-static bool parse_vid_gen_request(const json& body,
-                                  ServerRuntime& runtime,
-                                  VidGenJobRequest& request,
-                                  std::string& error_message) {
-    request.gen_params = *runtime.default_gen_params;
-
-    refresh_lora_cache(runtime);
-    if (!request.gen_params.from_json_str(body.dump(), [&](const std::string& path) {
-            return get_lora_full_path(runtime, path);
-        })) {
-        error_message = "invalid generation parameters";
-        return false;
-    }
-
-    std::string output_format = body.value("output_format", "webm");
-    int output_compression    = body.value("output_compression", 100);
-    if (!assign_output_options(request, output_format, output_compression, error_message)) {
-        return false;
-    }
-    // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
-    if (!request.gen_params.resolve_and_validate(VID_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
-        error_message = "invalid generation parameters";
-        return false;
-    }
-    return true;
-}
-
-void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
-    ServerRuntime* runtime = &rt;
-
-    svr.Get("/sdcpp/v1/capabilities", [runtime](const httplib::Request&, httplib::Response& res) {
-        res.status = 200;
-        res.set_content(make_capabilities_json(*runtime).dump(), "application/json");
-    });
-
-    svr.Post("/sdcpp/v1/img_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
-        try {
-            if (req.body.empty()) {
-                res.status = 400;
-                res.set_content(R"({"error":"empty body"})", "application/json");
-                return;
-            }
-            if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
-                res.status = 400;
-                res.set_content(json({{"error", unsupported_generation_mode_error(IMG_GEN)}}).dump(), "application/json");
-                return;
-            }
-
-            json body = json::parse(req.body);
-            ImgGenJobRequest request;
-            std::string error_message;
-            if (!parse_img_gen_request(body, *runtime, request, error_message)) {
-                res.status = 400;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            AsyncJobManager& manager                = *runtime->async_job_manager;
-            std::shared_ptr<AsyncGenerationJob> job = std::make_shared<AsyncGenerationJob>();
-            job->kind                               = AsyncJobKind::ImgGen;
-            job->status                             = AsyncJobStatus::Queued;
-            job->created_at                         = unix_timestamp_now();
-            job->img_gen                            = std::move(request);
-
-            {
-                std::lock_guard<std::mutex> lock(manager.mutex);
-                purge_expired_jobs(manager);
-                if (count_pending_jobs(manager) >= manager.max_pending_jobs) {
-                    res.status = 429;
-                    res.set_content(R"({"error":"job queue is full"})", "application/json");
-                    return;
-                }
-                job->id               = make_async_job_id(manager);
-                manager.jobs[job->id] = job;
-                manager.queue.push_back(job->id);
-            }
-
-            manager.cv.notify_one();
-
-            json out;
-            out["id"]       = job->id;
-            out["kind"]     = async_job_kind_name(job->kind);
-            out["status"]   = async_job_status_name(job->status);
-            out["created"]  = job->created_at;
-            out["poll_url"] = "/sdcpp/v1/jobs/" + job->id;
-
-            res.status = 202;
-            res.set_content(out.dump(), "application/json");
-        } catch (const json::parse_error& e) {
-            res.status = 400;
-            res.set_content(json({{"error", "invalid json"}, {"message", e.what()}}).dump(), "application/json");
-        } catch (const std::exception& e) {
-            res.status = 500;
-            res.set_content(json({{"error", "server_error"}, {"message", e.what()}}).dump(), "application/json");
-        }
-    });
-
-    svr.Post("/sdcpp/v1/vid_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
-        try {
-            if (req.body.empty()) {
-                res.status = 400;
-                res.set_content(R"({"error":"empty body"})", "application/json");
-                return;
-            }
-            if (!runtime_supports_generation_mode(*runtime, VID_GEN)) {
-                res.status = 400;
-                res.set_content(json({{"error", unsupported_generation_mode_error(VID_GEN)}}).dump(), "application/json");
-                return;
-            }
-
-            json body = json::parse(req.body);
-            VidGenJobRequest request;
-            std::string error_message;
-            if (!parse_vid_gen_request(body, *runtime, request, error_message)) {
-                res.status = 400;
-                res.set_content(json({{"error", error_message}}).dump(), "application/json");
-                return;
-            }
-
-            AsyncJobManager& manager                = *runtime->async_job_manager;
-            std::shared_ptr<AsyncGenerationJob> job = std::make_shared<AsyncGenerationJob>();
-            job->kind                               = AsyncJobKind::VidGen;
-            job->status                             = AsyncJobStatus::Queued;
-            job->created_at                         = unix_timestamp_now();
-            job->vid_gen                            = std::move(request);
-
-            {
-                std::lock_guard<std::mutex> lock(manager.mutex);
-                purge_expired_jobs(manager);
-                if (count_pending_jobs(manager) >= manager.max_pending_jobs) {
-                    res.status = 429;
-                    res.set_content(R"({"error":"job queue is full"})", "application/json");
-                    return;
-                }
-                job->id               = make_async_job_id(manager);
-                manager.jobs[job->id] = job;
-                manager.queue.push_back(job->id);
-            }
-
-            manager.cv.notify_one();
-
-            json out;
-            out["id"]       = job->id;
-            out["kind"]     = async_job_kind_name(job->kind);
-            out["status"]   = async_job_status_name(job->status);
-            out["created"]  = job->created_at;
-            out["poll_url"] = "/sdcpp/v1/jobs/" + job->id;
-
-            res.status = 202;
-            res.set_content(out.dump(), "application/json");
-        } catch (const json::parse_error& e) {
-            res.status = 400;
-            res.set_content(json({{"error", "invalid json"}, {"message", e.what()}}).dump(), "application/json");
-        } catch (const std::exception& e) {
-            res.status = 500;
-            res.set_content(json({{"error", "server_error"}, {"message", e.what()}}).dump(), "application/json");
-        }
-    });
-
-    svr.Get(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+))", [runtime](const httplib::Request& req, httplib::Response& res) {
-        AsyncJobManager& manager = *runtime->async_job_manager;
-        std::lock_guard<std::mutex> lock(manager.mutex);
-        purge_expired_jobs(manager);
-
-        std::string job_id = req.matches[1];
-        auto it            = manager.jobs.find(job_id);
-        if (it == manager.jobs.end()) {
-            if (manager.expired_jobs.find(job_id) != manager.expired_jobs.end()) {
-                res.status = 410;
-                res.set_content(R"({"error":"job expired"})", "application/json");
-            } else {
-                res.status = 404;
-                res.set_content(R"({"error":"job not found"})", "application/json");
-            }
-            return;
-        }
-
-        res.status = 200;
-        res.set_content(make_async_job_json(manager, *it->second).dump(), "application/json");
-    });
-
-    svr.Post(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+)/cancel)", [runtime](const httplib::Request& req, httplib::Response& res) {
-        AsyncJobManager& manager = *runtime->async_job_manager;
-        std::lock_guard<std::mutex> lock(manager.mutex);
-        purge_expired_jobs(manager);
-
-        std::string job_id = req.matches[1];
-        auto it            = manager.jobs.find(job_id);
-        if (it == manager.jobs.end()) {
-            if (manager.expired_jobs.find(job_id) != manager.expired_jobs.end()) {
-                res.status = 410;
-                res.set_content(R"({"error":"job expired"})", "application/json");
-            } else {
-                res.status = 404;
-                res.set_content(R"({"error":"job not found"})", "application/json");
-            }
-            return;
-        }
-
-        auto& job = *it->second;
-        if (job.status == AsyncJobStatus::Queued) {
-            if (!cancel_queued_job(manager, job)) {
-                res.status = 409;
-                res.set_content(R"({"error":"job queue state changed before cancellation"})", "application/json");
-                return;
-            }
-            res.status = 200;
-            res.set_content(make_async_job_json(manager, job).dump(), "application/json");
-            return;
-        }
-
-        if (job.status == AsyncJobStatus::Generating) {
-            res.status = 409;
-            res.set_content(R"({"error":"job is currently generating and cannot be interrupted yet"})", "application/json");
-            return;
-        }
-
-        res.status = 200;
-        res.set_content(make_async_job_json(manager, job).dump(), "application/json");
-    });
-}

examples/server/runtime.cpp

@@ -1,333 +0,0 @@
-#include "runtime.h"
-
-#include <algorithm>
-#include <cctype>
-#include <chrono>
-#include <cstdlib>
-#include <filesystem>
-#include <mutex>
-#include <regex>
-#include <sstream>
-
-#include "common/common.h"
-#include "common/log.h"
-
-namespace fs = std::filesystem;
-
-static std::string lower_ascii(std::string value) {
-    std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
-        return static_cast<char>(std::tolower(c));
-    });
-    return value;
-}
-
-static bool is_supported_model_ext(const fs::path& p) {
-    auto ext = lower_ascii(p.extension().string());
-    return ext == ".gguf" || ext == ".pt" || ext == ".pth" || ext == ".safetensors";
-}
-
-static const std::string k_base64_chars =
-    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-    "abcdefghijklmnopqrstuvwxyz"
-    "0123456789+/";
-
-std::string base64_encode(const std::vector<uint8_t>& bytes) {
-    std::string ret;
-    int val  = 0;
-    int valb = -6;
-    for (uint8_t c : bytes) {
-        val = (val << 8) + c;
-        valb += 8;
-        while (valb >= 0) {
-            ret.push_back(k_base64_chars[(val >> valb) & 0x3F]);
-            valb -= 6;
-        }
-    }
-    if (valb > -6) {
-        ret.push_back(k_base64_chars[((val << 8) >> (valb + 8)) & 0x3F]);
-    }
-    while (ret.size() % 4) {
-        ret.push_back('=');
-    }
-    return ret;
-}
-
-std::string normalize_output_format(std::string output_format) {
-    std::transform(output_format.begin(), output_format.end(), output_format.begin(),
-                   [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
-    return output_format;
-}
-
-std::vector<std::string> supported_img_output_formats(bool allow_webp) {
-    std::vector<std::string> formats = {"png", "jpeg"};
-#ifdef SD_USE_WEBP
-    if (allow_webp) {
-        formats.push_back("webp");
-    }
-#else
-    (void)allow_webp;
-#endif
-    return formats;
-}
-
-std::vector<std::string> supported_vid_output_formats() {
-    std::vector<std::string> formats;
-#ifdef SD_USE_WEBM
-    formats.push_back("webm");
-#endif
-#ifdef SD_USE_WEBP
-    formats.push_back("webp");
-#endif
-    formats.push_back("avi");
-    return formats;
-}
-
-static std::string valid_vid_output_formats_message() {
-    const std::vector<std::string> formats = supported_vid_output_formats();
-
-    std::string message = "invalid output_format, must be one of [";
-    for (size_t i = 0; i < formats.size(); ++i) {
-        if (i > 0) {
-            message += ", ";
-        }
-        message += formats[i];
-    }
-    message += "]";
-    return message;
-}
-
-bool assign_output_options(ImgGenJobRequest& request,
-                           std::string output_format,
-                           int output_compression,
-                           bool allow_webp,
-                           std::string& error_message) {
-    request.output_format      = normalize_output_format(std::move(output_format));
-    request.output_compression = std::clamp(output_compression, 0, 100);
-
-    const std::vector<std::string> valid_formats = supported_img_output_formats(allow_webp);
-    const bool valid_format                      = std::find(valid_formats.begin(),
-                                                             valid_formats.end(),
-                                                             request.output_format) != valid_formats.end();
-    if (!valid_format) {
-        error_message = "invalid output_format, must be one of [";
-        for (size_t i = 0; i < valid_formats.size(); ++i) {
-            if (i > 0) {
-                error_message += ", ";
-            }
-            error_message += valid_formats[i];
-        }
-        error_message += "]";
-        return false;
-    }
-
-    return true;
-}
-
-bool assign_output_options(VidGenJobRequest& request,
-                           std::string output_format,
-                           int output_compression,
-                           std::string& error_message) {
-    request.output_format      = normalize_output_format(std::move(output_format));
-    request.output_compression = std::clamp(output_compression, 0, 100);
-
-    if (request.output_format == "avi") {
-        return true;
-    }
-
-    if (request.output_format == "webm") {
-#ifdef SD_USE_WEBM
-        return true;
-#else
-        error_message = valid_vid_output_formats_message();
-        return false;
-#endif
-    }
-
-    if (request.output_format == "webp") {
-#ifdef SD_USE_WEBP
-        return true;
-#else
-        error_message = valid_vid_output_formats_message();
-        return false;
-#endif
-    }
-
-    error_message = valid_vid_output_formats_message();
-    return false;
-}
-
-std::string video_mime_type(const std::string& output_format) {
-    if (output_format == "webm") {
-        return "video/webm";
-    }
-    if (output_format == "webp") {
-        return "image/webp";
-    }
-    return "video/x-msvideo";
-}
-
-bool runtime_supports_generation_mode(const ServerRuntime& runtime, SDMode mode) {
-    if (mode == VID_GEN) {
-        return sd_ctx_supports_video_generation(runtime.sd_ctx);
-    }
-    if (mode == IMG_GEN) {
-        return sd_ctx_supports_image_generation(runtime.sd_ctx);
-    }
-    return true;
-}
-
-std::string unsupported_generation_mode_error(SDMode mode) {
-    if (mode == VID_GEN) {
-        return "loaded model does not support vid_gen";
-    }
-    if (mode == IMG_GEN) {
-        return "loaded model does not support img_gen";
-    }
-    return "loaded model does not support requested mode";
-}
-
-ArgOptions SDSvrParams::get_options() {
-    ArgOptions options;
-
-    options.string_options = {
-        {"-l", "--listen-ip", "server listen ip (default: 127.0.0.1)", &listen_ip},
-        {"", "--serve-html-path", "path to HTML file to serve at root (optional)", &serve_html_path},
-    };
-
-    options.int_options = {
-        {"", "--listen-port", "server listen port (default: 1234)", &listen_port},
-    };
-
-    options.bool_options = {
-        {"-v", "--verbose", "print extra info", true, &verbose},
-        {"", "--color", "colors the logging tags according to level", true, &color},
-    };
-
-    auto on_help_arg = [&](int, const char**, int, bool& valid) {
-        normal_exit = true;
-        valid       = true;
-        return -1;
-    };
-
-    options.manual_options = {
-        {"-h", "--help", "show this help message and exit", on_help_arg},
-    };
-    return options;
-}
-
-bool SDSvrParams::validate() {
-    if (listen_ip.empty()) {
-        LOG_ERROR("error: the following arguments are required: listen_ip");
-        return false;
-    }
-
-    if (listen_port < 0 || listen_port > 65535) {
-        LOG_ERROR("error: listen_port should be in the range [0, 65535]");
-        return false;
-    }
-
-    if (!serve_html_path.empty() && !fs::exists(serve_html_path)) {
-        LOG_ERROR("error: serve_html_path file does not exist: %s", serve_html_path.c_str());
-        return false;
-    }
-    return true;
-}
-
-bool SDSvrParams::resolve_and_validate() {
-    if (!validate()) {
-        return false;
-    }
-    return true;
-}
-
-std::string SDSvrParams::to_string() const {
-    std::ostringstream oss;
-    oss << "SDSvrParams {\n"
-        << "  listen_ip: " << listen_ip << ",\n"
-        << "  listen_port: \"" << listen_port << "\",\n"
-        << "  serve_html_path: \"" << serve_html_path << "\",\n"
-        << "}";
-    return oss.str();
-}
-
-void refresh_lora_cache(ServerRuntime& rt) {
-    std::vector<LoraEntry> new_cache;
-
-    fs::path lora_dir = rt.ctx_params->lora_model_dir;
-    if (fs::exists(lora_dir) && fs::is_directory(lora_dir)) {
-        for (auto& entry : fs::recursive_directory_iterator(lora_dir, fs::directory_options::skip_permission_denied)) {
-            if (!entry.is_regular_file()) {
-                continue;
-            }
-            const fs::path& p = entry.path();
-            if (!is_supported_model_ext(p)) {
-                continue;
-            }
-
-            LoraEntry lora_entry;
-            lora_entry.name     = p.stem().u8string();
-            lora_entry.fullpath = p.u8string();
-            std::string rel     = p.lexically_relative(lora_dir).u8string();
-            std::replace(rel.begin(), rel.end(), '\\', '/');
-            lora_entry.path = rel;
-
-            new_cache.push_back(std::move(lora_entry));
-        }
-    }
-
-    std::sort(new_cache.begin(), new_cache.end(), [](const LoraEntry& a, const LoraEntry& b) {
-        return a.path < b.path;
-    });
-
-    {
-        std::lock_guard<std::mutex> lock(*rt.lora_mutex);
-        *rt.lora_cache = std::move(new_cache);
-    }
-}
-
-std::string get_lora_full_path(ServerRuntime& rt, const std::string& path) {
-    std::lock_guard<std::mutex> lock(*rt.lora_mutex);
-    auto it = std::find_if(rt.lora_cache->begin(), rt.lora_cache->end(),
-                           [&](const LoraEntry& entry) { return entry.path == path; });
-    return it != rt.lora_cache->end() ? it->fullpath : "";
-}
-
-void refresh_upscaler_cache(ServerRuntime& rt) {
-    std::vector<UpscalerEntry> new_cache;
-
-    fs::path upscaler_dir = rt.ctx_params->hires_upscalers_dir;
-    if (fs::exists(upscaler_dir) && fs::is_directory(upscaler_dir)) {
-        for (auto& entry : fs::directory_iterator(upscaler_dir)) {
-            if (!entry.is_regular_file()) {
-                continue;
-            }
-            const fs::path& p = entry.path();
-            if (!is_supported_model_ext(p)) {
-                continue;
-            }
-
-            UpscalerEntry upscaler_entry;
-            upscaler_entry.name       = p.stem().u8string();
-            upscaler_entry.fullpath   = fs::absolute(p).lexically_normal().u8string();
-            upscaler_entry.model_name = "ESRGAN_4x";
-            upscaler_entry.path       = p.filename().u8string();
-
-            new_cache.push_back(std::move(upscaler_entry));
-        }
-    }
-
-    std::sort(new_cache.begin(), new_cache.end(), [](const UpscalerEntry& a, const UpscalerEntry& b) {
-        return a.name < b.name;
-    });
-
-    {
-        std::lock_guard<std::mutex> lock(*rt.upscaler_mutex);
-        *rt.upscaler_cache = std::move(new_cache);
-    }
-}
-
-int64_t unix_timestamp_now() {
-    return std::chrono::duration_cast<std::chrono::seconds>(
-               std::chrono::system_clock::now().time_since_epoch())
-        .count();
-}

examples/server/runtime.h

@@ -1,100 +0,0 @@
-#pragma once
-
-#include <algorithm>
-#include <cstdint>
-#include <mutex>
-#include <string>
-#include <vector>
-
-#include <json.hpp>
-#include "common/common.h"
-#include "common/resource_owners.hpp"
-#include "stable-diffusion.h"
-
-using json = nlohmann::json;
-
-struct ArgOptions;
-struct SDContextParams;
-struct AsyncJobManager;
-
-struct SDSvrParams {
-    std::string listen_ip = "127.0.0.1";
-    int listen_port       = 1234;
-    std::string serve_html_path;
-    bool normal_exit = false;
-    bool verbose     = false;
-    bool color       = false;
-
-    ArgOptions get_options();
-    bool validate();
-    bool resolve_and_validate();
-    std::string to_string() const;
-};
-
-struct LoraEntry {
-    std::string name;
-    std::string path;
-    std::string fullpath;
-};
-
-struct UpscalerEntry {
-    std::string name;
-    std::string path;
-    std::string fullpath;
-    std::string model_name;
-    int scale = 4;
-};
-
-struct ServerRuntime {
-    sd_ctx_t* sd_ctx;
-    std::mutex* sd_ctx_mutex;
-    const SDSvrParams* svr_params;
-    const SDContextParams* ctx_params;
-    const SDGenerationParams* default_gen_params;
-    std::vector<LoraEntry>* lora_cache;
-    std::mutex* lora_mutex;
-    std::vector<UpscalerEntry>* upscaler_cache;
-    std::mutex* upscaler_mutex;
-    AsyncJobManager* async_job_manager;
-};
-
-struct ImgGenJobRequest {
-    SDGenerationParams gen_params;
-    std::string output_format = "png";
-    int output_compression    = 100;
-
-    sd_img_gen_params_t to_sd_img_gen_params_t() {
-        return gen_params.to_sd_img_gen_params_t();
-    }
-};
-
-struct VidGenJobRequest {
-    SDGenerationParams gen_params;
-    std::string output_format = "webm";
-    int output_compression    = 100;
-
-    sd_vid_gen_params_t to_sd_vid_gen_params_t() {
-        return gen_params.to_sd_vid_gen_params_t();
-    }
-};
-
-std::string base64_encode(const std::vector<uint8_t>& bytes);
-std::string normalize_output_format(std::string output_format);
-std::vector<std::string> supported_img_output_formats(bool allow_webp = true);
-std::vector<std::string> supported_vid_output_formats();
-bool assign_output_options(ImgGenJobRequest& request,
-                           std::string output_format,
-                           int output_compression,
-                           bool allow_webp,
-                           std::string& error_message);
-bool assign_output_options(VidGenJobRequest& request,
-                           std::string output_format,
-                           int output_compression,
-                           std::string& error_message);
-std::string video_mime_type(const std::string& output_format);
-bool runtime_supports_generation_mode(const ServerRuntime& runtime, SDMode mode);
-std::string unsupported_generation_mode_error(SDMode mode);
-void refresh_lora_cache(ServerRuntime& rt);
-std::string get_lora_full_path(ServerRuntime& rt, const std::string& path);
-void refresh_upscaler_cache(ServerRuntime& rt);
-int64_t unix_timestamp_now();

format-code.sh

@@ -1,13 +1,4 @@
-for f in src/*.cpp src/*.h src/*.hpp \
+for f in src/*.cpp src/*.h src/*.hpp src/vocab/*.h src/vocab/*.cpp examples/cli/*.cpp examples/common/*.hpp examples/cli/*.h examples/server/*.cpp; do
-         src/conditioning/*.cpp src/conditioning/*.h src/conditioning/*.hpp \
-         src/core/*.cpp src/core/*.h src/core/*.hpp \
-         src/extensions/*.cpp src/extensions/*.h src/extensions/*.hpp \
-         src/runtime/*.cpp src/runtime/*.h src/runtime/*.hpp \
-         src/model/*/*.cpp src/model/*/*.h src/model/*/*.hpp \
-         src/tokenizers/*.h src/tokenizers/*.cpp src/tokenizers/vocab/*.h src/tokenizers/vocab/*.cpp \
-         src/model_io/*.h src/model_io/*.cpp examples/cli/*.cpp examples/cli/*.h examples/server/*.cpp \
-         examples/common/*.hpp examples/common/*.h examples/common/*.cpp; do
-  [[ -e "$f" ]] || continue
   [[ "$f" == vocab* ]] && continue
   echo "formatting '$f'"
   # if [ "$f" != "stable-diffusion.h" ]; then

ggml

@@ -1 +1 @@
-Subproject commit 0ce7ad348a3151e1da9f65d962044546bcaad421
+Subproject commit 404fcb9d7c96989569e68c9e7881ee3465a05c50

include/stable-diffusion.h

@@ -50,10 +50,6 @@ enum sample_method_t {
     TCD_SAMPLE_METHOD,
     RES_MULTISTEP_SAMPLE_METHOD,
     RES_2S_SAMPLE_METHOD,
-    ER_SDE_SAMPLE_METHOD,
-    EULER_CFG_PP_SAMPLE_METHOD,
-    EULER_A_CFG_PP_SAMPLE_METHOD,
-    EULER_GE_SAMPLE_METHOD,
     SAMPLE_METHOD_COUNT
 };
 
@@ -69,7 +65,6 @@ enum scheduler_t {
     KL_OPTIMAL_SCHEDULER,
     LCM_SCHEDULER,
     BONG_TANGENT_SCHEDULER,
-    LTX2_SCHEDULER,
     SCHEDULER_COUNT
 };
 
@@ -126,8 +121,7 @@ enum sd_type_t {
     // SD_TYPE_IQ4_NL_8_8 = 38,
     SD_TYPE_MXFP4 = 39,  // MXFP4 (1 block)
     SD_TYPE_NVFP4 = 40,  // NVFP4 (4 blocks, E4M3 scale)
-    SD_TYPE_Q1_0  = 41,
+    SD_TYPE_COUNT = 41,
-    SD_TYPE_COUNT = 42,
 };
 
 enum sd_log_level_t {
@@ -154,13 +148,11 @@ enum lora_apply_mode_t {
 
 typedef struct {
     bool enabled;
-    bool temporal_tiling;
     int tile_size_x;
     int tile_size_y;
     float target_overlap;
     float rel_size_x;
     float rel_size_y;
-    const char* extra_tiling_args;
 } sd_tiling_params_t;
 
 typedef struct {
@@ -168,14 +160,6 @@ typedef struct {
     const char* path;
 } sd_embedding_t;
 
-enum sd_vae_format_t {
-    SD_VAE_FORMAT_AUTO = -1,
-    SD_VAE_FORMAT_FLUX,
-    SD_VAE_FORMAT_SD3,
-    SD_VAE_FORMAT_FLUX2,
-    SD_VAE_FORMAT_COUNT,
-};
-
 typedef struct {
     const char* model_path;
     const char* clip_l_path;
@@ -186,10 +170,7 @@ typedef struct {
     const char* llm_vision_path;
     const char* diffusion_model_path;
     const char* high_noise_diffusion_model_path;
-    const char* uncond_diffusion_model_path;
-    const char* embeddings_connectors_path;
     const char* vae_path;
-    const char* audio_vae_path;
     const char* taesd_path;
     const char* control_net_path;
     const sd_embedding_t* embeddings;
@@ -221,20 +202,8 @@ typedef struct {
     bool chroma_use_t5_mask;
     int chroma_t5_mask_pad;
     bool qwen_image_zero_cond_t;
-    enum sd_vae_format_t vae_format;
-    float max_vram;  // GiB budget for graph-cut segmented param offload (0 = disabled, -1 = auto free VRAM minus 1 GiB)
-    bool stream_layers;  // Enable residency+prefetch streaming on top of --max-vram (no effect without --max-vram)
-    const char* backend;
-    const char* params_backend;
 } sd_ctx_params_t;
 
-typedef struct {
-    uint32_t sample_rate;
-    uint32_t channels;
-    uint64_t sample_count;
-    float* data;
-} sd_audio_t;
-
 typedef struct {
     uint32_t width;
     uint32_t height;
@@ -267,7 +236,6 @@ typedef struct {
     float* custom_sigmas;
     int custom_sigmas_count;
     float flow_shift;
-    const char* extra_sample_args;
 } sd_sample_params_t;
 
 typedef struct {
@@ -320,34 +288,6 @@ typedef struct {
     const char* path;
 } sd_lora_t;
 
-enum sd_hires_upscaler_t {
-    SD_HIRES_UPSCALER_NONE,
-    SD_HIRES_UPSCALER_LATENT,
-    SD_HIRES_UPSCALER_LATENT_NEAREST,
-    SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT,
-    SD_HIRES_UPSCALER_LATENT_ANTIALIASED,
-    SD_HIRES_UPSCALER_LATENT_BICUBIC,
-    SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED,
-    SD_HIRES_UPSCALER_LANCZOS,
-    SD_HIRES_UPSCALER_NEAREST,
-    SD_HIRES_UPSCALER_MODEL,
-    SD_HIRES_UPSCALER_COUNT,
-};
-
-typedef struct {
-    bool enabled;
-    enum sd_hires_upscaler_t upscaler;
-    const char* model_path;
-    float scale;
-    int target_width;
-    int target_height;
-    int steps;
-    float denoising_strength;
-    int upscale_tile_size;
-    float* custom_sigmas;
-    int custom_sigmas_count;
-} sd_hires_params_t;
-
 typedef struct {
     const sd_lora_t* loras;
     uint32_t lora_count;
@@ -371,7 +311,6 @@ typedef struct {
     sd_pm_params_t pm_params;
     sd_tiling_params_t vae_tiling_params;
     sd_cache_params_t cache;
-    sd_hires_params_t hires;
 } sd_img_gen_params_t;
 
 typedef struct {
@@ -392,11 +331,9 @@ typedef struct {
     float strength;
     int64_t seed;
     int video_frames;
-    int fps;
     float vace_strength;
     sd_tiling_params_t vae_tiling_params;
     sd_cache_params_t cache;
-    sd_hires_params_t hires;
 } sd_vid_gen_params_t;
 
 typedef struct sd_ctx_t sd_ctx_t;
@@ -410,8 +347,6 @@ SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
 SD_API void sd_set_preview_callback(sd_preview_cb_t cb, enum preview_t mode, int interval, bool denoised, bool noisy, void* data);
 SD_API int32_t sd_get_num_physical_cores();
 SD_API const char* sd_get_system_info();
-SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx);
-SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx);
 
 SD_API const char* sd_type_name(enum sd_type_t type);
 SD_API enum sd_type_t str_to_sd_type(const char* str);
@@ -427,18 +362,14 @@ SD_API const char* sd_preview_name(enum preview_t preview);
 SD_API enum preview_t str_to_preview(const char* str);
 SD_API const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode);
 SD_API enum lora_apply_mode_t str_to_lora_apply_mode(const char* str);
-SD_API const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler);
-SD_API enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str);
 
 SD_API void sd_cache_params_init(sd_cache_params_t* cache_params);
-SD_API void sd_hires_params_init(sd_hires_params_t* hires_params);
 
 SD_API void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params);
 SD_API char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params);
 
 SD_API sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params);
 SD_API void free_sd_ctx(sd_ctx_t* sd_ctx);
-SD_API void free_sd_audio(sd_audio_t* audio);
 
 SD_API void sd_sample_params_init(sd_sample_params_t* sample_params);
 SD_API char* sd_sample_params_to_str(const sd_sample_params_t* sample_params);
@@ -451,11 +382,7 @@ SD_API char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_para
 SD_API sd_image_t* generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* sd_img_gen_params);
 
 SD_API void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params);
-SD_API bool generate_video(sd_ctx_t* sd_ctx,
+SD_API sd_image_t* generate_video(sd_ctx_t* sd_ctx, const sd_vid_gen_params_t* sd_vid_gen_params, int* num_frames_out);
-                           const sd_vid_gen_params_t* sd_vid_gen_params,
-                           sd_image_t** frames_out,
-                           int* num_frames_out,
-                           sd_audio_t** audio_out);
 
 typedef struct upscaler_ctx_t upscaler_ctx_t;
 
@@ -463,9 +390,7 @@ SD_API upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path,
                                         bool offload_params_to_cpu,
                                         bool direct,
                                         int n_threads,
-                                        int tile_size,
+                                        int tile_size);
-                                        const char* backend,
-                                        const char* params_backend);
 SD_API void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx);
 
 SD_API sd_image_t upscale(upscaler_ctx_t* upscaler_ctx,

script/convert_fp8_scale_to_bf16.py

@@ -1,283 +0,0 @@
-#!/usr/bin/env python
-import argparse
-import json
-import math
-import os
-import struct
-from collections import Counter
-from pathlib import Path
-
-import torch
-from safetensors import safe_open
-
-
-FLOAT_DTYPES = {
-    "BF16",
-    "F16",
-    "F32",
-    "F64",
-    "F8_E4M3",
-    "F8_E4M3FN",
-    "F8_E5M2",
-}
-
-FP8_DTYPES = {
-    "F8_E4M3",
-    "F8_E4M3FN",
-    "F8_E5M2",
-}
-
-DTYPE_SIZES = {
-    "BOOL": 1,
-    "U8": 1,
-    "I8": 1,
-    "F8_E4M3": 1,
-    "F8_E4M3FN": 1,
-    "F8_E5M2": 1,
-    "U16": 2,
-    "I16": 2,
-    "F16": 2,
-    "BF16": 2,
-    "U32": 4,
-    "I32": 4,
-    "F32": 4,
-    "U64": 8,
-    "I64": 8,
-    "F64": 8,
-}
-
-
-def read_safetensors_header(path: Path):
-    with path.open("rb") as f:
-        header_len = struct.unpack("<Q", f.read(8))[0]
-        header = f.read(header_len).decode("utf-8").rstrip()
-    return json.loads(header)
-
-
-def numel(shape):
-    return math.prod(shape) if shape else 1
-
-
-def scale_key_for_weight(name: str):
-    if name.endswith(".weight"):
-        return name[:-len(".weight")] + ".weight_scale"
-    if name.endswith("weight"):
-        return name + "_scale"
-    return None
-
-
-def tensor_nbytes(dtype: str, shape):
-    return numel(shape) * DTYPE_SIZES[dtype]
-
-
-def build_output_plan(header):
-    entries = {k: v for k, v in header.items() if k != "__metadata__"}
-    paired_scale_keys = set()
-    plan = []
-
-    for name, info in entries.items():
-        scale_key = scale_key_for_weight(name)
-        if info["dtype"] in FP8_DTYPES and scale_key in entries:
-            paired_scale_keys.add(scale_key)
-
-    for name, info in entries.items():
-        if name in paired_scale_keys:
-            continue
-
-        dtype = info["dtype"]
-        shape = info["shape"]
-        scale_key = scale_key_for_weight(name)
-
-        if dtype in FP8_DTYPES and scale_key in entries:
-            scale_info = entries[scale_key]
-            plan.append(
-                {
-                    "name": name,
-                    "source_dtype": dtype,
-                    "output_dtype": "BF16",
-                    "shape": shape,
-                    "mode": "fp8_scaled_weight",
-                    "scale_key": scale_key,
-                }
-            )
-            continue
-
-        if dtype in FLOAT_DTYPES:
-            plan.append(
-                {
-                    "name": name,
-                    "source_dtype": dtype,
-                    "output_dtype": "BF16",
-                    "shape": shape,
-                    "mode": "float_to_bf16",
-                }
-            )
-        else:
-            plan.append(
-                {
-                    "name": name,
-                    "source_dtype": dtype,
-                    "output_dtype": dtype,
-                    "shape": shape,
-                    "mode": "copy",
-                }
-            )
-
-    metadata = dict(header.get("__metadata__", {}) or {})
-    metadata["format"] = "pt"
-    metadata["conversion"] = "fp8_weight_scale_to_bf16"
-
-    output_header = {"__metadata__": metadata}
-    offset = 0
-    for item in plan:
-        size = tensor_nbytes(item["output_dtype"], item["shape"])
-        output_header[item["name"]] = {
-            "dtype": item["output_dtype"],
-            "shape": item["shape"],
-            "data_offsets": [offset, offset + size],
-        }
-        offset += size
-
-    return plan, output_header, offset
-
-
-def write_tensor_bytes(out, tensor):
-    tensor = tensor.detach().cpu().contiguous()
-    if tensor.numel() == 0:
-        return
-    if tensor.dtype == torch.bfloat16:
-        tensor.view(torch.uint16).numpy().tofile(out)
-    elif tensor.dtype in (getattr(torch, "float8_e4m3fn", None), getattr(torch, "float8_e5m2", None)):
-        tensor.view(torch.uint8).numpy().tofile(out)
-    else:
-        tensor.numpy().tofile(out)
-
-
-def scale_view_for_chunk(scale, chunk, first_dim_start=0, first_dim_end=None):
-    scale = scale.to(torch.float32)
-
-    if scale.numel() == 1:
-        return scale.reshape((1,) * chunk.ndim)
-
-    if chunk.ndim > 0 and scale.ndim == 1:
-        if first_dim_end is not None and scale.shape[0] >= first_dim_end:
-            scale = scale[first_dim_start:first_dim_end]
-        if scale.shape[0] == chunk.shape[0]:
-            return scale.reshape((scale.shape[0],) + (1,) * (chunk.ndim - 1))
-
-    return scale
-
-
-def write_scaled_fp8_weight(out, weight, scale, chunk_rows):
-    if weight.ndim == 0:
-        result = weight.to(torch.float32) * scale_view_for_chunk(scale, weight)
-        write_tensor_bytes(out, result.to(torch.bfloat16))
-        return
-
-    rows = weight.shape[0]
-    for start in range(0, rows, chunk_rows):
-        end = min(start + chunk_rows, rows)
-        chunk = weight[start:end].to(torch.float32)
-        scale_view = scale_view_for_chunk(scale, chunk, start, end)
-        result = chunk * scale_view
-        write_tensor_bytes(out, result.to(torch.bfloat16))
-
-
-def write_float_as_bf16(out, tensor, chunk_rows):
-    if tensor.dtype == torch.bfloat16:
-        write_tensor_bytes(out, tensor)
-        return
-
-    if tensor.ndim == 0:
-        write_tensor_bytes(out, tensor.to(torch.bfloat16))
-        return
-
-    rows = tensor.shape[0]
-    for start in range(0, rows, chunk_rows):
-        end = min(start + chunk_rows, rows)
-        write_tensor_bytes(out, tensor[start:end].to(torch.bfloat16))
-
-
-def convert(input_path: Path, output_path: Path, chunk_rows: int, dry_run: bool):
-    header = read_safetensors_header(input_path)
-    plan, output_header, data_size = build_output_plan(header)
-
-    source_counts = Counter(item["source_dtype"] for item in plan)
-    output_counts = Counter(item["output_dtype"] for item in plan)
-    scaled_count = sum(item["mode"] == "fp8_scaled_weight" for item in plan)
-    dropped_scales = sum(item["mode"] == "fp8_scaled_weight" for item in plan)
-    header_bytes = json.dumps(output_header, separators=(",", ":")).encode("utf-8")
-    expected_size = 8 + len(header_bytes) + data_size
-
-    print(f"input:  {input_path}")
-    print(f"output: {output_path}")
-    print(f"tensors written: {len(plan)}")
-    print(f"scaled fp8 weights dequantized: {scaled_count}")
-    print(f"weight_scale tensors dropped: {dropped_scales}")
-    print(f"source dtypes: {dict(sorted(source_counts.items()))}")
-    print(f"output dtypes: {dict(sorted(output_counts.items()))}")
-    print(f"expected output size: {expected_size / (1024 ** 3):.2f} GiB")
-
-    if dry_run:
-        return
-
-    if output_path.exists():
-        raise FileExistsError(f"{output_path} already exists; pass --overwrite to replace it")
-
-    tmp_path = output_path.with_suffix(output_path.suffix + ".tmp")
-    if tmp_path.exists():
-        raise FileExistsError(f"{tmp_path} already exists; remove it or choose another output")
-
-    with safe_open(str(input_path), framework="pt", device="cpu") as sf, tmp_path.open("wb") as out:
-        out.write(struct.pack("<Q", len(header_bytes)))
-        out.write(header_bytes)
-
-        for index, item in enumerate(plan, 1):
-            name = item["name"]
-            print(f"[{index:04d}/{len(plan):04d}] {name} -> {item['output_dtype']}")
-
-            tensor = sf.get_tensor(name)
-            if item["mode"] == "fp8_scaled_weight":
-                scale = sf.get_tensor(item["scale_key"])
-                write_scaled_fp8_weight(out, tensor, scale, chunk_rows)
-            elif item["mode"] == "float_to_bf16":
-                write_float_as_bf16(out, tensor, chunk_rows)
-            else:
-                write_tensor_bytes(out, tensor)
-
-        actual_size = out.tell()
-
-    if actual_size != expected_size:
-        tmp_path.unlink(missing_ok=True)
-        raise RuntimeError(f"wrote {actual_size} bytes, expected {expected_size} bytes")
-
-    tmp_path.replace(output_path)
-    print("done")
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Convert an fp8 safetensors checkpoint with weight_scale tensors to bf16."
-    )
-    parser.add_argument("--input", default="ideogram4_fp8.safetensors", type=Path)
-    parser.add_argument("--output", default="ideogram4_bf16.safetensors", type=Path)
-    parser.add_argument("--chunk-rows", default=1024, type=int)
-    parser.add_argument("--dry-run", action="store_true")
-    parser.add_argument("--overwrite", action="store_true")
-    args = parser.parse_args()
-
-    input_path = args.input.resolve()
-    output_path = args.output.resolve()
-
-    if args.chunk_rows < 1:
-        raise ValueError("--chunk-rows must be >= 1")
-    if not input_path.exists():
-        raise FileNotFoundError(input_path)
-    if args.overwrite and output_path.exists():
-        output_path.unlink()
-
-    convert(input_path, output_path, args.chunk_rows, args.dry_run)
-
-
-if __name__ == "__main__":
-    main()

src/anima.hpp

@@ -1,61 +1,18 @@
-#ifndef __SD_MODEL_DIFFUSION_ANIMA_HPP__
+#ifndef __ANIMA_HPP__
-#define __SD_MODEL_DIFFUSION_ANIMA_HPP__
+#define __ANIMA_HPP__
 
-#include <algorithm>
 #include <cmath>
 #include <memory>
 #include <utility>
 #include <vector>
 
-#include "model/common/block.hpp"
+#include "common_block.hpp"
-#include "model/common/rope.hpp"
+#include "flux.hpp"
-#include "model/diffusion/flux.hpp"
+#include "rope.hpp"
-#include "model/diffusion/model.hpp"
 
 namespace Anima {
     constexpr int ANIMA_GRAPH_SIZE = 65536;
 
-    struct AnimaConfig {
-        int64_t in_channels       = 16;
-        int64_t out_channels      = 16;
-        int64_t hidden_size       = 2048;
-        int64_t text_embed_dim    = 1024;
-        int64_t num_heads         = 16;
-        int64_t head_dim          = 128;
-        int patch_size            = 2;
-        int64_t num_layers        = 28;
-        std::vector<int> axes_dim = {44, 42, 42};
-        int theta                 = 10000;
-
-        static AnimaConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, const std::string& prefix) {
-            AnimaConfig config;
-            int64_t detected_layers = 0;
-            std::string layer_tag   = prefix.empty() ? "blocks." : prefix + ".blocks.";
-            for (const auto& [name, _] : tensor_storage_map) {
-                size_t pos = name.find(layer_tag);
-                if (pos == std::string::npos) {
-                    continue;
-                }
-                size_t start = pos + layer_tag.size();
-                size_t end   = name.find('.', start);
-                if (end == std::string::npos) {
-                    continue;
-                }
-                int64_t layer_id = atoll(name.substr(start, end - start).c_str());
-                detected_layers  = std::max(detected_layers, layer_id + 1);
-            }
-            if (detected_layers > 0) {
-                config.num_layers = detected_layers;
-                LOG_DEBUG("anima: num_layers = %" PRId64 ", hidden_size = %" PRId64 ", num_heads = %" PRId64 ", head_dim = %" PRId64,
-                          config.num_layers,
-                          config.hidden_size,
-                          config.num_heads,
-                          config.head_dim);
-            }
-            return config;
-        }
-    };
-
     __STATIC_INLINE__ ggml_tensor* apply_gate(ggml_context* ctx,
                                               ggml_tensor* x,
                                               ggml_tensor* gate) {
@@ -460,22 +417,31 @@ namespace Anima {
 
     struct AnimaNet : public GGMLBlock {
     public:
-        AnimaConfig config;
+        int64_t in_channels       = 16;
+        int64_t out_channels      = 16;
+        int64_t hidden_size       = 2048;
+        int64_t text_embed_dim    = 1024;
+        int64_t num_heads         = 16;
+        int64_t head_dim          = 128;
+        int patch_size            = 2;
+        int64_t num_layers        = 28;
+        std::vector<int> axes_dim = {44, 42, 42};
+        int theta                 = 10000;
 
     public:
         AnimaNet() = default;
-        explicit AnimaNet(AnimaConfig config)
+        explicit AnimaNet(int64_t num_layers)
-            : config(config) {
+            : num_layers(num_layers) {
-            blocks["x_embedder"]       = std::make_shared<XEmbedder>((config.in_channels + 1) * config.patch_size * config.patch_size, config.hidden_size);
+            blocks["x_embedder"]       = std::make_shared<XEmbedder>((in_channels + 1) * patch_size * patch_size, hidden_size);
-            blocks["t_embedder"]       = std::make_shared<TimestepEmbedder>(config.hidden_size, config.hidden_size * 3);
+            blocks["t_embedder"]       = std::make_shared<TimestepEmbedder>(hidden_size, hidden_size * 3);
-            blocks["t_embedding_norm"] = std::make_shared<RMSNorm>(config.hidden_size, 1e-6f);
+            blocks["t_embedding_norm"] = std::make_shared<RMSNorm>(hidden_size, 1e-6f);
-            for (int i = 0; i < config.num_layers; i++) {
+            for (int i = 0; i < num_layers; i++) {
-                blocks["blocks." + std::to_string(i)] = std::make_shared<TransformerBlock>(config.hidden_size,
+                blocks["blocks." + std::to_string(i)] = std::make_shared<TransformerBlock>(hidden_size,
-                                                                                           config.text_embed_dim,
+                                                                                           text_embed_dim,
-                                                                                           config.num_heads,
+                                                                                           num_heads,
-                                                                                           config.head_dim);
+                                                                                           head_dim);
             }
-            blocks["final_layer"] = std::make_shared<FinalLayer>(config.hidden_size, config.patch_size, config.out_channels);
+            blocks["final_layer"] = std::make_shared<FinalLayer>(hidden_size, patch_size, out_channels);
             blocks["llm_adapter"] = std::make_shared<LLMAdapter>(1024, 1024, 1024, 6, 16);
         }
 
@@ -502,11 +468,11 @@ namespace Anima {
             auto padding_mask = ggml_ext_zeros(ctx->ggml_ctx, x->ne[0], x->ne[1], 1, x->ne[3]);
             x                 = ggml_concat(ctx->ggml_ctx, x, padding_mask, 2);  // [N, C + 1, H, W]
 
-            x = DiT::pad_and_patchify(ctx, x, config.patch_size, config.patch_size);  // [N, h*w, (C+1)*ph*pw]
+            x = DiT::pad_and_patchify(ctx, x, patch_size, patch_size);  // [N, h*w, (C+1)*ph*pw]
 
             x = x_embedder->forward(ctx, x);
 
-            auto timestep_proj     = ggml_ext_timestep_embedding(ctx->ggml_ctx, timestep, static_cast<int>(config.hidden_size));
+            auto timestep_proj     = ggml_ext_timestep_embedding(ctx->ggml_ctx, timestep, static_cast<int>(hidden_size));
             auto temb              = t_embedder->forward(ctx, timestep_proj);
             auto embedded_timestep = t_embedding_norm->forward(ctx, timestep_proj);
 
@@ -533,40 +499,53 @@ namespace Anima {
                 encoder_hidden_states = adapted_context;
             }
 
-            sd::ggml_graph_cut::mark_graph_cut(x, "anima.prelude", "x");
+            for (int i = 0; i < num_layers; i++) {
-            sd::ggml_graph_cut::mark_graph_cut(embedded_timestep, "anima.prelude", "embedded_timestep");
-            sd::ggml_graph_cut::mark_graph_cut(temb, "anima.prelude", "temb");
-            sd::ggml_graph_cut::mark_graph_cut(encoder_hidden_states, "anima.prelude", "context");
-
-            for (int i = 0; i < config.num_layers; i++) {
                 auto block = std::dynamic_pointer_cast<TransformerBlock>(blocks["blocks." + std::to_string(i)]);
                 x          = block->forward(ctx, x, encoder_hidden_states, embedded_timestep, temb, image_pe);
-                sd::ggml_graph_cut::mark_graph_cut(x, "anima.blocks." + std::to_string(i), "x");
             }
 
             x = final_layer->forward(ctx, x, embedded_timestep, temb);  // [N, h*w, ph*pw*C]
 
-            x = DiT::unpatchify_and_crop(ctx->ggml_ctx, x, H, W, config.patch_size, config.patch_size, false);  // [N, C, H, W]
+            x = DiT::unpatchify_and_crop(ctx->ggml_ctx, x, H, W, patch_size, patch_size, false);  // [N, C, H, W]
 
             return x;
         }
     };
 
-    struct AnimaRunner : public DiffusionModelRunner {
+    struct AnimaRunner : public GGMLRunner {
     public:
         std::vector<float> image_pe_vec;
         std::vector<float> adapter_q_pe_vec;
         std::vector<float> adapter_k_pe_vec;
-        AnimaConfig config;
         AnimaNet net;
 
         AnimaRunner(ggml_backend_t backend,
-                    ggml_backend_t params_backend,
+                    bool offload_params_to_cpu,
                     const String2TensorStorage& tensor_storage_map = {},
                     const std::string prefix                       = "model.diffusion_model")
-            : DiffusionModelRunner(backend, params_backend, prefix),
+            : GGMLRunner(backend, offload_params_to_cpu) {
-              config(AnimaConfig::detect_from_weights(tensor_storage_map, prefix + ".net")) {
+            int64_t num_layers    = 0;
-            net = AnimaNet(config);
+            std::string layer_tag = prefix + ".net.blocks.";
+            for (const auto& kv : tensor_storage_map) {
+                const std::string& tensor_name = kv.first;
+                size_t pos                     = tensor_name.find(layer_tag);
+                if (pos == std::string::npos) {
+                    continue;
+                }
+                size_t start = pos + layer_tag.size();
+                size_t end   = tensor_name.find('.', start);
+                if (end == std::string::npos) {
+                    continue;
+                }
+                int64_t layer_id = atoll(tensor_name.substr(start, end - start).c_str());
+                num_layers       = std::max(num_layers, layer_id + 1);
+            }
+            if (num_layers <= 0) {
+                num_layers = 28;
+            }
+            LOG_INFO("anima net layers: %" PRId64, num_layers);
+
+            net = AnimaNet(num_layers);
             net.init(params_ctx, tensor_storage_map, prefix + ".net");
         }
 
@@ -574,7 +553,7 @@ namespace Anima {
             return "anima";
         }
 
-        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string prefix) {
             net.get_param_tensors(tensors, prefix + ".net");
         }
 
@@ -613,8 +592,7 @@ namespace Anima {
                                           {},
                                           empty_ref_latents,
                                           false,
-                                          1.0f,
+                                          1.0f);
-                                          false);
 
             std::vector<float> axis_thetas = {
                 static_cast<float>(theta) * calc_ntk_factor(t_extrapolation_ratio, axes_dim[0]),
@@ -637,22 +615,22 @@ namespace Anima {
             GGML_ASSERT(x->ne[3] == 1);
             ggml_cgraph* gf = new_graph_custom(ANIMA_GRAPH_SIZE);
 
-            int64_t pad_h = (config.patch_size - x->ne[1] % config.patch_size) % config.patch_size;
+            int64_t pad_h = (net.patch_size - x->ne[1] % net.patch_size) % net.patch_size;
-            int64_t pad_w = (config.patch_size - x->ne[0] % config.patch_size) % config.patch_size;
+            int64_t pad_w = (net.patch_size - x->ne[0] % net.patch_size) % net.patch_size;
             int64_t h_pad = x->ne[1] + pad_h;
             int64_t w_pad = x->ne[0] + pad_w;
 
             image_pe_vec          = gen_anima_image_pe_vec(1,
                                                            static_cast<int>(h_pad),
                                                            static_cast<int>(w_pad),
-                                                           static_cast<int>(config.patch_size),
+                                                           static_cast<int>(net.patch_size),
-                                                           config.theta,
+                                                           net.theta,
-                                                           config.axes_dim,
+                                                           net.axes_dim,
                                                            4.0f,
                                                            4.0f,
                                                            1.0f);
-            int64_t image_pos_len = static_cast<int64_t>(image_pe_vec.size()) / (2 * 2 * (config.head_dim / 2));
+            int64_t image_pos_len = static_cast<int64_t>(image_pe_vec.size()) / (2 * 2 * (net.head_dim / 2));
-            auto image_pe         = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.head_dim / 2, image_pos_len);
+            auto image_pe         = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, net.head_dim / 2, image_pos_len);
             set_backend_tensor_data(image_pe, image_pe_vec.data());
 
             ggml_tensor* adapter_q_pe = nullptr;
@@ -699,20 +677,7 @@ namespace Anima {
             };
             return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
-
-        sd::Tensor<float> compute(int n_threads,
-                                  const DiffusionParams& diffusion_params) override {
-            GGML_ASSERT(diffusion_params.x != nullptr);
-            GGML_ASSERT(diffusion_params.timesteps != nullptr);
-            const auto* extra = diffusion_extra_as<AnimaDiffusionExtra>(diffusion_params);
-            return compute(n_threads,
-                           *diffusion_params.x,
-                           *diffusion_params.timesteps,
-                           tensor_or_empty(diffusion_params.context),
-                           tensor_or_empty(extra->t5_ids),
-                           tensor_or_empty(extra->t5_weights));
-        }
     };
 }  // namespace Anima
 
-#endif  // __SD_MODEL_DIFFUSION_ANIMA_HPP__
+#endif  // __ANIMA_HPP__

src/auto_encoder_kl.hpp

@@ -1,7 +1,7 @@
-#ifndef __SD_MODEL_VAE_AUTO_ENCODER_KL_HPP__
+#ifndef __AUTO_ENCODER_KL_HPP__
-#define __SD_MODEL_VAE_AUTO_ENCODER_KL_HPP__
+#define __AUTO_ENCODER_KL_HPP__
 
-#include "model/vae/vae.hpp"
+#include "vae.hpp"
 
 /*================================================== AutoEncoderKL ===================================================*/
 
@@ -328,7 +328,6 @@ public:
         auto conv_out    = std::dynamic_pointer_cast<Conv2d>(blocks["conv_out"]);
 
         auto h = conv_in->forward(ctx, x);  // [N, ch, h, w]
-        // sd::ggml_graph_cut::mark_graph_cut(h, "vae.encoder.prelude", "h");
 
         // downsampling
         size_t num_resolutions = ch_mult.size();
@@ -338,14 +337,12 @@ public:
                 auto down_block  = std::dynamic_pointer_cast<ResnetBlock>(blocks[name]);
 
                 h = down_block->forward(ctx, h);
-                // sd::ggml_graph_cut::mark_graph_cut(h, "vae.encoder.down." + std::to_string(i) + ".block." + std::to_string(j), "h");
             }
             if (i != num_resolutions - 1) {
                 std::string name = "down." + std::to_string(i) + ".downsample";
                 auto down_sample = std::dynamic_pointer_cast<DownSampleBlock>(blocks[name]);
 
                 h = down_sample->forward(ctx, h);
-                // sd::ggml_graph_cut::mark_graph_cut(h, "vae.encoder.down." + std::to_string(i) + ".downsample", "h");
             }
         }
 
@@ -353,7 +350,6 @@ public:
         h = mid_block_1->forward(ctx, h);
         h = mid_attn_1->forward(ctx, h);
         h = mid_block_2->forward(ctx, h);  // [N, block_in, h, w]
-        // sd::ggml_graph_cut::mark_graph_cut(h, "vae.encoder.mid", "h");
 
         // end
         h = norm_out->forward(ctx, h);
@@ -454,7 +450,6 @@ public:
 
         // conv_in
         auto h = conv_in->forward(ctx, z);  // [N, block_in, h, w]
-        // sd::ggml_graph_cut::mark_graph_cut(h, "vae.decoder.prelude", "h");
 
         // middle
         h = mid_block_1->forward(ctx, h);
@@ -462,7 +457,6 @@ public:
 
         h = mid_attn_1->forward(ctx, h);
         h = mid_block_2->forward(ctx, h);  // [N, block_in, h, w]
-        // sd::ggml_graph_cut::mark_graph_cut(h, "vae.decoder.mid", "h");
 
         // upsampling
         int num_resolutions = static_cast<int>(ch_mult.size());
@@ -472,14 +466,12 @@ public:
                 auto up_block    = std::dynamic_pointer_cast<ResnetBlock>(blocks[name]);
 
                 h = up_block->forward(ctx, h);
-                // sd::ggml_graph_cut::mark_graph_cut(h, "vae.decoder.up." + std::to_string(i) + ".block." + std::to_string(j), "h");
             }
             if (i != 0) {
                 std::string name = "up." + std::to_string(i) + ".upsample";
                 auto up_sample   = std::dynamic_pointer_cast<UpSampleBlock>(blocks[name]);
 
                 h = up_sample->forward(ctx, h);
-                // sd::ggml_graph_cut::mark_graph_cut(h, "vae.decoder.up." + std::to_string(i) + ".upsample", "h");
             }
         }
 
@@ -509,39 +501,14 @@ protected:
         bool double_z            = true;
     } dd_config;
 
-    static std::string get_tensor_name(const std::string& prefix, const std::string& name) {
-        return prefix.empty() ? name : prefix + "." + name;
-    }
-
-    void detect_decoder_ch(const String2TensorStorage& tensor_storage_map,
-                           const std::string& prefix,
-                           int& decoder_ch) {
-        auto conv_in_iter = tensor_storage_map.find(get_tensor_name(prefix, "decoder.conv_in.weight"));
-        if (conv_in_iter != tensor_storage_map.end() && conv_in_iter->second.n_dims >= 4 && conv_in_iter->second.ne[3] > 0) {
-            int last_ch_mult             = dd_config.ch_mult.back();
-            int64_t conv_in_out_channels = conv_in_iter->second.ne[3];
-            if (last_ch_mult > 0 && conv_in_out_channels % last_ch_mult == 0) {
-                decoder_ch = static_cast<int>(conv_in_out_channels / last_ch_mult);
-                LOG_INFO("vae decoder: ch = %d", decoder_ch);
-            } else {
-                LOG_WARN("vae decoder: failed to infer ch from %s (%" PRId64 " / %d)",
-                         get_tensor_name(prefix, "decoder.conv_in.weight").c_str(),
-                         conv_in_out_channels,
-                         last_ch_mult);
-            }
-        }
-    }
-
 public:
-    AutoEncoderKLModel(SDVersion version                              = VERSION_SD1,
+    AutoEncoderKLModel(SDVersion version          = VERSION_SD1,
-                       bool decode_only                               = true,
+                       bool decode_only           = true,
-                       bool use_linear_projection                     = false,
+                       bool use_linear_projection = false,
-                       bool use_video_decoder                         = false,
+                       bool use_video_decoder     = false)
-                       const String2TensorStorage& tensor_storage_map = {},
-                       const std::string& prefix                      = "")
         : version(version), decode_only(decode_only), use_video_decoder(use_video_decoder) {
         if (sd_version_is_dit(version)) {
-            if (sd_version_uses_flux2_vae(version)) {
+            if (sd_version_is_flux2(version)) {
                 dd_config.z_channels = 32;
                 embed_dim            = 32;
             } else {
@@ -552,9 +519,7 @@ public:
         if (use_video_decoder) {
             use_quant = false;
         }
-        int decoder_ch = dd_config.ch;
+        blocks["decoder"] = std::shared_ptr<GGMLBlock>(new Decoder(dd_config.ch,
-        detect_decoder_ch(tensor_storage_map, prefix, decoder_ch);
-        blocks["decoder"] = std::shared_ptr<GGMLBlock>(new Decoder(decoder_ch,
                                                                    dd_config.out_ch,
                                                                    dd_config.ch_mult,
                                                                    dd_config.num_res_blocks,
@@ -586,7 +551,7 @@ public:
 
     ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* z) {
         // z: [N, z_channels, h, w]
-        if (sd_version_uses_flux2_vae(version)) {
+        if (sd_version_is_flux2(version)) {
             // [N, C*p*p, h, w] -> [N, C, h*p, w*p]
             int64_t p = 2;
 
@@ -607,7 +572,6 @@ public:
         if (use_quant) {
             auto post_quant_conv = std::dynamic_pointer_cast<Conv2d>(blocks["post_quant_conv"]);
             z                    = post_quant_conv->forward(ctx, z);  // [N, z_channels, h, w]
-            // sd::ggml_graph_cut::mark_graph_cut(z, "vae.decode.prelude", "z");
         }
         auto decoder = std::dynamic_pointer_cast<Decoder>(blocks["decoder"]);
 
@@ -625,9 +589,8 @@ public:
         if (use_quant) {
             auto quant_conv = std::dynamic_pointer_cast<Conv2d>(blocks["quant_conv"]);
             z               = quant_conv->forward(ctx, z);  // [N, 2*embed_dim, h/8, w/8]
-            // sd::ggml_graph_cut::mark_graph_cut(z, "vae.encode.final", "z");
         }
-        if (sd_version_uses_flux2_vae(version)) {
+        if (sd_version_is_flux2(version)) {
             z = ggml_ext_chunk(ctx->ggml_ctx, z, 2, 2)[0];
 
             // [N, C, H, W] -> [N, C*p*p, H/p, W/p]
@@ -650,7 +613,7 @@ public:
 
     int get_encoder_output_channels() {
         int factor = dd_config.double_z ? 2 : 1;
-        if (sd_version_uses_flux2_vae(version)) {
+        if (sd_version_is_flux2(version)) {
             return dd_config.z_channels * 4;
         }
         return dd_config.z_channels * factor;
@@ -664,13 +627,13 @@ struct AutoEncoderKL : public VAE {
     AutoEncoderKLModel ae;
 
     AutoEncoderKL(ggml_backend_t backend,
-                  ggml_backend_t params_backend,
+                  bool offload_params_to_cpu,
                   const String2TensorStorage& tensor_storage_map,
                   const std::string prefix,
                   bool decode_only       = false,
                   bool use_video_decoder = false,
                   SDVersion version      = VERSION_SD1)
-        : decode_only(decode_only), VAE(version, backend, params_backend) {
+        : decode_only(decode_only), VAE(version, backend, offload_params_to_cpu) {
         if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
             scale_factor = 0.18215f;
             shift_factor = 0.f;
@@ -680,10 +643,10 @@ struct AutoEncoderKL : public VAE {
         } else if (sd_version_is_sd3(version)) {
             scale_factor = 1.5305f;
             shift_factor = 0.0609f;
-        } else if (sd_version_is_flux(version) || sd_version_is_z_image(version) || sd_version_is_longcat(version)) {
+        } else if (sd_version_is_flux(version) || sd_version_is_z_image(version)) {
             scale_factor = 0.3611f;
             shift_factor = 0.1159f;
-        } else if (sd_version_uses_flux2_vae(version)) {
+        } else if (sd_version_is_flux2(version)) {
             scale_factor = 1.0f;
             shift_factor = 0.f;
         }
@@ -699,7 +662,7 @@ struct AutoEncoderKL : public VAE {
                 break;
             }
         }
-        ae = AutoEncoderKLModel(version, decode_only, use_linear_projection, use_video_decoder, tensor_storage_map, prefix);
+        ae = AutoEncoderKLModel(version, decode_only, use_linear_projection, use_video_decoder);
         ae.init(params_ctx, tensor_storage_map, prefix);
     }
 
@@ -757,7 +720,7 @@ struct AutoEncoderKL : public VAE {
     }
 
     sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output, std::shared_ptr<RNG> rng) override {
-        if (sd_version_uses_flux2_vae(version)) {
+        if (sd_version_is_flux2(version)) {
             return vae_output;
         } else if (version == VERSION_SD1_PIX2PIX) {
             return sd::ops::chunk(vae_output, 2, 2)[0];
@@ -768,7 +731,7 @@ struct AutoEncoderKL : public VAE {
 
     std::pair<sd::Tensor<float>, sd::Tensor<float>> get_latents_mean_std(const sd::Tensor<float>& latents, int channel_dim) {
         GGML_ASSERT(channel_dim >= 0 && static_cast<size_t>(channel_dim) < static_cast<size_t>(latents.dim()));
-        if (sd_version_uses_flux2_vae(version)) {
+        if (sd_version_is_flux2(version)) {
             GGML_ASSERT(latents.shape()[channel_dim] == 128);
             std::vector<int64_t> stats_shape(static_cast<size_t>(latents.dim()), 1);
             stats_shape[static_cast<size_t>(channel_dim)] = latents.shape()[channel_dim];
@@ -814,7 +777,7 @@ struct AutoEncoderKL : public VAE {
     }
 
     sd::Tensor<float> diffusion_to_vae_latents(const sd::Tensor<float>& latents) override {
-        if (sd_version_uses_flux2_vae(version)) {
+        if (sd_version_is_flux2(version)) {
             int channel_dim                = 2;
             auto [mean_tensor, std_tensor] = get_latents_mean_std(latents, channel_dim);
             return (latents * std_tensor) / scale_factor + mean_tensor;
@@ -823,7 +786,7 @@ struct AutoEncoderKL : public VAE {
     }
 
     sd::Tensor<float> vae_to_diffusion_latents(const sd::Tensor<float>& latents) override {
-        if (sd_version_uses_flux2_vae(version)) {
+        if (sd_version_is_flux2(version)) {
             int channel_dim                = 2;
             auto [mean_tensor, std_tensor] = get_latents_mean_std(latents, channel_dim);
             return ((latents - mean_tensor) * scale_factor) / std_tensor;
@@ -886,4 +849,4 @@ struct AutoEncoderKL : public VAE {
     };
 };
 
-#endif  // __SD_MODEL_VAE_AUTO_ENCODER_KL_HPP__
+#endif  // __AUTO_ENCODER_KL_HPP__

src/cache_dit.hpp

@@ -1,5 +1,5 @@
-#ifndef __SD_RUNTIME_CACHE_DIT_HPP__
+#ifndef __CACHE_DIT_HPP__
-#define __SD_RUNTIME_CACHE_DIT_HPP__
+#define __CACHE_DIT_HPP__
 
 #include <algorithm>
 #include <cmath>
@@ -8,9 +8,9 @@
 #include <unordered_map>
 #include <vector>
 
-#include "core/ggml_extend.hpp"
+#include "condition_cache_utils.hpp"
-#include "core/tensor.hpp"
+#include "ggml_extend.hpp"
-#include "runtime/condition_cache_utils.hpp"
+#include "tensor.hpp"
 
 struct DBCacheConfig {
     bool enabled                        = false;
@@ -893,4 +893,4 @@ struct CacheDitConditionState {
     }
 };
 
-#endif  // __SD_RUNTIME_CACHE_DIT_HPP__
+#endif

src/clip.hpp

@@ -1,13 +1,461 @@
-#ifndef __SD_MODEL_TE_CLIP_HPP__
+#ifndef __CLIP_HPP__
-#define __SD_MODEL_TE_CLIP_HPP__
+#define __CLIP_HPP__
 
-#include "core/ggml_extend.hpp"
+#include "ggml_extend.hpp"
 #include "model.h"
-#include "tokenizers/clip_tokenizer.h"
+#include "tokenize_util.h"
+#include "vocab/vocab.h"
+
+/*================================================== CLIPTokenizer ===================================================*/
+
+__STATIC_INLINE__ std::vector<std::pair<int, std::u32string>> bytes_to_unicode() {
+    std::vector<std::pair<int, std::u32string>> byte_unicode_pairs;
+    std::set<int> byte_set;
+    for (int b = static_cast<int>('!'); b <= static_cast<int>('~'); ++b) {
+        byte_set.insert(b);
+        byte_unicode_pairs.push_back(std::pair<int, std::u32string>(b, unicode_value_to_utf32(b)));
+    }
+    for (int b = 161; b <= 172; ++b) {
+        byte_set.insert(b);
+        byte_unicode_pairs.push_back(std::pair<int, std::u32string>(b, unicode_value_to_utf32(b)));
+    }
+    for (int b = 174; b <= 255; ++b) {
+        byte_set.insert(b);
+        byte_unicode_pairs.push_back(std::pair<int, std::u32string>(b, unicode_value_to_utf32(b)));
+    }
+    int n = 0;
+    for (int b = 0; b < 256; ++b) {
+        if (byte_set.find(b) == byte_set.end()) {
+            byte_unicode_pairs.push_back(std::pair<int, std::u32string>(b, unicode_value_to_utf32(n + 256)));
+            ++n;
+        }
+    }
+    // LOG_DEBUG("byte_unicode_pairs %d", byte_unicode_pairs.size());
+    return byte_unicode_pairs;
+}
+
+// Ref: https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py
+
+typedef std::function<bool(std::string&, std::vector<int32_t>&)> on_new_token_cb_t;
+
+class CLIPTokenizer {
+private:
+    std::map<int, std::u32string> byte_encoder;
+    std::map<std::u32string, int> byte_decoder;
+    std::map<std::u32string, int> encoder;
+    std::map<int, std::u32string> decoder;
+    std::map<std::pair<std::u32string, std::u32string>, int> bpe_ranks;
+    std::regex pat;
+    int encoder_len;
+    int bpe_len;
+
+    std::vector<std::string> special_tokens;
+
+public:
+    const std::string UNK_TOKEN = "<|endoftext|>";
+    const std::string BOS_TOKEN = "<|startoftext|>";
+    const std::string EOS_TOKEN = "<|endoftext|>";
+    const std::string PAD_TOKEN = "<|endoftext|>";
+
+    const int UNK_TOKEN_ID = 49407;
+    const int BOS_TOKEN_ID = 49406;
+    const int EOS_TOKEN_ID = 49407;
+    const int PAD_TOKEN_ID = 49407;
+
+private:
+    static std::string strip(const std::string& str) {
+        std::string::size_type start = str.find_first_not_of(" \t\n\r\v\f");
+        std::string::size_type end   = str.find_last_not_of(" \t\n\r\v\f");
+
+        if (start == std::string::npos) {
+            // String contains only whitespace characters
+            return "";
+        }
+
+        return str.substr(start, end - start + 1);
+    }
+
+    static std::string whitespace_clean(std::string text) {
+        text = std::regex_replace(text, std::regex(R"(\s+)"), " ");
+        text = strip(text);
+        return text;
+    }
+
+    static std::set<std::pair<std::u32string, std::u32string>> get_pairs(const std::vector<std::u32string>& subwords) {
+        std::set<std::pair<std::u32string, std::u32string>> pairs;
+        if (subwords.size() == 0) {
+            return pairs;
+        }
+        std::u32string prev_subword = subwords[0];
+        for (int i = 1; i < subwords.size(); i++) {
+            std::u32string subword = subwords[i];
+            std::pair<std::u32string, std::u32string> pair(prev_subword, subword);
+            pairs.insert(pair);
+            prev_subword = subword;
+        }
+        return pairs;
+    }
+
+    bool is_special_token(const std::string& token) {
+        for (auto& special_token : special_tokens) {
+            if (special_token == token) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+public:
+    CLIPTokenizer(int pad_token_id = 49407, const std::string& merges_utf8_str = "")
+        : PAD_TOKEN_ID(pad_token_id) {
+        if (merges_utf8_str.size() > 0) {
+            load_from_merges(merges_utf8_str);
+        } else {
+            load_from_merges(load_clip_merges());
+        }
+        add_special_token("<|startoftext|>");
+        add_special_token("<|endoftext|>");
+    }
+
+    void load_from_merges(const std::string& merges_utf8_str) {
+        auto byte_unicode_pairs = bytes_to_unicode();
+        // printf("byte_unicode_pairs have %lu pairs \n", byte_unicode_pairs.size());
+        byte_encoder = std::map<int, std::u32string>(byte_unicode_pairs.begin(), byte_unicode_pairs.end());
+        for (auto& pair : byte_unicode_pairs) {
+            byte_decoder[pair.second] = pair.first;
+        }
+        // for (auto & pair: byte_unicode_pairs) {
+        //     std::cout << pair.first << ": " << pair.second << std::endl;
+        // }
+        std::vector<std::u32string> merges;
+        size_t start = 0;
+        size_t pos;
+        std::u32string merges_utf32_str = utf8_to_utf32(merges_utf8_str);
+        while ((pos = merges_utf32_str.find('\n', start)) != std::string::npos) {
+            merges.push_back(merges_utf32_str.substr(start, pos - start));
+            start = pos + 1;
+        }
+        // LOG_DEBUG("merges size %llu", merges.size());
+        GGML_ASSERT(merges.size() == 48895);
+        merges = std::vector<std::u32string>(merges.begin() + 1, merges.end());
+        std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
+        for (const auto& merge : merges) {
+            size_t space_pos = merge.find(' ');
+            merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
+            // LOG_DEBUG("%s", utf32_to_utf8(merge.substr(space_pos + 1)).c_str());
+            // printf("%s :: %s | %s \n", utf32_to_utf8(merge).c_str(), utf32_to_utf8(merge.substr(0, space_pos)).c_str(),
+            //                     utf32_to_utf8(merge.substr(space_pos + 1)).c_str());
+        }
+        std::vector<std::u32string> vocab;
+        for (const auto& pair : byte_unicode_pairs) {
+            vocab.push_back(pair.second);
+        }
+        for (const auto& pair : byte_unicode_pairs) {
+            vocab.push_back(pair.second + utf8_to_utf32("</w>"));
+        }
+        for (const auto& merge : merge_pairs) {
+            vocab.push_back(merge.first + merge.second);
+        }
+        vocab.push_back(utf8_to_utf32("<|startoftext|>"));
+        vocab.push_back(utf8_to_utf32("<|endoftext|>"));
+        LOG_DEBUG("vocab size: %llu", vocab.size());
+        int i = 0;
+        for (const auto& token : vocab) {
+            encoder[token] = i;
+            decoder[i]     = token;
+            i++;
+        }
+        encoder_len = i;
+
+        auto it = encoder.find(utf8_to_utf32("img</w>"));
+        if (it != encoder.end()) {
+            LOG_DEBUG("trigger word img already in vocab");
+        } else {
+            LOG_DEBUG("trigger word img not in vocab yet");
+        }
+
+        int rank = 0;
+        for (const auto& merge : merge_pairs) {
+            bpe_ranks[merge] = rank++;
+        }
+        bpe_len = rank;
+    };
+
+    void add_token(const std::string& text) {
+        std::u32string token = utf8_to_utf32(text);
+        auto it              = encoder.find(token);
+        if (it != encoder.end()) {
+            encoder[token]       = encoder_len;
+            decoder[encoder_len] = token;
+            encoder_len++;
+        }
+    }
+
+    void add_special_token(const std::string& token) {
+        special_tokens.push_back(token);
+    }
+
+    std::u32string bpe(const std::u32string& token) {
+        std::vector<std::u32string> word;
+
+        for (int i = 0; i < token.size() - 1; i++) {
+            word.emplace_back(1, token[i]);
+        }
+        word.push_back(token.substr(token.size() - 1) + utf8_to_utf32("</w>"));
+
+        std::set<std::pair<std::u32string, std::u32string>> pairs = get_pairs(word);
+
+        if (pairs.empty()) {
+            return token + utf8_to_utf32("</w>");
+        }
+
+        while (true) {
+            auto min_pair_iter = std::min_element(pairs.begin(),
+                                                  pairs.end(),
+                                                  [&](const std::pair<std::u32string, std::u32string>& a,
+                                                      const std::pair<std::u32string, std::u32string>& b) {
+                                                      if (bpe_ranks.find(a) == bpe_ranks.end()) {
+                                                          return false;
+                                                      } else if (bpe_ranks.find(b) == bpe_ranks.end()) {
+                                                          return true;
+                                                      }
+                                                      return bpe_ranks.at(a) < bpe_ranks.at(b);
+                                                  });
+
+            const std::pair<std::u32string, std::u32string>& bigram = *min_pair_iter;
+
+            if (bpe_ranks.find(bigram) == bpe_ranks.end()) {
+                break;
+            }
+
+            std::u32string first  = bigram.first;
+            std::u32string second = bigram.second;
+            std::vector<std::u32string> new_word;
+            int32_t i = 0;
+
+            while (i < word.size()) {
+                auto it = std::find(word.begin() + i, word.end(), first);
+                if (it == word.end()) {
+                    new_word.insert(new_word.end(), word.begin() + i, word.end());
+                    break;
+                }
+                new_word.insert(new_word.end(), word.begin() + i, it);
+                i = static_cast<int32_t>(std::distance(word.begin(), it));
+
+                if (word[i] == first && i < static_cast<int32_t>(word.size()) - 1 && word[i + 1] == second) {
+                    new_word.push_back(first + second);
+                    i += 2;
+                } else {
+                    new_word.push_back(word[i]);
+                    i += 1;
+                }
+            }
+
+            word = new_word;
+
+            if (word.size() == 1) {
+                break;
+            }
+            pairs = get_pairs(word);
+        }
+
+        std::u32string result;
+        for (int i = 0; i < word.size(); i++) {
+            result += word[i];
+            if (i != word.size() - 1) {
+                result += utf8_to_utf32(" ");
+            }
+        }
+
+        return result;
+    }
+
+    std::vector<int> tokenize(std::string text,
+                              on_new_token_cb_t on_new_token_cb,
+                              size_t max_length = 0,
+                              bool padding      = false) {
+        std::vector<int32_t> tokens = encode(text, on_new_token_cb);
+
+        tokens.insert(tokens.begin(), BOS_TOKEN_ID);
+        if (max_length > 0) {
+            if (tokens.size() > max_length - 1) {
+                tokens.resize(max_length - 1);
+                tokens.push_back(EOS_TOKEN_ID);
+            } else {
+                tokens.push_back(EOS_TOKEN_ID);
+                if (padding) {
+                    tokens.insert(tokens.end(), max_length - tokens.size(), PAD_TOKEN_ID);
+                }
+            }
+        }
+
+        return tokens;
+    }
+
+    void pad_tokens(std::vector<int>& tokens,
+                    std::vector<float>& weights,
+                    size_t max_length = 0,
+                    bool padding      = false) {
+        if (max_length > 0 && padding) {
+            size_t n = static_cast<size_t>(std::ceil(tokens.size() * 1.0 / (max_length - 2)));
+            if (n == 0) {
+                n = 1;
+            }
+            size_t length = max_length * n;
+            LOG_DEBUG("token length: %llu", length);
+            std::vector<int> new_tokens;
+            std::vector<float> new_weights;
+            new_tokens.push_back(BOS_TOKEN_ID);
+            new_weights.push_back(1.0);
+            int token_idx = 0;
+            for (int i = 1; i < length; i++) {
+                if (token_idx >= tokens.size()) {
+                    break;
+                }
+                if (i % max_length == 0) {
+                    new_tokens.push_back(BOS_TOKEN_ID);
+                    new_weights.push_back(1.0);
+                } else if (i % max_length == max_length - 1) {
+                    new_tokens.push_back(EOS_TOKEN_ID);
+                    new_weights.push_back(1.0);
+                } else {
+                    new_tokens.push_back(tokens[token_idx]);
+                    new_weights.push_back(weights[token_idx]);
+                    token_idx++;
+                }
+            }
+
+            new_tokens.push_back(EOS_TOKEN_ID);
+            new_weights.push_back(1.0);
+            tokens  = new_tokens;
+            weights = new_weights;
+
+            if (padding) {
+                tokens.insert(tokens.end(), length - tokens.size(), PAD_TOKEN_ID);
+                weights.insert(weights.end(), length - weights.size(), 1.0);
+            }
+        }
+    }
+
+    std::string clean_up_tokenization(std::string& text) {
+        std::regex pattern(R"( ,)");
+        // Replace " ," with ","
+        std::string result = std::regex_replace(text, pattern, ",");
+        return result;
+    }
+
+    std::string decode(const std::vector<int>& tokens) {
+        std::string text = "";
+        for (int t : tokens) {
+            if (t == 49406 || t == 49407)
+                continue;
+            std::u32string ts = decoder[t];
+            // printf("%d, %s \n", t,  utf32_to_utf8(ts).c_str());
+            std::string s = utf32_to_utf8(ts);
+            if (s.length() >= 4) {
+                if (ends_with(s, "</w>")) {
+                    text += s.replace(s.length() - 4, s.length() - 1, "") + " ";
+                } else {
+                    text += s;
+                }
+            } else {
+                text += " " + s;
+            }
+        }
+        // std::vector<unsigned char> bytes;
+        // for (auto c : text){
+        //     bytes.push_back(byte_decoder[c]);
+        // }
+
+        // std::string s((char *)bytes.data());
+        // std::string s = "";
+        text = clean_up_tokenization(text);
+        return trim(text);
+    }
+
+    std::vector<std::string> token_split(const std::string& text) {
+        std::regex pat(R"('s|'t|'re|'ve|'m|'ll|'d|[[:alpha:]]+|[[:digit:]]|[^[:space:][:alpha:][:digit:]]+)",
+                       std::regex::icase);
+        std::sregex_iterator iter(text.begin(), text.end(), pat);
+        std::sregex_iterator end;
+
+        std::vector<std::string> result;
+        for (; iter != end; ++iter) {
+            result.emplace_back(iter->str());
+        }
+
+        return result;
+    }
+
+    std::vector<int> encode(std::string text, on_new_token_cb_t on_new_token_cb) {
+        std::string original_text = text;
+        std::vector<int32_t> bpe_tokens;
+        text = whitespace_clean(text);
+        std::transform(text.begin(), text.end(), text.begin(), [](unsigned char c) { return std::tolower(c); });
+
+        std::string str = text;
+        std::vector<std::string> token_strs;
+
+        auto splited_texts = split_with_special_tokens(text, special_tokens);
+
+        for (auto& splited_text : splited_texts) {
+            LOG_DEBUG("token %s", splited_text.c_str());
+            if (is_special_token(splited_text)) {
+                LOG_DEBUG("special %s", splited_text.c_str());
+                bool skip = on_new_token_cb(splited_text, bpe_tokens);
+                if (skip) {
+                    token_strs.push_back(splited_text);
+                    continue;
+                }
+                continue;
+            }
+
+            auto tokens = token_split(splited_text);
+            for (auto& token : tokens) {
+                if (on_new_token_cb != nullptr) {
+                    bool skip = on_new_token_cb(token, bpe_tokens);
+                    if (skip) {
+                        token_strs.push_back(token);
+                        continue;
+                    }
+                }
+
+                std::string token_str = token;
+                std::u32string utf32_token;
+                for (int i = 0; i < token_str.length(); i++) {
+                    unsigned char b = token_str[i];
+                    utf32_token += byte_encoder[b];
+                }
+                auto bpe_strs = bpe(utf32_token);
+                size_t start  = 0;
+                size_t pos;
+                while ((pos = bpe_strs.find(' ', start)) != std::u32string::npos) {
+                    auto bpe_str = bpe_strs.substr(start, pos - start);
+                    bpe_tokens.push_back(encoder[bpe_str]);
+                    token_strs.push_back(utf32_to_utf8(bpe_str));
+
+                    start = pos + 1;
+                }
+                auto bpe_str = bpe_strs.substr(start, bpe_strs.size() - start);
+                bpe_tokens.push_back(encoder[bpe_str]);
+                token_strs.push_back(utf32_to_utf8(bpe_str));
+            }
+        }
+        // std::stringstream ss;
+        // ss << "[";
+        // for (auto token : token_strs) {
+        //     ss << "\"" << token << "\", ";
+        // }
+        // ss << "]";
+        // LOG_DEBUG("split prompt \"%s\" to tokens %s", original_text.c_str(), ss.str().c_str());
+        // printf("split prompt \"%s\" to tokens %s \n", original_text.c_str(), ss.str().c_str());
+        return bpe_tokens;
+    }
+};
 
 /*================================================ FrozenCLIPEmbedder ================================================*/
 
-// Ref: https://github.com/huggingface/transformers/blob/main/src/transformers/model/clip/modeling_clip.py
+// Ref: https://github.com/huggingface/transformers/blob/main/src/transformers/models/clip/modeling_clip.py
 
 struct CLIPMLP : public GGMLBlock {
 protected:
@@ -95,9 +543,8 @@ public:
 
     ggml_tensor* forward(GGMLRunnerContext* ctx,
                          ggml_tensor* x,
-                         ggml_tensor* mask                   = nullptr,
+                         ggml_tensor* mask = nullptr,
-                         int clip_skip                       = -1,
+                         int clip_skip     = -1) {
-                         const std::string& graph_cut_prefix = "") {
         // x: [N, n_token, d_model]
         int layer_idx = n_layer - 1;
         // LOG_DEBUG("clip_skip %d", clip_skip);
@@ -113,9 +560,6 @@ public:
             std::string name = "layers." + std::to_string(i);
             auto layer       = std::dynamic_pointer_cast<CLIPLayer>(blocks[name]);
             x                = layer->forward(ctx, x, mask);  // [N, n_token, d_model]
-            if (!graph_cut_prefix.empty()) {
-                sd::ggml_graph_cut::mark_graph_cut(x, graph_cut_prefix + ".layers." + std::to_string(i), "x");
-            }
             // LOG_DEBUG("layer %d", i);
         }
         return x;
@@ -308,8 +752,7 @@ public:
         auto final_layer_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["final_layer_norm"]);
 
         auto x = embeddings->forward(ctx, input_ids, tkn_embeddings);  // [N, n_token, hidden_size]
-        sd::ggml_graph_cut::mark_graph_cut(x, "clip_text.prelude", "x");
+        x      = encoder->forward(ctx, x, mask, return_pooled ? -1 : clip_skip);
-        x = encoder->forward(ctx, x, mask, return_pooled ? -1 : clip_skip, "clip_text");
         if (return_pooled || with_final_ln) {
             x = final_layer_norm->forward(ctx, x);
         }
@@ -373,8 +816,7 @@ public:
 
         auto x = embeddings->forward(ctx, pixel_values);  // [N, num_positions, embed_dim]
         x      = pre_layernorm->forward(ctx, x);
-        sd::ggml_graph_cut::mark_graph_cut(x, "clip_vision.prelude", "x");
+        x      = encoder->forward(ctx, x, nullptr, clip_skip);
-        x = encoder->forward(ctx, x, nullptr, clip_skip, "clip_vision");
 
         auto last_hidden_state = x;
 
@@ -469,13 +911,13 @@ struct CLIPTextModelRunner : public GGMLRunner {
     std::vector<float> attention_mask_vec;
 
     CLIPTextModelRunner(ggml_backend_t backend,
-                        ggml_backend_t params_backend,
+                        bool offload_params_to_cpu,
                         const String2TensorStorage& tensor_storage_map,
                         const std::string prefix,
                         CLIPVersion version = OPENAI_CLIP_VIT_L_14,
                         bool with_final_ln  = true,
                         bool force_clip_f32 = false)
-        : GGMLRunner(backend, params_backend) {
+        : GGMLRunner(backend, offload_params_to_cpu) {
         bool proj_in = false;
         for (const auto& [name, tensor_storage] : tensor_storage_map) {
             if (!starts_with(name, prefix)) {
@@ -579,4 +1021,4 @@ struct CLIPTextModelRunner : public GGMLRunner {
     }
 };
 
-#endif  // __SD_MODEL_TE_CLIP_HPP__
+#endif  // __CLIP_HPP__

src/common_block.hpp

@@ -1,9 +1,7 @@
-#ifndef __SD_MODEL_COMMON_BLOCK_HPP__
+#ifndef __COMMON_BLOCK_HPP__
-#define __SD_MODEL_COMMON_BLOCK_HPP__
+#define __COMMON_BLOCK_HPP__
 
-#include "core/ggml_extend.hpp"
+#include "ggml_extend.hpp"
-#include "core/util.h"
-#include "ggml-backend.h"
 
 class DownSampleBlock : public GGMLBlock {
 protected:
@@ -227,37 +225,6 @@ public:
     }
 };
 
-struct Mlp : public GGMLBlock {
-public:
-    Mlp(int64_t in_features,
-        int64_t hidden_features = -1,
-        int64_t out_features    = -1,
-        bool bias               = true) {
-        // act_layer is always lambda: nn.GELU(approximate="tanh")
-        // norm_layer is always None
-        // use_conv is always False
-        if (hidden_features == -1) {
-            hidden_features = in_features;
-        }
-        if (out_features == -1) {
-            out_features = in_features;
-        }
-        blocks["fc1"] = std::shared_ptr<GGMLBlock>(new Linear(in_features, hidden_features, bias));
-        blocks["fc2"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_features, out_features, bias));
-    }
-
-    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
-        // x: [N, n_token, in_features]
-        auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
-        auto fc2 = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
-
-        x = fc1->forward(ctx, x);
-        x = ggml_ext_gelu(ctx->ggml_ctx, x, true);
-        x = fc2->forward(ctx, x);
-        return x;
-    }
-};
-
 class FeedForward : public GGMLBlock {
 public:
     enum class Activation {
@@ -281,6 +248,9 @@ public:
         float scale         = 1.f;
         if (precision_fix) {
             scale = 1.f / 128.f;
+#ifdef SD_USE_VULKAN
+            force_prec_f32 = true;
+#endif
         }
         // The purpose of the scale here is to prevent NaN issues in certain situations.
         // For example, when using Vulkan without enabling force_prec_f32,
@@ -294,9 +264,6 @@ public:
 
         auto net_0 = std::dynamic_pointer_cast<UnaryBlock>(blocks["net.0"]);
         auto net_2 = std::dynamic_pointer_cast<Linear>(blocks["net.2"]);
-        if (sd_backend_is(ctx->backend, "Vulkan")) {
-            net_2->set_force_prec_f32(true);
-        }
 
         x = net_0->forward(ctx, x);  // [ne3, ne2, ne1, inner_dim]
         x = net_2->forward(ctx, x);  // [ne3, ne2, ne1, dim_out]
@@ -310,7 +277,6 @@ protected:
     int64_t context_dim;
     int64_t n_head;
     int64_t d_head;
-    bool xtra_dim = false;
 
 public:
     CrossAttention(int64_t query_dim,
@@ -322,11 +288,7 @@ public:
           query_dim(query_dim),
           context_dim(context_dim) {
         int64_t inner_dim = d_head * n_head;
-        if (context_dim == 320 && d_head == 320) {
+
-            // LOG_DEBUG("CrossAttention: temp set dim to 1024 for sdxs_09");
-            xtra_dim    = true;
-            context_dim = 1024;
-        }
         blocks["to_q"] = std::shared_ptr<GGMLBlock>(new Linear(query_dim, inner_dim, false));
         blocks["to_k"] = std::shared_ptr<GGMLBlock>(new Linear(context_dim, inner_dim, false));
         blocks["to_v"] = std::shared_ptr<GGMLBlock>(new Linear(context_dim, inner_dim, false));
@@ -351,16 +313,10 @@ public:
         int64_t n_context = context->ne[1];
         int64_t inner_dim = d_head * n_head;
 
-        auto q = to_q->forward(ctx, x);  // [N, n_token, inner_dim]
+        auto q = to_q->forward(ctx, x);        // [N, n_token, inner_dim]
-        if (xtra_dim) {
-            // LOG_DEBUG("CrossAttention: temp set dim to 1024 for sdxs_09");
-            context->ne[0] = 1024;  // patch dim
-        }
         auto k = to_k->forward(ctx, context);  // [N, n_context, inner_dim]
         auto v = to_v->forward(ctx, context);  // [N, n_context, inner_dim]
-        if (xtra_dim) {
+
-            context->ne[0] = 320;  // reset dim to orig
-        }
         x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, nullptr, false, ctx->flash_attn_enabled);  // [N, n_token, inner_dim]
 
         x = to_out_0->forward(ctx, x);  // [N, n_token, query_dim]
@@ -634,4 +590,4 @@ public:
     }
 };
 
-#endif  // __SD_MODEL_COMMON_BLOCK_HPP__
+#endif  // __COMMON_BLOCK_HPP__

src/common_dit.hpp

@@ -1,7 +1,7 @@
-#ifndef __SD_MODEL_DIFFUSION_DIT_HPP__
+#ifndef __COMMON_DIT_HPP__
-#define __SD_MODEL_DIFFUSION_DIT_HPP__
+#define __COMMON_DIT_HPP__
 
-#include "core/ggml_extend.hpp"
+#include "ggml_extend.hpp"
 
 namespace DiT {
     inline ggml_tensor* patchify(ggml_context* ctx,
@@ -103,64 +103,6 @@ namespace DiT {
         x         = ggml_ext_slice(ctx, x, 0, 0, W);               // [N, C, H, W]
         return x;
     }
-
-    inline ggml_tensor* patchify(ggml_context* ctx,
-                                 ggml_tensor* x,
-                                 int pt,
-                                 int ph,
-                                 int pw,
-                                 int64_t N = 1) {
-        // x: [N*C, T, H, W]
-        // return: [N, h*w, C*pt*ph*pw]
-        int64_t C     = x->ne[3] / N;
-        int64_t T     = x->ne[2];
-        int64_t H     = x->ne[1];
-        int64_t W     = x->ne[0];
-        int64_t t_len = T / pt;
-        int64_t h_len = H / ph;
-        int64_t w_len = W / pw;
-
-        GGML_ASSERT(C * N == x->ne[3]);
-        GGML_ASSERT(t_len * pt == T && h_len * ph == H && w_len * pw == W);
-
-        x = ggml_reshape_4d(ctx, x, pw * w_len, ph * h_len, pt, t_len * C * N);      // [N*C*t_len, pt, h_len*ph, w_len*pw]
-        x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3));          // [N*C*t_len, h_len*ph, pt, w_len*pw]
-        x = ggml_reshape_4d(ctx, x, pw * w_len, pt, ph, h_len * t_len * C * N);      // [N*C*t_len*h_len, ph, pt, w_len*pw]
-        x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3));          // [N*C*t_len*h_len, pt, ph, w_len*pw]
-        x = ggml_reshape_4d(ctx, x, pw, w_len, ph * pt, h_len * t_len * C * N);      // [N*C*t_len*h_len, pt*ph, w_len, pw]
-        x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3));          // [N*C*t_len*h_len, w_len, pt*ph, pw]
-        x = ggml_reshape_4d(ctx, x, pw * ph * pt, w_len * h_len * t_len, C, N);      // [N, C, t_len*h_len*w_len, pt*ph*pw]
-        x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3));          // [N, t_len*h_len*w_len, C, pt*ph*pw]
-        x = ggml_reshape_4d(ctx, x, pw * ph * pt * C, w_len * h_len * t_len, N, 1);  // [N, t_len*h_len*w_len, C*pt*ph*pw]
-        return x;
-    }
-
-    inline ggml_tensor* unpatchify(ggml_context* ctx,
-                                   ggml_tensor* x,
-                                   int64_t t_len,
-                                   int64_t h_len,
-                                   int64_t w_len,
-                                   int pt,
-                                   int ph,
-                                   int pw) {
-        // x: [N, t_len*h_len*w_len, pt*ph*pw*C]
-        // return: [N*C, t_len*pt, h_len*ph, w_len*pw]
-        int64_t N = x->ne[3];
-        int64_t C = x->ne[0] / pt / ph / pw;
-
-        GGML_ASSERT(C * pt * ph * pw == x->ne[0]);
-
-        x = ggml_reshape_4d(ctx, x, C, pw * ph * pt, w_len * h_len * t_len, N);  // [N, t_len*h_len*w_len, pt*ph*pw, C]
-        x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 1, 2, 0, 3));      // [N, C, t_len*h_len*w_len, pt*ph*pw]
-        x = ggml_reshape_4d(ctx, x, pw, ph * pt, w_len, h_len * t_len * C * N);  // [N*C*t_len*h_len, w_len, pt*ph, pw]
-        x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3));      // [N*C*t_len*h_len, pt*ph, w_len, pw]
-        x = ggml_reshape_4d(ctx, x, pw * w_len, ph, pt, h_len * t_len * C * N);  // [N*C*t_len*h_len, pt, ph, w_len*pw]
-        x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3));      // [N*C*t_len*h_len, ph, pt, w_len*pw]
-        x = ggml_reshape_4d(ctx, x, pw * w_len, pt, ph * h_len, t_len * C * N);  // [N*C*t_len, h_len*ph, pt, w_len*pw]
-        x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3));      // [N*C*t_len, pt, h_len*ph, w_len*pw]
-        x = ggml_reshape_4d(ctx, x, pw * w_len, ph * h_len, pt * t_len, C * N);  // [N*C, t_len*pt, h_len*ph, w_len*pw]
-        return x;
-    }
 }  // namespace DiT
 
-#endif  // __SD_MODEL_DIFFUSION_DIT_HPP__
+#endif  // __COMMON_DIT_HPP__

src/condition_cache_utils.hpp

@@ -1,9 +1,9 @@
-#ifndef __SD_RUNTIME_CONDITION_CACHE_UTILS_HPP__
+#ifndef __CONDITION_CACHE_UTILS_HPP__
-#define __SD_RUNTIME_CONDITION_CACHE_UTILS_HPP__
+#define __CONDITION_CACHE_UTILS_HPP__
 
 #include <vector>
 
-#include "core/tensor.hpp"
+#include "tensor.hpp"
 
 namespace sd {
 
@@ -61,4 +61,4 @@ namespace sd {
 
 }  // namespace sd
 
-#endif  // __SD_RUNTIME_CONDITION_CACHE_UTILS_HPP__
+#endif  // __CONDITION_CACHE_UTILS_HPP__

src/control.hpp

@@ -1,8 +1,8 @@
-#ifndef __SD_MODEL_DIFFUSION_CONTROL_HPP__
+#ifndef __CONTROL_HPP__
-#define __SD_MODEL_DIFFUSION_CONTROL_HPP__
+#define __CONTROL_HPP__
 
-#include "model/common/block.hpp"
+#include "common_block.hpp"
-#include "model_loader.h"
+#include "model.h"
 
 #define CONTROL_NET_GRAPH_SIZE 1536
 
@@ -319,10 +319,10 @@ struct ControlNet : public GGMLRunner {
     bool guided_hint_cached = false;
 
     ControlNet(ggml_backend_t backend,
-               ggml_backend_t params_backend,
+               bool offload_params_to_cpu,
                const String2TensorStorage& tensor_storage_map = {},
                SDVersion version                              = VERSION_SD1)
-        : GGMLRunner(backend, params_backend), control_net(version) {
+        : GGMLRunner(backend, offload_params_to_cpu), control_net(version) {
         control_net.init(params_ctx, tensor_storage_map, "");
     }
 
@@ -457,11 +457,7 @@ struct ControlNet : public GGMLRunner {
 
     bool load_from_file(const std::string& file_path, int n_threads) {
         LOG_INFO("loading control net from '%s'", file_path.c_str());
-        if (!alloc_params_buffer()) {
+        alloc_params_buffer();
-            LOG_ERROR("control net model buffer allocation failed");
-            return false;
-        }
-
         std::map<std::string, ggml_tensor*> tensors;
         control_net.get_param_tensors(tensors);
         std::set<std::string> ignore_tensors;
@@ -484,4 +480,4 @@ struct ControlNet : public GGMLRunner {
     }
 };
 
-#endif  // __SD_MODEL_DIFFUSION_CONTROL_HPP__
+#endif  // __CONTROL_HPP__

src/convert.cpp

@@ -1,138 +0,0 @@
-#include <cstring>
-#include <mutex>
-#include <regex>
-#include <vector>
-
-#include "model_io/gguf_io.h"
-#include "model_io/safetensors_io.h"
-#include "model_loader.h"
-#include "util.h"
-
-#include "ggml_extend_backend.h"
-
-static ggml_type get_export_tensor_type(ModelLoader& model_loader,
-                                        const TensorStorage& tensor_storage,
-                                        ggml_type type,
-                                        const TensorTypeRules& tensor_type_rules) {
-    const std::string& name = tensor_storage.name;
-    ggml_type tensor_type   = tensor_storage.type;
-    ggml_type dst_type      = type;
-
-    for (const auto& tensor_type_rule : tensor_type_rules) {
-        std::regex pattern(tensor_type_rule.first);
-        if (std::regex_search(name, pattern)) {
-            dst_type = tensor_type_rule.second;
-            break;
-        }
-    }
-
-    if (model_loader.tensor_should_be_converted(tensor_storage, dst_type)) {
-        tensor_type = dst_type;
-    }
-
-    return tensor_type;
-}
-
-static bool load_tensors_for_export(ModelLoader& model_loader,
-                                    ggml_context* ggml_ctx,
-                                    ggml_type type,
-                                    const TensorTypeRules& tensor_type_rules,
-                                    std::vector<TensorWriteInfo>& tensors) {
-    std::mutex tensor_mutex;
-    auto on_new_tensor_cb = [&](const TensorStorage& tensor_storage, ggml_tensor** dst_tensor) -> bool {
-        const std::string& name = tensor_storage.name;
-        ggml_type tensor_type   = get_export_tensor_type(model_loader, tensor_storage, type, tensor_type_rules);
-
-        std::lock_guard<std::mutex> lock(tensor_mutex);
-        ggml_tensor* tensor = ggml_new_tensor(ggml_ctx, tensor_type, tensor_storage.n_dims, tensor_storage.ne);
-        if (tensor == nullptr) {
-            LOG_ERROR("ggml_new_tensor failed");
-            return false;
-        }
-        ggml_set_name(tensor, name.c_str());
-
-        if (!tensor->data) {
-            GGML_ASSERT(ggml_nelements(tensor) == 0);
-            // Avoid crashing writers by setting a dummy pointer for zero-sized tensors.
-            LOG_DEBUG("setting dummy pointer for zero-sized tensor %s", name.c_str());
-            tensor->data = ggml_get_mem_buffer(ggml_ctx);
-        }
-
-        TensorWriteInfo write_info;
-        write_info.tensor = tensor;
-        write_info.n_dims = tensor_storage.n_dims;
-        for (int i = 0; i < tensor_storage.n_dims; ++i) {
-            write_info.ne[i] = tensor_storage.ne[i];
-        }
-
-        *dst_tensor = tensor;
-        tensors.push_back(std::move(write_info));
-
-        return true;
-    };
-
-    bool success = model_loader.load_tensors(on_new_tensor_cb);
-    LOG_INFO("load tensors done");
-    return success;
-}
-
-bool convert(const char* input_path,
-             const char* vae_path,
-             const char* output_path,
-             sd_type_t output_type,
-             const char* tensor_type_rules,
-             bool convert_name) {
-    ModelLoader model_loader;
-
-    if (!model_loader.init_from_file(input_path)) {
-        LOG_ERROR("init model loader from file failed: '%s'", input_path);
-        return false;
-    }
-
-    if (vae_path != nullptr && strlen(vae_path) > 0) {
-        if (!model_loader.init_from_file(vae_path, "vae.")) {
-            LOG_ERROR("init model loader from file failed: '%s'", vae_path);
-            return false;
-        }
-    }
-    if (convert_name) {
-        model_loader.convert_tensors_name();
-    }
-
-    ggml_type type             = (ggml_type)output_type;
-    bool output_is_safetensors = ends_with(output_path, ".safetensors");
-    TensorTypeRules type_rules = parse_tensor_type_rules(tensor_type_rules);
-
-    auto backend    = sd_backend_cpu_init();
-    size_t mem_size = 1 * 1024 * 1024;  // for padding
-    mem_size += model_loader.get_tensor_storage_map().size() * ggml_tensor_overhead();
-    mem_size += model_loader.get_params_mem_size(backend, type);
-    LOG_INFO("model tensors mem size: %.2fMB", mem_size / 1024.f / 1024.f);
-    ggml_context* ggml_ctx = ggml_init({mem_size, nullptr, false});
-
-    if (ggml_ctx == nullptr) {
-        LOG_ERROR("ggml_init failed for converter");
-        ggml_backend_free(backend);
-        return false;
-    }
-
-    std::vector<TensorWriteInfo> tensors;
-    bool success = load_tensors_for_export(model_loader, ggml_ctx, type, type_rules, tensors);
-    ggml_backend_free(backend);
-
-    std::string error;
-    if (success) {
-        if (output_is_safetensors) {
-            success = write_safetensors_file(output_path, tensors, &error);
-        } else {
-            success = write_gguf_file(output_path, tensors, &error);
-        }
-    }
-
-    if (!success && !error.empty()) {
-        LOG_ERROR("%s", error.c_str());
-    }
-
-    ggml_free(ggml_ctx);
-    return success;
-}

src/core/ggml_extend_backend.cpp

@@ -1,656 +0,0 @@
-#include "core/ggml_extend_backend.h"
-
-#include <algorithm>
-#include <cctype>
-#include <cstdlib>
-#include <mutex>
-#include <sstream>
-#include <stdexcept>
-#include <vector>
-
-#include "core/util.h"
-#include "stable-diffusion.h"
-
-static std::string trim_copy(const std::string& value) {
-    size_t begin = 0;
-    while (begin < value.size() && std::isspace(static_cast<unsigned char>(value[begin]))) {
-        ++begin;
-    }
-    size_t end = value.size();
-    while (end > begin && std::isspace(static_cast<unsigned char>(value[end - 1]))) {
-        --end;
-    }
-    return value.substr(begin, end - begin);
-}
-
-static std::string lower_copy(std::string value) {
-    std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
-        return static_cast<char>(std::tolower(c));
-    });
-    return value;
-}
-
-static std::vector<std::string> split_copy(const std::string& value, char delimiter) {
-    std::vector<std::string> parts;
-    std::string part;
-    std::istringstream stream(value);
-    while (std::getline(stream, part, delimiter)) {
-        parts.push_back(part);
-    }
-    return parts;
-}
-
-static bool is_default_backend_token(const std::string& name) {
-    const std::string lower = lower_copy(trim_copy(name));
-    return lower.empty() || lower == "default" || lower == "auto";
-}
-
-static bool parse_backend_module(const std::string& raw_name, SDBackendModule* module) {
-    std::string name = lower_copy(trim_copy(raw_name));
-    name.erase(std::remove(name.begin(), name.end(), '-'), name.end());
-    name.erase(std::remove(name.begin(), name.end(), '_'), name.end());
-
-    if (name == "diffusion" || name == "model" || name == "unet" || name == "dit") {
-        *module = SDBackendModule::DIFFUSION;
-        return true;
-    }
-    if (name == "te" || name == "clip" || name == "text" || name == "textencoder" || name == "textencoders" || name == "conditioner" || name == "cond" || name == "llm" || name == "t5" || name == "t5xxl") {
-        *module = SDBackendModule::TE;
-        return true;
-    }
-    if (name == "clipvision" || name == "vision") {
-        *module = SDBackendModule::CLIP_VISION;
-        return true;
-    }
-    if (name == "vae" || name == "firststage" || name == "autoencoder" || name == "tae") {
-        *module = SDBackendModule::VAE;
-        return true;
-    }
-    if (name == "controlnet" || name == "control") {
-        *module = SDBackendModule::CONTROL_NET;
-        return true;
-    }
-    if (name == "photomaker" || name == "photomakerid" || name == "pmid" || name == "photo") {
-        *module = SDBackendModule::PHOTOMAKER;
-        return true;
-    }
-    if (name == "upscaler" || name == "esrgan" || name == "hires") {
-        *module = SDBackendModule::UPSCALER;
-        return true;
-    }
-    return false;
-}
-
-static std::string module_assignment_name(const SDBackendAssignment& assignment, SDBackendModule module) {
-    auto it = assignment.module_names.find(module);
-    if (it != assignment.module_names.end()) {
-        return it->second;
-    }
-    return assignment.default_name;
-}
-
-static std::string backend_cache_key(ggml_backend_t backend) {
-    if (backend == nullptr) {
-        return "";
-    }
-    ggml_backend_dev_t dev = ggml_backend_get_device(backend);
-    if (dev != nullptr) {
-        return lower_copy(ggml_backend_dev_name(dev));
-    }
-    const char* backend_name = ggml_backend_name(backend);
-    return backend_name != nullptr ? lower_copy(backend_name) : "";
-}
-
-static std::string resolve_first_device_by_type(enum ggml_backend_dev_type type) {
-    ggml_backend_dev_t dev = ggml_backend_dev_by_type(type);
-    if (dev == nullptr) {
-        return "";
-    }
-    return ggml_backend_dev_name(dev);
-}
-
-static ggml_backend_buffer_t ggml_backend_tensor_buffer(const struct ggml_tensor* tensor) {
-    if (tensor == nullptr) {
-        return nullptr;
-    }
-
-    return tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-}
-
-static bool ggml_backend_tensor_is_host_accessible(const struct ggml_tensor* tensor) {
-    if (tensor == nullptr || tensor->data == nullptr) {
-        return false;
-    }
-
-    ggml_backend_buffer_t buffer = ggml_backend_tensor_buffer(tensor);
-    return buffer == nullptr || ggml_backend_buffer_is_host(buffer);
-}
-
-static size_t ggml_backend_tensor_offset(const struct ggml_tensor* tensor, int64_t i0, int64_t i1, int64_t i2, int64_t i3) {
-    return static_cast<size_t>(i0 * tensor->nb[0] + i1 * tensor->nb[1] + i2 * tensor->nb[2] + i3 * tensor->nb[3]);
-}
-
-template <typename T>
-static void ggml_backend_tensor_write_scalar(const struct ggml_tensor* tensor, int64_t i0, int64_t i1, int64_t i2, int64_t i3, T value) {
-    const size_t offset = ggml_backend_tensor_offset(tensor, i0, i1, i2, i3);
-
-    if (ggml_backend_tensor_is_host_accessible(tensor)) {
-        auto* dst = reinterpret_cast<T*>(reinterpret_cast<char*>(tensor->data) + offset);
-        *dst      = value;
-        return;
-    }
-
-    ggml_backend_tensor_set(const_cast<struct ggml_tensor*>(tensor), &value, offset, sizeof(T));
-}
-
-static void ggml_set_f32_nd(const struct ggml_tensor* tensor, int64_t i0, int64_t i1, int64_t i2, int64_t i3, float value) {
-    switch (tensor->type) {
-        case GGML_TYPE_I8:
-            ggml_backend_tensor_write_scalar(tensor, i0, i1, i2, i3, static_cast<int8_t>(value));
-            break;
-        case GGML_TYPE_I16:
-            ggml_backend_tensor_write_scalar(tensor, i0, i1, i2, i3, static_cast<int16_t>(value));
-            break;
-        case GGML_TYPE_I32:
-            ggml_backend_tensor_write_scalar(tensor, i0, i1, i2, i3, static_cast<int32_t>(value));
-            break;
-        case GGML_TYPE_F16:
-            ggml_backend_tensor_write_scalar(tensor, i0, i1, i2, i3, ggml_fp32_to_fp16(value));
-            break;
-        case GGML_TYPE_BF16:
-            ggml_backend_tensor_write_scalar(tensor, i0, i1, i2, i3, ggml_fp32_to_bf16(value));
-            break;
-        case GGML_TYPE_F32:
-            ggml_backend_tensor_write_scalar(tensor, i0, i1, i2, i3, value);
-            break;
-        default:
-            GGML_ABORT("fatal error");
-    }
-}
-
-void ggml_ext_im_set_f32_1d(const struct ggml_tensor* tensor, int i, float value) {
-    if (!ggml_is_contiguous(tensor)) {
-        int64_t id[4] = {0, 0, 0, 0};
-        ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
-        ggml_set_f32_nd(tensor, id[0], id[1], id[2], id[3], value);
-        return;
-    }
-
-    switch (tensor->type) {
-        case GGML_TYPE_I8:
-            ggml_backend_tensor_write_scalar(tensor, i, 0, 0, 0, static_cast<int8_t>(value));
-            break;
-        case GGML_TYPE_I16:
-            ggml_backend_tensor_write_scalar(tensor, i, 0, 0, 0, static_cast<int16_t>(value));
-            break;
-        case GGML_TYPE_I32:
-            ggml_backend_tensor_write_scalar(tensor, i, 0, 0, 0, static_cast<int32_t>(value));
-            break;
-        case GGML_TYPE_F16:
-            ggml_backend_tensor_write_scalar(tensor, i, 0, 0, 0, ggml_fp32_to_fp16(value));
-            break;
-        case GGML_TYPE_BF16:
-            ggml_backend_tensor_write_scalar(tensor, i, 0, 0, 0, ggml_fp32_to_bf16(value));
-            break;
-        case GGML_TYPE_F32:
-            ggml_backend_tensor_write_scalar(tensor, i, 0, 0, 0, value);
-            break;
-        default:
-            GGML_ABORT("fatal error");
-    }
-}
-
-static void ggml_backend_load_all_once() {
-    // If the registry already has devices and the CPU backend is present,
-    // assume either static registration or explicit host-side preloading has
-    // completed and avoid rescanning the default paths.
-    if (ggml_backend_dev_count() > 0 && ggml_backend_reg_by_name("CPU") != nullptr) {
-        return;
-    }
-    // In dynamic-backend mode the backend modules are discovered at runtime,
-    // so we must load them before asking for the CPU backend or its proc table.
-    // If the host preloaded only a subset of backends, allow one default-path
-    // scan so missing modules can still be discovered.
-    static std::once_flag once;
-    std::call_once(once, []() {
-        if (ggml_backend_dev_count() > 0 && ggml_backend_reg_by_name("CPU") != nullptr) {
-            return;
-        }
-        ggml_backend_load_all();
-    });
-}
-
-bool sd_backend_is(ggml_backend_t backend, const std::string& name) {
-    if (!backend) {
-        return false;
-    }
-    ggml_backend_dev_t dev = ggml_backend_get_device(backend);
-    if (!dev) {
-        return false;
-    }
-    std::string dev_name = ggml_backend_dev_name(dev);
-    return lower_copy(dev_name).find(lower_copy(name)) != std::string::npos;
-}
-
-static std::string get_default_backend_name() {
-    ggml_backend_load_all_once();
-    // should pick the same backend preference as ggml_backend_init_best
-    std::string name = resolve_first_device_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);
-    if (!name.empty()) {
-        return name;
-    }
-    name = resolve_first_device_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU);
-    if (!name.empty()) {
-        return name;
-    }
-    return resolve_first_device_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
-}
-
-static std::string sd_resolve_backend_name(const std::string& name) {
-    ggml_backend_load_all_once();
-    std::string requested = trim_copy(name);
-    std::string lower     = lower_copy(requested);
-
-    if (is_default_backend_token(lower)) {
-        return get_default_backend_name();
-    }
-    if (lower == "gpu") {
-        std::string result = resolve_first_device_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);
-        if (!result.empty()) {
-            return result;
-        }
-        return resolve_first_device_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU);
-    }
-
-    const size_t device_count = ggml_backend_dev_count();
-    for (size_t i = 0; i < device_count; ++i) {
-        ggml_backend_dev_t dev = ggml_backend_dev_get(i);
-        std::string dev_name   = ggml_backend_dev_name(dev);
-        if (lower_copy(dev_name) == lower) {
-            return dev_name;
-        }
-    }
-
-    for (size_t i = 0; i < device_count; ++i) {
-        ggml_backend_dev_t dev = ggml_backend_dev_get(i);
-        std::string dev_name   = ggml_backend_dev_name(dev);
-        std::string dev_lower  = lower_copy(dev_name);
-        if (dev_lower.rfind(lower, 0) == 0) {
-            return dev_name;
-        }
-    }
-
-    return "";
-}
-
-static bool backend_name_exists(const std::string& name) {
-    return !sd_resolve_backend_name(name).empty();
-}
-
-static ggml_backend_t init_named_backend(const std::string& name) {
-    ggml_backend_load_all_once();
-    LOG_DEBUG("Initializing backend: %s", name.c_str());
-    if (trim_copy(name).empty()) {
-        return ggml_backend_init_best();
-    }
-
-    std::string resolved = sd_resolve_backend_name(name);
-    if (resolved.empty()) {
-        return nullptr;
-    }
-    return ggml_backend_init_by_name(resolved.c_str(), nullptr);
-}
-
-bool sd_backend_is_cpu(ggml_backend_t backend) {
-    if (backend == nullptr) {
-        return false;
-    }
-    auto dev = ggml_backend_get_device(backend);
-    return dev != nullptr && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU;
-}
-
-ggml_backend_t sd_backend_cpu_init() {
-    ggml_backend_load_all_once();
-    return ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
-}
-
-bool sd_backend_cpu_set_n_threads(ggml_backend_t backend, int n_threads) {
-    if (backend == nullptr) {
-        return false;
-    }
-    auto dev = ggml_backend_get_device(backend);
-    if (dev != nullptr && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {
-        auto reg                           = ggml_backend_dev_backend_reg(dev);
-        auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t)ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
-        if (ggml_backend_set_n_threads_fn != nullptr) {
-            ggml_backend_set_n_threads_fn(backend, n_threads);
-            return true;
-        }
-    }
-    return false;
-}
-
-const char* sd_get_system_info() {
-    static std::string cache_info = []() -> std::string {
-        ggml_backend_load_all_once();
-        std::stringstream ss;
-        ss << "System Info: \n";
-        auto dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
-        if (dev != nullptr) {
-            auto reg                          = ggml_backend_dev_backend_reg(dev);
-            auto ggml_backend_get_features_fn = (ggml_backend_get_features_t)ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features");
-            if (ggml_backend_get_features_fn != nullptr) {
-                ggml_backend_feature* feat = ggml_backend_get_features_fn(reg);
-                while (feat->name && feat->value) {
-                    ss << "   " << feat->name << " = " << feat->value << " | ";
-                    feat++;
-                }
-            } else {
-                LOG_WARN("unable to get CPU features");
-            }
-        } else {
-            LOG_WARN("unable to get CPU features");
-        }
-        return ss.str();
-    }();
-    return cache_info.c_str();
-}
-
-static ggml_backend_t sd_get_default_backend() {
-    ggml_backend_load_all_once();
-    static std::once_flag once;
-    std::call_once(once, []() {
-        size_t dev_count = ggml_backend_dev_count();
-        if (dev_count == 0) {
-            LOG_ERROR("No devices found!");
-        } else {
-            LOG_DEBUG("Found %zu backend devices:", dev_count);
-            for (size_t i = 0; i < dev_count; ++i) {
-                auto dev = ggml_backend_dev_get(i);
-                LOG_DEBUG("#%zu: %s", i, ggml_backend_dev_name(dev));
-            }
-        }
-    });
-
-    ggml_backend_t backend   = nullptr;
-    const char* SD_VK_DEVICE = getenv("SD_VK_DEVICE");
-    if (SD_VK_DEVICE != nullptr) {
-        std::string sd_vk_device_str = SD_VK_DEVICE;
-        try {
-            unsigned long long device  = std::stoull(sd_vk_device_str);
-            std::string vk_device_name = "Vulkan" + std::to_string(device);
-            if (backend_name_exists(vk_device_name)) {
-                LOG_INFO("Selecting %s as main device by env var SD_VK_DEVICE", vk_device_name.c_str());
-                backend = init_named_backend(vk_device_name);
-                if (!backend) {
-                    LOG_WARN("Device %s requested by SD_VK_DEVICE failed to init. Falling back to the default device.", vk_device_name.c_str());
-                }
-            } else {
-                LOG_WARN("Device %s requested by SD_VK_DEVICE was not found. Falling back to the default device.", vk_device_name.c_str());
-            }
-        } catch (const std::invalid_argument&) {
-            LOG_WARN("SD_VK_DEVICE environment variable is not a valid integer (%s). Falling back to the default device.", SD_VK_DEVICE);
-        } catch (const std::out_of_range&) {
-            LOG_WARN("SD_VK_DEVICE environment variable value is out of range for `unsigned long long` type (%s). Falling back to the default device.", SD_VK_DEVICE);
-        }
-    }
-
-    if (!backend) {
-        std::string dev_name = get_default_backend_name();
-        backend              = init_named_backend(dev_name);
-        if (!backend && !dev_name.empty()) {
-            LOG_WARN("device %s failed to init", dev_name.c_str());
-        }
-    }
-
-    if (!backend) {
-        LOG_WARN("loading CPU backend");
-        backend = sd_backend_cpu_init();
-    }
-
-    if (sd_backend_is_cpu(backend)) {
-        LOG_DEBUG("Using CPU backend");
-    }
-
-    return backend;
-}
-
-static bool sd_parse_backend_assignment(const std::string& spec, SDBackendAssignment* assignment, std::string* error) {
-    if (assignment == nullptr) {
-        return false;
-    }
-
-    *assignment          = {};
-    const std::string in = trim_copy(spec);
-    if (in.empty()) {
-        return true;
-    }
-
-    for (const std::string& raw_part : split_copy(in, ',')) {
-        const std::string part = trim_copy(raw_part);
-        if (part.empty()) {
-            continue;
-        }
-
-        const size_t eq = part.find('=');
-        if (eq == std::string::npos) {
-            assignment->set_default(part);
-            continue;
-        }
-
-        const std::string key   = trim_copy(part.substr(0, eq));
-        const std::string value = trim_copy(part.substr(eq + 1));
-        if (key.empty() || value.empty()) {
-            if (error != nullptr) {
-                *error = "invalid backend assignment '" + part + "'";
-            }
-            return false;
-        }
-
-        const std::string key_lower = lower_copy(key);
-        if (key_lower == "all" || key_lower == "default" || key_lower == "*") {
-            assignment->set_default(value);
-            continue;
-        }
-
-        SDBackendModule module = SDBackendModule::DIFFUSION;
-        if (!parse_backend_module(key, &module)) {
-            if (error != nullptr) {
-                *error = "unknown backend module '" + key + "'";
-            }
-            return false;
-        }
-        assignment->set_module(module, value);
-    }
-    return true;
-}
-
-bool SDBackendAssignment::empty() const {
-    return default_name.empty() && module_names.empty();
-}
-
-std::string SDBackendAssignment::get(SDBackendModule module) const {
-    return module_assignment_name(*this, module);
-}
-
-void SDBackendAssignment::set_default(const std::string& name) {
-    default_name = trim_copy(name);
-}
-
-void SDBackendAssignment::set_module(SDBackendModule module, const std::string& name) {
-    module_names[module] = trim_copy(name);
-}
-
-void SDBackendHandleDeleter::operator()(ggml_backend_t backend) const {
-    ggml_backend_free(backend);
-}
-
-SDBackendManager::~SDBackendManager() {
-    reset();
-}
-
-void SDBackendManager::reset() {
-    backends_.clear();
-    runtime_assignment_ = {};
-    params_assignment_  = {};
-}
-
-ggml_backend_t SDBackendManager::runtime_backend(SDBackendModule module) {
-    return init_cached_backend(runtime_assignment_.get(module));
-}
-
-ggml_backend_t SDBackendManager::params_backend(SDBackendModule module) {
-    std::string name = params_assignment_.get(module);
-    if (name.empty()) {
-        return runtime_backend(module);
-    }
-    return init_cached_backend(name);
-}
-
-bool SDBackendManager::runtime_backend_is_cpu(SDBackendModule module) {
-    return sd_backend_is_cpu(runtime_backend(module));
-}
-
-bool SDBackendManager::params_backend_is_cpu(SDBackendModule module) {
-    return sd_backend_is_cpu(params_backend(module));
-}
-
-bool SDBackendManager::runtime_backend_supports_host_buffer(SDBackendModule module) {
-    ggml_backend_t backend = runtime_backend(module);
-    if (backend == nullptr) {
-        return false;
-    }
-    if (sd_backend_is_cpu(backend)) {
-        return true;
-    }
-    ggml_backend_dev_t dev = ggml_backend_get_device(backend);
-    if (dev == nullptr) {
-        return false;
-    }
-    ggml_backend_dev_props props;
-    ggml_backend_dev_get_props(dev, &props);
-    return props.caps.buffer_from_host_ptr;
-}
-
-bool SDBackendManager::init(const char* backend_spec,
-                            const char* params_backend_spec,
-                            bool offload_params_to_cpu,
-                            bool keep_clip_on_cpu,
-                            bool keep_vae_on_cpu,
-                            bool keep_control_net_on_cpu,
-                            std::string* error) {
-    reset();
-
-    if (!sd_parse_backend_assignment(SAFE_STR(backend_spec), &runtime_assignment_, error)) {
-        return false;
-    }
-    if (!sd_parse_backend_assignment(SAFE_STR(params_backend_spec), &params_assignment_, error)) {
-        return false;
-    }
-
-    if (runtime_assignment_.empty()) {
-        if (keep_clip_on_cpu) {
-            runtime_assignment_.set_module(SDBackendModule::TE, "cpu");
-        }
-        if (keep_vae_on_cpu) {
-            runtime_assignment_.set_module(SDBackendModule::VAE, "cpu");
-        }
-        if (keep_control_net_on_cpu) {
-            runtime_assignment_.set_module(SDBackendModule::CONTROL_NET, "cpu");
-        }
-    }
-
-    if (params_assignment_.empty() && offload_params_to_cpu) {
-        params_assignment_.set_default("cpu");
-    }
-
-    return validate(error);
-}
-
-bool SDBackendManager::validate(std::string* error) const {
-    auto validate_name = [&](const std::string& name) -> bool {
-        if (is_default_backend_token(name)) {
-            return true;
-        }
-        if (!sd_resolve_backend_name(name).empty()) {
-            return true;
-        }
-        if (error != nullptr) {
-            *error = "backend '" + name + "' was not found";
-        }
-        return false;
-    };
-
-    if (!validate_name(runtime_assignment_.default_name) ||
-        !validate_name(params_assignment_.default_name)) {
-        return false;
-    }
-    for (const auto& kv : runtime_assignment_.module_names) {
-        if (!validate_name(kv.second)) {
-            return false;
-        }
-    }
-    for (const auto& kv : params_assignment_.module_names) {
-        if (!validate_name(kv.second)) {
-            return false;
-        }
-    }
-    return true;
-}
-
-ggml_backend_t SDBackendManager::init_cached_backend(const std::string& name) {
-    std::string resolved   = sd_resolve_backend_name(name);
-    std::string key        = lower_copy(resolved);
-    ggml_backend_t backend = nullptr;
-
-    if (!key.empty()) {
-        auto it = backends_.find(key);
-        if (it != backends_.end()) {
-            return it->second.get();
-        }
-    } else if (!is_default_backend_token(name)) {
-        LOG_ERROR("backend '%s' was not found", name.c_str());
-        return nullptr;
-    }
-
-    backend = is_default_backend_token(name) ? sd_get_default_backend() : init_named_backend(resolved);
-    if (backend == nullptr) {
-        LOG_ERROR("failed to initialize backend '%s'", name.c_str());
-        return nullptr;
-    }
-
-    std::string actual_key = backend_cache_key(backend);
-    if (actual_key.empty()) {
-        actual_key = !key.empty() ? key : lower_copy(trim_copy(name));
-    }
-
-    auto it = backends_.find(actual_key);
-    if (it != backends_.end()) {
-        ggml_backend_free(backend);
-        return it->second.get();
-    }
-
-    SDBackendHandle handle(backend);
-    backends_.emplace(actual_key, std::move(handle));
-    return backend;
-}
-
-const char* sd_backend_module_name(SDBackendModule module) {
-    switch (module) {
-        case SDBackendModule::DIFFUSION:
-            return "diffusion";
-        case SDBackendModule::TE:
-            return "te";
-        case SDBackendModule::CLIP_VISION:
-            return "clip_vision";
-        case SDBackendModule::VAE:
-            return "vae";
-        case SDBackendModule::CONTROL_NET:
-            return "controlnet";
-        case SDBackendModule::PHOTOMAKER:
-            return "photomaker";
-        case SDBackendModule::UPSCALER:
-            return "upscaler";
-    }
-    return "unknown";
-}

src/core/ggml_extend_backend.h

@@ -1,79 +0,0 @@
-#ifndef __SD_CORE_GGML_EXTEND_BACKEND_H__
-#define __SD_CORE_GGML_EXTEND_BACKEND_H__
-
-#include <cstdint>
-#include <cstring>
-#include <memory>
-#include <string>
-#include <unordered_map>
-
-#include "ggml-backend.h"
-#include "ggml.h"
-
-enum class SDBackendModule {
-    DIFFUSION,
-    TE,
-    CLIP_VISION,
-    VAE,
-    CONTROL_NET,
-    PHOTOMAKER,
-    UPSCALER,
-};
-
-struct SDBackendAssignment {
-    std::string default_name;
-    std::unordered_map<SDBackendModule, std::string> module_names;
-
-    bool empty() const;
-    std::string get(SDBackendModule module) const;
-    void set_default(const std::string& name);
-    void set_module(SDBackendModule module, const std::string& name);
-};
-
-struct SDBackendHandleDeleter {
-    void operator()(ggml_backend_t backend) const;
-};
-
-using SDBackendHandle = std::unique_ptr<struct ggml_backend, SDBackendHandleDeleter>;
-
-class SDBackendManager {
-private:
-    SDBackendAssignment runtime_assignment_;
-    SDBackendAssignment params_assignment_;
-    std::unordered_map<std::string, SDBackendHandle> backends_;
-
-public:
-    SDBackendManager() = default;
-    ~SDBackendManager();
-
-    SDBackendManager(const SDBackendManager&)            = delete;
-    SDBackendManager& operator=(const SDBackendManager&) = delete;
-
-    bool init(const char* backend_spec,
-              const char* params_backend_spec,
-              bool offload_params_to_cpu,
-              bool keep_clip_on_cpu,
-              bool keep_vae_on_cpu,
-              bool keep_control_net_on_cpu,
-              std::string* error);
-    void reset();
-
-    ggml_backend_t runtime_backend(SDBackendModule module);
-    ggml_backend_t params_backend(SDBackendModule module);
-
-    bool runtime_backend_is_cpu(SDBackendModule module);
-    bool params_backend_is_cpu(SDBackendModule module);
-    bool runtime_backend_supports_host_buffer(SDBackendModule module);
-
-private:
-    bool validate(std::string* error) const;
-    ggml_backend_t init_cached_backend(const std::string& name);
-};
-
-bool sd_backend_is(ggml_backend_t backend, const std::string& name);
-bool sd_backend_is_cpu(ggml_backend_t backend);
-ggml_backend_t sd_backend_cpu_init();
-bool sd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads);
-const char* sd_backend_module_name(SDBackendModule module);
-void ggml_ext_im_set_f32_1d(const struct ggml_tensor* tensor, int i, float value);
-#endif  // __SD_CORE_GGML_EXTEND_BACKEND_H__

src/core/ggml_graph_cut.cpp

@@ -1,806 +0,0 @@
-#include "core/ggml_graph_cut.h"
-
-#include <algorithm>
-#include <cstring>
-#include <map>
-#include <set>
-#include <sstream>
-#include <stack>
-#include <unordered_map>
-
-#include "core/util.h"
-#include "ggml-alloc.h"
-#include "ggml-backend.h"
-
-#include "ggml/src/ggml-impl.h"
-
-namespace sd::ggml_graph_cut {
-
-    static constexpr double MAX_VRAM_BYTES_PER_GIB = 1024.0 * 1024.0 * 1024.0;
-
-    static std::string graph_cut_tensor_display_name(const ggml_tensor* tensor) {
-        if (tensor == nullptr) {
-            return "<null>";
-        }
-        if (tensor->name[0] != '\0') {
-            return tensor->name;
-        }
-        return sd_format("<tensor@%p>", (const void*)tensor);
-    }
-
-    static int graph_leaf_index(ggml_cgraph* gf, const ggml_tensor* tensor) {
-        GGML_ASSERT(gf != nullptr);
-        GGML_ASSERT(tensor != nullptr);
-        for (int i = 0; i < gf->n_leafs; ++i) {
-            if (gf->leafs[i] == tensor) {
-                return i;
-            }
-        }
-        return -1;
-    }
-
-    static bool is_params_tensor(const std::unordered_set<const ggml_tensor*>& params_tensor_set,
-                                 const ggml_tensor* tensor) {
-        if (tensor == nullptr) {
-            return false;
-        }
-        return params_tensor_set.find(tensor) != params_tensor_set.end();
-    }
-
-    static int graph_node_index_by_name(ggml_cgraph* gf, const char* name) {
-        GGML_ASSERT(gf != nullptr);
-        if (name == nullptr || name[0] == '\0') {
-            return -1;
-        }
-        const int n_nodes = ggml_graph_n_nodes(gf);
-        for (int i = 0; i < n_nodes; ++i) {
-            ggml_tensor* node = ggml_graph_node(gf, i);
-            if (node != nullptr && std::strcmp(node->name, name) == 0) {
-                return i;
-            }
-        }
-        return -1;
-    }
-
-    static Plan::InputShape input_shape(const ggml_tensor* tensor) {
-        Plan::InputShape shape;
-        if (tensor == nullptr) {
-            return shape;
-        }
-        shape.type = tensor->type;
-        for (int i = 0; i < GGML_MAX_DIMS; ++i) {
-            shape.ne[static_cast<size_t>(i)] = tensor->ne[i];
-        }
-        return shape;
-    }
-
-    static size_t graph_cut_segment_vram_bytes(const Segment& segment) {
-        return segment.compute_buffer_size +
-               segment.input_param_bytes +
-               segment.input_previous_cut_bytes +
-               segment.output_bytes;
-    }
-
-    size_t max_vram_gib_to_bytes(float max_vram) {
-        if (max_vram <= 0.f) {
-            return 0;
-        }
-        return static_cast<size_t>(static_cast<double>(max_vram) * MAX_VRAM_BYTES_PER_GIB);
-    }
-
-    static float max_vram_bytes_to_gib(size_t max_vram_bytes) {
-        return static_cast<float>(static_cast<double>(max_vram_bytes) / MAX_VRAM_BYTES_PER_GIB);
-    }
-
-    static size_t resolve_auto_max_vram_bytes(float spare_vram, ggml_backend_t backend) {
-        if (backend == nullptr) {
-            LOG_WARN("--max-vram < 0 requested, but no backend is available; disabling graph splitting");
-            return 0;
-        }
-
-        ggml_backend_dev_t dev = ggml_backend_get_device(backend);
-        if (dev == nullptr) {
-            LOG_WARN("--max-vram < 0 requested, but no backend device is available; disabling graph splitting");
-            return 0;
-        }
-        if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {
-            LOG_WARN("--max-vram < 0 requested, but the main backend is CPU; disabling graph splitting");
-            return 0;
-        }
-
-        size_t free_vram  = 0;
-        size_t total_vram = 0;
-        ggml_backend_dev_memory(dev, &free_vram, &total_vram);
-        size_t spare_bytes = static_cast<size_t>(MAX_VRAM_BYTES_PER_GIB * spare_vram);
-
-        if (free_vram <= spare_bytes) {
-            LOG_WARN("--max-vram < 0 requested, but free VRAM is %.2f GiB; reserving %.2f GiB leaves no graph budget",
-                     free_vram / MAX_VRAM_BYTES_PER_GIB, spare_vram);
-            return 0;
-        }
-
-        const size_t max_vram_bytes = free_vram - spare_bytes;
-        LOG_INFO("--max-vram < 0 auto-detected %.2f GiB free VRAM (%.2f GiB total), reserving %.2f GiB; using %.2f GiB",
-                 free_vram / MAX_VRAM_BYTES_PER_GIB,
-                 total_vram / MAX_VRAM_BYTES_PER_GIB,
-                 spare_vram,
-                 max_vram_bytes / MAX_VRAM_BYTES_PER_GIB);
-        return max_vram_bytes;
-    }
-
-    float resolve_max_vram_gib(float max_vram, ggml_backend_t backend) {
-        if (max_vram >= 0.f) {
-            return max_vram;
-        }
-        return max_vram_bytes_to_gib(resolve_auto_max_vram_bytes(-max_vram, backend));
-    }
-
-    static Segment make_segment_seed(const Plan& plan,
-                                     size_t start_segment_index,
-                                     size_t end_segment_index) {
-        GGML_ASSERT(start_segment_index < plan.segments.size());
-        GGML_ASSERT(end_segment_index < plan.segments.size());
-        GGML_ASSERT(start_segment_index <= end_segment_index);
-
-        Segment seed;
-        const auto& start_segment  = plan.segments[start_segment_index];
-        const auto& target_segment = plan.segments[end_segment_index];
-        std::unordered_set<int> seen_output_node_indices;
-        for (size_t seg_idx = start_segment_index; seg_idx <= end_segment_index; ++seg_idx) {
-            for (int output_node_index : plan.segments[seg_idx].output_node_indices) {
-                if (seen_output_node_indices.insert(output_node_index).second) {
-                    seed.output_node_indices.push_back(output_node_index);
-                }
-            }
-        }
-        if (start_segment_index == end_segment_index) {
-            seed.group_name = target_segment.group_name;
-        } else {
-            seed.group_name = sd_format("%s..%s",
-                                        start_segment.group_name.c_str(),
-                                        target_segment.group_name.c_str());
-        }
-        return seed;
-    }
-
-    static void build_segment(ggml_cgraph* gf,
-                              Plan& plan,
-                              Segment& segment,
-                              const std::unordered_map<const ggml_tensor*, int>& producer_index,
-                              std::unordered_set<int>& available_cut_output_node_indices,
-                              ggml_backend_t backend,
-                              const std::unordered_set<const ggml_tensor*>& params_tensor_set,
-                              const char* log_desc) {
-        std::set<int> internal_nodes;
-        std::unordered_set<const ggml_tensor*> input_seen;
-        std::vector<Segment::InputRef> input_refs;
-
-        std::stack<ggml_tensor*> work_stack;
-        for (int output_node_index : segment.output_node_indices) {
-            ggml_tensor* output = ggml_graph_node(gf, output_node_index);
-            if (output != nullptr) {
-                work_stack.push(output);
-            }
-        }
-
-        while (!work_stack.empty()) {
-            ggml_tensor* tensor = work_stack.top();
-            work_stack.pop();
-
-            if (tensor == nullptr) {
-                continue;
-            }
-
-            auto producer_it = producer_index.find(tensor);
-            if (producer_it == producer_index.end()) {
-                if (input_seen.insert(tensor).second) {
-                    Segment::InputRef input_ref;
-                    input_ref.type         = is_params_tensor(params_tensor_set, tensor) ? Segment::INPUT_PARAM : Segment::INPUT_EXTERNAL;
-                    input_ref.display_name = graph_cut_tensor_display_name(tensor);
-                    input_ref.leaf_index   = graph_leaf_index(gf, tensor);
-                    input_refs.push_back(std::move(input_ref));
-                }
-                continue;
-            }
-
-            int node_idx = producer_it->second;
-            if (available_cut_output_node_indices.find(node_idx) != available_cut_output_node_indices.end()) {
-                if (input_seen.insert(tensor).second) {
-                    Segment::InputRef input_ref;
-                    input_ref.type         = Segment::INPUT_PREVIOUS_CUT;
-                    input_ref.display_name = graph_cut_tensor_display_name(tensor);
-                    input_ref.node_index   = node_idx;
-                    input_refs.push_back(std::move(input_ref));
-                }
-                continue;
-            }
-
-            if (!internal_nodes.insert(node_idx).second) {
-                continue;
-            }
-
-            ggml_tensor* node = ggml_graph_node(gf, node_idx);
-            for (int src_idx = 0; src_idx < GGML_MAX_SRC; ++src_idx) {
-                if (node->src[src_idx] != nullptr) {
-                    work_stack.push(node->src[src_idx]);
-                }
-            }
-        }
-
-        if (!internal_nodes.empty()) {
-            segment.internal_node_indices.assign(internal_nodes.begin(), internal_nodes.end());
-        }
-
-        std::sort(input_refs.begin(),
-                  input_refs.end(),
-                  [](const Segment::InputRef& a, const Segment::InputRef& b) {
-                      if (a.type != b.type) {
-                          return a.type < b.type;
-                      }
-                      return a.display_name < b.display_name;
-                  });
-        segment.input_refs = input_refs;
-        for (const auto& input : input_refs) {
-            ggml_tensor* current_input = input_tensor(gf, input);
-            size_t tensor_bytes        = current_input == nullptr
-                                             ? 0
-                                             : (input.type == Segment::INPUT_PREVIOUS_CUT
-                                                    ? cache_tensor_bytes(current_input)
-                                                    : ggml_nbytes(current_input));
-            switch (input.type) {
-                case Segment::INPUT_PREVIOUS_CUT:
-                    segment.input_previous_cut_bytes += tensor_bytes;
-                    break;
-                case Segment::INPUT_PARAM:
-                    segment.input_param_bytes += tensor_bytes;
-                    break;
-                case Segment::INPUT_EXTERNAL:
-                default:
-                    segment.input_external_bytes += tensor_bytes;
-                    break;
-            }
-        }
-        for (int output_node_index : segment.output_node_indices) {
-            ggml_tensor* output = ggml_graph_node(gf, output_node_index);
-            segment.output_bytes += cache_tensor_bytes(output);
-        }
-        segment.compute_buffer_size = measure_segment_compute_buffer(backend, gf, segment, log_desc);
-
-        for (int output_node_index : segment.output_node_indices) {
-            available_cut_output_node_indices.insert(output_node_index);
-        }
-        plan.segments.push_back(std::move(segment));
-    }
-
-    bool is_graph_cut_tensor(const ggml_tensor* tensor) {
-        if (tensor == nullptr || tensor->name[0] == '\0') {
-            return false;
-        }
-        return std::strncmp(tensor->name, GGML_RUNNER_CUT_PREFIX, std::strlen(GGML_RUNNER_CUT_PREFIX)) == 0;
-    }
-
-    std::string make_graph_cut_name(const std::string& group, const std::string& output) {
-        return std::string(GGML_RUNNER_CUT_PREFIX) + group + "|" + output;
-    }
-
-    void mark_graph_cut(ggml_tensor* tensor, const std::string& group, const std::string& output) {
-        if (tensor == nullptr) {
-            return;
-        }
-        auto name = make_graph_cut_name(group, output);
-        ggml_set_name(tensor, name.c_str());
-    }
-
-    int leaf_count(ggml_cgraph* gf) {
-        GGML_ASSERT(gf != nullptr);
-        return gf->n_leafs;
-    }
-
-    ggml_tensor* leaf_tensor(ggml_cgraph* gf, int leaf_index) {
-        GGML_ASSERT(gf != nullptr);
-        if (leaf_index < 0 || leaf_index >= gf->n_leafs) {
-            return nullptr;
-        }
-        return gf->leafs[leaf_index];
-    }
-
-    ggml_backend_buffer_t tensor_buffer(const ggml_tensor* tensor) {
-        if (tensor == nullptr) {
-            return nullptr;
-        }
-        return tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-    }
-
-    ggml_tensor* cache_source_tensor(ggml_tensor* tensor) {
-        if (tensor == nullptr) {
-            return nullptr;
-        }
-        if (tensor_buffer(tensor) == nullptr && tensor->src[0] != nullptr &&
-            ggml_nelements(tensor->src[0]) == ggml_nelements(tensor) &&
-            ggml_nbytes(tensor->src[0]) == ggml_nbytes(tensor)) {
-            return cache_source_tensor(tensor->src[0]);
-        }
-        return tensor->view_src ? tensor->view_src : tensor;
-    }
-
-    size_t cache_tensor_bytes(const ggml_tensor* tensor) {
-        if (tensor == nullptr) {
-            return 0;
-        }
-        const ggml_tensor* cache_src = tensor->view_src ? tensor->view_src : tensor;
-        return ggml_nbytes(cache_src);
-    }
-
-    bool plan_matches_graph(ggml_cgraph* gf, const Plan& plan) {
-        GGML_ASSERT(gf != nullptr);
-        if (ggml_graph_n_nodes(gf) != plan.n_nodes || gf->n_leafs != plan.n_leafs) {
-            return false;
-        }
-        for (const auto& input_shape_ref : plan.input_shapes) {
-            if (input_shape_ref.leaf_index < 0 || input_shape_ref.leaf_index >= gf->n_leafs) {
-                return false;
-            }
-            ggml_tensor* leaf = gf->leafs[input_shape_ref.leaf_index];
-            if (leaf == nullptr || input_shape_ref.type != leaf->type) {
-                return false;
-            }
-            for (int d = 0; d < GGML_MAX_DIMS; ++d) {
-                if (input_shape_ref.ne[static_cast<size_t>(d)] != leaf->ne[d]) {
-                    return false;
-                }
-            }
-        }
-        return true;
-    }
-
-    ggml_tensor* output_tensor(ggml_cgraph* gf, const Segment& segment, size_t output_index) {
-        GGML_ASSERT(gf != nullptr);
-        if (output_index >= segment.output_node_indices.size()) {
-            return nullptr;
-        }
-        int node_index = segment.output_node_indices[output_index];
-        if (node_index < 0 || node_index >= ggml_graph_n_nodes(gf)) {
-            return nullptr;
-        }
-        return ggml_graph_node(gf, node_index);
-    }
-
-    ggml_tensor* input_tensor(ggml_cgraph* gf, const Segment::InputRef& input_ref) {
-        GGML_ASSERT(gf != nullptr);
-        if (input_ref.type == Segment::INPUT_PREVIOUS_CUT) {
-            if (input_ref.node_index < 0 || input_ref.node_index >= ggml_graph_n_nodes(gf)) {
-                return nullptr;
-            }
-            return ggml_graph_node(gf, input_ref.node_index);
-        }
-        if (input_ref.leaf_index < 0 || input_ref.leaf_index >= gf->n_leafs) {
-            return nullptr;
-        }
-        return leaf_tensor(gf, input_ref.leaf_index);
-    }
-
-    std::vector<ggml_tensor*> param_tensors(ggml_cgraph* gf, const Segment& segment) {
-        GGML_ASSERT(gf != nullptr);
-        std::vector<ggml_tensor*> tensors;
-        std::unordered_set<ggml_tensor*> seen_tensors;
-        tensors.reserve(segment.input_refs.size());
-        seen_tensors.reserve(segment.input_refs.size());
-        for (const auto& input_ref : segment.input_refs) {
-            if (input_ref.type != Segment::INPUT_PARAM) {
-                continue;
-            }
-            ggml_tensor* tensor = input_tensor(gf, input_ref);
-            if (tensor == nullptr) {
-                continue;
-            }
-            if (seen_tensors.insert(tensor).second) {
-                tensors.push_back(tensor);
-            }
-        }
-        return tensors;
-    }
-
-    std::vector<ggml_tensor*> runtime_param_tensors(ggml_cgraph* gf, const Segment& segment, const char* log_desc) {
-        std::vector<ggml_tensor*> tensors = param_tensors(gf, segment);
-        std::vector<ggml_tensor*> filtered_tensors;
-        filtered_tensors.reserve(tensors.size());
-        for (ggml_tensor* tensor : tensors) {
-            if (tensor_buffer(tensor) == nullptr) {
-                LOG_WARN("%s graph cut skipping param input without buffer: segment=%s tensor=%s",
-                         log_desc == nullptr ? "unknown" : log_desc,
-                         segment.group_name.c_str(),
-                         tensor->name);
-                continue;
-            }
-            filtered_tensors.push_back(tensor);
-        }
-        return filtered_tensors;
-    }
-
-    std::unordered_set<std::string> collect_future_input_names(ggml_cgraph* gf,
-                                                               const Plan& plan,
-                                                               size_t current_segment_index) {
-        GGML_ASSERT(gf != nullptr);
-        std::unordered_set<std::string> future_input_names;
-        for (size_t seg_idx = current_segment_index + 1; seg_idx < plan.segments.size(); ++seg_idx) {
-            const auto& segment = plan.segments[seg_idx];
-            for (const auto& input_ref : segment.input_refs) {
-                if (input_ref.type != Segment::INPUT_PREVIOUS_CUT) {
-                    continue;
-                }
-                ggml_tensor* current_input = input_tensor(gf, input_ref);
-                if (current_input != nullptr && current_input->name[0] != '\0') {
-                    future_input_names.insert(current_input->name);
-                }
-            }
-        }
-        return future_input_names;
-    }
-
-    ggml_cgraph* build_segment_graph(ggml_cgraph* gf,
-                                     const Segment& segment,
-                                     ggml_context** graph_ctx_out) {
-        GGML_ASSERT(gf != nullptr);
-        GGML_ASSERT(graph_ctx_out != nullptr);
-
-        const size_t graph_size = segment.internal_node_indices.size() + segment.input_refs.size() + 8;
-        ggml_init_params params = {
-            /*.mem_size   =*/ggml_graph_overhead_custom(graph_size, false) + 1024,
-            /*.mem_buffer =*/nullptr,
-            /*.no_alloc   =*/true,
-        };
-        ggml_context* graph_ctx = ggml_init(params);
-        GGML_ASSERT(graph_ctx != nullptr);
-        ggml_cgraph* segment_graph = ggml_new_graph_custom(graph_ctx, graph_size, false);
-        GGML_ASSERT(segment_graph != nullptr);
-
-        for (const auto& input : segment.input_refs) {
-            ggml_tensor* current_input = input_tensor(gf, input);
-            if (current_input == nullptr) {
-                continue;
-            }
-            GGML_ASSERT(segment_graph->n_leafs < segment_graph->size);
-            segment_graph->leafs[segment_graph->n_leafs++] = current_input;
-        }
-
-        for (int output_node_index : segment.output_node_indices) {
-            ggml_tensor* output = ggml_graph_node(gf, output_node_index);
-            if (output == nullptr) {
-                continue;
-            }
-            ggml_set_output(output);
-        }
-        for (int node_idx : segment.internal_node_indices) {
-            ggml_graph_add_node(segment_graph, ggml_graph_node(gf, node_idx));
-        }
-        *graph_ctx_out = graph_ctx;
-        return segment_graph;
-    }
-
-    size_t measure_segment_compute_buffer(ggml_backend_t backend,
-                                          ggml_cgraph* gf,
-                                          const Segment& segment,
-                                          const char* log_desc) {
-        GGML_ASSERT(backend != nullptr);
-        GGML_ASSERT(gf != nullptr);
-        if (segment.internal_node_indices.empty()) {
-            return 0;
-        }
-
-        ggml_context* graph_ctx    = nullptr;
-        ggml_cgraph* segment_graph = build_segment_graph(gf, segment, &graph_ctx);
-        ggml_gallocr_t allocr      = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));
-
-        size_t sizes[1] = {0};
-        ggml_gallocr_reserve_n_size(
-            allocr,
-            segment_graph,
-            nullptr,
-            nullptr,
-            sizes);
-        size_t buffer_size = sizes[0];
-
-        ggml_gallocr_free(allocr);
-        ggml_free(graph_ctx);
-        return buffer_size;
-    }
-
-    Plan build_plan(ggml_backend_t backend,
-                    ggml_cgraph* gf,
-                    const std::unordered_set<const ggml_tensor*>& params_tensor_set,
-                    const char* log_desc) {
-        GGML_ASSERT(backend != nullptr);
-        GGML_ASSERT(gf != nullptr);
-        Plan plan;
-        plan.available    = true;
-        const int n_nodes = ggml_graph_n_nodes(gf);
-        if (n_nodes <= 0) {
-            return plan;
-        }
-        plan.n_nodes = n_nodes;
-        plan.n_leafs = gf->n_leafs;
-        for (int i = 0; i < gf->n_leafs; ++i) {
-            ggml_tensor* leaf = gf->leafs[i];
-            if (is_params_tensor(params_tensor_set, leaf)) {
-                continue;
-            }
-            auto shape       = input_shape(leaf);
-            shape.leaf_index = i;
-            plan.input_shapes.push_back(shape);
-        }
-
-        std::unordered_map<const ggml_tensor*, int> producer_index;
-        producer_index.reserve(static_cast<size_t>(n_nodes));
-        for (int i = 0; i < n_nodes; ++i) {
-            producer_index[ggml_graph_node(gf, i)] = i;
-        }
-
-        std::vector<Segment> grouped_segments;
-        std::unordered_map<std::string, size_t> group_to_segment;
-        for (int i = 0; i < n_nodes; ++i) {
-            ggml_tensor* node = ggml_graph_node(gf, i);
-            if (!is_graph_cut_tensor(node)) {
-                continue;
-            }
-
-            plan.has_cuts = true;
-            std::string full_name(node->name);
-            std::string payload = full_name.substr(std::strlen(GGML_RUNNER_CUT_PREFIX));
-            size_t sep          = payload.find('|');
-            std::string group   = sep == std::string::npos ? payload : payload.substr(0, sep);
-
-            auto it = group_to_segment.find(group);
-            if (it == group_to_segment.end()) {
-                Segment segment;
-                segment.group_name = group;
-                segment.output_node_indices.push_back(i);
-                group_to_segment[group] = grouped_segments.size();
-                grouped_segments.push_back(std::move(segment));
-            } else {
-                auto& segment = grouped_segments[it->second];
-                segment.output_node_indices.push_back(i);
-            }
-        }
-
-        if (!plan.has_cuts) {
-            return plan;
-        }
-
-        std::unordered_set<int> available_cut_output_node_indices;
-        available_cut_output_node_indices.reserve(static_cast<size_t>(n_nodes));
-        for (auto& segment : grouped_segments) {
-            build_segment(gf,
-                          plan,
-                          segment,
-                          producer_index,
-                          available_cut_output_node_indices,
-                          backend,
-                          params_tensor_set,
-                          log_desc);
-        }
-
-        int final_output_index = graph_node_index_by_name(gf, "ggml_runner_final_result_tensor");
-        if (final_output_index < 0) {
-            final_output_index = n_nodes - 1;
-        }
-        ggml_tensor* final_output = final_output_index >= 0 ? ggml_graph_node(gf, final_output_index) : nullptr;
-        if (final_output != nullptr && available_cut_output_node_indices.find(final_output_index) == available_cut_output_node_indices.end()) {
-            Segment final_segment;
-            final_segment.group_name = "ggml_runner.final";
-            final_segment.output_node_indices.push_back(final_output_index);
-            build_segment(gf,
-                          plan,
-                          final_segment,
-                          producer_index,
-                          available_cut_output_node_indices,
-                          backend,
-                          params_tensor_set,
-                          log_desc);
-        }
-
-        return plan;
-    }
-
-    Plan apply_max_vram_budget(ggml_cgraph* gf,
-                               const Plan& base_plan,
-                               size_t max_graph_vram_bytes,
-                               ggml_backend_t backend,
-                               const std::unordered_set<const ggml_tensor*>& params_tensor_set,
-                               const char* log_desc) {
-        GGML_ASSERT(backend != nullptr);
-        GGML_ASSERT(gf != nullptr);
-        int64_t t_budget_begin = ggml_time_ms();
-        if (max_graph_vram_bytes == 0 || !base_plan.has_cuts || base_plan.segments.size() <= 1) {
-            return base_plan;
-        }
-
-        const int n_nodes = ggml_graph_n_nodes(gf);
-        std::unordered_map<const ggml_tensor*, int> producer_index;
-        producer_index.reserve(static_cast<size_t>(n_nodes));
-        for (int i = 0; i < n_nodes; ++i) {
-            producer_index[ggml_graph_node(gf, i)] = i;
-        }
-
-        Plan merged_plan;
-        merged_plan.available = true;
-        merged_plan.has_cuts  = base_plan.has_cuts;
-        merged_plan.valid     = base_plan.valid;
-        merged_plan.n_nodes   = base_plan.n_nodes;
-        merged_plan.n_leafs   = base_plan.n_leafs;
-
-        std::unordered_set<int> available_cut_output_node_indices;
-        available_cut_output_node_indices.reserve(static_cast<size_t>(n_nodes));
-
-        size_t start_segment_index = 0;
-        while (start_segment_index < base_plan.segments.size()) {
-            Plan single_plan;
-            auto single_available_cut_output_node_indices = available_cut_output_node_indices;
-            auto single_seed                              = make_segment_seed(base_plan,
-                                                                              start_segment_index,
-                                                                              start_segment_index);
-            build_segment(gf,
-                          single_plan,
-                          single_seed,
-                          producer_index,
-                          single_available_cut_output_node_indices,
-                          backend,
-                          params_tensor_set,
-                          log_desc);
-            GGML_ASSERT(!single_plan.segments.empty());
-
-            size_t best_end_segment_index = start_segment_index;
-            bool can_merge_next_segment   = graph_cut_segment_vram_bytes(single_plan.segments.back()) <= max_graph_vram_bytes;
-
-            while (can_merge_next_segment && best_end_segment_index + 1 < base_plan.segments.size()) {
-                const size_t next_end_segment_index = best_end_segment_index + 1;
-                Plan candidate_plan;
-                auto candidate_available_cut_output_node_indices = available_cut_output_node_indices;
-                auto candidate_seed                              = make_segment_seed(base_plan,
-                                                                                     start_segment_index,
-                                                                                     next_end_segment_index);
-                build_segment(gf,
-                              candidate_plan,
-                              candidate_seed,
-                              producer_index,
-                              candidate_available_cut_output_node_indices,
-                              backend,
-                              params_tensor_set,
-                              log_desc);
-                GGML_ASSERT(!candidate_plan.segments.empty());
-
-                const auto& candidate_segment = candidate_plan.segments.back();
-                if (graph_cut_segment_vram_bytes(candidate_segment) > max_graph_vram_bytes) {
-                    break;
-                }
-
-                best_end_segment_index = next_end_segment_index;
-            }
-
-            auto best_seed = make_segment_seed(base_plan,
-                                               start_segment_index,
-                                               best_end_segment_index);
-            build_segment(gf,
-                          merged_plan,
-                          best_seed,
-                          producer_index,
-                          available_cut_output_node_indices,
-                          backend,
-                          params_tensor_set,
-                          log_desc);
-            start_segment_index = best_end_segment_index + 1;
-        }
-
-        if (log_desc != nullptr && merged_plan.segments.size() != base_plan.segments.size()) {
-            LOG_INFO("%s graph cut max_vram=%.2f MB merged %zu segments -> %zu segments",
-                     log_desc,
-                     max_graph_vram_bytes / 1024.0 / 1024.0,
-                     base_plan.segments.size(),
-                     merged_plan.segments.size());
-        }
-
-        if (log_desc != nullptr) {
-            LOG_DEBUG("%s graph cut max_vram budget merge took %lld ms",
-                      log_desc,
-                      ggml_time_ms() - t_budget_begin);
-        }
-
-        return merged_plan;
-    }
-
-    Plan resolve_plan(ggml_backend_t backend,
-                      ggml_cgraph* gf,
-                      PlanCache* cache,
-                      size_t max_graph_vram_bytes,
-                      const std::unordered_set<const ggml_tensor*>& params_tensor_set,
-                      const char* log_desc) {
-        GGML_ASSERT(backend != nullptr);
-        GGML_ASSERT(gf != nullptr);
-        GGML_ASSERT(cache != nullptr);
-
-        int64_t t_prepare_begin = ggml_time_ms();
-        Plan base_plan;
-        int64_t t_plan_begin = ggml_time_ms();
-        if (cache->graph_cut_plan.available && plan_matches_graph(gf, cache->graph_cut_plan)) {
-            base_plan = cache->graph_cut_plan;
-        } else {
-            base_plan                                = build_plan(backend, gf, params_tensor_set, log_desc);
-            cache->graph_cut_plan                    = base_plan;
-            cache->graph_cut_plan.available          = true;
-            cache->budgeted_graph_cut_plan.available = false;
-            if (log_desc != nullptr) {
-                LOG_INFO("%s build cached graph cut plan done (taking %lld ms)", log_desc, ggml_time_ms() - t_plan_begin);
-            }
-        }
-
-        Plan resolved_plan = base_plan;
-        if (max_graph_vram_bytes > 0 && base_plan.has_cuts) {
-            if (cache->budgeted_graph_cut_plan.available &&
-                cache->budgeted_graph_cut_plan_max_vram_bytes == max_graph_vram_bytes &&
-                plan_matches_graph(gf, cache->budgeted_graph_cut_plan)) {
-                resolved_plan = cache->budgeted_graph_cut_plan;
-            } else {
-                resolved_plan                                 = apply_max_vram_budget(gf,
-                                                                                      base_plan,
-                                                                                      max_graph_vram_bytes,
-                                                                                      backend,
-                                                                                      params_tensor_set,
-                                                                                      log_desc);
-                cache->budgeted_graph_cut_plan                = resolved_plan;
-                cache->budgeted_graph_cut_plan.available      = true;
-                cache->budgeted_graph_cut_plan_max_vram_bytes = max_graph_vram_bytes;
-            }
-        }
-        return resolved_plan;
-    }
-
-    void annotate_residency(Plan& plan, size_t max_graph_vram_bytes) {
-        // Cached plans may be reused with a smaller live budget.
-        for (auto& seg : plan.segments) {
-            seg.residency = SegmentResidency::STREAMED;
-        }
-        if (max_graph_vram_bytes == 0 || plan.segments.size() < 2) {
-            return;
-        }
-
-        bool any_param_bearing = false;
-        for (const auto& seg : plan.segments) {
-            if (seg.input_param_bytes > 0) {
-                any_param_bearing = true;
-                break;
-            }
-        }
-        if (!any_param_bearing) {
-            return;
-        }
-
-        // Leave room for the largest active streamed segment.
-        size_t worst_streamed_footprint = 0;
-        for (const auto& seg : plan.segments) {
-            const size_t seg_footprint = seg.input_param_bytes +
-                                         seg.compute_buffer_size +
-                                         seg.output_bytes +
-                                         seg.input_previous_cut_bytes +
-                                         seg.input_external_bytes;
-            if (seg_footprint > worst_streamed_footprint) {
-                worst_streamed_footprint = seg_footprint;
-            }
-        }
-        constexpr size_t safety = 512ull * 1024 * 1024;
-        const size_t reserved   = safety + worst_streamed_footprint;
-
-        if (max_graph_vram_bytes <= reserved) {
-            return;
-        }
-        const size_t available = max_graph_vram_bytes - reserved;
-
-        size_t cumulative = 0;
-        for (auto& seg : plan.segments) {
-            if (cumulative + seg.input_param_bytes > available) {
-                break;
-            }
-            seg.residency = SegmentResidency::RESIDENT;
-            cumulative += seg.input_param_bytes;
-        }
-    }
-
-}  // namespace sd::ggml_graph_cut

src/core/ggml_graph_cut.h

@@ -1,117 +0,0 @@
-#ifndef __SD_CORE_GGML_GRAPH_CUT_H__
-#define __SD_CORE_GGML_GRAPH_CUT_H__
-
-#include <array>
-#include <cstdint>
-#include <string>
-#include <unordered_set>
-#include <vector>
-
-#include "ggml-backend.h"
-#include "ggml.h"
-
-namespace sd::ggml_graph_cut {
-
-    // Streaming residency for a segment's params.
-    enum class SegmentResidency : uint8_t {
-        STREAMED = 0,
-        RESIDENT = 1,
-    };
-
-    struct Segment {
-        enum InputType {
-            INPUT_EXTERNAL = 0,
-            INPUT_PREVIOUS_CUT,
-            INPUT_PARAM,
-        };
-
-        struct InputRef {
-            InputType type = INPUT_EXTERNAL;
-            std::string display_name;
-            int leaf_index = -1;
-            int node_index = -1;
-        };
-
-        size_t compute_buffer_size      = 0;
-        size_t output_bytes             = 0;
-        size_t input_external_bytes     = 0;
-        size_t input_previous_cut_bytes = 0;
-        size_t input_param_bytes        = 0;
-        std::string group_name;
-        std::vector<int> internal_node_indices;
-        std::vector<int> output_node_indices;
-        std::vector<InputRef> input_refs;
-        SegmentResidency residency = SegmentResidency::STREAMED;
-    };
-
-    struct Plan {
-        struct InputShape {
-            int leaf_index                        = -1;
-            ggml_type type                        = GGML_TYPE_COUNT;
-            std::array<int64_t, GGML_MAX_DIMS> ne = {0, 0, 0, 0};
-        };
-
-        bool available = false;
-        bool has_cuts  = false;
-        bool valid     = true;
-        int n_nodes    = 0;
-        int n_leafs    = 0;
-        std::vector<InputShape> input_shapes;
-        std::vector<Segment> segments;
-    };
-
-    struct PlanCache {
-        Plan graph_cut_plan;
-        Plan budgeted_graph_cut_plan;
-        size_t budgeted_graph_cut_plan_max_vram_bytes = 0;
-    };
-
-    static constexpr const char* GGML_RUNNER_CUT_PREFIX = "ggml_runner_cut:";
-
-    bool is_graph_cut_tensor(const ggml_tensor* tensor);
-    std::string make_graph_cut_name(const std::string& group, const std::string& output);
-    void mark_graph_cut(ggml_tensor* tensor, const std::string& group, const std::string& output);
-    int leaf_count(ggml_cgraph* gf);
-    ggml_tensor* leaf_tensor(ggml_cgraph* gf, int leaf_index);
-    ggml_backend_buffer_t tensor_buffer(const ggml_tensor* tensor);
-    ggml_tensor* cache_source_tensor(ggml_tensor* tensor);
-    size_t cache_tensor_bytes(const ggml_tensor* tensor);
-    bool plan_matches_graph(ggml_cgraph* gf, const Plan& plan);
-    ggml_tensor* output_tensor(ggml_cgraph* gf, const Segment& segment, size_t output_index);
-    ggml_tensor* input_tensor(ggml_cgraph* gf, const Segment::InputRef& input_ref);
-    std::vector<ggml_tensor*> param_tensors(ggml_cgraph* gf, const Segment& segment);
-    std::vector<ggml_tensor*> runtime_param_tensors(ggml_cgraph* gf, const Segment& segment, const char* log_desc);
-    std::unordered_set<std::string> collect_future_input_names(ggml_cgraph* gf,
-                                                               const Plan& plan,
-                                                               size_t current_segment_index);
-    ggml_cgraph* build_segment_graph(ggml_cgraph* gf,
-                                     const Segment& segment,
-                                     ggml_context** graph_ctx_out);
-    size_t measure_segment_compute_buffer(ggml_backend_t backend,
-                                          ggml_cgraph* gf,
-                                          const Segment& segment,
-                                          const char* log_desc);
-    size_t max_vram_gib_to_bytes(float max_vram);
-    float resolve_max_vram_gib(float max_vram, ggml_backend_t backend);
-    Plan build_plan(ggml_backend_t backend,
-                    ggml_cgraph* gf,
-                    const std::unordered_set<const ggml_tensor*>& params_tensor_set,
-                    const char* log_desc);
-    Plan apply_max_vram_budget(ggml_cgraph* gf,
-                               const Plan& base_plan,
-                               size_t max_graph_vram_bytes,
-                               ggml_backend_t backend,
-                               const std::unordered_set<const ggml_tensor*>& params_tensor_set,
-                               const char* log_desc);
-    Plan resolve_plan(ggml_backend_t backend,
-                      ggml_cgraph* gf,
-                      PlanCache* cache,
-                      size_t max_graph_vram_bytes,
-                      const std::unordered_set<const ggml_tensor*>& params_tensor_set,
-                      const char* log_desc);
-
-    // Mark leading segments resident when they fit after streamed-segment headroom.
-    void annotate_residency(Plan& plan, size_t max_graph_vram_bytes);
-}  // namespace sd::ggml_graph_cut
-
-#endif  // __SD_CORE_GGML_GRAPH_CUT_H__

src/core/layer_registry.cpp

@@ -1,132 +0,0 @@
-#include "core/layer_registry.h"
-
-#include <utility>
-
-#include "core/util.h"
-
-namespace sd::layer_registry {
-
-    void LayerRegistry::register_layer(const std::string& name, ggml_tensor* tensor) {
-        auto& info = layers_[name];
-        info.tensors.push_back(tensor);
-        info.bytes += ggml_nbytes(tensor);
-    }
-
-    bool LayerRegistry::move_layer_to_gpu(const std::string& name) {
-        auto it = layers_.find(name);
-        if (it == layers_.end())
-            return false;
-
-        LayerInfo& info = it->second;
-        if (info.on_gpu)
-            return true;
-        if (gpu_backend_ == nullptr || cpu_backend_ == nullptr) {
-            LOG_ERROR("layer_registry: backends not set; cannot move '%s' to GPU",
-                      name.c_str());
-            return false;
-        }
-        if (info.tensors.empty()) {
-            info.on_gpu = true;
-            return true;
-        }
-
-        // 1. Build a no_alloc context big enough to hold one twin tensor per CPU
-        //    tensor, plus a little overhead.
-        const size_t ctx_size = info.tensors.size() * ggml_tensor_overhead() + 1024;
-        ggml_init_params ctx_params{ctx_size, /*mem_buffer=*/nullptr, /*no_alloc=*/true};
-        ggml_context* twin_ctx = ggml_init(ctx_params);
-        if (twin_ctx == nullptr) {
-            LOG_ERROR("layer_registry: failed to allocate twin context for '%s'",
-                      name.c_str());
-            return false;
-        }
-
-        // 2. Create one GPU twin per CPU tensor. The twin shares the original
-        //    name so any name-based lookup keeps working.
-        std::vector<ggml_tensor*> gpu_twins;
-        gpu_twins.reserve(info.tensors.size());
-        for (ggml_tensor* cpu_t : info.tensors) {
-            ggml_tensor* twin = ggml_dup_tensor(twin_ctx, cpu_t);
-            if (cpu_t->name[0] != '\0') {
-                ggml_set_name(twin, cpu_t->name);
-            }
-            gpu_twins.push_back(twin);
-        }
-
-        // 3. Back the twins with a GPU buffer in one alloc call.
-        ggml_backend_buffer_t gpu_buffer = ggml_backend_alloc_ctx_tensors(twin_ctx, gpu_backend_);
-        if (gpu_buffer == nullptr) {
-            LOG_ERROR("layer_registry: failed to allocate GPU buffer for '%s'",
-                      name.c_str());
-            ggml_free(twin_ctx);
-            return false;
-        }
-
-        // 4. H2D copy + sync.
-        for (size_t i = 0; i < info.tensors.size(); ++i) {
-            ggml_backend_tensor_copy(info.tensors[i], gpu_twins[i]);
-        }
-        ggml_backend_synchronize(gpu_backend_);
-
-        // 5. Swap buffer/data/extra so the originals now point at GPU memory.
-        for (size_t i = 0; i < info.tensors.size(); ++i) {
-            std::swap(info.tensors[i]->buffer, gpu_twins[i]->buffer);
-            std::swap(info.tensors[i]->data, gpu_twins[i]->data);
-            std::swap(info.tensors[i]->extra, gpu_twins[i]->extra);
-        }
-
-        info.gpu_twins  = std::move(gpu_twins);
-        info.twin_ctx   = twin_ctx;
-        info.gpu_buffer = gpu_buffer;
-        info.on_gpu     = true;
-        return true;
-    }
-
-    bool LayerRegistry::move_layer_to_cpu(const std::string& name) {
-        auto it = layers_.find(name);
-        if (it == layers_.end())
-            return false;
-
-        LayerInfo& info = it->second;
-        if (!info.on_gpu)
-            return true;
-        if (info.tensors.size() != info.gpu_twins.size()) {
-            LOG_ERROR("layer_registry: twin/tensor count mismatch for '%s'",
-                      name.c_str());
-            return false;
-        }
-
-        // 1. Swap back: originals point at CPU memory again.
-        for (size_t i = 0; i < info.tensors.size(); ++i) {
-            if (info.gpu_twins[i] == nullptr)
-                continue;
-            std::swap(info.tensors[i]->buffer, info.gpu_twins[i]->buffer);
-            std::swap(info.tensors[i]->data, info.gpu_twins[i]->data);
-            std::swap(info.tensors[i]->extra, info.gpu_twins[i]->extra);
-        }
-
-        // 2. Free the GPU buffer + twin context.
-        if (info.gpu_buffer != nullptr) {
-            ggml_backend_buffer_free(info.gpu_buffer);
-            info.gpu_buffer = nullptr;
-        }
-        if (info.twin_ctx != nullptr) {
-            ggml_free(info.twin_ctx);
-            info.twin_ctx = nullptr;
-        }
-        info.gpu_twins.clear();
-        info.on_gpu = false;
-        return true;
-    }
-
-    bool LayerRegistry::is_layer_on_gpu(const std::string& name) const {
-        auto it = layers_.find(name);
-        return it != layers_.end() && it->second.on_gpu;
-    }
-
-    size_t LayerRegistry::get_layer_size(const std::string& name) const {
-        auto it = layers_.find(name);
-        return it != layers_.end() ? it->second.bytes : 0;
-    }
-
-}  // namespace sd::layer_registry

src/core/layer_registry.h

@@ -1,50 +0,0 @@
-#ifndef __SD_CORE_LAYER_REGISTRY_H__
-#define __SD_CORE_LAYER_REGISTRY_H__
-
-#include <map>
-#include <set>
-#include <string>
-#include <vector>
-
-#include "ggml-backend.h"
-#include "ggml.h"
-
-namespace sd::layer_registry {
-
-    struct LayerInfo {
-        std::vector<ggml_tensor*> tensors;
-        std::vector<ggml_tensor*> gpu_twins;
-        ggml_context* twin_ctx           = nullptr;
-        ggml_backend_buffer_t gpu_buffer = nullptr;
-        bool on_gpu                      = false;
-        size_t bytes                     = 0;
-    };
-
-    class LayerRegistry {
-    public:
-        LayerRegistry() = default;
-        LayerRegistry(ggml_backend_t gpu_backend, ggml_backend_t cpu_backend)
-            : gpu_backend_(gpu_backend), cpu_backend_(cpu_backend) {}
-
-        void set_backends(ggml_backend_t gpu_backend, ggml_backend_t cpu_backend) {
-            gpu_backend_ = gpu_backend;
-            cpu_backend_ = cpu_backend;
-        }
-        void register_layer(const std::string& name, ggml_tensor* tensor);
-        bool move_layer_to_gpu(const std::string& name);
-        bool move_layer_to_cpu(const std::string& name);
-        bool is_layer_on_gpu(const std::string& name) const;
-        size_t get_layer_size(const std::string& name) const;
-        size_t get_layer_count() const { return layers_.size(); }
-
-        const std::map<std::string, LayerInfo>& layers() const { return layers_; }
-
-    private:
-        ggml_backend_t gpu_backend_ = nullptr;
-        ggml_backend_t cpu_backend_ = nullptr;
-        std::map<std::string, LayerInfo> layers_;
-    };
-
-}  // namespace sd::layer_registry
-
-#endif  // __SD_CORE_LAYER_REGISTRY_H__

src/diffusion_model.hpp

@@ -0,0 +1,519 @@
+#ifndef __DIFFUSION_MODEL_H__
+#define __DIFFUSION_MODEL_H__
+
+#include <optional>
+#include "anima.hpp"
+#include "flux.hpp"
+#include "mmdit.hpp"
+#include "qwen_image.hpp"
+#include "tensor_ggml.hpp"
+#include "unet.hpp"
+#include "wan.hpp"
+#include "z_image.hpp"
+
+struct DiffusionParams {
+    const sd::Tensor<float>* x                        = nullptr;
+    const sd::Tensor<float>* timesteps                = nullptr;
+    const sd::Tensor<float>* context                  = nullptr;
+    const sd::Tensor<float>* c_concat                 = nullptr;
+    const sd::Tensor<float>* y                        = nullptr;
+    const sd::Tensor<int32_t>* t5_ids                 = nullptr;
+    const sd::Tensor<float>* t5_weights               = nullptr;
+    const sd::Tensor<float>* guidance                 = nullptr;
+    const std::vector<sd::Tensor<float>>* ref_latents = nullptr;
+    bool increase_ref_index                           = false;
+    int num_video_frames                              = -1;
+    const std::vector<sd::Tensor<float>>* controls    = nullptr;
+    float control_strength                            = 0.f;
+    const sd::Tensor<float>* vace_context             = nullptr;
+    float vace_strength                               = 1.f;
+    const std::vector<int>* skip_layers               = nullptr;
+};
+
+template <typename T>
+static inline const sd::Tensor<T>& tensor_or_empty(const sd::Tensor<T>* tensor) {
+    static const sd::Tensor<T> kEmpty;
+    return tensor != nullptr ? *tensor : kEmpty;
+}
+
+struct DiffusionModel {
+    virtual std::string get_desc()                                               = 0;
+    virtual sd::Tensor<float> compute(int n_threads,
+                                      const DiffusionParams& diffusion_params)   = 0;
+    virtual void alloc_params_buffer()                                           = 0;
+    virtual void free_params_buffer()                                            = 0;
+    virtual void free_compute_buffer()                                           = 0;
+    virtual void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) = 0;
+    virtual size_t get_params_buffer_size()                                      = 0;
+    virtual void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter){};
+    virtual int64_t get_adm_in_channels()                            = 0;
+    virtual void set_flash_attention_enabled(bool enabled)           = 0;
+    virtual void set_circular_axes(bool circular_x, bool circular_y) = 0;
+};
+
+struct UNetModel : public DiffusionModel {
+    UNetModelRunner unet;
+
+    UNetModel(ggml_backend_t backend,
+              bool offload_params_to_cpu,
+              const String2TensorStorage& tensor_storage_map = {},
+              SDVersion version                              = VERSION_SD1)
+        : unet(backend, offload_params_to_cpu, tensor_storage_map, "model.diffusion_model", version) {
+    }
+
+    std::string get_desc() override {
+        return unet.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        unet.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        unet.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        unet.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+        unet.get_param_tensors(tensors, "model.diffusion_model");
+    }
+
+    size_t get_params_buffer_size() override {
+        return unet.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        unet.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return unet.unet.adm_in_channels;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        unet.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        unet.set_circular_axes(circular_x, circular_y);
+    }
+
+    sd::Tensor<float> compute(int n_threads,
+                              const DiffusionParams& diffusion_params) override {
+        GGML_ASSERT(diffusion_params.x != nullptr);
+        GGML_ASSERT(diffusion_params.timesteps != nullptr);
+        static const std::vector<sd::Tensor<float>> empty_controls;
+        return unet.compute(n_threads,
+                            *diffusion_params.x,
+                            *diffusion_params.timesteps,
+                            tensor_or_empty(diffusion_params.context),
+                            tensor_or_empty(diffusion_params.c_concat),
+                            tensor_or_empty(diffusion_params.y),
+                            diffusion_params.num_video_frames,
+                            diffusion_params.controls ? *diffusion_params.controls : empty_controls,
+                            diffusion_params.control_strength);
+    }
+};
+
+struct MMDiTModel : public DiffusionModel {
+    MMDiTRunner mmdit;
+
+    MMDiTModel(ggml_backend_t backend,
+               bool offload_params_to_cpu,
+               const String2TensorStorage& tensor_storage_map = {})
+        : mmdit(backend, offload_params_to_cpu, tensor_storage_map, "model.diffusion_model") {
+    }
+
+    std::string get_desc() override {
+        return mmdit.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        mmdit.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        mmdit.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        mmdit.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+        mmdit.get_param_tensors(tensors, "model.diffusion_model");
+    }
+
+    size_t get_params_buffer_size() override {
+        return mmdit.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        mmdit.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return 768 + 1280;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        mmdit.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        mmdit.set_circular_axes(circular_x, circular_y);
+    }
+
+    sd::Tensor<float> compute(int n_threads,
+                              const DiffusionParams& diffusion_params) override {
+        GGML_ASSERT(diffusion_params.x != nullptr);
+        GGML_ASSERT(diffusion_params.timesteps != nullptr);
+        static const std::vector<int> empty_skip_layers;
+        return mmdit.compute(n_threads,
+                             *diffusion_params.x,
+                             *diffusion_params.timesteps,
+                             tensor_or_empty(diffusion_params.context),
+                             tensor_or_empty(diffusion_params.y),
+                             diffusion_params.skip_layers ? *diffusion_params.skip_layers : empty_skip_layers);
+    }
+};
+
+struct FluxModel : public DiffusionModel {
+    Flux::FluxRunner flux;
+
+    FluxModel(ggml_backend_t backend,
+              bool offload_params_to_cpu,
+              const String2TensorStorage& tensor_storage_map = {},
+              SDVersion version                              = VERSION_FLUX,
+              bool use_mask                                  = false)
+        : flux(backend, offload_params_to_cpu, tensor_storage_map, "model.diffusion_model", version, use_mask) {
+    }
+
+    std::string get_desc() override {
+        return flux.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        flux.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        flux.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        flux.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+        flux.get_param_tensors(tensors, "model.diffusion_model");
+    }
+
+    size_t get_params_buffer_size() override {
+        return flux.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        flux.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return 768;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        flux.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        flux.set_circular_axes(circular_x, circular_y);
+    }
+
+    sd::Tensor<float> compute(int n_threads,
+                              const DiffusionParams& diffusion_params) override {
+        GGML_ASSERT(diffusion_params.x != nullptr);
+        GGML_ASSERT(diffusion_params.timesteps != nullptr);
+        static const std::vector<sd::Tensor<float>> empty_ref_latents;
+        static const std::vector<int> empty_skip_layers;
+        return flux.compute(n_threads,
+                            *diffusion_params.x,
+                            *diffusion_params.timesteps,
+                            tensor_or_empty(diffusion_params.context),
+                            tensor_or_empty(diffusion_params.c_concat),
+                            tensor_or_empty(diffusion_params.y),
+                            tensor_or_empty(diffusion_params.guidance),
+                            diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents,
+                            diffusion_params.increase_ref_index,
+                            diffusion_params.skip_layers ? *diffusion_params.skip_layers : empty_skip_layers);
+    }
+};
+
+struct AnimaModel : public DiffusionModel {
+    std::string prefix;
+    Anima::AnimaRunner anima;
+
+    AnimaModel(ggml_backend_t backend,
+               bool offload_params_to_cpu,
+               const String2TensorStorage& tensor_storage_map = {},
+               const std::string prefix                       = "model.diffusion_model")
+        : prefix(prefix), anima(backend, offload_params_to_cpu, tensor_storage_map, prefix) {
+    }
+
+    std::string get_desc() override {
+        return anima.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        anima.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        anima.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        anima.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+        anima.get_param_tensors(tensors, prefix);
+    }
+
+    size_t get_params_buffer_size() override {
+        return anima.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        anima.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return 768;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        anima.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        anima.set_circular_axes(circular_x, circular_y);
+    }
+
+    sd::Tensor<float> compute(int n_threads,
+                              const DiffusionParams& diffusion_params) override {
+        GGML_ASSERT(diffusion_params.x != nullptr);
+        GGML_ASSERT(diffusion_params.timesteps != nullptr);
+        return anima.compute(n_threads,
+                             *diffusion_params.x,
+                             *diffusion_params.timesteps,
+                             tensor_or_empty(diffusion_params.context),
+                             tensor_or_empty(diffusion_params.t5_ids),
+                             tensor_or_empty(diffusion_params.t5_weights));
+    }
+};
+
+struct WanModel : public DiffusionModel {
+    std::string prefix;
+    WAN::WanRunner wan;
+
+    WanModel(ggml_backend_t backend,
+             bool offload_params_to_cpu,
+             const String2TensorStorage& tensor_storage_map = {},
+             const std::string prefix                       = "model.diffusion_model",
+             SDVersion version                              = VERSION_WAN2)
+        : prefix(prefix), wan(backend, offload_params_to_cpu, tensor_storage_map, prefix, version) {
+    }
+
+    std::string get_desc() override {
+        return wan.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        wan.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        wan.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        wan.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+        wan.get_param_tensors(tensors, prefix);
+    }
+
+    size_t get_params_buffer_size() override {
+        return wan.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        wan.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return 768;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        wan.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        wan.set_circular_axes(circular_x, circular_y);
+    }
+
+    sd::Tensor<float> compute(int n_threads,
+                              const DiffusionParams& diffusion_params) override {
+        GGML_ASSERT(diffusion_params.x != nullptr);
+        GGML_ASSERT(diffusion_params.timesteps != nullptr);
+        return wan.compute(n_threads,
+                           *diffusion_params.x,
+                           *diffusion_params.timesteps,
+                           tensor_or_empty(diffusion_params.context),
+                           tensor_or_empty(diffusion_params.y),
+                           tensor_or_empty(diffusion_params.c_concat),
+                           sd::Tensor<float>(),
+                           tensor_or_empty(diffusion_params.vace_context),
+                           diffusion_params.vace_strength);
+    }
+};
+
+struct QwenImageModel : public DiffusionModel {
+    std::string prefix;
+    Qwen::QwenImageRunner qwen_image;
+
+    QwenImageModel(ggml_backend_t backend,
+                   bool offload_params_to_cpu,
+                   const String2TensorStorage& tensor_storage_map = {},
+                   const std::string prefix                       = "model.diffusion_model",
+                   SDVersion version                              = VERSION_QWEN_IMAGE,
+                   bool zero_cond_t                               = false)
+        : prefix(prefix), qwen_image(backend, offload_params_to_cpu, tensor_storage_map, prefix, version, zero_cond_t) {
+    }
+
+    std::string get_desc() override {
+        return qwen_image.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        qwen_image.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        qwen_image.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        qwen_image.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+        qwen_image.get_param_tensors(tensors, prefix);
+    }
+
+    size_t get_params_buffer_size() override {
+        return qwen_image.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        qwen_image.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return 768;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        qwen_image.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        qwen_image.set_circular_axes(circular_x, circular_y);
+    }
+
+    sd::Tensor<float> compute(int n_threads,
+                              const DiffusionParams& diffusion_params) override {
+        GGML_ASSERT(diffusion_params.x != nullptr);
+        GGML_ASSERT(diffusion_params.timesteps != nullptr);
+        static const std::vector<sd::Tensor<float>> empty_ref_latents;
+        return qwen_image.compute(n_threads,
+                                  *diffusion_params.x,
+                                  *diffusion_params.timesteps,
+                                  tensor_or_empty(diffusion_params.context),
+                                  diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents,
+                                  true);
+    }
+};
+
+struct ZImageModel : public DiffusionModel {
+    std::string prefix;
+    ZImage::ZImageRunner z_image;
+
+    ZImageModel(ggml_backend_t backend,
+                bool offload_params_to_cpu,
+                const String2TensorStorage& tensor_storage_map = {},
+                const std::string prefix                       = "model.diffusion_model",
+                SDVersion version                              = VERSION_Z_IMAGE)
+        : prefix(prefix), z_image(backend, offload_params_to_cpu, tensor_storage_map, prefix, version) {
+    }
+
+    std::string get_desc() override {
+        return z_image.get_desc();
+    }
+
+    void alloc_params_buffer() override {
+        z_image.alloc_params_buffer();
+    }
+
+    void free_params_buffer() override {
+        z_image.free_params_buffer();
+    }
+
+    void free_compute_buffer() override {
+        z_image.free_compute_buffer();
+    }
+
+    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+        z_image.get_param_tensors(tensors, prefix);
+    }
+
+    size_t get_params_buffer_size() override {
+        return z_image.get_params_buffer_size();
+    }
+
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override {
+        z_image.set_weight_adapter(adapter);
+    }
+
+    int64_t get_adm_in_channels() override {
+        return 768;
+    }
+
+    void set_flash_attention_enabled(bool enabled) {
+        z_image.set_flash_attention_enabled(enabled);
+    }
+
+    void set_circular_axes(bool circular_x, bool circular_y) override {
+        z_image.set_circular_axes(circular_x, circular_y);
+    }
+
+    sd::Tensor<float> compute(int n_threads,
+                              const DiffusionParams& diffusion_params) override {
+        GGML_ASSERT(diffusion_params.x != nullptr);
+        GGML_ASSERT(diffusion_params.timesteps != nullptr);
+        static const std::vector<sd::Tensor<float>> empty_ref_latents;
+        return z_image.compute(n_threads,
+                               *diffusion_params.x,
+                               *diffusion_params.timesteps,
+                               tensor_or_empty(diffusion_params.context),
+                               diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents,
+                               true);
+    }
+};
+
+#endif

src/easycache.hpp

@@ -1,15 +1,15 @@
-#ifndef __SD_RUNTIME_EASYCACHE_HPP__
+#ifndef __EASYCACHE_HPP__
-#define __SD_RUNTIME_EASYCACHE_HPP__
+#define __EASYCACHE_HPP__
 
 #include <cmath>
 #include <limits>
 #include <unordered_map>
 #include <vector>
 
-#include "core/ggml_extend.hpp"
+#include "condition_cache_utils.hpp"
-#include "core/tensor.hpp"
+#include "denoiser.hpp"
-#include "runtime/condition_cache_utils.hpp"
+#include "ggml_extend.hpp"
-#include "runtime/denoiser.hpp"
+#include "tensor.hpp"
 
 struct EasyCacheConfig {
     bool enabled          = false;
@@ -258,4 +258,4 @@ struct EasyCacheState {
     }
 };
 
-#endif  // __SD_RUNTIME_EASYCACHE_HPP__
+#endif

src/esrgan.hpp

@@ -1,8 +1,8 @@
-#ifndef __SD_MODEL_UPSCALER_ESRGAN_HPP__
+#ifndef __ESRGAN_HPP__
-#define __SD_MODEL_UPSCALER_ESRGAN_HPP__
+#define __ESRGAN_HPP__
 
-#include "core/ggml_extend.hpp"
+#include "ggml_extend.hpp"
-#include "model_loader.h"
+#include "model.h"
 
 /*
     ===================================    ESRGAN  ===================================
@@ -124,33 +124,27 @@ public:
         auto conv_hr    = std::dynamic_pointer_cast<Conv2d>(blocks["conv_hr"]);
         auto conv_last  = std::dynamic_pointer_cast<Conv2d>(blocks["conv_last"]);
 
-        auto feat = conv_first->forward(ctx, x);
+        auto feat      = conv_first->forward(ctx, x);
-        sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.prelude", "feat");
         auto body_feat = feat;
         for (int i = 0; i < num_block; i++) {
             std::string name = "body." + std::to_string(i);
             auto block       = std::dynamic_pointer_cast<RRDB>(blocks[name]);
 
             body_feat = block->forward(ctx, body_feat);
-            sd::ggml_graph_cut::mark_graph_cut(body_feat, "esrgan.body." + std::to_string(i), "feat");
         }
         body_feat = conv_body->forward(ctx, body_feat);
         feat      = ggml_add(ctx->ggml_ctx, feat, body_feat);
-        sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.body.out", "feat");
         // upsample
         if (scale >= 2) {
             auto conv_up1 = std::dynamic_pointer_cast<Conv2d>(blocks["conv_up1"]);
             feat          = lrelu(ctx, conv_up1->forward(ctx, ggml_upscale(ctx->ggml_ctx, feat, 2, GGML_SCALE_MODE_NEAREST)));
-            sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.up1", "feat");
             if (scale == 4) {
                 auto conv_up2 = std::dynamic_pointer_cast<Conv2d>(blocks["conv_up2"]);
                 feat          = lrelu(ctx, conv_up2->forward(ctx, ggml_upscale(ctx->ggml_ctx, feat, 2, GGML_SCALE_MODE_NEAREST)));
-                sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.up2", "feat");
             }
         }
         // for all scales
         auto out = conv_last->forward(ctx, lrelu(ctx, conv_hr->forward(ctx, feat)));
-        sd::ggml_graph_cut::mark_graph_cut(out, "esrgan.final", "out");
         return out;
     }
 };
@@ -161,10 +155,10 @@ struct ESRGAN : public GGMLRunner {
     int tile_size = 128;  // avoid cuda OOM for 4gb VRAM
 
     ESRGAN(ggml_backend_t backend,
-           ggml_backend_t params_backend,
+           bool offload_params_to_cpu,
            int tile_size                                  = 128,
            const String2TensorStorage& tensor_storage_map = {})
-        : GGMLRunner(backend, params_backend) {
+        : GGMLRunner(backend, offload_params_to_cpu) {
         this->tile_size = tile_size;
     }
 
@@ -270,11 +264,7 @@ struct ESRGAN : public GGMLRunner {
         rrdb_net = std::make_unique<RRDBNet>(detected_scale, detected_num_block, detected_num_in_ch, detected_num_out_ch, detected_num_feat, detected_num_grow_ch);
         rrdb_net->init(params_ctx, {}, "");
 
-        if (!alloc_params_buffer()) {
+        alloc_params_buffer();
-            LOG_ERROR("esrgan model buffer allocation failed");
-            return false;
-        }
-
         std::map<std::string, ggml_tensor*> esrgan_tensors;
         rrdb_net->get_param_tensors(esrgan_tensors);
 
@@ -372,4 +362,4 @@ struct ESRGAN : public GGMLRunner {
     }
 };
 
-#endif  // __SD_MODEL_UPSCALER_ESRGAN_HPP__
+#endif  // __ESRGAN_HPP__

src/extensions/generation_extension.h

@@ -1,73 +0,0 @@
-#ifndef __SD_EXTENSIONS_GENERATION_EXTENSION_H__
-#define __SD_EXTENSIONS_GENERATION_EXTENSION_H__
-
-#include <functional>
-#include <map>
-#include <memory>
-#include <set>
-#include <string>
-
-#include "conditioning/conditioner.hpp"
-#include "core/ggml_extend_backend.h"
-#include "model_loader.h"
-#include "stable-diffusion.h"
-
-struct GenerationExtensionInitContext {
-    const sd_ctx_params_t* params;
-    SDVersion version;
-    const String2TensorStorage& tensor_storage_map;
-    ModelLoader& model_loader;
-    int n_threads;
-    std::function<bool(SDBackendModule)> ensure_backend_pair;
-    std::function<ggml_backend_t(SDBackendModule)> backend_for;
-    std::function<ggml_backend_t(SDBackendModule)> params_backend_for;
-};
-
-struct GenerationExtensionTensorContext {
-    std::map<std::string, ggml_tensor*>& tensors;
-    std::map<std::string, ggml_tensor*>& mmap_able_tensors;
-    std::function<bool(SDBackendModule)> module_can_mmap;
-};
-
-struct GenerationExtensionConditionContext {
-    Conditioner* conditioner;
-    ConditionerParams& condition_params;
-    const sd_pm_params_t& pm_params;
-    std::map<std::string, ggml_tensor*>& tensors;
-    SDVersion version;
-    int n_threads;
-    int total_steps;
-    bool free_params_immediately;
-};
-
-struct GenerationExtension {
-    virtual ~GenerationExtension() = default;
-
-    virtual const char* name() const = 0;
-    virtual bool is_enabled() const {
-        return false;
-    }
-    virtual bool init(const GenerationExtensionInitContext&) {
-        return true;
-    }
-    virtual void collect_param_tensors(GenerationExtensionTensorContext&) {}
-    virtual void add_ignore_tensors(std::set<std::string>&) const {}
-    virtual bool alloc_params_buffer() {
-        return true;
-    }
-    virtual size_t get_params_buffer_size() const {
-        return 0;
-    }
-    virtual void reset_runtime_condition() {}
-    virtual bool prepare_condition(GenerationExtensionConditionContext&) {
-        return false;
-    }
-    virtual const SDCondition& before_condition(int step,
-                                                const SDCondition& condition) const {
-        return condition;
-    }
-};
-
-std::shared_ptr<GenerationExtension> create_photomaker_extension();
-
-#endif

src/extensions/photomaker_extension.cpp

@@ -1,325 +0,0 @@
-#include "extensions/generation_extension.h"
-
-#include <algorithm>
-#include <cstring>
-#include <tuple>
-#include <utility>
-
-#include "core/tensor_ggml.hpp"
-#include "core/util.h"
-#include "model/adapter/lora.hpp"
-#include "model/adapter/pmid.hpp"
-
-static std::tuple<std::vector<int>, std::vector<float>, std::vector<bool>>
-tokenize_photomaker_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
-                            const std::string& text,
-                            int trigger_token_count,
-                            int32_t image_token) {
-    auto tokens_and_weights           = clip_conditioner.tokenize(text);
-    std::vector<int> source_tokens    = std::move(tokens_and_weights.first);
-    std::vector<float> source_weights = std::move(tokens_and_weights.second);
-
-    if (!source_tokens.empty() && source_tokens.front() == clip_conditioner.tokenizer.BOS_TOKEN_ID) {
-        source_tokens.erase(source_tokens.begin());
-        source_weights.erase(source_weights.begin());
-    }
-    if (!source_tokens.empty() && source_tokens.back() == clip_conditioner.tokenizer.EOS_TOKEN_ID) {
-        source_tokens.pop_back();
-        source_weights.pop_back();
-    }
-
-    std::vector<int> tokens;
-    std::vector<float> weights;
-    int32_t class_idx = -1;
-    for (size_t i = 0; i < source_tokens.size(); i++) {
-        int token = source_tokens[i];
-        if (token == image_token) {
-            if (!tokens.empty()) {
-                class_idx          = static_cast<int32_t>(tokens.size()) - 1;
-                int class_token    = tokens.back();
-                float class_weight = weights.back();
-                for (int j = 1; j < trigger_token_count; j++) {
-                    tokens.push_back(class_token);
-                    weights.push_back(class_weight);
-                }
-            }
-            continue;
-        }
-        tokens.push_back(token);
-        weights.push_back(source_weights[i]);
-    }
-
-    clip_conditioner.tokenizer.pad_tokens(tokens,
-                                          &weights,
-                                          nullptr,
-                                          clip_conditioner.text_model->model.n_token,
-                                          clip_conditioner.text_model->model.n_token,
-                                          true);
-    std::vector<bool> class_token_mask;
-    for (int i = 0; i < tokens.size(); i++) {
-        class_token_mask.push_back(class_idx + 1 <= i && i < class_idx + 1 + trigger_token_count);
-    }
-
-    return std::make_tuple(tokens, weights, class_token_mask);
-}
-
-static std::tuple<SDCondition, std::vector<bool>>
-get_photomaker_condition_with_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
-                                      int n_threads,
-                                      const ConditionerParams& conditioner_params,
-                                      const std::string& trigger_word,
-                                      int trigger_token_count) {
-    auto image_tokens = clip_conditioner.convert_token_to_id(trigger_word);
-    GGML_ASSERT(image_tokens.size() == 1);
-    auto tokens_and_weights         = tokenize_photomaker_trigger(clip_conditioner,
-                                                                  conditioner_params.text,
-                                                                  trigger_token_count,
-                                                                  image_tokens[0]);
-    std::vector<int>& tokens        = std::get<0>(tokens_and_weights);
-    std::vector<float>& weights     = std::get<1>(tokens_and_weights);
-    std::vector<bool>& trigger_mask = std::get<2>(tokens_and_weights);
-    auto cond                       = clip_conditioner.get_learned_condition_common(n_threads,
-                                                                                    tokens,
-                                                                                    weights,
-                                                                                    conditioner_params.clip_skip,
-                                                                                    conditioner_params.width,
-                                                                                    conditioner_params.height,
-                                                                                    conditioner_params.zero_out_masked);
-    return std::make_tuple(std::move(cond), trigger_mask);
-}
-
-static std::string remove_photomaker_trigger_from_prompt(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
-                                                         const std::string& prompt,
-                                                         const std::string& trigger_word) {
-    auto image_tokens = clip_conditioner.convert_token_to_id(trigger_word);
-    GGML_ASSERT(image_tokens.size() == 1);
-    auto tokens_and_weights  = clip_conditioner.tokenize(prompt);
-    std::vector<int>& tokens = tokens_and_weights.first;
-    auto it                  = std::find(tokens.begin(), tokens.end(), image_tokens[0]);
-    GGML_ASSERT(it != tokens.end());
-    tokens.erase(it);
-    return clip_conditioner.decode(tokens);
-}
-
-struct PhotoMakerExtension : public GenerationExtension {
-    std::shared_ptr<PhotoMakerIDEncoder> pmid_model;
-    std::shared_ptr<LoraModel> pmid_lora;
-    bool enabled = false;
-    std::string model_path;
-    std::string trigger_word = "img";
-    SDCondition id_condition;
-    int start_merge_step = -1;
-
-    const char* name() const override {
-        return "photomaker";
-    }
-
-    bool is_enabled() const override {
-        return enabled;
-    }
-
-    bool init(const GenerationExtensionInitContext& ctx) override {
-        model_path = SAFE_STR(ctx.params->photo_maker_path);
-        if (model_path.empty()) {
-            return true;
-        }
-
-        if (!ctx.ensure_backend_pair(SDBackendModule::PHOTOMAKER)) {
-            return false;
-        }
-
-        PMVersion pm_version = std::strstr(model_path.c_str(), "v2") != nullptr ? PM_VERSION_2 : PM_VERSION_1;
-        pmid_model           = std::make_shared<PhotoMakerIDEncoder>(ctx.backend_for(SDBackendModule::PHOTOMAKER),
-                                                           ctx.params_backend_for(SDBackendModule::PHOTOMAKER),
-                                                           ctx.tensor_storage_map,
-                                                           "pmid",
-                                                           ctx.version,
-                                                           pm_version);
-        if (pm_version == PM_VERSION_2) {
-            LOG_INFO("using PhotoMaker Version 2");
-        }
-
-        pmid_lora               = std::make_shared<LoraModel>("pmid",
-                                                ctx.backend_for(SDBackendModule::PHOTOMAKER),
-                                                ctx.params_backend_for(SDBackendModule::PHOTOMAKER),
-                                                model_path,
-                                                "",
-                                                ctx.version);
-        auto lora_tensor_filter = [&](const std::string& tensor_name) {
-            return starts_with(tensor_name, "lora.model");
-        };
-        if (!pmid_lora->load_from_file(ctx.n_threads, lora_tensor_filter)) {
-            LOG_WARN("load photomaker lora tensors from %s failed", model_path.c_str());
-            return false;
-        }
-
-        LOG_INFO("loading stacked ID embedding (PHOTOMAKER) model file from '%s'", model_path.c_str());
-        if (!ctx.model_loader.init_from_file_and_convert_name(model_path, "pmid.")) {
-            LOG_WARN("loading stacked ID embedding from '%s' failed", model_path.c_str());
-            return true;
-        }
-
-        enabled = true;
-        return true;
-    }
-
-    void collect_param_tensors(GenerationExtensionTensorContext& ctx) override {
-        if (!enabled || pmid_model == nullptr) {
-            return;
-        }
-
-        std::map<std::string, ggml_tensor*> temp;
-        pmid_model->get_param_tensors(temp, "pmid");
-        bool do_mmap = ctx.module_can_mmap(SDBackendModule::PHOTOMAKER);
-        for (const auto& [key, tensor] : temp) {
-            ctx.tensors[key] = tensor;
-            if (do_mmap) {
-                ctx.mmap_able_tensors[key] = tensor;
-            }
-        }
-    }
-
-    void add_ignore_tensors(std::set<std::string>& ignore_tensors) const override {
-        if (!enabled) {
-            return;
-        }
-        ignore_tensors.insert("pmid.unet.");
-    }
-
-    bool alloc_params_buffer() override {
-        if (!enabled || pmid_model == nullptr) {
-            return true;
-        }
-        return pmid_model->alloc_params_buffer();
-    }
-
-    size_t get_params_buffer_size() const override {
-        if (!enabled || pmid_model == nullptr) {
-            return 0;
-        }
-        return pmid_model->get_params_buffer_size();
-    }
-
-    void reset_runtime_condition() override {
-        id_condition     = {};
-        start_merge_step = -1;
-    }
-
-    bool prepare_condition(GenerationExtensionConditionContext& ctx) override {
-        reset_runtime_condition();
-        if (!enabled || pmid_model == nullptr || pmid_lora == nullptr) {
-            return false;
-        }
-
-        if (!pmid_lora->applied) {
-            int64_t t0 = ggml_time_ms();
-            pmid_lora->apply(ctx.tensors, ctx.version, ctx.n_threads);
-            int64_t t1         = ggml_time_ms();
-            pmid_lora->applied = true;
-            LOG_INFO("pmid_lora apply completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
-            if (ctx.free_params_immediately) {
-                pmid_lora->free_params_buffer();
-            }
-        }
-
-        bool pmv2 = pmid_model->get_version() == PM_VERSION_2;
-        if (ctx.pm_params.id_images_count <= 0 || ctx.pm_params.id_images == nullptr) {
-            LOG_WARN("Provided PhotoMaker model file, but NO input ID images");
-            LOG_WARN("Turn off PhotoMaker for this request");
-            return false;
-        }
-        auto* clip_conditioner = dynamic_cast<FrozenCLIPEmbedderWithCustomWords*>(ctx.conditioner);
-        if (clip_conditioner == nullptr) {
-            LOG_WARN("PhotoMaker requires FrozenCLIPEmbedderWithCustomWords conditioner");
-            LOG_WARN("Turn off PhotoMaker for this request");
-            return false;
-        }
-
-        int clip_image_size        = 224;
-        pmid_model->style_strength = ctx.pm_params.style_strength;
-        sd::Tensor<float> id_image_tensor;
-        for (int i = 0; i < ctx.pm_params.id_images_count; i++) {
-            auto id_image           = sd_image_to_tensor(ctx.pm_params.id_images[i]);
-            auto processed_id_image = clip_preprocess(id_image, clip_image_size, clip_image_size);
-            if (id_image_tensor.empty()) {
-                id_image_tensor = processed_id_image;
-            } else {
-                id_image_tensor = sd::ops::concat(id_image_tensor, processed_id_image, 3);
-            }
-        }
-
-        int64_t t0                        = ggml_time_ms();
-        int trigger_token_count           = pmv2 ? 2 * ctx.pm_params.id_images_count : ctx.pm_params.id_images_count;
-        auto cond_tup                     = get_photomaker_condition_with_trigger(*clip_conditioner,
-                                                                                  ctx.n_threads,
-                                                                                  ctx.condition_params,
-                                                                                  trigger_word,
-                                                                                  trigger_token_count);
-        SDCondition prepared_id_condition = std::get<0>(cond_tup);
-        auto class_tokens_mask            = std::get<1>(cond_tup);
-        if (std::find(class_tokens_mask.begin(), class_tokens_mask.end(), true) == class_tokens_mask.end()) {
-            LOG_WARN("PhotoMaker trigger word '%s' was not found in prompt", trigger_word.c_str());
-            LOG_WARN("Turn off PhotoMaker for this request");
-            return false;
-        }
-
-        sd::Tensor<float> id_embeds;
-        if (pmv2 && ctx.pm_params.id_embed_path != nullptr) {
-            try {
-                id_embeds = sd::load_tensor_from_file_as_tensor<float>(ctx.pm_params.id_embed_path);
-            } catch (const std::exception&) {
-                id_embeds = {};
-            }
-        }
-        if (pmv2 && id_embeds.empty()) {
-            LOG_WARN("Provided PhotoMaker images, but NO valid ID embeds file for PM v2");
-            LOG_WARN("Turn off PhotoMaker for this request");
-            return false;
-        }
-        if (pmv2 && ctx.pm_params.id_images_count != id_embeds.shape()[1]) {
-            LOG_WARN("PhotoMaker image count (%d) does NOT match ID embeds (%d). You should run face_detect.py again.",
-                     ctx.pm_params.id_images_count,
-                     static_cast<int>(id_embeds.shape()[1]));
-            LOG_WARN("Turn off PhotoMaker for this request");
-            return false;
-        }
-
-        auto res = pmid_model->compute(ctx.n_threads,
-                                       id_image_tensor,
-                                       prepared_id_condition.c_crossattn,
-                                       id_embeds,
-                                       class_tokens_mask);
-        if (res.empty()) {
-            LOG_ERROR("Photomaker ID Stacking failed");
-            LOG_WARN("Turn off PhotoMaker for this request");
-            return false;
-        }
-
-        prepared_id_condition.c_crossattn = std::move(res);
-        int64_t t1                        = ggml_time_ms();
-        id_condition                      = std::move(prepared_id_condition);
-        start_merge_step                  = int(ctx.pm_params.style_strength / 100.f * ctx.total_steps);
-        ctx.condition_params.text         = remove_photomaker_trigger_from_prompt(*clip_conditioner,
-                                                                                  ctx.condition_params.text,
-                                                                                  trigger_word);
-        LOG_INFO("Photomaker ID Stacking, taking %" PRId64 " ms", t1 - t0);
-        LOG_INFO("PHOTOMAKER: start_merge_step: %d", start_merge_step);
-
-        if (ctx.free_params_immediately) {
-            pmid_model->free_params_buffer();
-        }
-        return true;
-    }
-
-    const SDCondition& before_condition(int step,
-                                        const SDCondition& condition) const override {
-        if (!id_condition.empty() && start_merge_step != -1 && step > start_merge_step) {
-            return id_condition;
-        }
-        return condition;
-    }
-};
-
-std::shared_ptr<GenerationExtension> create_photomaker_extension() {
-    return std::make_shared<PhotoMakerExtension>();
-}

src/flux.hpp

@@ -1,167 +1,17 @@
-#ifndef __SD_MODEL_DIFFUSION_FLUX_HPP__
+#ifndef __FLUX_HPP__
-#define __SD_MODEL_DIFFUSION_FLUX_HPP__
+#define __FLUX_HPP__
 
 #include <memory>
 #include <vector>
 
-#include "model/common/rope.hpp"
+#include "common_dit.hpp"
-#include "model/diffusion/dit.hpp"
+#include "model.h"
-#include "model/diffusion/model.hpp"
+#include "rope.hpp"
-#include "model_loader.h"
 
 #define FLUX_GRAPH_SIZE 10240
 
 namespace Flux {
 
-    struct ChromaRadianceConfig {
-        int64_t nerf_hidden_size = 64;
-        int nerf_mlp_ratio       = 4;
-        int nerf_depth           = 4;
-        int nerf_max_freqs       = 8;
-        bool use_x0              = false;
-        bool fake_patch_size_x2  = false;
-    };
-
-    struct FluxConfig {
-        SDVersion version         = VERSION_FLUX;
-        bool is_chroma            = false;
-        int patch_size            = 2;
-        int64_t in_channels       = 64;
-        int64_t out_channels      = 64;
-        int64_t vec_in_dim        = 768;
-        int64_t context_in_dim    = 4096;
-        int64_t hidden_size       = 3072;
-        float mlp_ratio           = 4.0f;
-        int num_heads             = 24;
-        int depth                 = 19;
-        int depth_single_blocks   = 38;
-        std::vector<int> axes_dim = {16, 56, 56};
-        int axes_dim_sum          = 128;
-        int theta                 = 10000;
-        bool qkv_bias             = true;
-        bool guidance_embed       = true;
-        int64_t in_dim            = 64;
-        bool disable_bias         = false;
-        bool share_modulation     = false;
-        bool semantic_txt_norm    = false;
-        bool use_yak_mlp          = false;
-        bool use_mlp_silu_act     = false;
-        float ref_index_scale     = 1.f;
-        ChromaRadianceConfig chroma_radiance_params;
-
-        static FluxConfig detect_from_weights(const String2TensorStorage& tensor_storage_map,
-                                              const std::string& prefix,
-                                              SDVersion version = VERSION_FLUX) {
-            FluxConfig config;
-            config.version             = version;
-            config.guidance_embed      = false;
-            config.depth               = 0;
-            config.depth_single_blocks = 0;
-            if (version == VERSION_FLUX_FILL) {
-                config.in_channels = 384;
-            } else if (version == VERSION_FLUX_CONTROLS) {
-                config.in_channels = 128;
-            } else if (version == VERSION_FLEX_2) {
-                config.in_channels = 196;
-            } else if (version == VERSION_CHROMA_RADIANCE) {
-                config.in_channels = 3;
-                config.patch_size  = 16;
-            } else if (version == VERSION_OVIS_IMAGE) {
-                config.semantic_txt_norm = true;
-                config.use_yak_mlp       = true;
-                config.vec_in_dim        = 0;
-            } else if (sd_version_is_flux2(version)) {
-                config.in_channels      = 128;
-                config.patch_size       = 1;
-                config.out_channels     = 128;
-                config.mlp_ratio        = 3.f;
-                config.theta            = 2000;
-                config.axes_dim         = {32, 32, 32, 32};
-                config.vec_in_dim       = 0;
-                config.qkv_bias         = false;
-                config.disable_bias     = true;
-                config.share_modulation = true;
-                config.ref_index_scale  = 10.f;
-                config.use_mlp_silu_act = true;
-            } else if (sd_version_is_longcat(version)) {
-                config.context_in_dim = 3584;
-                config.vec_in_dim     = 0;
-            }
-
-            int64_t head_dim                   = 0;
-            int64_t actual_radiance_patch_size = -1;
-            for (const auto& [name, tensor_storage] : tensor_storage_map) {
-                if (!starts_with(name, prefix)) {
-                    continue;
-                }
-                if (name.find("guidance_in.in_layer.weight") != std::string::npos) {
-                    config.guidance_embed = true;
-                }
-                if (name.find("__x0__") != std::string::npos) {
-                    LOG_DEBUG("using x0 prediction");
-                    config.chroma_radiance_params.use_x0 = true;
-                }
-                if (name.find("__32x32__") != std::string::npos) {
-                    LOG_DEBUG("using patch size 32");
-                    config.patch_size = 32;
-                }
-                if (name.find("img_in_patch.weight") != std::string::npos) {
-                    actual_radiance_patch_size = tensor_storage.ne[0];
-                    LOG_DEBUG("actual radiance patch size: %" PRId64, actual_radiance_patch_size);
-                }
-                if (name.find("distilled_guidance_layer.in_proj.weight") != std::string::npos) {
-                    config.is_chroma = true;
-                }
-                size_t db = name.find("double_blocks.");
-                if (db != std::string::npos) {
-                    std::string block_name = name.substr(db);
-                    int block_depth        = atoi(block_name.substr(14, block_name.find(".", 14)).c_str());
-                    if (block_depth + 1 > config.depth) {
-                        config.depth = block_depth + 1;
-                    }
-                }
-                size_t sb = name.find("single_blocks.");
-                if (sb != std::string::npos) {
-                    std::string block_name = name.substr(sb);
-                    int block_depth        = atoi(block_name.substr(14, block_name.find(".", 14)).c_str());
-                    if (block_depth + 1 > config.depth_single_blocks) {
-                        config.depth_single_blocks = block_depth + 1;
-                    }
-                }
-                if (ends_with(name, "txt_in.weight")) {
-                    config.context_in_dim = tensor_storage.ne[0];
-                    config.hidden_size    = tensor_storage.ne[1];
-                }
-                if (ends_with(name, "single_blocks.0.norm.key_norm.scale")) {
-                    head_dim = tensor_storage.ne[0];
-                }
-                if (ends_with(name, "double_blocks.0.txt_attn.norm.key_norm.scale")) {
-                    head_dim = tensor_storage.ne[0];
-                }
-            }
-            if (actual_radiance_patch_size > 0 && actual_radiance_patch_size != config.patch_size) {
-                GGML_ASSERT(config.patch_size == 2 * actual_radiance_patch_size);
-                LOG_DEBUG("using fake x2 patch size");
-                config.chroma_radiance_params.fake_patch_size_x2 = true;
-            }
-            if (head_dim > 0) {
-                config.num_heads = static_cast<int>(config.hidden_size / head_dim);
-            }
-            config.axes_dim_sum = 0;
-            for (int axis_dim : config.axes_dim) {
-                config.axes_dim_sum += axis_dim;
-            }
-            LOG_DEBUG("flux: depth = %d, depth_single_blocks = %d, guidance_embed = %s, context_in_dim = %" PRId64 ", hidden_size = %" PRId64 ", num_heads = %d",
-                      config.depth,
-                      config.depth_single_blocks,
-                      config.guidance_embed ? "true" : "false",
-                      config.context_in_dim,
-                      config.hidden_size,
-                      config.num_heads);
-            return config;
-        }
-    };
-
     struct MLPEmbedder : public UnaryBlock {
     public:
         MLPEmbedder(int64_t in_dim, int64_t hidden_dim, bool bias = true) {
@@ -596,6 +446,7 @@ namespace Flux {
             if (use_yak_mlp || use_mlp_silu_act) {
                 mlp_mult_factor = 2;
             }
+
             blocks["linear1"]  = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, hidden_size * 3 + mlp_hidden_dim * mlp_mult_factor, mlp_proj_bias));
             blocks["linear2"]  = std::shared_ptr<GGMLBlock>(new Linear(hidden_size + mlp_hidden_dim, hidden_size, mlp_proj_bias));
             blocks["norm"]     = std::shared_ptr<GGMLBlock>(new QKNorm(head_dim));
@@ -872,90 +723,127 @@ namespace Flux {
         }
     };
 
+    struct ChromaRadianceParams {
+        int64_t nerf_hidden_size = 64;
+        int nerf_mlp_ratio       = 4;
+        int nerf_depth           = 4;
+        int nerf_max_freqs       = 8;
+        bool use_x0              = false;
+        bool fake_patch_size_x2  = false;
+    };
+
+    struct FluxParams {
+        SDVersion version         = VERSION_FLUX;
+        bool is_chroma            = false;
+        int patch_size            = 2;
+        int64_t in_channels       = 64;
+        int64_t out_channels      = 64;
+        int64_t vec_in_dim        = 768;
+        int64_t context_in_dim    = 4096;
+        int64_t hidden_size       = 3072;
+        float mlp_ratio           = 4.0f;
+        int num_heads             = 24;
+        int depth                 = 19;
+        int depth_single_blocks   = 38;
+        std::vector<int> axes_dim = {16, 56, 56};
+        int axes_dim_sum          = 128;
+        int theta                 = 10000;
+        bool qkv_bias             = true;
+        bool guidance_embed       = true;
+        int64_t in_dim            = 64;
+        bool disable_bias         = false;
+        bool share_modulation     = false;
+        bool semantic_txt_norm    = false;
+        bool use_yak_mlp          = false;
+        bool use_mlp_silu_act     = false;
+        float ref_index_scale     = 1.f;
+        ChromaRadianceParams chroma_radiance_params;
+    };
+
     struct Flux : public GGMLBlock {
     public:
-        FluxConfig config;
+        FluxParams params;
         Flux() {}
-        Flux(FluxConfig config)
+        Flux(FluxParams params)
-            : config(config) {
+            : params(params) {
-            if (config.version == VERSION_CHROMA_RADIANCE) {
+            if (params.version == VERSION_CHROMA_RADIANCE) {
-                std::pair<int, int> kernel_size = {config.patch_size, config.patch_size};
+                std::pair<int, int> kernel_size = {params.patch_size, params.patch_size};
-                if (config.chroma_radiance_params.fake_patch_size_x2) {
+                if (params.chroma_radiance_params.fake_patch_size_x2) {
-                    kernel_size = {config.patch_size / 2, config.patch_size / 2};
+                    kernel_size = {params.patch_size / 2, params.patch_size / 2};
                 }
                 std::pair<int, int> stride = kernel_size;
 
-                blocks["img_in_patch"] = std::make_shared<Conv2d>(config.in_channels,
+                blocks["img_in_patch"] = std::make_shared<Conv2d>(params.in_channels,
-                                                                  config.hidden_size,
+                                                                  params.hidden_size,
                                                                   kernel_size,
                                                                   stride);
             } else {
-                blocks["img_in"] = std::make_shared<Linear>(config.in_channels, config.hidden_size, !config.disable_bias);
+                blocks["img_in"] = std::make_shared<Linear>(params.in_channels, params.hidden_size, !params.disable_bias);
             }
-            if (config.is_chroma) {
+            if (params.is_chroma) {
-                blocks["distilled_guidance_layer"] = std::make_shared<ChromaApproximator>(config.in_dim, config.hidden_size);
+                blocks["distilled_guidance_layer"] = std::make_shared<ChromaApproximator>(params.in_dim, params.hidden_size);
             } else {
-                blocks["time_in"] = std::make_shared<MLPEmbedder>(256, config.hidden_size, !config.disable_bias);
+                blocks["time_in"] = std::make_shared<MLPEmbedder>(256, params.hidden_size, !params.disable_bias);
-                if (config.vec_in_dim > 0) {
+                if (params.vec_in_dim > 0) {
-                    blocks["vector_in"] = std::make_shared<MLPEmbedder>(config.vec_in_dim, config.hidden_size, !config.disable_bias);
+                    blocks["vector_in"] = std::make_shared<MLPEmbedder>(params.vec_in_dim, params.hidden_size, !params.disable_bias);
                 }
-                if (config.guidance_embed) {
+                if (params.guidance_embed) {
-                    blocks["guidance_in"] = std::make_shared<MLPEmbedder>(256, config.hidden_size, !config.disable_bias);
+                    blocks["guidance_in"] = std::make_shared<MLPEmbedder>(256, params.hidden_size, !params.disable_bias);
                 }
             }
-            if (config.semantic_txt_norm) {
+            if (params.semantic_txt_norm) {
-                blocks["txt_norm"] = std::make_shared<RMSNorm>(config.context_in_dim);
+                blocks["txt_norm"] = std::make_shared<RMSNorm>(params.context_in_dim);
             }
-            blocks["txt_in"] = std::make_shared<Linear>(config.context_in_dim, config.hidden_size, !config.disable_bias);
+            blocks["txt_in"] = std::make_shared<Linear>(params.context_in_dim, params.hidden_size, !params.disable_bias);
 
-            for (int i = 0; i < config.depth; i++) {
+            for (int i = 0; i < params.depth; i++) {
-                blocks["double_blocks." + std::to_string(i)] = std::make_shared<DoubleStreamBlock>(config.hidden_size,
+                blocks["double_blocks." + std::to_string(i)] = std::make_shared<DoubleStreamBlock>(params.hidden_size,
-                                                                                                   config.num_heads,
+                                                                                                   params.num_heads,
-                                                                                                   config.mlp_ratio,
+                                                                                                   params.mlp_ratio,
                                                                                                    i,
-                                                                                                   config.qkv_bias,
+                                                                                                   params.qkv_bias,
-                                                                                                   config.is_chroma,
+                                                                                                   params.is_chroma,
-                                                                                                   config.share_modulation,
+                                                                                                   params.share_modulation,
-                                                                                                   !config.disable_bias,
+                                                                                                   !params.disable_bias,
-                                                                                                   config.use_yak_mlp,
+                                                                                                   params.use_yak_mlp,
-                                                                                                   config.use_mlp_silu_act);
+                                                                                                   params.use_mlp_silu_act);
             }
 
-            for (int i = 0; i < config.depth_single_blocks; i++) {
+            for (int i = 0; i < params.depth_single_blocks; i++) {
-                blocks["single_blocks." + std::to_string(i)] = std::make_shared<SingleStreamBlock>(config.hidden_size,
+                blocks["single_blocks." + std::to_string(i)] = std::make_shared<SingleStreamBlock>(params.hidden_size,
-                                                                                                   config.num_heads,
+                                                                                                   params.num_heads,
-                                                                                                   config.mlp_ratio,
+                                                                                                   params.mlp_ratio,
                                                                                                    i,
                                                                                                    0.f,
-                                                                                                   config.is_chroma,
+                                                                                                   params.is_chroma,
-                                                                                                   config.share_modulation,
+                                                                                                   params.share_modulation,
-                                                                                                   !config.disable_bias,
+                                                                                                   !params.disable_bias,
-                                                                                                   config.use_yak_mlp,
+                                                                                                   params.use_yak_mlp,
-                                                                                                   config.use_mlp_silu_act);
+                                                                                                   params.use_mlp_silu_act);
             }
 
-            if (config.version == VERSION_CHROMA_RADIANCE) {
+            if (params.version == VERSION_CHROMA_RADIANCE) {
-                blocks["nerf_image_embedder"] = std::make_shared<NerfEmbedder>(config.in_channels,
+                blocks["nerf_image_embedder"] = std::make_shared<NerfEmbedder>(params.in_channels,
-                                                                               config.chroma_radiance_params.nerf_hidden_size,
+                                                                               params.chroma_radiance_params.nerf_hidden_size,
-                                                                               config.chroma_radiance_params.nerf_max_freqs);
+                                                                               params.chroma_radiance_params.nerf_max_freqs);
 
-                for (int i = 0; i < config.chroma_radiance_params.nerf_depth; i++) {
+                for (int i = 0; i < params.chroma_radiance_params.nerf_depth; i++) {
-                    blocks["nerf_blocks." + std::to_string(i)] = std::make_shared<NerfGLUBlock>(config.hidden_size,
+                    blocks["nerf_blocks." + std::to_string(i)] = std::make_shared<NerfGLUBlock>(params.hidden_size,
-                                                                                                config.chroma_radiance_params.nerf_hidden_size,
+                                                                                                params.chroma_radiance_params.nerf_hidden_size,
-                                                                                                config.chroma_radiance_params.nerf_mlp_ratio);
+                                                                                                params.chroma_radiance_params.nerf_mlp_ratio);
                 }
 
-                blocks["nerf_final_layer_conv"] = std::make_shared<NerfFinalLayerConv>(config.chroma_radiance_params.nerf_hidden_size,
+                blocks["nerf_final_layer_conv"] = std::make_shared<NerfFinalLayerConv>(params.chroma_radiance_params.nerf_hidden_size,
-                                                                                       config.in_channels);
+                                                                                       params.in_channels);
 
             } else {
-                blocks["final_layer"] = std::make_shared<LastLayer>(config.hidden_size, 1, config.out_channels, config.is_chroma, !config.disable_bias);
+                blocks["final_layer"] = std::make_shared<LastLayer>(params.hidden_size, 1, params.out_channels, params.is_chroma, !params.disable_bias);
             }
 
-            if (config.share_modulation) {
+            if (params.share_modulation) {
-                blocks["double_stream_modulation_img"] = std::make_shared<Modulation>(config.hidden_size, true, !config.disable_bias);
+                blocks["double_stream_modulation_img"] = std::make_shared<Modulation>(params.hidden_size, true, !params.disable_bias);
-                blocks["double_stream_modulation_txt"] = std::make_shared<Modulation>(config.hidden_size, true, !config.disable_bias);
+                blocks["double_stream_modulation_txt"] = std::make_shared<Modulation>(params.hidden_size, true, !params.disable_bias);
-                blocks["single_stream_modulation"]     = std::make_shared<Modulation>(config.hidden_size, false, !config.disable_bias);
+                blocks["single_stream_modulation"]     = std::make_shared<Modulation>(params.hidden_size, false, !params.disable_bias);
             }
         }
 
@@ -978,7 +866,7 @@ namespace Flux {
 
             ggml_tensor* vec;
             ggml_tensor* txt_img_mask = nullptr;
-            if (config.is_chroma) {
+            if (params.is_chroma) {
                 int64_t mod_index_length = 344;
                 auto approx              = std::dynamic_pointer_cast<ChromaApproximator>(blocks["distilled_guidance_layer"]);
                 auto distill_timestep    = ggml_ext_timestep_embedding(ctx->ggml_ctx, timesteps, 16, 10000, 1000.f);
@@ -1006,7 +894,7 @@ namespace Flux {
             } else {
                 auto time_in = std::dynamic_pointer_cast<MLPEmbedder>(blocks["time_in"]);
                 vec          = time_in->forward(ctx, ggml_ext_timestep_embedding(ctx->ggml_ctx, timesteps, 256, 10000, 1000.f));
-                if (config.guidance_embed) {
+                if (params.guidance_embed) {
                     GGML_ASSERT(guidance != nullptr);
                     auto guidance_in = std::dynamic_pointer_cast<MLPEmbedder>(blocks["guidance_in"]);
                     // bf16 and fp16 result is different
@@ -1014,7 +902,7 @@ namespace Flux {
                     vec       = ggml_add(ctx->ggml_ctx, vec, guidance_in->forward(ctx, g_in));
                 }
 
-                if (config.vec_in_dim > 0) {
+                if (params.vec_in_dim > 0) {
                     auto vector_in = std::dynamic_pointer_cast<MLPEmbedder>(blocks["vector_in"]);
                     vec            = ggml_add(ctx->ggml_ctx, vec, vector_in->forward(ctx, y));
                 }
@@ -1023,7 +911,7 @@ namespace Flux {
             std::vector<ModulationOut> ds_img_mods;
             std::vector<ModulationOut> ds_txt_mods;
             std::vector<ModulationOut> ss_mods;
-            if (config.share_modulation) {
+            if (params.share_modulation) {
                 auto double_stream_modulation_img = std::dynamic_pointer_cast<Modulation>(blocks["double_stream_modulation_img"]);
                 auto double_stream_modulation_txt = std::dynamic_pointer_cast<Modulation>(blocks["double_stream_modulation_txt"]);
                 auto single_stream_modulation     = std::dynamic_pointer_cast<Modulation>(blocks["single_stream_modulation"]);
@@ -1033,18 +921,15 @@ namespace Flux {
                 ss_mods     = single_stream_modulation->forward(ctx, vec);
             }
 
-            if (config.semantic_txt_norm) {
+            if (params.semantic_txt_norm) {
                 auto semantic_txt_norm = std::dynamic_pointer_cast<RMSNorm>(blocks["txt_norm"]);
 
                 txt = semantic_txt_norm->forward(ctx, txt);
             }
 
             txt = txt_in->forward(ctx, txt);
-            sd::ggml_graph_cut::mark_graph_cut(img, "flux.prelude", "img");
-            sd::ggml_graph_cut::mark_graph_cut(txt, "flux.prelude", "txt");
-            sd::ggml_graph_cut::mark_graph_cut(vec, "flux.prelude", "vec");
 
-            for (int i = 0; i < config.depth; i++) {
+            for (int i = 0; i < params.depth; i++) {
                 if (skip_layers.size() > 0 && std::find(skip_layers.begin(), skip_layers.end(), i) != skip_layers.end()) {
                     continue;
                 }
@@ -1054,19 +939,16 @@ namespace Flux {
                 auto img_txt = block->forward(ctx, img, txt, vec, pe, txt_img_mask, ds_img_mods, ds_txt_mods);
                 img          = img_txt.first;   // [N, n_img_token, hidden_size]
                 txt          = img_txt.second;  // [N, n_txt_token, hidden_size]
-                sd::ggml_graph_cut::mark_graph_cut(img, "flux.double_blocks." + std::to_string(i), "img");
-                sd::ggml_graph_cut::mark_graph_cut(txt, "flux.double_blocks." + std::to_string(i), "txt");
             }
 
             auto txt_img = ggml_concat(ctx->ggml_ctx, txt, img, 1);  // [N, n_txt_token + n_img_token, hidden_size]
-            for (int i = 0; i < config.depth_single_blocks; i++) {
+            for (int i = 0; i < params.depth_single_blocks; i++) {
-                if (skip_layers.size() > 0 && std::find(skip_layers.begin(), skip_layers.end(), i + config.depth) != skip_layers.end()) {
+                if (skip_layers.size() > 0 && std::find(skip_layers.begin(), skip_layers.end(), i + params.depth) != skip_layers.end()) {
                     continue;
                 }
                 auto block = std::dynamic_pointer_cast<SingleStreamBlock>(blocks["single_blocks." + std::to_string(i)]);
 
                 txt_img = block->forward(ctx, txt_img, vec, pe, txt_img_mask, ss_mods);
-                sd::ggml_graph_cut::mark_graph_cut(txt_img, "flux.single_blocks." + std::to_string(i), "txt_img");
             }
 
             img = ggml_view_3d(ctx->ggml_ctx,
@@ -1111,14 +993,14 @@ namespace Flux {
             int64_t W      = x->ne[0];
             int64_t H      = x->ne[1];
             int64_t C      = x->ne[2];
-            int patch_size = config.patch_size;
+            int patch_size = params.patch_size;
             int pad_h      = (patch_size - H % patch_size) % patch_size;
             int pad_w      = (patch_size - W % patch_size) % patch_size;
 
-            auto img      = DiT::pad_to_patch_size(ctx, x, config.patch_size, config.patch_size);
+            auto img      = DiT::pad_to_patch_size(ctx, x, params.patch_size, params.patch_size);
             auto orig_img = img;
 
-            if (config.chroma_radiance_params.fake_patch_size_x2) {
+            if (params.chroma_radiance_params.fake_patch_size_x2) {
                 // It's supposed to be using GGML_SCALE_MODE_NEAREST, but this seems more stable
                 // Maybe the implementation of nearest-neighbor interpolation in ggml behaves differently than the one in PyTorch?
                 // img = F.interpolate(img, size=(H//2, W//2), mode="nearest")
@@ -1149,7 +1031,7 @@ namespace Flux {
             auto nerf_hidden = ggml_reshape_2d(ctx->ggml_ctx, out, out->ne[0], out->ne[1] * out->ne[2]);  // [N*num_patches, hidden_size]
             auto img_dct     = nerf_image_embedder->forward(ctx, nerf_pixels, dct);                       // [N*num_patches, patch_size*patch_size, nerf_hidden_size]
 
-            for (int i = 0; i < config.chroma_radiance_params.nerf_depth; i++) {
+            for (int i = 0; i < params.chroma_radiance_params.nerf_depth; i++) {
                 auto block = std::dynamic_pointer_cast<NerfGLUBlock>(blocks["nerf_blocks." + std::to_string(i)]);
 
                 img_dct = block->forward(ctx, img_dct, nerf_hidden);
@@ -1161,7 +1043,7 @@ namespace Flux {
 
             out = nerf_final_layer_conv->forward(ctx, img_dct);  // [N, C, H, W]
 
-            if (config.chroma_radiance_params.use_x0) {
+            if (params.chroma_radiance_params.use_x0) {
                 out = _apply_x0_residual(ctx, out, orig_img, timestep);
             }
 
@@ -1185,14 +1067,14 @@ namespace Flux {
             int64_t W      = x->ne[0];
             int64_t H      = x->ne[1];
             int64_t C      = x->ne[2];
-            int patch_size = config.patch_size;
+            int patch_size = params.patch_size;
             int pad_h      = (patch_size - H % patch_size) % patch_size;
             int pad_w      = (patch_size - W % patch_size) % patch_size;
 
             auto img           = DiT::pad_and_patchify(ctx, x, patch_size, patch_size);
             int64_t img_tokens = img->ne[1];
 
-            if (config.version == VERSION_FLUX_FILL) {
+            if (params.version == VERSION_FLUX_FILL) {
                 GGML_ASSERT(c_concat != nullptr);
                 ggml_tensor* masked = ggml_view_4d(ctx->ggml_ctx, c_concat, c_concat->ne[0], c_concat->ne[1], C, 1, c_concat->nb[1], c_concat->nb[2], c_concat->nb[3], 0);
                 ggml_tensor* mask   = ggml_view_4d(ctx->ggml_ctx, c_concat, c_concat->ne[0], c_concat->ne[1], 8 * 8, 1, c_concat->nb[1], c_concat->nb[2], c_concat->nb[3], c_concat->nb[2] * C);
@@ -1201,7 +1083,7 @@ namespace Flux {
                 mask   = DiT::pad_and_patchify(ctx, mask, patch_size, patch_size);
 
                 img = ggml_concat(ctx->ggml_ctx, img, ggml_concat(ctx->ggml_ctx, masked, mask, 0), 0);
-            } else if (config.version == VERSION_FLEX_2) {
+            } else if (params.version == VERSION_FLEX_2) {
                 GGML_ASSERT(c_concat != nullptr);
                 ggml_tensor* masked  = ggml_view_4d(ctx->ggml_ctx, c_concat, c_concat->ne[0], c_concat->ne[1], C, 1, c_concat->nb[1], c_concat->nb[2], c_concat->nb[3], 0);
                 ggml_tensor* mask    = ggml_view_4d(ctx->ggml_ctx, c_concat, c_concat->ne[0], c_concat->ne[1], 1, 1, c_concat->nb[1], c_concat->nb[2], c_concat->nb[3], c_concat->nb[2] * C);
@@ -1212,7 +1094,7 @@ namespace Flux {
                 control = DiT::pad_and_patchify(ctx, control, patch_size, patch_size);
 
                 img = ggml_concat(ctx->ggml_ctx, img, ggml_concat(ctx->ggml_ctx, ggml_concat(ctx->ggml_ctx, masked, mask, 0), control, 0), 0);
-            } else if (config.version == VERSION_FLUX_CONTROLS) {
+            } else if (params.version == VERSION_FLUX_CONTROLS) {
                 GGML_ASSERT(c_concat != nullptr);
 
                 auto control = DiT::pad_and_patchify(ctx, c_concat, patch_size, patch_size);
@@ -1259,7 +1141,7 @@ namespace Flux {
             // pe: (L, d_head/2, 2, 2)
             // return: (N, C, H, W)
 
-            if (config.version == VERSION_CHROMA_RADIANCE) {
+            if (params.version == VERSION_CHROMA_RADIANCE) {
                 return forward_chroma_radiance(ctx,
                                                x,
                                                timestep,
@@ -1289,9 +1171,9 @@ namespace Flux {
         }
     };
 
-    struct FluxRunner : public DiffusionModelRunner {
+    struct FluxRunner : public GGMLRunner {
     public:
-        FluxConfig config;
+        FluxParams flux_params;
         Flux flux;
         std::vector<float> pe_vec;
         std::vector<float> mod_index_arange_vec;
@@ -1301,20 +1183,116 @@ namespace Flux {
         bool use_mask = false;
 
         FluxRunner(ggml_backend_t backend,
-                   ggml_backend_t params_backend,
+                   bool offload_params_to_cpu,
                    const String2TensorStorage& tensor_storage_map = {},
                    const std::string prefix                       = "",
                    SDVersion version                              = VERSION_FLUX,
                    bool use_mask                                  = false)
-            : DiffusionModelRunner(backend, params_backend, prefix),
+            : GGMLRunner(backend, offload_params_to_cpu), version(version), use_mask(use_mask) {
-              config(FluxConfig::detect_from_weights(tensor_storage_map, prefix, version)),
+            flux_params.version             = version;
-              version(version),
+            flux_params.guidance_embed      = false;
-              use_mask(use_mask) {
+            flux_params.depth               = 0;
-            if (config.is_chroma) {
+            flux_params.depth_single_blocks = 0;
+            if (version == VERSION_FLUX_FILL) {
+                flux_params.in_channels = 384;
+            } else if (version == VERSION_FLUX_CONTROLS) {
+                flux_params.in_channels = 128;
+            } else if (version == VERSION_FLEX_2) {
+                flux_params.in_channels = 196;
+            } else if (version == VERSION_CHROMA_RADIANCE) {
+                flux_params.in_channels = 3;
+                flux_params.patch_size  = 16;
+            } else if (version == VERSION_OVIS_IMAGE) {
+                flux_params.semantic_txt_norm = true;
+                flux_params.use_yak_mlp       = true;
+                flux_params.vec_in_dim        = 0;
+            } else if (sd_version_is_flux2(version)) {
+                flux_params.in_channels      = 128;
+                flux_params.patch_size       = 1;
+                flux_params.out_channels     = 128;
+                flux_params.mlp_ratio        = 3.f;
+                flux_params.theta            = 2000;
+                flux_params.axes_dim         = {32, 32, 32, 32};
+                flux_params.vec_in_dim       = 0;
+                flux_params.qkv_bias         = false;
+                flux_params.disable_bias     = true;
+                flux_params.share_modulation = true;
+                flux_params.ref_index_scale  = 10.f;
+                flux_params.use_mlp_silu_act = true;
+            }
+            int64_t head_dim                   = 0;
+            int64_t actual_radiance_patch_size = -1;
+            for (auto pair : tensor_storage_map) {
+                std::string tensor_name = pair.first;
+                if (!starts_with(tensor_name, prefix))
+                    continue;
+                if (tensor_name.find("guidance_in.in_layer.weight") != std::string::npos) {
+                    flux_params.guidance_embed = true;
+                }
+                if (tensor_name.find("__x0__") != std::string::npos) {
+                    LOG_DEBUG("using x0 prediction");
+                    flux_params.chroma_radiance_params.use_x0 = true;
+                }
+                if (tensor_name.find("__32x32__") != std::string::npos) {
+                    LOG_DEBUG("using patch size 32");
+                    flux_params.patch_size = 32;
+                }
+                if (tensor_name.find("img_in_patch.weight") != std::string::npos) {
+                    actual_radiance_patch_size = pair.second.ne[0];
+                    LOG_DEBUG("actual radiance patch size: %d", actual_radiance_patch_size);
+                }
+                if (tensor_name.find("distilled_guidance_layer.in_proj.weight") != std::string::npos) {
+                    // Chroma
+                    flux_params.is_chroma = true;
+                }
+                size_t db = tensor_name.find("double_blocks.");
+                if (db != std::string::npos) {
+                    tensor_name     = tensor_name.substr(db);  // remove prefix
+                    int block_depth = atoi(tensor_name.substr(14, tensor_name.find(".", 14)).c_str());
+                    if (block_depth + 1 > flux_params.depth) {
+                        flux_params.depth = block_depth + 1;
+                    }
+                }
+                size_t sb = tensor_name.find("single_blocks.");
+                if (sb != std::string::npos) {
+                    tensor_name     = tensor_name.substr(sb);  // remove prefix
+                    int block_depth = atoi(tensor_name.substr(14, tensor_name.find(".", 14)).c_str());
+                    if (block_depth + 1 > flux_params.depth_single_blocks) {
+                        flux_params.depth_single_blocks = block_depth + 1;
+                    }
+                }
+                if (ends_with(tensor_name, "txt_in.weight")) {
+                    flux_params.context_in_dim = pair.second.ne[0];
+                    flux_params.hidden_size    = pair.second.ne[1];
+                }
+                if (ends_with(tensor_name, "single_blocks.0.norm.key_norm.scale")) {
+                    head_dim = pair.second.ne[0];
+                }
+                if (ends_with(tensor_name, "double_blocks.0.txt_attn.norm.key_norm.scale")) {
+                    head_dim = pair.second.ne[0];
+                }
+            }
+            if (actual_radiance_patch_size > 0 && actual_radiance_patch_size != flux_params.patch_size) {
+                GGML_ASSERT(flux_params.patch_size == 2 * actual_radiance_patch_size);
+                LOG_DEBUG("using fake x2 patch size");
+                flux_params.chroma_radiance_params.fake_patch_size_x2 = true;
+            }
+
+            flux_params.num_heads = static_cast<int>(flux_params.hidden_size / head_dim);
+
+            LOG_INFO("flux: depth = %d, depth_single_blocks = %d, guidance_embed = %s, context_in_dim = %" PRId64
+                     ", hidden_size = %" PRId64 ", num_heads = %d",
+                     flux_params.depth,
+                     flux_params.depth_single_blocks,
+                     flux_params.guidance_embed ? "true" : "false",
+                     flux_params.context_in_dim,
+                     flux_params.hidden_size,
+                     flux_params.num_heads);
+            if (flux_params.is_chroma) {
                 LOG_INFO("Using pruned modulation (Chroma)");
             }
 
-            flux = Flux(config);
+            flux = Flux(flux_params);
             flux.init(params_ctx, tensor_storage_map, prefix);
         }
 
@@ -1322,7 +1300,7 @@ namespace Flux {
             return "flux";
         }
 
-        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string prefix) {
             flux.get_param_tensors(tensors, prefix);
         }
 
@@ -1390,10 +1368,10 @@ namespace Flux {
             ggml_tensor* context   = make_optional_input(context_tensor);
             ggml_tensor* c_concat  = make_optional_input(c_concat_tensor);
             ggml_tensor* y         = make_optional_input(y_tensor);
-            if (config.guidance_embed || config.is_chroma) {
+            if (flux_params.guidance_embed || flux_params.is_chroma) {
                 if (!guidance_tensor.empty()) {
                     this->guidance_tensor = guidance_tensor;
-                    if (config.is_chroma) {
+                    if (flux_params.is_chroma) {
                         this->guidance_tensor.fill_(0.f);
                     }
                 }
@@ -1411,7 +1389,7 @@ namespace Flux {
             ggml_tensor* mod_index_arange = nullptr;
             ggml_tensor* dct              = nullptr;  // for chroma radiance
 
-            if (config.is_chroma) {
+            if (flux_params.is_chroma) {
                 if (!use_mask) {
                     y = nullptr;
                 }
@@ -1428,31 +1406,31 @@ namespace Flux {
             } else if (version == VERSION_OVIS_IMAGE) {
                 txt_arange_dims = {1, 2};
             }
+
             pe_vec      = Rope::gen_flux_pe(static_cast<int>(x->ne[1]),
                                             static_cast<int>(x->ne[0]),
-                                            config.patch_size,
+                                            flux_params.patch_size,
                                             static_cast<int>(x->ne[3]),
                                             static_cast<int>(context->ne[1]),
                                             txt_arange_dims,
                                             ref_latents,
                                             increase_ref_index,
-                                            config.ref_index_scale,
+                                            flux_params.ref_index_scale,
-                                            config.theta,
+                                            flux_params.theta,
                                             circular_y_enabled,
                                             circular_x_enabled,
-                                            config.axes_dim,
+                                            flux_params.axes_dim);
-                                            sd_version_is_longcat(version));
+            int pos_len = static_cast<int>(pe_vec.size() / flux_params.axes_dim_sum / 2);
-            int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             // LOG_DEBUG("pos_len %d", pos_len);
-            auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
+            auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, flux_params.axes_dim_sum / 2, pos_len);
             // pe->data = pe_vec.data();
             // print_ggml_tensor(pe);
             // pe->data = nullptr;
             set_backend_tensor_data(pe, pe_vec.data());
 
             if (version == VERSION_CHROMA_RADIANCE) {
-                int patch_size     = config.patch_size;
+                int patch_size     = flux_params.patch_size;
-                int nerf_max_freqs = config.chroma_radiance_params.nerf_max_freqs;
+                int nerf_max_freqs = flux_params.chroma_radiance_params.nerf_max_freqs;
                 dct_vec            = fetch_dct_pos(patch_size, nerf_max_freqs);
                 dct                = ggml_new_tensor_2d(compute_ctx, GGML_TYPE_F32, nerf_max_freqs * nerf_max_freqs, patch_size * patch_size);
                 // dct->data = dct_vec.data();
@@ -1504,25 +1482,6 @@ namespace Flux {
             return result;
         }
 
-        sd::Tensor<float> compute(int n_threads,
-                                  const DiffusionParams& diffusion_params) override {
-            GGML_ASSERT(diffusion_params.x != nullptr);
-            GGML_ASSERT(diffusion_params.timesteps != nullptr);
-            const auto* extra = diffusion_extra_as<FluxDiffusionExtra>(diffusion_params);
-            static const std::vector<sd::Tensor<float>> empty_ref_latents;
-            static const std::vector<int> empty_skip_layers;
-            return compute(n_threads,
-                           *diffusion_params.x,
-                           *diffusion_params.timesteps,
-                           tensor_or_empty(diffusion_params.context),
-                           tensor_or_empty(diffusion_params.c_concat),
-                           tensor_or_empty(diffusion_params.y),
-                           tensor_or_empty(extra->guidance),
-                           diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents,
-                           diffusion_params.increase_ref_index,
-                           extra->skip_layers ? *extra->skip_layers : empty_skip_layers);
-        }
-
         void test() {
             ggml_init_params params;
             params.mem_size   = static_cast<size_t>(1024 * 1024) * 1024;  // 1GB
@@ -1580,7 +1539,7 @@ namespace Flux {
 
         static void load_from_file_and_test(const std::string& file_path) {
             // ggml_backend_t backend = ggml_backend_cuda_init(0);
-            ggml_backend_t backend    = sd_backend_cpu_init();
+            ggml_backend_t backend    = ggml_backend_cpu_init();
             ggml_type model_data_type = GGML_TYPE_COUNT;
 
             ModelLoader model_loader;
@@ -1599,17 +1558,13 @@ namespace Flux {
             }
 
             std::shared_ptr<FluxRunner> flux = std::make_shared<FluxRunner>(backend,
-                                                                            backend,
+                                                                            false,
                                                                             tensor_storage_map,
                                                                             "model.diffusion_model",
                                                                             VERSION_FLUX2,
                                                                             false);
 
-            if (!flux->alloc_params_buffer()) {
+            flux->alloc_params_buffer();
-                LOG_ERROR("flux model allocation failed");
-                return;
-            }
-
             std::map<std::string, ggml_tensor*> tensors;
             flux->get_param_tensors(tensors, "model.diffusion_model");
 
@@ -1627,4 +1582,4 @@ namespace Flux {
 
 }  // namespace Flux
 
-#endif  // __SD_MODEL_DIFFUSION_FLUX_HPP__
+#endif  // __FLUX_HPP__

src/gguf_reader.hpp

@@ -1,13 +1,13 @@
-#ifndef __SD_MODEL_IO_GGUF_READER_EXT_H__
+#ifndef __GGUF_READER_HPP__
-#define __SD_MODEL_IO_GGUF_READER_EXT_H__
+#define __GGUF_READER_HPP__
 
 #include <cstdint>
 #include <fstream>
 #include <string>
 #include <vector>
 
-#include "core/util.h"
 #include "ggml.h"
+#include "util.h"
 
 struct GGUFTensorInfo {
     std::string name;
@@ -59,9 +59,6 @@ private:
         if (!safe_read(fin, key_len))
             return false;
 
-        if (key_len > 4096)
-            return false;
-
         std::string key(key_len, '\0');
         if (!safe_read(fin, (char*)key.data(), key_len))
             return false;
@@ -231,4 +228,4 @@ public:
     size_t data_offset() const { return data_offset_; }
 };
 
-#endif  // __SD_MODEL_IO_GGUF_READER_EXT_H__
+#endif  // __GGUF_READER_HPP__

src/gits_noise.inl

@@ -0,0 +1,349 @@
+#ifndef GITS_NOISE_INL
+#define GITS_NOISE_INL
+
+const std::vector<std::vector<float>> GITS_NOISE_0_80 = {
+    { 14.61464119f, 7.49001646f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 6.77309084f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 3.07277966f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 2.05039096f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 8.75849152f, 7.49001646f, 5.85520077f, 2.05039096f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 8.75849152f, 7.49001646f, 5.85520077f, 3.07277966f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 11.54541874f, 8.75849152f, 7.49001646f, 5.85520077f, 3.07277966f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.23089790f, 10.90732002f, 8.75849152f, 7.49001646f, 5.85520077f, 3.07277966f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 10.90732002f, 8.75849152f, 7.49001646f, 5.85520077f, 3.07277966f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 10.90732002f, 9.24142551f, 8.30717278f, 7.49001646f, 5.85520077f, 3.07277966f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 10.90732002f, 9.24142551f, 8.30717278f, 7.49001646f, 6.14220476f, 4.86714602f, 3.07277966f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.31284904f, 9.24142551f, 8.30717278f, 7.49001646f, 6.14220476f, 4.86714602f, 3.07277966f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.90732002f, 10.31284904f, 9.24142551f, 8.30717278f, 7.49001646f, 6.14220476f, 4.86714602f, 3.07277966f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.90732002f, 10.31284904f, 9.24142551f, 8.75849152f, 8.30717278f, 7.49001646f, 6.14220476f, 4.86714602f, 3.07277966f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.90732002f, 10.31284904f, 9.75859547f, 9.24142551f, 8.75849152f, 8.30717278f, 7.49001646f, 6.14220476f, 4.86714602f, 3.19567990f, 1.98035145f, 0.86115354f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.90732002f, 10.31284904f, 9.75859547f, 9.24142551f, 8.75849152f, 8.30717278f, 7.49001646f, 6.14220476f, 4.86714602f, 3.19567990f, 1.98035145f, 0.86115354f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.90732002f, 10.31284904f, 9.75859547f, 9.24142551f, 8.75849152f, 8.30717278f, 7.88507891f, 7.49001646f, 6.77309084f, 5.85520077f, 4.65472794f, 3.07277966f, 1.84880662f, 0.83188516f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_0_85 = {
+    { 14.61464119f, 7.49001646f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 1.84880662f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 6.77309084f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.11996698f, 3.07277966f, 1.24153244f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.09240818f, 2.84484982f, 0.95350921f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 8.75849152f, 7.49001646f, 5.09240818f, 2.84484982f, 0.95350921f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 8.75849152f, 7.49001646f, 5.58536053f, 3.19567990f, 1.84880662f, 0.803307f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 11.54541874f, 8.75849152f, 7.49001646f, 5.58536053f, 3.19567990f, 1.84880662f, 0.803307f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 11.54541874f, 8.75849152f, 7.49001646f, 6.14220476f, 4.65472794f, 3.07277966f, 1.84880662f, 0.803307f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.23089790f, 10.90732002f, 8.75849152f, 7.49001646f, 6.14220476f, 4.65472794f, 3.07277966f, 1.84880662f, 0.803307f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.23089790f, 10.90732002f, 9.24142551f, 8.30717278f, 7.49001646f, 6.14220476f, 4.65472794f, 3.07277966f, 1.84880662f, 0.803307f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 10.90732002f, 9.24142551f, 8.30717278f, 7.49001646f, 6.14220476f, 4.65472794f, 3.07277966f, 1.84880662f, 0.803307f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.31284904f, 9.24142551f, 8.30717278f, 7.49001646f, 6.14220476f, 4.65472794f, 3.07277966f, 1.84880662f, 0.803307f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.31284904f, 9.24142551f, 8.30717278f, 7.49001646f, 6.14220476f, 4.86714602f, 3.60512662f, 2.63833880f, 1.56271636f, 0.72133851f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.31284904f, 9.24142551f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 4.65472794f, 3.46139455f, 2.45070267f, 1.56271636f, 0.72133851f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.31284904f, 9.24142551f, 8.75849152f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 4.65472794f, 3.46139455f, 2.45070267f, 1.56271636f, 0.72133851f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.90732002f, 10.31284904f, 9.24142551f, 8.75849152f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 4.65472794f, 3.46139455f, 2.45070267f, 1.56271636f, 0.72133851f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.90732002f, 10.31284904f, 9.75859547f, 9.24142551f, 8.75849152f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 4.65472794f, 3.46139455f, 2.45070267f, 1.56271636f, 0.72133851f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.90732002f, 10.31284904f, 9.75859547f, 9.24142551f, 8.75849152f, 8.30717278f, 7.88507891f, 7.49001646f, 6.77309084f, 5.85520077f, 4.65472794f, 3.46139455f, 2.45070267f, 1.56271636f, 0.72133851f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_0_90 = {
+    { 14.61464119f, 6.77309084f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 3.07277966f, 0.95350921f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 2.54230714f, 0.89115214f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 4.86714602f, 2.54230714f, 0.89115214f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.09240818f, 3.07277966f, 1.61558151f, 0.69515091f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 8.75849152f, 7.11996698f, 4.86714602f, 3.07277966f, 1.61558151f, 0.69515091f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 8.75849152f, 7.49001646f, 5.85520077f, 4.45427561f, 2.95596409f, 1.61558151f, 0.69515091f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 8.75849152f, 7.49001646f, 5.85520077f, 4.45427561f, 3.19567990f, 2.19988537f, 1.24153244f, 0.57119018f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 10.90732002f, 8.75849152f, 7.49001646f, 5.85520077f, 4.45427561f, 3.19567990f, 2.19988537f, 1.24153244f, 0.57119018f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 11.54541874f, 9.24142551f, 8.30717278f, 7.49001646f, 5.85520077f, 4.45427561f, 3.19567990f, 2.19988537f, 1.24153244f, 0.57119018f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 11.54541874f, 9.24142551f, 8.30717278f, 7.49001646f, 6.14220476f, 4.86714602f, 3.75677586f, 2.84484982f, 1.84880662f, 1.08895338f, 0.52423614f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.23089790f, 10.90732002f, 9.24142551f, 8.30717278f, 7.49001646f, 6.14220476f, 4.86714602f, 3.75677586f, 2.84484982f, 1.84880662f, 1.08895338f, 0.52423614f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.23089790f, 10.90732002f, 9.24142551f, 8.30717278f, 7.49001646f, 6.44769001f, 5.58536053f, 4.45427561f, 3.32507086f, 2.45070267f, 1.61558151f, 0.95350921f, 0.45573691f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 10.90732002f, 9.24142551f, 8.30717278f, 7.49001646f, 6.44769001f, 5.58536053f, 4.45427561f, 3.32507086f, 2.45070267f, 1.61558151f, 0.95350921f, 0.45573691f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 10.90732002f, 9.24142551f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 4.86714602f, 3.91689563f, 3.07277966f, 2.27973175f, 1.56271636f, 0.95350921f, 0.45573691f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.31284904f, 9.24142551f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 4.86714602f, 3.91689563f, 3.07277966f, 2.27973175f, 1.56271636f, 0.95350921f, 0.45573691f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.31284904f, 9.24142551f, 8.75849152f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 4.86714602f, 3.91689563f, 3.07277966f, 2.27973175f, 1.56271636f, 0.95350921f, 0.45573691f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.96784878f, 12.23089790f, 11.54541874f, 10.31284904f, 9.24142551f, 8.75849152f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 5.09240818f, 4.45427561f, 3.60512662f, 2.95596409f, 2.19988537f, 1.51179266f, 0.89115214f, 0.43325692f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_0_95 = {
+    { 14.61464119f, 6.77309084f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 2.84484982f, 0.89115214f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 2.36326075f, 0.803307f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 2.95596409f, 1.56271636f, 0.64427125f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 4.86714602f, 2.95596409f, 1.56271636f, 0.64427125f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 4.86714602f, 3.07277966f, 1.91321158f, 1.08895338f, 0.50118381f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.45427561f, 3.07277966f, 1.91321158f, 1.08895338f, 0.50118381f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 8.75849152f, 7.49001646f, 5.85520077f, 4.45427561f, 3.07277966f, 1.91321158f, 1.08895338f, 0.50118381f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 8.75849152f, 7.49001646f, 5.85520077f, 4.45427561f, 3.19567990f, 2.19988537f, 1.41535246f, 0.803307f, 0.38853383f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 8.75849152f, 7.49001646f, 5.85520077f, 4.65472794f, 3.46139455f, 2.63833880f, 1.84880662f, 1.24153244f, 0.72133851f, 0.34370604f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 10.90732002f, 8.75849152f, 7.49001646f, 5.85520077f, 4.65472794f, 3.46139455f, 2.63833880f, 1.84880662f, 1.24153244f, 0.72133851f, 0.34370604f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 10.90732002f, 8.75849152f, 7.49001646f, 6.14220476f, 4.86714602f, 3.75677586f, 2.95596409f, 2.19988537f, 1.56271636f, 1.05362725f, 0.64427125f, 0.32104823f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 10.90732002f, 8.75849152f, 7.49001646f, 6.44769001f, 5.58536053f, 4.65472794f, 3.60512662f, 2.95596409f, 2.19988537f, 1.56271636f, 1.05362725f, 0.64427125f, 0.32104823f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 11.54541874f, 9.24142551f, 8.30717278f, 7.49001646f, 6.44769001f, 5.58536053f, 4.65472794f, 3.60512662f, 2.95596409f, 2.19988537f, 1.56271636f, 1.05362725f, 0.64427125f, 0.32104823f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 11.54541874f, 9.24142551f, 8.30717278f, 7.49001646f, 6.44769001f, 5.58536053f, 4.65472794f, 3.75677586f, 3.07277966f, 2.45070267f, 1.78698075f, 1.24153244f, 0.83188516f, 0.50118381f, 0.22545385f, 0.02916753f },
+    { 14.61464119f, 12.96784878f, 11.54541874f, 9.24142551f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 5.09240818f, 4.45427561f, 3.60512662f, 2.95596409f, 2.36326075f, 1.72759056f, 1.24153244f, 0.83188516f, 0.50118381f, 0.22545385f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.23089790f, 10.90732002f, 9.24142551f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 5.09240818f, 4.45427561f, 3.60512662f, 2.95596409f, 2.36326075f, 1.72759056f, 1.24153244f, 0.83188516f, 0.50118381f, 0.22545385f, 0.02916753f },
+    { 14.61464119f, 13.76078796f, 12.23089790f, 10.90732002f, 9.24142551f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 5.09240818f, 4.45427561f, 3.75677586f, 3.07277966f, 2.45070267f, 1.91321158f, 1.46270394f, 1.05362725f, 0.72133851f, 0.43325692f, 0.19894916f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_1_00 = {
+    { 14.61464119f, 1.56271636f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 0.95350921f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 2.36326075f, 0.803307f, 0.02916753f },
+    { 14.61464119f, 7.11996698f, 3.07277966f, 1.56271636f, 0.59516323f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 2.84484982f, 1.41535246f, 0.57119018f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 2.84484982f, 1.61558151f, 0.86115354f, 0.38853383f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 4.86714602f, 2.84484982f, 1.61558151f, 0.86115354f, 0.38853383f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 4.86714602f, 3.07277966f, 1.98035145f, 1.24153244f, 0.72133851f, 0.34370604f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.45427561f, 3.07277966f, 1.98035145f, 1.24153244f, 0.72133851f, 0.34370604f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.45427561f, 3.19567990f, 2.27973175f, 1.51179266f, 0.95350921f, 0.54755926f, 0.25053367f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.45427561f, 3.19567990f, 2.36326075f, 1.61558151f, 1.08895338f, 0.72133851f, 0.41087446f, 0.17026083f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 8.75849152f, 7.49001646f, 5.85520077f, 4.45427561f, 3.19567990f, 2.36326075f, 1.61558151f, 1.08895338f, 0.72133851f, 0.41087446f, 0.17026083f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 8.75849152f, 7.49001646f, 5.85520077f, 4.65472794f, 3.60512662f, 2.84484982f, 2.12350607f, 1.56271636f, 1.08895338f, 0.72133851f, 0.41087446f, 0.17026083f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 8.75849152f, 7.49001646f, 5.85520077f, 4.65472794f, 3.60512662f, 2.84484982f, 2.19988537f, 1.61558151f, 1.162866f, 0.803307f, 0.50118381f, 0.27464288f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 8.75849152f, 7.49001646f, 5.85520077f, 4.65472794f, 3.75677586f, 3.07277966f, 2.45070267f, 1.84880662f, 1.36964464f, 1.01931262f, 0.72133851f, 0.45573691f, 0.25053367f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 8.75849152f, 7.49001646f, 6.14220476f, 5.09240818f, 4.26497746f, 3.46139455f, 2.84484982f, 2.19988537f, 1.67050016f, 1.24153244f, 0.92192322f, 0.64427125f, 0.43325692f, 0.25053367f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 8.75849152f, 7.49001646f, 6.14220476f, 5.09240818f, 4.26497746f, 3.60512662f, 2.95596409f, 2.45070267f, 1.91321158f, 1.51179266f, 1.12534678f, 0.83188516f, 0.59516323f, 0.38853383f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 9.24142551f, 8.30717278f, 7.49001646f, 6.14220476f, 5.09240818f, 4.26497746f, 3.60512662f, 2.95596409f, 2.45070267f, 1.91321158f, 1.51179266f, 1.12534678f, 0.83188516f, 0.59516323f, 0.38853383f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 12.23089790f, 9.24142551f, 8.30717278f, 7.49001646f, 6.77309084f, 5.85520077f, 5.09240818f, 4.26497746f, 3.60512662f, 2.95596409f, 2.45070267f, 1.91321158f, 1.51179266f, 1.12534678f, 0.83188516f, 0.59516323f, 0.38853383f, 0.22545385f, 0.09824532f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_1_05 = {
+    { 14.61464119f, 0.95350921f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 0.89115214f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 2.05039096f, 0.72133851f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 2.84484982f, 1.28281462f, 0.52423614f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 3.07277966f, 1.61558151f, 0.803307f, 0.34370604f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 2.84484982f, 1.56271636f, 0.803307f, 0.34370604f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 2.84484982f, 1.61558151f, 0.95350921f, 0.52423614f, 0.22545385f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.07277966f, 1.98035145f, 1.24153244f, 0.74807048f, 0.41087446f, 0.17026083f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.19567990f, 2.27973175f, 1.51179266f, 0.95350921f, 0.59516323f, 0.34370604f, 0.13792117f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 5.09240818f, 3.46139455f, 2.45070267f, 1.61558151f, 1.08895338f, 0.72133851f, 0.45573691f, 0.25053367f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.09240818f, 3.46139455f, 2.45070267f, 1.61558151f, 1.08895338f, 0.72133851f, 0.45573691f, 0.25053367f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.45427561f, 3.19567990f, 2.36326075f, 1.61558151f, 1.08895338f, 0.72133851f, 0.45573691f, 0.25053367f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.45427561f, 3.19567990f, 2.45070267f, 1.72759056f, 1.24153244f, 0.86115354f, 0.59516323f, 0.38853383f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.65472794f, 3.60512662f, 2.84484982f, 2.19988537f, 1.61558151f, 1.162866f, 0.83188516f, 0.59516323f, 0.38853383f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.65472794f, 3.60512662f, 2.84484982f, 2.19988537f, 1.67050016f, 1.28281462f, 0.95350921f, 0.72133851f, 0.52423614f, 0.34370604f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.65472794f, 3.60512662f, 2.95596409f, 2.36326075f, 1.84880662f, 1.41535246f, 1.08895338f, 0.83188516f, 0.61951244f, 0.45573691f, 0.32104823f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.65472794f, 3.60512662f, 2.95596409f, 2.45070267f, 1.91321158f, 1.51179266f, 1.20157266f, 0.95350921f, 0.74807048f, 0.57119018f, 0.43325692f, 0.29807833f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 8.30717278f, 7.11996698f, 5.85520077f, 4.65472794f, 3.60512662f, 2.95596409f, 2.45070267f, 1.91321158f, 1.51179266f, 1.20157266f, 0.95350921f, 0.74807048f, 0.57119018f, 0.43325692f, 0.29807833f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 8.30717278f, 7.11996698f, 5.85520077f, 4.65472794f, 3.60512662f, 2.95596409f, 2.45070267f, 1.98035145f, 1.61558151f, 1.32549286f, 1.08895338f, 0.86115354f, 0.69515091f, 0.54755926f, 0.41087446f, 0.29807833f, 0.19894916f, 0.09824532f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_1_10 = {
+    { 14.61464119f, 0.89115214f, 0.02916753f },
+    { 14.61464119f, 2.36326075f, 0.72133851f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 1.61558151f, 0.57119018f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 2.45070267f, 1.08895338f, 0.45573691f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 2.95596409f, 1.56271636f, 0.803307f, 0.34370604f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 3.07277966f, 1.61558151f, 0.89115214f, 0.4783645f, 0.19894916f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 3.07277966f, 1.84880662f, 1.08895338f, 0.64427125f, 0.34370604f, 0.13792117f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 2.84484982f, 1.61558151f, 0.95350921f, 0.54755926f, 0.27464288f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 2.95596409f, 1.91321158f, 1.24153244f, 0.803307f, 0.4783645f, 0.25053367f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.07277966f, 2.05039096f, 1.41535246f, 0.95350921f, 0.64427125f, 0.41087446f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.19567990f, 2.27973175f, 1.61558151f, 1.12534678f, 0.803307f, 0.54755926f, 0.36617002f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.32507086f, 2.45070267f, 1.72759056f, 1.24153244f, 0.89115214f, 0.64427125f, 0.45573691f, 0.32104823f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 5.09240818f, 3.60512662f, 2.84484982f, 2.05039096f, 1.51179266f, 1.08895338f, 0.803307f, 0.59516323f, 0.43325692f, 0.29807833f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 5.09240818f, 3.60512662f, 2.84484982f, 2.12350607f, 1.61558151f, 1.24153244f, 0.95350921f, 0.72133851f, 0.54755926f, 0.41087446f, 0.29807833f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 5.85520077f, 4.45427561f, 3.19567990f, 2.45070267f, 1.84880662f, 1.41535246f, 1.08895338f, 0.83188516f, 0.64427125f, 0.50118381f, 0.36617002f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 5.85520077f, 4.45427561f, 3.19567990f, 2.45070267f, 1.91321158f, 1.51179266f, 1.20157266f, 0.95350921f, 0.74807048f, 0.59516323f, 0.45573691f, 0.34370604f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 5.85520077f, 4.45427561f, 3.46139455f, 2.84484982f, 2.19988537f, 1.72759056f, 1.36964464f, 1.08895338f, 0.86115354f, 0.69515091f, 0.54755926f, 0.43325692f, 0.34370604f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.45427561f, 3.46139455f, 2.84484982f, 2.19988537f, 1.72759056f, 1.36964464f, 1.08895338f, 0.86115354f, 0.69515091f, 0.54755926f, 0.43325692f, 0.34370604f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 11.54541874f, 7.49001646f, 5.85520077f, 4.45427561f, 3.46139455f, 2.84484982f, 2.19988537f, 1.72759056f, 1.36964464f, 1.08895338f, 0.89115214f, 0.72133851f, 0.59516323f, 0.4783645f, 0.38853383f, 0.29807833f, 0.22545385f, 0.17026083f, 0.09824532f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_1_15 = {
+    { 14.61464119f, 0.83188516f, 0.02916753f },
+    { 14.61464119f, 1.84880662f, 0.59516323f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 1.56271636f, 0.52423614f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 1.91321158f, 0.83188516f, 0.34370604f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.45070267f, 1.24153244f, 0.59516323f, 0.25053367f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.51179266f, 0.803307f, 0.41087446f, 0.17026083f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.56271636f, 0.89115214f, 0.50118381f, 0.25053367f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 3.07277966f, 1.84880662f, 1.12534678f, 0.72133851f, 0.43325692f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 3.07277966f, 1.91321158f, 1.24153244f, 0.803307f, 0.52423614f, 0.34370604f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 2.95596409f, 1.91321158f, 1.24153244f, 0.803307f, 0.52423614f, 0.34370604f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.07277966f, 2.05039096f, 1.36964464f, 0.95350921f, 0.69515091f, 0.4783645f, 0.32104823f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.07277966f, 2.12350607f, 1.51179266f, 1.08895338f, 0.803307f, 0.59516323f, 0.43325692f, 0.29807833f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.07277966f, 2.12350607f, 1.51179266f, 1.08895338f, 0.803307f, 0.59516323f, 0.45573691f, 0.34370604f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.07277966f, 2.19988537f, 1.61558151f, 1.24153244f, 0.95350921f, 0.74807048f, 0.59516323f, 0.45573691f, 0.34370604f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.19567990f, 2.45070267f, 1.78698075f, 1.32549286f, 1.01931262f, 0.803307f, 0.64427125f, 0.50118381f, 0.38853383f, 0.29807833f, 0.22545385f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.19567990f, 2.45070267f, 1.78698075f, 1.32549286f, 1.01931262f, 0.803307f, 0.64427125f, 0.52423614f, 0.41087446f, 0.32104823f, 0.25053367f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.19567990f, 2.45070267f, 1.84880662f, 1.41535246f, 1.12534678f, 0.89115214f, 0.72133851f, 0.59516323f, 0.4783645f, 0.38853383f, 0.32104823f, 0.25053367f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.86714602f, 3.19567990f, 2.45070267f, 1.84880662f, 1.41535246f, 1.12534678f, 0.89115214f, 0.72133851f, 0.59516323f, 0.50118381f, 0.41087446f, 0.34370604f, 0.29807833f, 0.25053367f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_1_20 = {
+    { 14.61464119f, 0.803307f, 0.02916753f },
+    { 14.61464119f, 1.56271636f, 0.52423614f, 0.02916753f },
+    { 14.61464119f, 2.36326075f, 0.92192322f, 0.36617002f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.24153244f, 0.59516323f, 0.25053367f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.05039096f, 0.95350921f, 0.45573691f, 0.17026083f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.45070267f, 1.24153244f, 0.64427125f, 0.29807833f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.45070267f, 1.36964464f, 0.803307f, 0.45573691f, 0.25053367f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.61558151f, 0.95350921f, 0.59516323f, 0.36617002f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.67050016f, 1.08895338f, 0.74807048f, 0.50118381f, 0.32104823f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.95596409f, 1.84880662f, 1.24153244f, 0.83188516f, 0.59516323f, 0.41087446f, 0.27464288f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 3.07277966f, 1.98035145f, 1.36964464f, 0.95350921f, 0.69515091f, 0.50118381f, 0.36617002f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 3.46139455f, 2.36326075f, 1.56271636f, 1.08895338f, 0.803307f, 0.59516323f, 0.45573691f, 0.34370604f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 6.77309084f, 3.46139455f, 2.45070267f, 1.61558151f, 1.162866f, 0.86115354f, 0.64427125f, 0.50118381f, 0.38853383f, 0.29807833f, 0.22545385f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.65472794f, 3.07277966f, 2.12350607f, 1.51179266f, 1.08895338f, 0.83188516f, 0.64427125f, 0.50118381f, 0.38853383f, 0.29807833f, 0.22545385f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.65472794f, 3.07277966f, 2.12350607f, 1.51179266f, 1.08895338f, 0.83188516f, 0.64427125f, 0.50118381f, 0.41087446f, 0.32104823f, 0.25053367f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.65472794f, 3.07277966f, 2.12350607f, 1.51179266f, 1.08895338f, 0.83188516f, 0.64427125f, 0.50118381f, 0.41087446f, 0.34370604f, 0.27464288f, 0.22545385f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.65472794f, 3.07277966f, 2.19988537f, 1.61558151f, 1.20157266f, 0.92192322f, 0.72133851f, 0.57119018f, 0.45573691f, 0.36617002f, 0.29807833f, 0.25053367f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.65472794f, 3.07277966f, 2.19988537f, 1.61558151f, 1.24153244f, 0.95350921f, 0.74807048f, 0.59516323f, 0.4783645f, 0.38853383f, 0.32104823f, 0.27464288f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 7.49001646f, 4.65472794f, 3.07277966f, 2.19988537f, 1.61558151f, 1.24153244f, 0.95350921f, 0.74807048f, 0.59516323f, 0.50118381f, 0.41087446f, 0.34370604f, 0.29807833f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_1_25 = {
+    { 14.61464119f, 0.72133851f, 0.02916753f },
+    { 14.61464119f, 1.56271636f, 0.50118381f, 0.02916753f },
+    { 14.61464119f, 2.05039096f, 0.803307f, 0.32104823f, 0.02916753f },
+    { 14.61464119f, 2.36326075f, 0.95350921f, 0.43325692f, 0.17026083f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.24153244f, 0.59516323f, 0.27464288f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 3.07277966f, 1.51179266f, 0.803307f, 0.43325692f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.36326075f, 1.24153244f, 0.72133851f, 0.41087446f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.45070267f, 1.36964464f, 0.83188516f, 0.52423614f, 0.34370604f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.61558151f, 0.98595673f, 0.64427125f, 0.43325692f, 0.27464288f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.67050016f, 1.08895338f, 0.74807048f, 0.52423614f, 0.36617002f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.72759056f, 1.162866f, 0.803307f, 0.59516323f, 0.45573691f, 0.34370604f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.95596409f, 1.84880662f, 1.24153244f, 0.86115354f, 0.64427125f, 0.4783645f, 0.36617002f, 0.27464288f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.95596409f, 1.84880662f, 1.28281462f, 0.92192322f, 0.69515091f, 0.52423614f, 0.41087446f, 0.32104823f, 0.25053367f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.95596409f, 1.91321158f, 1.32549286f, 0.95350921f, 0.72133851f, 0.54755926f, 0.43325692f, 0.34370604f, 0.27464288f, 0.22545385f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.95596409f, 1.91321158f, 1.32549286f, 0.95350921f, 0.72133851f, 0.57119018f, 0.45573691f, 0.36617002f, 0.29807833f, 0.25053367f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.95596409f, 1.91321158f, 1.32549286f, 0.95350921f, 0.74807048f, 0.59516323f, 0.4783645f, 0.38853383f, 0.32104823f, 0.27464288f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 3.07277966f, 2.05039096f, 1.41535246f, 1.05362725f, 0.803307f, 0.61951244f, 0.50118381f, 0.41087446f, 0.34370604f, 0.29807833f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 3.07277966f, 2.05039096f, 1.41535246f, 1.05362725f, 0.803307f, 0.64427125f, 0.52423614f, 0.43325692f, 0.36617002f, 0.32104823f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 3.07277966f, 2.05039096f, 1.46270394f, 1.08895338f, 0.83188516f, 0.66947293f, 0.54755926f, 0.45573691f, 0.38853383f, 0.34370604f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_1_30 = {
+    { 14.61464119f, 0.72133851f, 0.02916753f },
+    { 14.61464119f, 1.24153244f, 0.43325692f, 0.02916753f },
+    { 14.61464119f, 1.56271636f, 0.59516323f, 0.22545385f, 0.02916753f },
+    { 14.61464119f, 1.84880662f, 0.803307f, 0.36617002f, 0.13792117f, 0.02916753f },
+    { 14.61464119f, 2.36326075f, 1.01931262f, 0.52423614f, 0.25053367f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.36964464f, 0.74807048f, 0.41087446f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 3.07277966f, 1.56271636f, 0.89115214f, 0.54755926f, 0.34370604f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 3.07277966f, 1.61558151f, 0.95350921f, 0.61951244f, 0.41087446f, 0.27464288f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.45070267f, 1.36964464f, 0.83188516f, 0.54755926f, 0.36617002f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.45070267f, 1.41535246f, 0.92192322f, 0.64427125f, 0.45573691f, 0.34370604f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.6383388f, 1.56271636f, 1.01931262f, 0.72133851f, 0.50118381f, 0.36617002f, 0.27464288f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.61558151f, 1.05362725f, 0.74807048f, 0.54755926f, 0.41087446f, 0.32104823f, 0.25053367f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.61558151f, 1.08895338f, 0.77538133f, 0.57119018f, 0.43325692f, 0.34370604f, 0.27464288f, 0.22545385f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.61558151f, 1.08895338f, 0.803307f, 0.59516323f, 0.45573691f, 0.36617002f, 0.29807833f, 0.25053367f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.61558151f, 1.08895338f, 0.803307f, 0.59516323f, 0.4783645f, 0.38853383f, 0.32104823f, 0.27464288f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.72759056f, 1.162866f, 0.83188516f, 0.64427125f, 0.50118381f, 0.41087446f, 0.34370604f, 0.29807833f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.72759056f, 1.162866f, 0.83188516f, 0.64427125f, 0.52423614f, 0.43325692f, 0.36617002f, 0.32104823f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.78698075f, 1.24153244f, 0.92192322f, 0.72133851f, 0.57119018f, 0.45573691f, 0.38853383f, 0.34370604f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.84484982f, 1.78698075f, 1.24153244f, 0.92192322f, 0.72133851f, 0.57119018f, 0.4783645f, 0.41087446f, 0.36617002f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_1_35 = {
+    { 14.61464119f, 0.69515091f, 0.02916753f },
+    { 14.61464119f, 0.95350921f, 0.34370604f, 0.02916753f },
+    { 14.61464119f, 1.56271636f, 0.57119018f, 0.19894916f, 0.02916753f },
+    { 14.61464119f, 1.61558151f, 0.69515091f, 0.29807833f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.84880662f, 0.83188516f, 0.43325692f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.162866f, 0.64427125f, 0.36617002f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.36964464f, 0.803307f, 0.50118381f, 0.32104823f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.41535246f, 0.83188516f, 0.54755926f, 0.36617002f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.56271636f, 0.95350921f, 0.64427125f, 0.45573691f, 0.32104823f, 0.22545385f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.56271636f, 0.95350921f, 0.64427125f, 0.45573691f, 0.34370604f, 0.25053367f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 3.07277966f, 1.61558151f, 1.01931262f, 0.72133851f, 0.52423614f, 0.38853383f, 0.29807833f, 0.22545385f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 3.07277966f, 1.61558151f, 1.01931262f, 0.72133851f, 0.52423614f, 0.41087446f, 0.32104823f, 0.25053367f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 3.07277966f, 1.61558151f, 1.05362725f, 0.74807048f, 0.54755926f, 0.43325692f, 0.34370604f, 0.27464288f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 3.07277966f, 1.72759056f, 1.12534678f, 0.803307f, 0.59516323f, 0.45573691f, 0.36617002f, 0.29807833f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 3.07277966f, 1.72759056f, 1.12534678f, 0.803307f, 0.59516323f, 0.4783645f, 0.38853383f, 0.32104823f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.45070267f, 1.51179266f, 1.01931262f, 0.74807048f, 0.57119018f, 0.45573691f, 0.36617002f, 0.32104823f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.6383388f, 1.61558151f, 1.08895338f, 0.803307f, 0.61951244f, 0.50118381f, 0.41087446f, 0.34370604f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.6383388f, 1.61558151f, 1.08895338f, 0.803307f, 0.64427125f, 0.52423614f, 0.43325692f, 0.36617002f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 5.85520077f, 2.6383388f, 1.61558151f, 1.08895338f, 0.803307f, 0.64427125f, 0.52423614f, 0.45573691f, 0.38853383f, 0.34370604f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_1_40 = {
+    { 14.61464119f, 0.59516323f, 0.02916753f },
+    { 14.61464119f, 0.95350921f, 0.34370604f, 0.02916753f },
+    { 14.61464119f, 1.08895338f, 0.43325692f, 0.13792117f, 0.02916753f },
+    { 14.61464119f, 1.56271636f, 0.64427125f, 0.27464288f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.61558151f, 0.803307f, 0.43325692f, 0.22545385f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.05039096f, 0.95350921f, 0.54755926f, 0.34370604f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.24153244f, 0.72133851f, 0.43325692f, 0.27464288f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.24153244f, 0.74807048f, 0.50118381f, 0.34370604f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.28281462f, 0.803307f, 0.52423614f, 0.36617002f, 0.27464288f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.28281462f, 0.803307f, 0.54755926f, 0.38853383f, 0.29807833f, 0.22545385f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.41535246f, 0.86115354f, 0.59516323f, 0.43325692f, 0.32104823f, 0.25053367f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.51179266f, 0.95350921f, 0.64427125f, 0.45573691f, 0.34370604f, 0.27464288f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.51179266f, 0.95350921f, 0.64427125f, 0.4783645f, 0.36617002f, 0.29807833f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.56271636f, 0.98595673f, 0.69515091f, 0.52423614f, 0.41087446f, 0.34370604f, 0.29807833f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.56271636f, 1.01931262f, 0.72133851f, 0.54755926f, 0.43325692f, 0.36617002f, 0.32104823f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.61558151f, 1.05362725f, 0.74807048f, 0.57119018f, 0.45573691f, 0.38853383f, 0.34370604f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.61558151f, 1.08895338f, 0.803307f, 0.61951244f, 0.50118381f, 0.41087446f, 0.36617002f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.61558151f, 1.08895338f, 0.803307f, 0.61951244f, 0.50118381f, 0.43325692f, 0.38853383f, 0.34370604f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.61558151f, 1.08895338f, 0.803307f, 0.64427125f, 0.52423614f, 0.45573691f, 0.41087446f, 0.36617002f, 0.34370604f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_1_45 = {
+    { 14.61464119f, 0.59516323f, 0.02916753f },
+    { 14.61464119f, 0.803307f, 0.25053367f, 0.02916753f },
+    { 14.61464119f, 0.95350921f, 0.34370604f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.24153244f, 0.54755926f, 0.25053367f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.56271636f, 0.72133851f, 0.36617002f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.61558151f, 0.803307f, 0.45573691f, 0.27464288f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.91321158f, 0.95350921f, 0.57119018f, 0.36617002f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.19988537f, 1.08895338f, 0.64427125f, 0.41087446f, 0.27464288f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.24153244f, 0.74807048f, 0.50118381f, 0.34370604f, 0.25053367f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.24153244f, 0.74807048f, 0.50118381f, 0.36617002f, 0.27464288f, 0.22545385f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.28281462f, 0.803307f, 0.54755926f, 0.41087446f, 0.32104823f, 0.25053367f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.28281462f, 0.803307f, 0.57119018f, 0.43325692f, 0.34370604f, 0.27464288f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.28281462f, 0.83188516f, 0.59516323f, 0.45573691f, 0.36617002f, 0.29807833f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.28281462f, 0.83188516f, 0.59516323f, 0.45573691f, 0.36617002f, 0.32104823f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.51179266f, 0.95350921f, 0.69515091f, 0.52423614f, 0.41087446f, 0.34370604f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.51179266f, 0.95350921f, 0.69515091f, 0.52423614f, 0.43325692f, 0.36617002f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.56271636f, 0.98595673f, 0.72133851f, 0.54755926f, 0.45573691f, 0.38853383f, 0.34370604f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.56271636f, 1.01931262f, 0.74807048f, 0.57119018f, 0.4783645f, 0.41087446f, 0.36617002f, 0.34370604f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.84484982f, 1.56271636f, 1.01931262f, 0.74807048f, 0.59516323f, 0.50118381f, 0.43325692f, 0.38853383f, 0.36617002f, 0.34370604f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f }
+};
+
+const std::vector<std::vector<float>> GITS_NOISE_1_50 = {
+    { 14.61464119f, 0.54755926f, 0.02916753f },
+    { 14.61464119f, 0.803307f, 0.25053367f, 0.02916753f },
+    { 14.61464119f, 0.86115354f, 0.32104823f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.24153244f, 0.54755926f, 0.25053367f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.56271636f, 0.72133851f, 0.36617002f, 0.19894916f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.61558151f, 0.803307f, 0.45573691f, 0.27464288f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.61558151f, 0.83188516f, 0.52423614f, 0.34370604f, 0.25053367f, 0.17026083f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.84880662f, 0.95350921f, 0.59516323f, 0.38853383f, 0.27464288f, 0.19894916f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.84880662f, 0.95350921f, 0.59516323f, 0.41087446f, 0.29807833f, 0.22545385f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 1.84880662f, 0.95350921f, 0.61951244f, 0.43325692f, 0.32104823f, 0.25053367f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.19988537f, 1.12534678f, 0.72133851f, 0.50118381f, 0.36617002f, 0.27464288f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.19988537f, 1.12534678f, 0.72133851f, 0.50118381f, 0.36617002f, 0.29807833f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.36326075f, 1.24153244f, 0.803307f, 0.57119018f, 0.43325692f, 0.34370604f, 0.29807833f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.36326075f, 1.24153244f, 0.803307f, 0.57119018f, 0.43325692f, 0.34370604f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.36326075f, 1.24153244f, 0.803307f, 0.59516323f, 0.45573691f, 0.36617002f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.36326075f, 1.24153244f, 0.803307f, 0.59516323f, 0.45573691f, 0.38853383f, 0.34370604f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.32549286f, 0.86115354f, 0.64427125f, 0.50118381f, 0.41087446f, 0.36617002f, 0.34370604f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.36964464f, 0.92192322f, 0.69515091f, 0.54755926f, 0.45573691f, 0.41087446f, 0.36617002f, 0.34370604f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f },
+    { 14.61464119f, 2.45070267f, 1.41535246f, 0.95350921f, 0.72133851f, 0.57119018f, 0.4783645f, 0.43325692f, 0.38853383f, 0.36617002f, 0.34370604f, 0.32104823f, 0.29807833f, 0.27464288f, 0.25053367f, 0.22545385f, 0.19894916f, 0.17026083f, 0.13792117f, 0.09824532f, 0.02916753f }
+};
+
+const std::vector<const std::vector<std::vector<float>>*> GITS_NOISE = {
+    &GITS_NOISE_0_80,
+    &GITS_NOISE_0_85,
+    &GITS_NOISE_0_90,
+    &GITS_NOISE_0_95,
+    &GITS_NOISE_1_00,
+    &GITS_NOISE_1_05,
+    &GITS_NOISE_1_10,
+    &GITS_NOISE_1_15,
+    &GITS_NOISE_1_20,
+    &GITS_NOISE_1_25,
+    &GITS_NOISE_1_30,
+    &GITS_NOISE_1_35,
+    &GITS_NOISE_1_40,
+    &GITS_NOISE_1_45,
+    &GITS_NOISE_1_50
+};
+
+#endif // GITS_NOISE_INL

src/latent-preview.h

@@ -1,140 +1,8 @@
 #include <algorithm>
 #include <cstddef>
 #include <cstdint>
-#include "core/tensor.hpp"
 #include "ggml.h"
-
+#include "tensor.hpp"
-const float ltxav_latent_rgb_proj[128][3] = {
-    {-0.0293802f, -0.0362516f, -0.0291386f},
-    {0.0117735f, 0.0223435f, 0.018856f},
-    {0.00922335f, 0.0145666f, 0.0038772f},
-    {0.0227299f, 0.0109122f, 0.0131384f},
-    {0.00192413f, 0.0024648f, 0.00689245f},
-    {-0.0105576f, -0.0135933f, -0.00873841f},
-    {-0.0310222f, -0.0396358f, -0.0408445f},
-    {0.0149737f, 0.0316323f, 0.03415f},
-    {0.0027752f, 0.00814889f, 0.0108575f},
-    {-0.000678017f, -0.00180589f, -0.0161684f},
-    {0.0153964f, 0.0159774f, 0.0186479f},
-    {-0.0222799f, -0.0202068f, -0.0181082f},
-    {0.0128696f, 0.00754416f, -0.00673279f},
-    {0.0142729f, 0.00448099f, -0.00193934f},
-    {-0.014066f, -0.0193755f, -0.0160104f},
-    {-0.0176785f, -0.015903f, -0.0152621f},
-    {0.0307381f, 0.0292082f, 0.0328668f},
-    {0.0332928f, 0.0368629f, 0.0440893f},
-    {0.0186304f, 0.0124069f, 0.0160734f},
-    {0.00477787f, -0.00315658f, -0.000145702f},
-    {0.0183099f, 0.0122593f, 0.00599732f},
-    {-0.0194551f, -0.0183924f, -0.0147465f},
-    {0.0025732f, 0.00442582f, 0.0173176f},
-    {-0.0169423f, -0.0293863f, -0.0225908f},
-    {-0.021228f, -0.0265094f, -0.0253049f},
-    {0.0327111f, 0.0187133f, 0.0266184f},
-    {-0.0226425f, -0.0313781f, -0.0414356f},
-    {-0.0163142f, -0.0146144f, -0.0171793f},
-    {0.0192183f, 0.0108411f, 0.00829186f},
-    {-0.032246f, -0.0274846f, -0.0287434f},
-    {0.00345399f, 0.0115567f, 0.015288f},
-    {0.000972292f, 0.00331303f, 0.0110501f},
-    {0.000939494f, -0.00705084f, -0.00979449f},
-    {0.0405155f, 0.0339534f, 0.0419513f},
-    {0.0198596f, 0.0186626f, 0.0213766f},
-    {-0.00982375f, -0.00880439f, -0.00470429f},
-    {-0.0313707f, -0.0258098f, -0.0211663f},
-    {0.0144159f, 0.0117896f, 0.0141573f},
-    {0.0164571f, 0.0149178f, 0.00921599f},
-    {0.0436184f, 0.0346583f, 0.0360647f},
-    {-0.00289744f, -0.000752502f, 0.000675415f},
-    {-0.00621715f, -0.000558851f, 0.0135814f},
-    {-0.00817579f, -0.0113584f, -0.00556793f},
-    {0.00965067f, 0.0178221f, 0.015821f},
-    {0.0211832f, 0.0180827f, 0.0154707f},
-    {-0.00412858f, -0.00374182f, 0.0029568f},
-    {-0.0175603f, -0.0226242f, -0.0279012f},
-    {-0.00437471f, -0.00668329f, 0.000164887f},
-    {-0.0355983f, -0.0419093f, -0.0383065f},
-    {0.0144314f, 0.0192514f, 0.0175639f},
-    {-0.0130693f, -0.00569884f, -0.00341647f},
-    {-0.00184689f, 0.00189034f, -0.00190561f},
-    {0.019457f, 0.00842282f, 0.0123738f},
-    {-0.00477146f, -0.00206932f, 0.00283336f},
-    {-0.0364544f, -0.0256141f, -0.0322336f},
-    {-0.0295634f, -0.0295048f, -0.021057f},
-    {0.0144484f, 0.0191862f, 0.0112445f},
-    {0.0536406f, 0.0582376f, 0.0570966f},
-    {0.0085178f, 0.00748455f, 0.00995162f},
-    {-0.0136637f, -0.0172914f, -0.0195978f},
-    {-0.0339128f, -0.0392692f, -0.0355216f},
-    {0.00612855f, 0.00568303f, -0.00212333f},
-    {-0.0029225f, 0.00668819f, 0.0122131f},
-    {0.00841843f, 0.000181587f, -0.00650644f},
-    {-0.00514432f, 0.0127043f, 0.0168049f},
-    {-0.00997384f, -0.00602262f, -0.0164031f},
-    {0.0233226f, 0.033254f, 0.0307266f},
-    {-0.0110201f, -0.0164169f, -0.0161829f},
-    {-0.0195952f, -0.0177943f, -0.0115377f},
-    {-0.00523918f, -0.00452043f, 0.00267397f},
-    {0.0313464f, 0.0288241f, 0.0262496f},
-    {0.0324018f, 0.0339792f, 0.0312209f},
-    {-0.0163247f, -0.0230503f, -0.0263239f},
-    {0.000420577f, -0.00535659f, -0.00663426f},
-    {-0.012897f, -0.00203767f, -0.000622678f},
-    {-0.0632956f, -0.0651325f, -0.0584479f},
-    {-0.00426634f, -0.0150098f, -0.00719348f},
-    {0.00476109f, 0.00674315f, 0.00895472f},
-    {0.0129384f, 0.0158352f, 0.00963773f},
-    {-0.0333379f, -0.0410522f, -0.0317462f},
-    {0.00344054f, 0.00275915f, 0.00355732f},
-    {0.0209062f, 0.0273453f, 0.0222967f},
-    {0.00827287f, 0.00223045f, 0.00325844f},
-    {-0.0149132f, -0.0183973f, -0.0199781f},
-    {-0.0100786f, -0.0103681f, -0.00218224f},
-    {-0.00791409f, -0.00405153f, -0.00599893f},
-    {0.0176126f, 0.00618342f, -6.6569e-05f},
-    {0.00942486f, -0.00206494f, -0.00580324f},
-    {0.00678093f, -0.00291742f, -0.000921195f},
-    {-0.0221992f, -0.00483162f, -0.000848514f},
-    {-0.0151587f, -0.0157166f, -0.0107302f},
-    {0.00909646f, 0.0171985f, 0.0169785f},
-    {0.0127224f, 0.0170612f, 0.0303428f},
-    {0.0196562f, 0.00212451f, 0.0127744f},
-    {0.0233013f, 0.0228994f, 0.0108387f},
-    {0.00520761f, 0.00992992f, 0.0066267f},
-    {-3.77736e-05f, 0.00460229f, -0.00475132f},
-    {-0.0311763f, -0.0453566f, -0.0486901f},
-    {0.0195798f, 0.0281246f, 0.0180102f},
-    {-0.0174149f, -0.0240867f, -0.0188785f},
-    {0.000104658f, 0.00659008f, 0.0144594f},
-    {-0.00311086f, -0.0241426f, -0.0244164f},
-    {0.0336462f, 0.0305173f, 0.0331101f},
-    {0.0613625f, 0.066561f, 0.0610198f},
-    {-0.0286757f, -0.0325401f, -0.0338036f},
-    {0.0141534f, 0.0188266f, 0.0253059f},
-    {-0.00548197f, -0.00170198f, 0.00561745f},
-    {-0.0117872f, -0.00763218f, -0.0145037f},
-    {-0.0253304f, -0.0245217f, -0.0144905f},
-    {-0.00393624f, 0.00350048f, 0.00765561f},
-    {0.0113625f, 0.00561576f, -0.0113672f},
-    {-0.0301278f, -0.0261472f, -0.0301903f},
-    {0.016863f, 0.0173781f, 0.0170916f},
-    {-0.00495108f, 0.00686749f, 0.00282767f},
-    {0.00125409f, -0.00378072f, -0.00264117f},
-    {-0.00264001f, -0.00529772f, -0.0113109f},
-    {-0.054888f, -0.0575461f, -0.0509146f},
-    {-0.019442f, -0.0232916f, -0.0258637f},
-    {0.0133362f, 0.0161808f, 0.00917951f},
-    {-0.0349002f, -0.0372642f, -0.0466206f},
-    {-0.00216926f, 0.00208738f, 0.00766492f},
-    {0.0268528f, 0.0301179f, 0.0228579f},
-    {0.0226176f, 0.021536f, 0.023152f},
-    {-0.0110646f, -0.00511349f, -0.0137346f},
-    {-0.0098424f, -0.00218176f, 0.00414545f},
-    {0.00200216f, 0.00441732f, -0.0136515f},
-    {0.00695946f, 0.00313109f, -0.00379435f},
-    {0.0188377f, 0.0144059f, 0.0229724f},
-};
-float ltxav_latent_rgb_bias[3] = {0.043849f, 0.0201085f, 0.0150286f};
 
 const float wan_21_latent_rgb_proj[16][3] = {
     {0.015123f, -0.148418f, 0.479828f},

src/lora.hpp

@@ -1,9 +1,8 @@
-#ifndef __SD_MODEL_ADAPTER_LORA_HPP__
+#ifndef __LORA_HPP__
-#define __SD_MODEL_ADAPTER_LORA_HPP__
+#define __LORA_HPP__
 
 #include <mutex>
-#include "core/ggml_extend.hpp"
+#include "ggml_extend.hpp"
-#include "model_loader.h"
 
 #define LORA_GRAPH_BASE_SIZE 10240
 
@@ -23,11 +22,10 @@ struct LoraModel : public GGMLRunner {
 
     LoraModel(const std::string& lora_id,
               ggml_backend_t backend,
-              ggml_backend_t params_backend,
               const std::string& file_path = "",
               std::string prefix           = "",
               SDVersion version            = VERSION_COUNT)
-        : lora_id(lora_id), file_path(file_path), GGMLRunner(backend, params_backend) {
+        : lora_id(lora_id), file_path(file_path), GGMLRunner(backend, false) {
         prefix = "lora." + prefix;
         if (!model_loader.init_from_file_and_convert_name(file_path, prefix, version)) {
             load_failed = true;
@@ -87,10 +85,7 @@ struct LoraModel : public GGMLRunner {
             lora_tensors[name] = real;
         }
 
-        if (!alloc_params_buffer()) {
+        alloc_params_buffer();
-            LOG_ERROR("lora model buffer allocation failed");
-            return false;
-        }
 
         dry_run = false;
         model_loader.load_tensors(on_new_tensor_cb, n_threads);
@@ -134,7 +129,7 @@ struct LoraModel : public GGMLRunner {
         }
     }
 
-    ggml_tensor* get_lora_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_backend_t backend) {
+    ggml_tensor* get_lora_weight_diff(const std::string& model_tensor_name, ggml_context* ctx) {
         ggml_tensor* updown = nullptr;
         int index           = 0;
         while (true) {
@@ -157,17 +152,17 @@ struct LoraModel : public GGMLRunner {
 
             auto iter = lora_tensors.find(lora_up_name);
             if (iter != lora_tensors.end()) {
-                lora_up = ggml_ext_cast_f32(ctx, backend, iter->second);
+                lora_up = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(lora_mid_name);
             if (iter != lora_tensors.end()) {
-                lora_mid = ggml_ext_cast_f32(ctx, backend, iter->second);
+                lora_mid = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(lora_down_name);
             if (iter != lora_tensors.end()) {
-                lora_down = ggml_ext_cast_f32(ctx, backend, iter->second);
+                lora_down = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             if (lora_up == nullptr || lora_down == nullptr) {
@@ -213,7 +208,7 @@ struct LoraModel : public GGMLRunner {
         return updown;
     }
 
-    ggml_tensor* get_raw_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_backend_t backend) {
+    ggml_tensor* get_raw_weight_diff(const std::string& model_tensor_name, ggml_context* ctx) {
         ggml_tensor* updown = nullptr;
         int index           = 0;
         while (true) {
@@ -230,7 +225,7 @@ struct LoraModel : public GGMLRunner {
 
             auto iter = lora_tensors.find(diff_name);
             if (iter != lora_tensors.end()) {
-                curr_updown = ggml_ext_cast_f32(ctx, backend, iter->second);
+                curr_updown = ggml_ext_cast_f32(ctx, iter->second);
             } else {
                 break;
             }
@@ -253,7 +248,7 @@ struct LoraModel : public GGMLRunner {
         return updown;
     }
 
-    ggml_tensor* get_loha_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_backend_t backend) {
+    ggml_tensor* get_loha_weight_diff(const std::string& model_tensor_name, ggml_context* ctx) {
         ggml_tensor* updown = nullptr;
         int index           = 0;
         while (true) {
@@ -281,33 +276,33 @@ struct LoraModel : public GGMLRunner {
 
             auto iter = lora_tensors.find(hada_1_down_name);
             if (iter != lora_tensors.end()) {
-                hada_1_down = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_1_down = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(hada_1_up_name);
             if (iter != lora_tensors.end()) {
-                hada_1_up = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_1_up = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(hada_1_mid_name);
             if (iter != lora_tensors.end()) {
-                hada_1_mid = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_1_mid = ggml_ext_cast_f32(ctx, iter->second);
                 hada_1_up  = ggml_cont(ctx, ggml_transpose(ctx, hada_1_up));
             }
 
             iter = lora_tensors.find(hada_2_down_name);
             if (iter != lora_tensors.end()) {
-                hada_2_down = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_2_down = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(hada_2_up_name);
             if (iter != lora_tensors.end()) {
-                hada_2_up = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_2_up = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(hada_2_mid_name);
             if (iter != lora_tensors.end()) {
-                hada_2_mid = ggml_ext_cast_f32(ctx, backend, iter->second);
+                hada_2_mid = ggml_ext_cast_f32(ctx, iter->second);
                 hada_2_up  = ggml_cont(ctx, ggml_transpose(ctx, hada_2_up));
             }
 
@@ -356,7 +351,7 @@ struct LoraModel : public GGMLRunner {
         return updown;
     }
 
-    ggml_tensor* get_lokr_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_backend_t backend) {
+    ggml_tensor* get_lokr_weight_diff(const std::string& model_tensor_name, ggml_context* ctx) {
         ggml_tensor* updown = nullptr;
         int index           = 0;
         while (true) {
@@ -383,24 +378,24 @@ struct LoraModel : public GGMLRunner {
 
             auto iter = lora_tensors.find(lokr_w1_name);
             if (iter != lora_tensors.end()) {
-                lokr_w1 = ggml_ext_cast_f32(ctx, backend, iter->second);
+                lokr_w1 = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             iter = lora_tensors.find(lokr_w2_name);
             if (iter != lora_tensors.end()) {
-                lokr_w2 = ggml_ext_cast_f32(ctx, backend, iter->second);
+                lokr_w2 = ggml_ext_cast_f32(ctx, iter->second);
             }
 
             int64_t rank = 1;
             if (lokr_w1 == nullptr) {
                 iter = lora_tensors.find(lokr_w1_a_name);
                 if (iter != lora_tensors.end()) {
-                    lokr_w1_a = ggml_ext_cast_f32(ctx, backend, iter->second);
+                    lokr_w1_a = ggml_ext_cast_f32(ctx, iter->second);
                 }
 
                 iter = lora_tensors.find(lokr_w1_b_name);
                 if (iter != lora_tensors.end()) {
-                    lokr_w1_b = ggml_ext_cast_f32(ctx, backend, iter->second);
+                    lokr_w1_b = ggml_ext_cast_f32(ctx, iter->second);
                 }
 
                 if (lokr_w1_a == nullptr || lokr_w1_b == nullptr) {
@@ -415,12 +410,12 @@ struct LoraModel : public GGMLRunner {
             if (lokr_w2 == nullptr) {
                 iter = lora_tensors.find(lokr_w2_a_name);
                 if (iter != lora_tensors.end()) {
-                    lokr_w2_a = ggml_ext_cast_f32(ctx, backend, iter->second);
+                    lokr_w2_a = ggml_ext_cast_f32(ctx, iter->second);
                 }
 
                 iter = lora_tensors.find(lokr_w2_b_name);
                 if (iter != lora_tensors.end()) {
-                    lokr_w2_b = ggml_ext_cast_f32(ctx, backend, iter->second);
+                    lokr_w2_b = ggml_ext_cast_f32(ctx, iter->second);
                 }
 
                 if (lokr_w2_a == nullptr || lokr_w2_b == nullptr) {
@@ -473,23 +468,23 @@ struct LoraModel : public GGMLRunner {
         return updown;
     }
 
-    ggml_tensor* get_weight_diff(const std::string& model_tensor_name, ggml_backend_t backend, ggml_context* ctx, ggml_tensor* model_tensor, bool with_lora_and_lokr = true) {
+    ggml_tensor* get_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_tensor* model_tensor, bool with_lora_and_lokr = true) {
         // lora
         ggml_tensor* diff = nullptr;
         if (with_lora_and_lokr) {
-            diff = get_lora_weight_diff(model_tensor_name, ctx, backend);
+            diff = get_lora_weight_diff(model_tensor_name, ctx);
         }
         // diff
         if (diff == nullptr) {
-            diff = get_raw_weight_diff(model_tensor_name, ctx, backend);
+            diff = get_raw_weight_diff(model_tensor_name, ctx);
         }
         // loha
         if (diff == nullptr) {
-            diff = get_loha_weight_diff(model_tensor_name, ctx, backend);
+            diff = get_loha_weight_diff(model_tensor_name, ctx);
         }
         // lokr
         if (diff == nullptr && with_lora_and_lokr) {
-            diff = get_lokr_weight_diff(model_tensor_name, ctx, backend);
+            diff = get_lokr_weight_diff(model_tensor_name, ctx);
         }
         if (diff != nullptr) {
             if (ggml_nelements(diff) < ggml_nelements(model_tensor)) {
@@ -507,7 +502,6 @@ struct LoraModel : public GGMLRunner {
     }
 
     ggml_tensor* get_out_diff(ggml_context* ctx,
-                              ggml_backend_t backend,
                               ggml_tensor* x,
                               WeightAdapter::ForwardParams forward_params,
                               const std::string& model_tensor_name) {
@@ -596,7 +590,7 @@ struct LoraModel : public GGMLRunner {
                 }
                 scale_value *= multiplier;
 
-                auto curr_out_diff = ggml_ext_lokr_forward(ctx, backend, x, lokr_w1, lokr_w1_a, lokr_w1_b, lokr_w2, lokr_w2_a, lokr_w2_b, is_conv2d, forward_params.conv2d, scale_value);
+                auto curr_out_diff = ggml_ext_lokr_forward(ctx, x, lokr_w1, lokr_w1_a, lokr_w1_b, lokr_w2, lokr_w2_a, lokr_w2_b, is_conv2d, forward_params.conv2d, scale_value);
                 if (out_diff == nullptr) {
                     out_diff = curr_out_diff;
                 } else {
@@ -767,27 +761,27 @@ struct LoraModel : public GGMLRunner {
             ggml_tensor* model_tensor     = it.second;
 
             // lora
-            ggml_tensor* diff = get_weight_diff(model_tensor_name, runtime_backend, compute_ctx, model_tensor);
+            ggml_tensor* diff = get_weight_diff(model_tensor_name, compute_ctx, model_tensor);
             if (diff == nullptr) {
                 continue;
             }
 
             ggml_tensor* original_tensor = model_tensor;
-            if (!sd_backend_is_cpu(runtime_backend) && ggml_backend_buffer_is_host(original_tensor->buffer)) {
+            if (!ggml_backend_is_cpu(runtime_backend) && ggml_backend_buffer_is_host(original_tensor->buffer)) {
                 model_tensor = ggml_dup_tensor(compute_ctx, model_tensor);
                 set_backend_tensor_data(model_tensor, original_tensor->data);
             }
 
             ggml_tensor* final_tensor;
             if (model_tensor->type != GGML_TYPE_F32 && model_tensor->type != GGML_TYPE_F16) {
-                final_tensor = ggml_ext_cast_f32(compute_ctx, runtime_backend, model_tensor);
+                final_tensor = ggml_ext_cast_f32(compute_ctx, model_tensor);
                 final_tensor = ggml_add_inplace(compute_ctx, final_tensor, diff);
                 final_tensor = ggml_cpy(compute_ctx, final_tensor, model_tensor);
             } else {
                 final_tensor = ggml_add_inplace(compute_ctx, model_tensor, diff);
             }
             ggml_build_forward_expand(gf, final_tensor);
-            if (!sd_backend_is_cpu(runtime_backend) && ggml_backend_buffer_is_host(original_tensor->buffer)) {
+            if (!ggml_backend_is_cpu(runtime_backend) && ggml_backend_buffer_is_host(original_tensor->buffer)) {
                 original_tensor_to_final_tensor[original_tensor] = final_tensor;
             }
         }
@@ -847,35 +841,34 @@ public:
         : lora_models(lora_models) {
     }
 
-    ggml_tensor* patch_weight(ggml_context* ctx, ggml_backend_t backend, ggml_tensor* weight, const std::string& weight_name, bool with_lora_and_lokr) {
+    ggml_tensor* patch_weight(ggml_context* ctx, ggml_tensor* weight, const std::string& weight_name, bool with_lora_and_lokr) {
         for (auto& lora_model : lora_models) {
-            ggml_tensor* diff = lora_model->get_weight_diff(weight_name, backend, ctx, weight, with_lora_and_lokr);
+            ggml_tensor* diff = lora_model->get_weight_diff(weight_name, ctx, weight, with_lora_and_lokr);
             if (diff == nullptr) {
                 continue;
             }
 
             if (weight->type != GGML_TYPE_F32 && weight->type != GGML_TYPE_F16) {
-                weight = ggml_ext_cast_f32(ctx, backend, weight);
+                weight = ggml_ext_cast_f32(ctx, weight);
             }
             weight = ggml_add(ctx, weight, diff);
         }
         return weight;
     }
 
-    ggml_tensor* patch_weight(ggml_context* ctx, ggml_backend_t backend, ggml_tensor* weight, const std::string& weight_name) override {
+    ggml_tensor* patch_weight(ggml_context* ctx, ggml_tensor* weight, const std::string& weight_name) override {
-        return patch_weight(ctx, backend, weight, weight_name, true);
+        return patch_weight(ctx, weight, weight_name, true);
     }
 
     ggml_tensor* forward_with_lora(ggml_context* ctx,
-                                   ggml_backend_t backend,
                                    ggml_tensor* x,
                                    ggml_tensor* w,
                                    ggml_tensor* b,
                                    const std::string& prefix,
                                    WeightAdapter::ForwardParams forward_params) override {
-        w = patch_weight(ctx, backend, w, prefix + "weight", false);
+        w = patch_weight(ctx, w, prefix + "weight", false);
         if (b) {
-            b = patch_weight(ctx, backend, b, prefix + "bias", false);
+            b = patch_weight(ctx, b, prefix + "bias", false);
         }
         ggml_tensor* out;
         if (forward_params.op_type == ForwardParams::op_type_t::OP_LINEAR) {
@@ -897,7 +890,7 @@ public:
                                    forward_params.conv2d.scale);
         }
         for (auto& lora_model : lora_models) {
-            ggml_tensor* out_diff = lora_model->get_out_diff(ctx, backend, x, forward_params, prefix + "weight");
+            ggml_tensor* out_diff = lora_model->get_out_diff(ctx, x, forward_params, prefix + "weight");
             if (out_diff == nullptr) {
                 continue;
             }
@@ -915,4 +908,4 @@ public:
     }
 };
 
-#endif  // __SD_MODEL_ADAPTER_LORA_HPP__
+#endif  // __LORA_HPP__