refactor: unify ggml operation helpers around runner context (#2314)

Co-authored-by: Maxime Dréan <maximedrean@proton.me>

.dockerignore

@@ -1,7 +1,8 @@
 build*/
 docs/
 test/
+tests/
 
 .cache/
 *.swp
-models/
+models/

.github/workflows/build.yml

@@ -16,6 +16,7 @@ on:
         ".github/workflows/**",
         ".dockerignore",
         "docker/Dockerfile*",
+        "scripts/ci-cuda-cache.sh",
         "**/CMakeLists.txt",
         "**/Makefile",
         "**/*.h",
@@ -23,9 +24,11 @@ on:
         "**/*.c",
         "**/*.cpp",
         "**/*.cu",
+        "**/*.cuh",
         "ggml",
         "examples/server/frontend",
         "examples/server/frontend/**",
+        "examples/server/recipes/*.json",
         "start-server.sh",
         "start-server.bat",
       ]
@@ -36,6 +39,7 @@ on:
         ".github/workflows/**",
         ".dockerignore",
         "docker/Dockerfile*",
+        "scripts/ci-cuda-cache.sh",
         "**/CMakeLists.txt",
         "**/Makefile",
         "**/*.h",
@@ -43,9 +47,11 @@ on:
         "**/*.c",
         "**/*.cpp",
         "**/*.cu",
+        "**/*.cuh",
         "ggml",
         "examples/server/frontend",
         "examples/server/frontend/**",
+        "examples/server/recipes/*.json",
         "start-server.sh",
         "start-server.bat",
       ]
@@ -53,6 +59,9 @@ on:
 env:
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
   cancel-in-progress: true
@@ -64,17 +73,17 @@ jobs:
     steps:
       - name: Clone
         id: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           submodules: recursive
 
       - name: Setup Node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
-          node-version: 20
+          node-version: 24
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v5
         with:
           version: 10.15.1
 
@@ -95,7 +104,8 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        shell: bash
+        run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
 
       - name: Fetch system info
         id: system-info
@@ -116,7 +126,7 @@ jobs:
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip
           path: |
@@ -128,17 +138,17 @@ jobs:
     steps:
       - name: Clone
         id: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           submodules: recursive
 
       - name: Setup Node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
-          node-version: 20
+          node-version: 24
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v5
         with:
           version: 10.15.1
 
@@ -159,7 +169,8 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        shell: bash
+        run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
 
       - name: Fetch system info
         id: system-info
@@ -180,7 +191,7 @@ jobs:
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-vulkan.zip
           path: |
@@ -228,25 +239,27 @@ jobs:
           submodules: recursive
 
       - name: Setup Node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
-          node-version: 20
+          node-version: 24
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v5
         with:
           version: 10.15.1
 
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        shell: bash
+        run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        id: buildx
+        uses: docker/setup-buildx-action@v4
 
       - name: Log in to the container registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -254,7 +267,7 @@ jobs:
 
       - name: Extract metadata for Docker
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
@@ -265,9 +278,24 @@ jobs:
           # if set to "true" but frees about 6 GB
           tool-cache: false
 
+      - name: Cache CUDA compilation
+        if: matrix.variant == 'cuda'
+        uses: actions/cache@v5
+        with:
+          path: ${{ runner.temp }}/cuda-ccache
+          key: cuda-ccache-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('docker/Dockerfile.cuda') }}-${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: |
+            cuda-ccache-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('docker/Dockerfile.cuda') }}-
+
+      - name: Restore CUDA compiler cache mount
+        if: matrix.variant == 'cuda'
+        env:
+          CUDA_CACHE_BUILDER: ${{ steps.buildx.outputs.name }}
+        run: bash scripts/ci-cuda-cache.sh restore
+
       - name: Build and push Docker image
         id: build-push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           platforms: ${{ matrix.platform }}
@@ -278,23 +306,29 @@ jobs:
           annotations: ${{ steps.meta.outputs.annotations }}
           build-args: ${{ matrix.build-args }}
 
+      - name: Export CUDA compiler cache mount
+        if: matrix.variant == 'cuda'
+        env:
+          CUDA_CACHE_BUILDER: ${{ steps.buildx.outputs.name }}
+        run: bash scripts/ci-cuda-cache.sh save
+
   macOS-latest-cmake:
     runs-on: macos-latest
 
     steps:
       - name: Clone
         id: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           submodules: recursive
 
       - name: Setup Node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
-          node-version: 20
+          node-version: 24
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v5
         with:
           version: 10.15.1
 
@@ -315,7 +349,8 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        shell: bash
+        run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
 
       - name: Fetch system info
         id: system-info
@@ -336,7 +371,7 @@ jobs:
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip
           path: |
@@ -354,35 +389,44 @@ jobs:
           - build: "cpu"
             defines: "-DGGML_NATIVE=OFF -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
           - build: "cuda12"
-            defines: "-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_ARCHITECTURES='61;70;75;80;86;89;90;100;120' -DCMAKE_CUDA_FLAGS='-Xcudafe \"--diag_suppress=177\" -Xcudafe \"--diag_suppress=550\"' -DGGML_NATIVE=OFF -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
+            defines: "-DSD_CUDA=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_ARCHITECTURES='61;70;75;80;86;89;90;100;120' -DCMAKE_CUDA_FLAGS='-Xcudafe \"--diag_suppress=177\" -Xcudafe \"--diag_suppress=550\"' -DGGML_NATIVE=OFF -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
           - build: "vulkan"
             defines: "-DSD_VULKAN=ON -DSD_BUILD_SHARED_LIBS=ON -DGGML_NATIVE=OFF -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
     steps:
       - name: Clone
         id: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           submodules: recursive
 
       - name: Setup Node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
-          node-version: 20
+          node-version: 24
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v5
         with:
           version: 10.15.1
 
       - name: Install cuda-toolkit
         id: cuda-toolkit
         if: ${{ matrix.build == 'cuda12' }}
-        uses: Jimver/cuda-toolkit@v0.2.22
+        uses: Jimver/cuda-toolkit@v0.2.35
         with:
           cuda: "12.8.1"
           method: "network"
           sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
 
+      - name: Cache CUDA compilation
+        if: matrix.build == 'cuda12'
+        uses: ggml-org/ccache-action@v1.2.21
+        with:
+          variant: ccache
+          install: binary
+          key: windows-cuda-12.8.1-x64
+          max-size: "2G"
+
       - name: Install Vulkan SDK
         id: get_vulkan
         if: ${{ matrix.build == 'vulkan' }}
@@ -392,13 +436,15 @@ jobs:
           Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
           Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
 
-      - name: Activate MSVC environment
-        id: msvc_dev_cmd
-        uses: ilammy/msvc-dev-cmd@v1
-
       - name: Build
         id: cmake_build
+        shell: pwsh
         run: |
+          $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"
+          $vsPath = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath
+          if (-not $vsPath) { throw "Visual Studio C++ tools not found" }
+          & "$vsPath\Common7\Tools\Launch-VsDevShell.ps1" -Arch amd64 -HostArch amd64 -SkipAutomaticLocation
+
           mkdir build
           cd build
           cmake .. -DCMAKE_CXX_FLAGS='/bigobj' -G Ninja -DCMAKE_C_COMPILER=cl.exe -DCMAKE_CXX_COMPILER=cl.exe -DCMAKE_BUILD_TYPE=Release ${{ matrix.defines }}
@@ -407,7 +453,8 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        shell: bash
+        run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
 
       - name: Pack artifacts
         id: pack_artifacts
@@ -427,6 +474,9 @@ jobs:
             ls .\build\bin
             throw "Can't find stable-diffusion.dll"
           }
+          if ("${{ matrix.build }}" -eq "cuda12") {
+            Copy-Item thirdparty/cutlass/LICENSE.txt (Join-Path (Split-Path -Parent $filePath) 'cutlass.txt')
+          }
           7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip $filePath .\start-server.sh .\start-server.bat
 
       - name: Copy and pack Cuda runtime
@@ -440,7 +490,7 @@ jobs:
 
       - name: Upload Cuda runtime
         if: ${{ matrix.build == 'cuda12' && (github.event_name == 'push' && github.ref == 'refs/heads/master' || github.event.inputs.create_release == 'true') }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: sd-cudart-sd-bin-win-cu12-x64.zip
           path: |
@@ -448,7 +498,7 @@ jobs:
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip
           path: |
@@ -462,29 +512,29 @@ jobs:
       gpu_targets: "gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
         with:
           submodules: recursive
 
       - name: Setup Node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
-          node-version: 20
+          node-version: 24
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v5
         with:
           version: 10.15.1
 
       - name: Cache ROCm Installation
         id: cache-rocm
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: C:\TheRock\build
           key: rocm-wheels-${{ env.ROCM_VERSION }}-${{ runner.os }}
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.16
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: windows-rocm-${{ env.ROCM_VERSION }}-x64
           evict-old-files: 1d
@@ -565,7 +615,8 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: pr-mpt/actions-commit-hash@v2
+        shell: bash
+        run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
 
       - name: Bundle HIP runtime libraries
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
@@ -586,7 +637,7 @@ jobs:
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip
           path: |
@@ -613,7 +664,7 @@ jobs:
           submodules: recursive
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.16
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: ubuntu-rocm-cmake-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
           evict-old-files: 1d
@@ -686,12 +737,12 @@ jobs:
       # setup-node installs into /opt/hostedtoolcache, which is removed above.
       # Keep Node/pnpm setup after disk cleanup so the server frontend can be embedded.
       - name: Setup Node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
-          node-version: 20
+          node-version: 24
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v5
         with:
           version: 10.15.1
 
@@ -720,7 +771,8 @@ jobs:
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: prompt/actions-commit-hash@v2
+        shell: bash
+        run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
 
       - name: Prepare artifacts
         id: prepare_artifacts
@@ -749,7 +801,7 @@ jobs:
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip
           path: |
@@ -760,6 +812,9 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    permissions:
+      contents: write
+
     needs:
       - ubuntu-latest-cmake
       - ubuntu-latest-cmake-vulkan
@@ -771,13 +826,13 @@ jobs:
 
     steps:
       - name: Clone
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
       - name: Download artifacts
         id: download-artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           path: ./artifact
           pattern: sd-*
@@ -790,36 +845,47 @@ jobs:
 
       - name: Get commit hash
         id: commit
-        uses: prompt/actions-commit-hash@v2
+        shell: bash
+        run: echo "short=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
 
       - name: Create release
         id: create_release
         if: ${{ github.event_name == 'workflow_dispatch' || github.ref_name == 'master' }}
-        uses: anzz1/action-create-release@v1
+        uses: actions/github-script@v9
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RELEASE_TAG: ${{ format('{0}-{1}-{2}', env.BRANCH_NAME, steps.commit_count.outputs.count, steps.commit.outputs.short) }}
         with:
-          tag_name: ${{ format('{0}-{1}-{2}', env.BRANCH_NAME, steps.commit_count.outputs.count, steps.commit.outputs.short) }}
+          script: |
+            const { data: release } = await github.rest.repos.createRelease({
+              ...context.repo,
+              tag_name: process.env.RELEASE_TAG,
+              name: process.env.RELEASE_TAG,
+              target_commitish: context.sha,
+              draft: false,
+              prerelease: false,
+            });
+            core.setOutput('id', release.id);
 
       - name: Upload release
         id: upload_release
         if: ${{ github.event_name == 'workflow_dispatch' || github.ref_name == 'master' }}
-        uses: actions/github-script@v3
+        uses: actions/github-script@v9
+        env:
+          RELEASE_ID: ${{ steps.create_release.outputs.id }}
         with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
             const path = require('path');
-            const fs = require('fs');
-            const release_id = '${{ steps.create_release.outputs.id }}';
-            for (let file of await fs.readdirSync('./artifact')) {
+            const fs = require('fs').promises;
+            const release_id = Number(process.env.RELEASE_ID);
+            for (const file of await fs.readdir('./artifact')) {
               if (path.extname(file) === '.zip') {
                 console.log('uploadReleaseAsset', file);
-                await github.repos.uploadReleaseAsset({
+                await github.rest.repos.uploadReleaseAsset({
                   owner: context.repo.owner,
                   repo: context.repo.repo,
                   release_id: release_id,
                   name: file,
-                  data: await fs.readFileSync(`./artifact/${file}`)
+                  data: await fs.readFile(`./artifact/${file}`)
                 });
               }
             }

.gitignore

@@ -2,6 +2,7 @@ build*/
 cmake-build-*/
 test/
 tests/
+logs/
 .vscode/
 .idea/
 .cache/

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "thirdparty/libwebm"]
 	path = thirdparty/libwebm
 	url = https://github.com/webmproject/libwebm.git
+[submodule "thirdparty/cutlass"]
+	path = thirdparty/cutlass
+	url = https://github.com/NVIDIA/cutlass.git

CMakeLists.txt

@@ -9,13 +9,28 @@ if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
 endif()
 
 if (MSVC)
+    cmake_host_system_information(RESULT SD_LOGICAL_CORES QUERY NUMBER_OF_LOGICAL_CORES)
+    if (SD_LOGICAL_CORES GREATER 8)
+        math(EXPR SD_DEFAULT_MSVC_MP_COUNT "${SD_LOGICAL_CORES} - 2")
+    elseif (SD_LOGICAL_CORES GREATER 2)
+        math(EXPR SD_DEFAULT_MSVC_MP_COUNT "${SD_LOGICAL_CORES} - 1")
+    else()
+        set(SD_DEFAULT_MSVC_MP_COUNT "${SD_LOGICAL_CORES}")
+    endif()
+    if (NOT SD_DEFAULT_MSVC_MP_COUNT GREATER 0)
+        set(SD_DEFAULT_MSVC_MP_COUNT 1)
+    endif()
+    set(SD_MSVC_MP_COUNT "${SD_DEFAULT_MSVC_MP_COUNT}" CACHE STRING "Maximum parallel MSVC compilations per project")
+    if (NOT SD_MSVC_MP_COUNT MATCHES "^[1-9][0-9]*$" OR SD_MSVC_MP_COUNT GREATER 65536)
+        message(FATAL_ERROR "SD_MSVC_MP_COUNT must be an integer between 1 and 65536")
+    endif()
     add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
     add_compile_definitions(_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING)
     # /MP is MSVC-only: icx rejects it outright once offloading is enabled.
     add_compile_options(
-        $<$<AND:$<COMPILE_LANGUAGE:C>,$<C_COMPILER_ID:MSVC>>:/MP>
+        $<$<AND:$<COMPILE_LANGUAGE:C>,$<C_COMPILER_ID:MSVC>>:/MP${SD_MSVC_MP_COUNT}>
         $<$<COMPILE_LANGUAGE:C>:/utf-8>
-        $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/MP>
+        $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/MP${SD_MSVC_MP_COUNT}>
         $<$<COMPILE_LANGUAGE:CXX>:/utf-8>
     )
 endif()
@@ -111,6 +126,12 @@ if(SD_CUDA)
 endif()
 
 if(SD_METAL)
+    if(NOT SD_USE_UPSTREAM_GGML)
+        if(CMAKE_VERSION VERSION_LESS 3.16)
+            message(FATAL_ERROR "SD_METAL with SD_USE_UPSTREAM_GGML=OFF requires CMake 3.16 or later for OBJC and OBJCXX support")
+        endif()
+        enable_language(OBJC OBJCXX)
+    endif()
     message("-- Use Metal as backend stable-diffusion")
     set(GGML_METAL ON)
 endif()
@@ -225,6 +246,8 @@ file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
     "src/core/*.h"
     "src/core/*.cpp"
     "src/core/*.hpp"
+    "src/core/cpu/*.h"
+    "src/core/cpu/*.cpp"
     "src/core/probe/*.h"
     "src/core/probe/*.cpp"
     "src/extensions/*.h"
@@ -343,6 +366,51 @@ if (SD_CUDA)
     # Keep the driver stub on downstream link lines when no driver is installed.
     target_link_libraries(${SD_LIB} PUBLIC CUDA::cuda_driver)
     set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_USE_CUDA)
+    if (NOT SD_USE_UPSTREAM_GGML)
+        include(cmake/cutlass.cmake)
+        enable_language(CUDA)
+        set_target_properties(${SD_LIB} PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON)
+        if (SD_CUDA_CUTLASS_PATH)
+            target_sources(${SD_LIB} PRIVATE src/core/cuda/cute_attention.cu src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu src/core/cuda/bf16_gemm.cu)
+            set_source_files_properties(src/core/cuda/cute_attention.cu src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu src/core/cuda/bf16_gemm.cu PROPERTIES
+                INCLUDE_DIRECTORIES "${SD_CUDA_CUTLASS_PATH}/include"
+                COMPILE_OPTIONS "--use_fast_math;--extended-lambda;--expt-relaxed-constexpr")
+            if (MSVC)
+                set_property(SOURCE src/core/cuda/cute_attention.cu src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu src/core/cuda/bf16_gemm.cu APPEND PROPERTY COMPILE_OPTIONS -Xcompiler=/Zc:preprocessor,/Zc:__cplusplus)
+            endif()
+            if (GGML_CUDA_FORCE_CUBLAS)
+                set_property(SOURCE src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu src/core/cuda/bf16_gemm.cu APPEND PROPERTY COMPILE_DEFINITIONS SD_CUDA_FORCE_CUBLAS)
+            endif()
+        endif()
+    endif()
+endif()
+set(SD_BACKEND_REGISTRATIONS)
+if (SD_METAL AND NOT SD_USE_UPSTREAM_GGML)
+    add_subdirectory(src/core/metal)
+    target_link_libraries(${SD_LIB} PRIVATE "-framework Foundation" "-framework Metal")
+    list(APPEND SD_BACKEND_REGISTRATIONS sd_ggml_register_metal_i8)
+endif()
+if (NOT SD_USE_UPSTREAM_GGML AND (SD_CUDA OR SD_HIPBLAS OR SD_MUSA))
+    if(SD_HIPBLAS)
+        include(cmake/hip.cmake)
+    endif()
+    add_subdirectory(src/core/cuda)
+    foreach(VENDOR CUDA HIP MUSA)
+        if(SD_${VENDOR} OR (VENDOR STREQUAL "HIP" AND SD_HIPBLAS))
+            string(TOLOWER "${VENDOR}" NAMESPACE)
+            list(APPEND SD_BACKEND_REGISTRATIONS "sd_${NAMESPACE}::register_extensions")
+        endif()
+    endforeach()
+endif()
+if (SD_VULKAN AND NOT SD_USE_UPSTREAM_GGML)
+    add_subdirectory(src/core/vulkan)
+    list(APPEND SD_BACKEND_REGISTRATIONS sd_ggml_register_vulkan_rope sd_ggml_register_vulkan_i8 sd_ggml_register_vulkan_modulate sd_ggml_register_vulkan_rms_affine)
+endif()
+if(SD_BACKEND_REGISTRATIONS)
+    string(REPLACE ";" ",\n    " SD_BACKEND_REGISTRATIONS "${SD_BACKEND_REGISTRATIONS}")
+    configure_file(cmake/backend_extensions.h.in generated/sd_backend_extensions.h @ONLY)
+    set_property(SOURCE src/core/ggml_extensions.cpp APPEND PROPERTY INCLUDE_DIRECTORIES "${CMAKE_CURRENT_BINARY_DIR}/generated")
+    set_property(SOURCE src/core/ggml_extensions.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_BACKEND_EXTENSIONS)
 endif()
 if (SD_HIPBLAS)
     find_package(hip REQUIRED CONFIG HINTS "$ENV{ROCM_PATH}" /opt/rocm)

CONTRIBUTING.md

@@ -29,6 +29,18 @@ PRs should include:
 - Linked issue/PR context when applicable.
 - Screenshots or sample outputs for UI/visual behavior changes.
 
+## Documentation
+
+Write model usage guides in `docs/` for users. Focus on supported weights,
+setup, runnable commands, parameter meanings, output behavior, and practical
+limits such as memory requirements. Include technical details only when they
+help users choose settings or troubleshoot a problem.
+
+Keep tensor layouts, internal class names, kernel implementation details,
+algorithm derivations, and development history out of usage guides. Keep task-specific
+profiling logs, numerical comparison procedures, and test artifacts local, and
+summarize relevant verification results in the PR description.
+
 ## README UI Listings
 
 The [UIs list in the README](README.md#uis) only includes open-source projects that use `stable-diffusion.cpp` as a backend. Do not submit PRs to add closed-source UIs.

README.md

@@ -59,6 +59,7 @@ API and command-line option may change frequently.***
     - [Z-Image](./docs/z_image.md)
     - [Z-Image L2P](./docs/z_image_l2p.md)
     - [MiniT2I](./docs/minit2i.md)
+    - [P-H3-Image](./docs/p_h3_image.md)
     - [SenseNova U1.5](./docs/sensenova_u1.md)
     - [Ovis-Image](./docs/ovis_image.md)
     - [Anima](./docs/anima.md)
@@ -87,6 +88,9 @@ API and command-line option may change frequently.***
     - [LTX-2.3/LTX-2.5](./docs/ltx2.md)
     - [HunyuanVideo 1.5](./docs/hunyuan_video.md)
     - [LingBot-Video](./docs/lingbot_video.md)
+  - Video Upscale Models
+    - [RelayVSR](./docs/relay_vsr.md)
+    - [SeedVR2](./docs/seedvr2.md)
   - [PhotoMaker](./docs/photo_maker.md) support.
   - [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL, including Plus)
   - Control Net support with SD 1.5
@@ -197,6 +201,9 @@ These projects use `stable-diffusion.cpp` as a backend for their image generatio
 - [Neural-Pixel](https://github.com/Luiz-Alcantara/Neural-Pixel)
 - [KoboldCpp](https://github.com/LostRuins/koboldcpp)
 - [Kokon](https://github.com/GabrieleLozupone/kokon)
+- [NodeTool](https://github.com/nodetool-ai/nodetool)
+- [Quo](https://github.com/quo-ai/quo)
+- [Prisme](https://github.com/d3Ita/prisme)
 
 ## Contributors
 

cmake/backend_extensions.h.in

@@ -0,0 +1,17 @@
+#ifndef __SD_BACKEND_EXTENSIONS_H__
+#define __SD_BACKEND_EXTENSIONS_H__
+
+#include "core/cuda/registration.h"
+
+void sd_ggml_register_metal_i8(ggml_backend_dev_t device);
+
+void sd_ggml_register_vulkan_rope(ggml_backend_dev_t device);
+void sd_ggml_register_vulkan_i8(ggml_backend_dev_t device);
+void sd_ggml_register_vulkan_rms_affine(ggml_backend_dev_t device);
+void sd_ggml_register_vulkan_modulate(ggml_backend_dev_t device);
+
+static void (*const sd_backend_extensions[])(ggml_backend_dev_t) = {
+    @SD_BACKEND_REGISTRATIONS@
+};
+
+#endif

cmake/cutlass.cmake

@@ -0,0 +1,20 @@
+set(sd_cutlass_default "${PROJECT_SOURCE_DIR}/thirdparty/cutlass")
+if(NOT DEFINED SD_CUDA_CUTLASS_PATH AND DEFINED GGML_CUDA_CUTLASS_PATH)
+    get_filename_component(sd_old_cutlass_default "${SD_GGML_SOURCE_DIR}/src/ggml-cuda/vendors/cutlass" ABSOLUTE)
+    if(NOT GGML_CUDA_CUTLASS_PATH STREQUAL sd_old_cutlass_default)
+        set(sd_cutlass_default "${GGML_CUDA_CUTLASS_PATH}")
+    endif()
+    message(STATUS "Migrating GGML_CUDA_CUTLASS_PATH to SD_CUDA_CUTLASS_PATH")
+endif()
+set(SD_CUDA_CUTLASS_PATH "${sd_cutlass_default}" CACHE PATH "sd: CUTLASS source directory for INT8 GEMM and CuTe attention, empty to disable")
+unset(GGML_CUDA_CUTLASS_PATH CACHE)
+if(SD_CUDA_CUTLASS_PATH)
+    if(NOT EXISTS "${SD_CUDA_CUTLASS_PATH}/include/cutlass/gemm/kernel/default_gemm_universal_with_visitor.h" OR
+       NOT EXISTS "${SD_CUDA_CUTLASS_PATH}/include/cute/tensor.hpp")
+        message(FATAL_ERROR
+            "CUTLASS headers not found at ${SD_CUDA_CUTLASS_PATH}. "
+            "Run git submodule update --init --recursive from the sd.cpp repository root, "
+            "set SD_CUDA_CUTLASS_PATH to a CUTLASS source tree (tested with 3.9.2), "
+            "or set SD_CUDA_CUTLASS_PATH to an empty string to disable CUTLASS.")
+    endif()
+endif()

cmake/embed_spirv.cmake

@@ -0,0 +1,3 @@
+file(READ "${INPUT}" SPIRV HEX)
+string(REGEX REPLACE "(..)(..)(..)(..)" "0x\\4\\3\\2\\1," SPIRV_WORDS "${SPIRV}")
+file(WRITE "${OUTPUT}" "#include <cstdint>\nnamespace sd_vulkan_shaders {\nstatic const uint32_t ${NAME}[] = {${SPIRV_WORDS}};\n}\n")

cmake/ggml.cmake

@@ -9,17 +9,19 @@ if(NOT TARGET ggml)
     add_library(ggml ALIAS ggml::ggml)
 endif()
 
-get_target_property(sd_ggml_imported ggml IMPORTED)
-if(sd_ggml_imported)
-    set(sd_ggml_private_include "${SD_GGML_SOURCE_DIR}/src")
-else()
-    get_target_property(sd_ggml_private_include ggml SOURCE_DIR)
+if(SD_USE_UPSTREAM_GGML)
+    get_target_property(sd_ggml_imported ggml IMPORTED)
+    if(sd_ggml_imported)
+        set(sd_ggml_private_include "${SD_GGML_SOURCE_DIR}/src")
+    else()
+        get_target_property(sd_ggml_private_include ggml SOURCE_DIR)
+    endif()
+    if(NOT EXISTS "${sd_ggml_private_include}/ggml-impl.h")
+        message(FATAL_ERROR "Set SD_GGML_SOURCE_DIR to the source tree matching the selected ggml library (ggml-impl.h is required).")
+    endif()
+    target_include_directories(${SD_LIB} PRIVATE "${sd_ggml_private_include}")
+    set_property(TARGET ${SD_LIB} PROPERTY SD_GGML_PRIVATE_INCLUDE_DIR "${sd_ggml_private_include}")
 endif()
-if(NOT EXISTS "${sd_ggml_private_include}/ggml-impl.h")
-    message(FATAL_ERROR "Set SD_GGML_SOURCE_DIR to the source tree matching the selected ggml library (ggml-impl.h is required).")
-endif()
-target_include_directories(${SD_LIB} PRIVATE "${sd_ggml_private_include}")
-set_property(TARGET ${SD_LIB} PROPERTY SD_GGML_PRIVATE_INCLUDE_DIR "${sd_ggml_private_include}")
 
 if(SD_USE_UPSTREAM_GGML)
     target_compile_definitions(${SD_LIB} PUBLIC SD_USE_UPSTREAM_GGML)

cmake/hip.cmake

@@ -0,0 +1,12 @@
+if(NOT WIN32 AND NOT CMAKE_CXX_COMPILER MATCHES "hipcc")
+    if(NOT CMAKE_HIP_ARCHITECTURES)
+        if(GPU_TARGETS)
+            set(CMAKE_HIP_ARCHITECTURES ${GPU_TARGETS})
+        elseif(AMDGPU_TARGETS)
+            set(CMAKE_HIP_ARCHITECTURES ${AMDGPU_TARGETS})
+        endif()
+    endif()
+    # HIP objects are linked by the parent target, so initialize its link rules here.
+    enable_language(HIP)
+    set_property(TARGET ${SD_LIB} PROPERTY HIP_ARCHITECTURES "${CMAKE_HIP_ARCHITECTURES}")
+endif()

docker/Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends build-essential
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
     gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
     rm /tmp/nodesource-repo.gpg.key && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
     apt-get update && \
     apt-get install -y --no-install-recommends nodejs && \
     npm install -g pnpm@10.15.1 && \

docker/Dockerfile.cuda

@@ -1,7 +1,9 @@
+# syntax=docker/dockerfile:1
+
 ARG CUDA_VERSION=12.6.3
 ARG UBUNTU_VERSION=24.04
 
-FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu${UBUNTU_VERSION} AS build
+FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS build
 
 # sd-server embeds the web UI at build time, so the build image needs Node/pnpm.
 RUN apt-get update && apt-get install -y --no-install-recommends build-essential git ccache cmake ca-certificates curl gnupg && \
@@ -9,7 +11,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends build-essential
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
     gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
     rm /tmp/nodesource-repo.gpg.key && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
     apt-get update && \
     apt-get install -y --no-install-recommends nodejs && \
     npm install -g pnpm@10.15.1 && \
@@ -25,8 +27,15 @@ ARG CUDA_ARCHITECTURES=""
 ARG GGML_CUDA_FA_ALL_QUANTS=""
 ARG GGML_CUDA_ENABLE_DYNAMIC_CPU_BACKENDS=ON
 
+ENV CCACHE_DIR=/root/.cache/ccache \
+    CCACHE_MAXSIZE=2G \
+    CCACHE_COMPILERCHECK=content
+
 RUN set -- \
-        -DSD_CUDA=ON; \
+        -DSD_CUDA=ON \
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache; \
     if [ "${GGML_CUDA_ENABLE_DYNAMIC_CPU_BACKENDS}" = "ON" ]; then \
         set -- "$@" \
             -DSD_BUILD_SHARED_LIBS=ON \
@@ -44,15 +53,20 @@ RUN set -- \
         set -- "$@" "-DGGML_CUDA_FA_ALL_QUANTS=${GGML_CUDA_FA_ALL_QUANTS}"; \
     fi; \
     cmake . -B ./build "$@"
-RUN cmake --build ./build --config Release -j$(nproc)
+RUN --mount=type=cache,target=/root/.cache/ccache \
+    ccache --zero-stats && \
+    cmake --build ./build --config Release -j$(nproc) && \
+    ccache --cleanup && \
+    ccache --show-stats
 
-FROM nvidia/cuda:${CUDA_VERSION}-cudnn-runtime-ubuntu${UBUNTU_VERSION} AS runtime
+FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION} AS runtime
 
 RUN apt-get update && \
     apt-get install --yes --no-install-recommends libgomp1 && \
     apt-get clean
 
 COPY --from=build /sd.cpp/build/bin /sd.cpp/bin
+COPY --from=build /sd.cpp/thirdparty/cutlass/LICENSE.txt /sd.cpp/bin/cutlass.txt
 RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \
     printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \
     chmod +x /sd-cli /sd-server

docker/Dockerfile.musa

@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends ccache cmake gi
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
     gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
     rm /tmp/nodesource-repo.gpg.key && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
     apt-get update && \
     apt-get install -y --no-install-recommends nodejs && \
     npm install -g pnpm@10.15.1 && \

docker/Dockerfile.rocm

@@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
     gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
     rm /tmp/nodesource-repo.gpg.key && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
     apt-get update && apt-get install -y --no-install-recommends nodejs && \
     npm install -g pnpm@10.15.1 && \
     apt-get clean && rm -rf /var/lib/apt/lists/*

docker/Dockerfile.sycl

@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends cmake ca-certif
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
     gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
     rm /tmp/nodesource-repo.gpg.key && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
     apt-get update && \
     apt-get install -y --no-install-recommends nodejs && \
     npm install -g pnpm@10.15.1 && \

docker/Dockerfile.vulkan

@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends build-essential
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
     gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
     rm /tmp/nodesource-repo.gpg.key && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
     apt-get update && \
     apt-get install -y --no-install-recommends nodejs && \
     npm install -g pnpm@10.15.1 && \

docs/backend.md

@@ -135,7 +135,10 @@ regardless of argument order, even with `--auto-fit on`.
 
 Auto-fit preserves explicit `--backend` assignments, including per-module
 assignments and device lists. For modules without a runtime assignment, it chooses
-the GPU with the largest available memory budget (the first device on a tie).
+the discrete GPU with the largest positive memory budget (the first device on a
+tie), falling back to integrated GPUs when no discrete GPU has a positive budget.
+An integrated GPU's shared system memory does not give it priority over a discrete
+GPU. Explicit assignments such as `--backend Vulkan1` still select that device.
 It then derives parameter placements from the model metadata, each module's
 compute devices, and the remaining memory budgets. The chosen backend
 specifications are printed.
@@ -247,6 +250,9 @@ These are backend-level defaults. GGML still chooses the actual kernel based on
 weight type, shape, and hardware; enabling direct convolution does not guarantee
 that every operation uses implicit GEMM. The CUDA-family implicit GEMM paths
 require F16 weights and can use ordinary direct kernels for other cases.
+For 3D convolution, a supported direct kernel is also used as a fallback when
+the im2col decomposition is unsupported for the actual tensor sizes, including
+when an intermediate reshape or matrix multiplication exceeds backend limits.
 This option does not change the separately created image upscaler context.
 
 Repeating `--conv-direct` replaces the previous assignment string. Existing

docs/build.md

@@ -2,20 +2,44 @@
 
 ## Get the Code
 
-```
+```shell
 git clone --recursive https://github.com/leejet/stable-diffusion.cpp
 cd stable-diffusion.cpp
 ```
 
-- If you have already cloned the repository, you can use the following command to update the repository to the latest code.
+If you already cloned the repository without submodules, initialize them from the repository root:
 
+```shell
+git submodule update --init --recursive
 ```
+
+To update an existing checkout and its submodules:
+
+```shell
 cd stable-diffusion.cpp
 git pull origin master
-git submodule init
-git submodule update
+git submodule update --init --recursive
 ```
 
+## MSVC compilation parallelism
+
+MSVC builds limit parallel C/C++ compilation within each project by default.
+For `n` logical processors, the limit is `n` when `n <= 2`, `n - 1` when
+`3 <= n <= 8`, and `n - 2` when `n > 8`. The default is calculated when the
+build directory is first configured with this setting. Override it with
+`-DSD_MSVC_MP_COUNT=4`, or pass `-USD_MSVC_MP_COUNT` to recalculate it.
+
+For Visual Studio builds, keep project-level parallelism at one to avoid
+multiplying the per-project limit:
+
+```shell
+cmake --build build --config Release --parallel 1
+```
+
+This setting controls MSVC's `/MP` option, not CUDA compiler parallelism or a
+hard CPU usage limit. Other build tools, such as Ninja, schedule compilation
+jobs separately; use `cmake --build build --parallel N` to limit those jobs.
+
 ## Selecting a GGML source tree
 
 By default, sd.cpp builds the patched GGML submodule in `ggml/`. To build with
@@ -27,8 +51,10 @@ cmake -S . -B build-upstream -DSD_USE_UPSTREAM_GGML=ON -DSD_GGML_SOURCE_DIR=../g
 cmake --build build-upstream --config Release
 ```
 
-The selected source tree supplies both the library and its private headers.
-Backend options such as `-DSD_CUDA=ON` apply to the selected tree as usual.
+Upstream compatibility mode uses the selected library and matching private
+headers. The patched GGML build uses public headers, including `ggml-ext.h` and
+`ggml-graph.h`. Backend options such as `-DSD_CUDA=ON` apply to the selected tree
+as usual.
 
 `SD_USE_UPSTREAM_GGML` defaults to `OFF`, which enables the patched GGML
 extensions. Set it to `ON` when using upstream GGML; it selects the compatibility
@@ -45,10 +71,12 @@ creating an inference context. Ordinary floating-point and shared GGML
 quantization types remain available, subject to backend operator support.
 
 `SD_USE_SYSTEM_GGML=ON` instead links an installed GGML CMake package, located
-with `ggml_DIR` or `CMAKE_PREFIX_PATH`. In that mode, `SD_GGML_SOURCE_DIR` must
-point to the matching source tree for private headers. The installed library
-must use the same ABI settings as sd.cpp, including `GGML_MAX_NAME`.
-Set `SD_USE_UPSTREAM_GGML=ON` as well if the installed package is upstream GGML.
+with `ggml_DIR` or `CMAKE_PREFIX_PATH`. An installed patched package must include
+the extension and graph APIs; its source tree is not required for private
+headers. The installed library must use the same ABI settings as sd.cpp,
+including `GGML_MAX_NAME`. For an upstream package, also set
+`SD_USE_UPSTREAM_GGML=ON` and point `SD_GGML_SOURCE_DIR` to its matching source
+tree for the compatibility layer's private headers.
 
 ## WebP and WebM Support in Examples
 
@@ -90,8 +118,8 @@ cmake --build . --config Release
 
 ## Build with CUDA
 
-Native SageAttention is included when using CUDA with patched GGML
-(`SD_USE_UPSTREAM_GGML=OFF`).
+The sd.cpp SageAttention extension is included when using CUDA with patched
+GGML (`SD_USE_UPSTREAM_GGML=OFF`).
 See [SageAttention](sage_attention.md) for GPU requirements and `--sage-attn` usage.
 
 This provides GPU acceleration using NVIDIA GPU. Make sure to have the CUDA toolkit installed. You can download it from your Linux distro's package manager (e.g. `apt install nvidia-cuda-toolkit`) or from here: [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads). Recommended to have at least 4 GB of VRAM.
@@ -102,6 +130,28 @@ cmake .. -DSD_CUDA=ON
 cmake --build . --config Release
 ```
 
+CUDA extensions use sd.cpp's `thirdparty/cutlass` submodule by default
+for eligible INT8 GEMM and CuTe attention operations. If its headers are missing,
+CMake stops with an initialization hint; it does not download them automatically.
+Run `git submodule update --init --recursive` from the repository root before
+configuring.
+
+To build with the existing CUDA implementations without CUTLASS, configure from
+the repository root with:
+
+```shell
+cmake -S . -B build -DSD_CUDA=ON -DSD_CUDA_CUTLASS_PATH:PATH=
+cmake --build build --config Release
+```
+
+Alternatively, set `SD_CUDA_CUTLASS_PATH` to another local CUTLASS source
+directory (tested with 3.9.2). Existing CMake caches retain their configured path;
+pass `-USD_CUDA_CUTLASS_PATH` when reconfiguring to restore the bundled default.
+Existing `GGML_CUDA_CUTLASS_PATH` caches are migrated on first configuration:
+the old bundled path becomes `thirdparty/cutlass`, while custom paths and an
+empty disabled value are preserved. An explicitly set `SD_CUDA_CUTLASS_PATH`
+takes precedence. Non-CUDA builds do not require CUTLASS.
+
 ## Build with HipBLAS
 
 This provides GPU acceleration using AMD GPU. Make sure to have the ROCm toolkit installed.

docs/docker.md

@@ -18,12 +18,26 @@ docker run --rm --init -v /path/to/models:/models -v /path/to/output/:/output -p
 
 ## Building using Docker
 
+Initialize the submodules from the repository root before building. Docker copies
+the local checkout, so its build context must include the submodule contents:
+
 ```shell
+git submodule update --init --recursive
 docker build -f docker/Dockerfile -t sd .
 ```
 
 ## Building variants using Docker
 
+### CUDA (NVIDIA GPU):
+
+```shell
+docker build -f docker/Dockerfile.cuda -t sd-cuda .
+```
+
+To select GPU architectures, pass `--build-arg "CUDA_ARCHITECTURES=89"` (for
+example, RTX 4090). Run the image on a host configured with NVIDIA Container
+Toolkit, using `docker run --rm --gpus all ... sd-cuda [args...]`.
+
 ### Vulkan:
 
 ```shell
@@ -67,4 +81,4 @@ docker run --rm --device=/dev/dri --device=/dev/kfd --group-add video \
 #   -v ./models:/models -v ./build:/output \
 #   sd -m /models/sd-v1-4.ckpt -v -o /output/output.png \
 #   -p "a lovely cat"
-```
+```

docs/edit.md

@@ -85,6 +85,9 @@ If presets are insufficient, you can manually configure the following parameters
 | `vlm_max_size` | int | Maximum VLM input size; interpreted according to `vlm_resize_mode`. | Integer |
 | `vlm_min_size` | int | Minimum VLM input size; interpreted according to `vlm_resize_mode`. | Integer |
 | `vlm_size` | int | Shortcut to set both VLM min and max size to the same value. | Integer |
+| `ref_image_size` | string | MiniMax-H3 reference image sizing before VAE encoding. `match` scales each reference down to the generation's pixel area; `max` uses a 2048 px short edge. Only references above the limit are downscaled; each axis is then snapped to the nearest 32 px multiple (an axis of at least 16 px grows by at most 16 px, and references are never narrower than 32 px). | `match`, `max` |
+
+`ref_image_size` applies to `vid_gen` with MiniMax-H3 Ref2VA. Reference latents are attended to at every sampling step, so `max` can be several times slower.
 
 ### Preset Default Values
 

docs/generation_metadata.md

@@ -0,0 +1,165 @@
+# Generation results and metadata
+
+`generate_image()`, `generate_video()`, `adetail_image()` and `upscale()` return
+owned results containing the media and its generation record. This is a breaking
+C API change: applications and bindings must update their calls and rebuild.
+
+```c
+sd_img_gen_params_t params;
+sd_img_gen_params_init(&params);
+params.prompt = "a small house";
+
+sd_image_generation_result_t* result = NULL;
+if (generate_image(ctx, &params, &result)) {
+    for (int i = 0; i < result->count; ++i) {
+        const sd_image_result_t* item = &result->items[i];
+        /* Consume item->image and item->generation_info together. */
+    }
+    free_sd_image_generation_result(result);
+}
+```
+
+`result_out` is required and is set to `NULL` on failure. A successful result
+owns its pixels and UTF-8 JSON strings. The video result additionally owns its
+audio and contains the effective `fps` and `frame_count`. Results remain valid
+after freeing the generation context. Use the matching library result destructor;
+do not separately free fields unless transferring ownership and clearing them.
+Both result destructors accept `NULL`.
+
+## Reproduction parameters
+
+The `sdcpp.generation/v1` JSON is a snapshot of generation parameters for the
+same loaded context and the same input images, masks, reference media and other
+assets. It is created after `GenerationRequest` resolves the request, before
+preprocessing or sampling. A failed call returns no result or metadata.
+
+Fields follow the C generation parameter structs: `prompt`, `negative_prompt`,
+`width`, `height`, `seed`, `sample_params`, `hires`, `cache`,
+`vae_tiling_params`, `image_preprocess`, and the image/video-specific options.
+`kind` distinguishes image and video parameters. There is no inferred `task`.
+Enums use their API string names. Seeds are decimal strings to preserve their
+full integer range. Omitted optional fields retain the parameter initializer's
+defaults, including automatic values represented by infinity in the C API.
+
+Disabled optional features are omitted entirely: `cache`, `hires`, VAE tiling,
+zero/nonpositive `shifted_timestep`, inactive SLG, false circular axes, and empty
+preprocessing/reference arguments. SLG layers remain present when `slg_uncond`
+is enabled even with zero SLG scale. Enabled caches store their mode's parameters;
+TaylorSeer retains the shared DBCache thresholds and limits used by the runtime.
+Image CFG at its neutral value of 1 is omitted and resolves back to 1 by default.
+
+Image strength, ControlNet/IP-Adapter strength and PhotoMaker settings are stored
+when their corresponding input images are supplied. PuLID requires an embedding
+path. Layer count is stored for layered generation; video VACE and high-noise
+settings require those models. High-noise settings omit scheduler, custom sigmas,
+flow shift and timestep shifting because that pass shares the base configuration.
+Explicit nonempty argument strings are retained without rewriting their contents.
+Core sampling values such as seed, sampler, scheduler, steps and guidance remain
+explicit even when equal to their defaults. ADetailer always retains its inpaint
+strength because its input crops are created later.
+Model-required VAE tiling is resolved before serialization, so forced tiling
+(such as MiniMax-H3) remains present even if the caller did not enable it.
+
+The request fixes the seed, dimensions, clip skip, guidance, sampler, scheduler
+and eta before taking the snapshot. `custom_sigmas` is stored only when supplied
+as an input. Automatically computed schedules, preprocessing transforms,
+conditioning/cache decisions, detector regions and output statistics are not
+stored: they are recomputed from the same parameters and inputs.
+
+Each image records the seed used to sample its latent. `batch_count` is omitted;
+replay image parameters with `batch_count = 1`. Layers or frames decoded from
+the same latent share its seed, so layered generation reproduces that group of
+layers. Video frames share the video request. Partial batch cancellation returns
+metadata only for completed outputs. Full cancellation returns no result.
+
+The current image pipeline prepares input latents once using the starting seed
+and shares them across a batch. With a stochastic VAE encoder, replaying a later
+image alone using its sampling seed can change those input latents. Per-image
+seeds therefore do not guarantee independent reproduction for these paths.
+
+To replay a record, initialize the matching C parameter struct, populate its
+fields from the JSON, restore the same input buffers, and call generation with
+the same context. Keep custom-sigma/SLG arrays and string storage alive for the
+call. Paths supplied as generation parameters (LoRAs, hires models and identity
+embeddings) are retained. `models` identifies successfully loaded context model
+files by component (`model`, `diffusion_model`, `clip_l`, `vae`, etc.), using
+filenames with extensions and omitting directories. Optional components disabled
+for the request and preview-only TAE are omitted. WebUI text includes `Model`
+(the diffusion model when supplied separately, otherwise the checkpoint, without
+the extension) and `VAE` for a separate active VAE. These names are identifiers,
+not hashes or sufficient configuration to rebuild a context; replay still requires
+the same model files and context settings. The JSON does not embed media buffers
+or model weights.
+
+When `--lora-model-dir` is configured, CLI/server output records express LoRA
+paths relative to that directory, preserving subdirectories and file extensions
+and using `/` separators. This applies to file metadata, API responses and
+ADetailer records. Resolve these paths against the same LoRA directory before
+passing them back to the C API. Core results retain the supplied loading paths.
+Files outside the directory retain `..` components; paths on another volume,
+or records without a configured LoRA directory, retain their original paths.
+
+## Postprocessing
+
+Upscale records `upscale.upscale_factor`. ADetailer records `adetailer` and
+`inpaint_params`, with a concrete starting seed chosen before detection. These
+operations require the same upscaler/detector context and source image on replay.
+They do not store detection results or per-region generation traces.
+
+The CLI/server append operation parameter snapshots to `postprocess` in call
+order. The source generation parameters remain unchanged. Frame postprocessing
+uses the same operation parameters for each video frame. Standalone
+postprocessing does not invent source diffusion parameters for an input image.
+
+## File metadata
+
+Image encoders accept the native record and derive WebUI infotext from it.
+Images use the existing metadata locations and combined text layout:
+`<WebUI infotext>, SDCPP: <native JSON>`. Standalone postprocessing without
+source diffusion parameters stores only native JSON. Video metadata stores
+only native JSON.
+
+| Container | Metadata location |
+| --- | --- |
+| PNG | `tEXt`, keyword `parameters` |
+| JPEG | COM segment, `parameters` label followed by a NUL byte |
+| WebP image / animated WebP | XMP property `sdcpp:parameters` |
+| AVI | RIFF INFO `ICMT` |
+| WebM | `COMMENT` tag |
+
+Inspect images or videos with `sd-cli -M metadata --image <file>`; the same
+command supports all containers above, with `--metadata-format json` for a
+structured report. Video metadata inspection does not require a model or video
+decoder.
+Matroska containers are also supported, including `.webm` outputs whose audio
+codec causes the muxer to select the Matroska document type.
+
+The XMP namespace is `https://github.com/leejet/stable-diffusion.cpp/ns/1.0/`.
+PNG and JPEG retain the existing UTF-8 text payload convention. Memory encoding
+and file saving use the same implementation. The core result and API
+`generation_info` fields remain pure JSON; the combined layout applies only
+to embedded image metadata.
+
+WebUI conversion is lossy: known samplers and schedules have explicit name
+mappings; unsupported ones use `SDCPP Sampler`/`SDCPP Schedule` fields. Hires
+keeps the base request size and separate configured hires size/steps. The native
+record retains the full generation parameters. Standalone postprocessing has no WebUI
+infotext because its original diffusion settings are unknown.
+
+WebUI prompt text appends ordinary LoRAs as `<lora:filename_without_extension:weight>`.
+Directory components are omitted to match WebUI's filename lookup; the native
+`loras` array retains relative paths and weights. High-noise-only LoRAs and names
+that cannot be represented by this tag syntax remain in the native record only.
+The native prompt is not modified, and postprocessing LoRAs are not added to the
+source generation prompt.
+
+JPEG requires the COM payload, including the label and terminating NUL bytes,
+to fit within 65,533 bytes. Larger records fail encoding with an error rather
+than being truncated. Use PNG or WebP for larger records.
+
+Disabling file metadata omits both image representations, but does not disable
+the core generation record or its API response. Native async image results and
+OpenAI image response items contain `generation_info`; video results contain a
+sequence-level `generation_info`. The SD API `info` object contains per-output
+`generation_info` alongside derived `infotexts` and each image's seed. Its outer
+`parameters` field continues to echo the request.

docs/ggml_extensions.md

@@ -0,0 +1,550 @@
+# ggml operator extensions
+
+The patched ggml exposes `ggml-ext.h` for application-owned operators, kernel
+overrides, and graph fusion rules. Registrations are in-process and scoped to a
+backend device. Register them before capability queries or graph allocation;
+callback code and user data must remain alive until all dependent graphs and
+backends have been destroyed. Registration names are unique, and duplicate
+registrations fail rather than replacing live implementations.
+
+## Operator definitions
+
+`ggml_ext_register_op` registers a versioned name, attribute size, and an output
+metadata inference callback. `ggml_ext_op_create` creates an ordinary ggml tensor
+with explicit input dependencies. The initial interface supports one contiguous,
+out-of-place output with an existing `ggml_type`.
+
+Attributes are copied into the unused portion of `op_params`. They are small,
+plain values; use `memcpy` when reading attributes that need stricter alignment.
+Auxiliary tensors must appear in `src`, not in an attribute pointer or user data.
+Registered operators use `GGML_OP_CUSTOM` internally, with an identifiable
+callback marker. Unsupported devices do not mistake them for CPU callbacks.
+These process-local descriptors are not a graph serialization format.
+
+## Kernel selection
+
+`ggml_ext_register_kernel(device, &desc)` associates a kernel with a built-in
+`ggml_op`, or with a registered custom operator. `supports` describes correctness
+constraints; `preferred` optionally decides whether to override an available
+native implementation. Higher priorities are considered first. When native
+execution is unsupported, a supported extension can run regardless of its
+preference callback.
+
+Capability queries include registered single-node implementations, so the
+scheduler can select a device even when its native implementation rejects the
+operator. Callbacks inspect metadata only and must not recursively call the
+public capability query for the same device and operator. A fusion-only
+registration does not claim single-node support.
+
+`ggml_ext_dev_supports_native_op` queries the original device implementation
+without extension dispatch. Fusion policies can use it to select a supported
+extension even when their performance preference normally favors native kernels.
+
+`prepare` creates optional host-side plan state and reports workspace bytes.
+It can run during memory measurement, before tensor addresses are assigned, and
+must not launch work or allocate device buffers. Workspace is allocated by ggml
+before execution and included in scheduler memory estimates and buffer-size
+reports. `execute` receives the backend, workspace buffer, and current tensor
+addresses. `destroy` releases prepared state. A preparation or execution error
+is propagated; execution never retries a native implementation after launch.
+
+Workspace is scratch storage shared by sequential extension steps in a plan;
+its capacity is the maximum request, not the sum across nodes. Contents are not
+preserved between calls. Queue work in backend execution order, and finish any
+private-stream use before subsequent backend work can access the workspace.
+Persistent state belongs in `prepare`/`destroy` state or explicit tensor inputs.
+
+Overriding a built-in operator preserves its numerical and aliasing contract,
+including allocator reuse of input storage. Explicit views/in-place nodes are
+not intercepted by this first interface. Rebuild the scheduler allocation after
+changing graph metadata, selection policy, or registrations.
+
+## Fusion
+
+A `match` callback returns the number of consecutive nodes it can replace.
+The scheduler freezes this selection before allocating tensors. ggml checks
+compute flags, external uses, output flags, and views; the initial implementation
+accepts a conservative set of pure operators with only the final node exposed
+as the fused output. The rule remains responsible for mathematical equivalence,
+dtype, shape, layout, and hardware constraints. Internal views are accepted when
+their storage belongs to the fused subgraph. In-place intermediates are accepted
+only when their storage owner is an earlier node in the same match. External
+consumers and output flags still prevent eliminating those results. The final
+output must own storage, use a terminal `CPY`, or write back to an earlier owning
+node within the match. Internal writeback must cover exactly the owner's full
+shape, type, strides and storage offset, without external consumers of that
+owner. Output flags propagated to that owner are retained. Terminal `CPY` keeps
+its original destination and write semantics. The extension must handle input/output storage overlap, including
+allocator reuse, with scratch space when necessary. Other in-place writes are
+not eligible for extension fusion when they target storage outside the match.
+
+An optional `optimize` callback runs before matching and allocation. Use
+`ggml_ext_graph_delay` to move a producer range after independent nodes. The
+helper rejects dependencies, unknown side effects, and writes that alias moved
+inputs or outputs. It can cross independent copies and known in-place operators
+when their storage does not alias the moved inputs or outputs. Owning dtype
+conversions can also be moved or crossed.
+Reordering must preserve edges and tensor metadata. Fusion legality is checked
+again after reordering.
+
+External inputs stay allocated through the final node. Native graph optimization
+and execution operate on the remaining contiguous ranges, so native fusion cannot
+consume nodes selected by an extension. Fused intermediates are excluded from
+allocation through the backend's `skip_alloc` callback and
+`ggml_gallocr_set_skipped_tensors`. Their metadata and graph edges remain available,
+but execution must not access their data or buffers. Storage backing the final
+output is retained, including an internal owner used by an in-place final write.
+The skipped set is replaced for each allocation graph; changing which tensors are skipped
+invalidates cached allocation assignments even when tensor shapes are unchanged.
+
+Automatic extension fusion is enabled in scheduler execution with one pipeline
+copy and without evaluation callbacks. Multiple pipeline copies and evaluation
+callbacks retain individual operations. Direct backend graph computation supports
+single-node extensions; it does not fuse an already allocated graph. Direct calls
+use temporary plans and synchronize before releasing them; scheduler plans and
+workspaces are reused across executions.
+
+Use the backend graph APIs or the scheduler. The legacy CPU `ggml_graph_compute`
+and backend `graph_plan_*` interfaces do not dispatch these extensions.
+
+## Backend services
+
+Packed ComfyUI INT4 linear layers use `sd.linear_w4_convrot.v1`, built by
+`ggml_ext_mul_mat_w4_convrot`. The operator consumes byte-packed I8 weights and the
+explicit output of `sd.quantize_i8_convrot.v1`; F32 callers get a quantization
+node during graph construction. Logical output dimensions are stored in the
+operator attributes, so backend copies of the packed activation do not need to
+retain the producer's op marker. W4A4 uses row scales; W4A8 additionally exposes
+the codebook, channel scales, and FP8 relative scales as tensor dependencies.
+CPU, CUDA/HIP/MUSA, and Vulkan register kernels through the same extension API
+as INT8. No ggml submodule changes are required.
+
+Services are discovered through `ggml_backend_reg_get_proc_address`:
+
+| Service | Purpose |
+| --- | --- |
+| `ggml_ext_cpu_parallel_v1` | Run an application callback on the CPU backend's configured thread pool, with its abort callback. |
+| `ggml_ext_cuda_context_v1` | Borrow the current CUDA device and stream, ordered with native work. |
+| `ggml_ext_cuda_device_v1` | Get the physical CUDA ordinal for capability checks before backend creation, including virtual ggml devices. |
+| `ggml_ext_cuda_blas_v1` | Borrow the current stream's cuBLAS handle during execution, with host scalar pointers. |
+| `ggml_ext_gpu_device_v1` | Query CUDA/HIP/MUSA runtime kind, physical ordinal, shared-memory limit, warp size, and INT8 BLAS capability. |
+| `ggml_ext_gpu_context_v1` | Borrow the matching runtime's device and execution stream. |
+| `ggml_ext_gpu_blas_v1` | Borrow the matching cuBLAS, hipBLAS, or muBLAS handle on that stream. |
+| `ggml_ext_metal_device_v1` | Borrow the Metal device for application pipeline compilation. |
+| `ggml_ext_metal_buffer_v1` | Resolve a backend buffer range to its Metal buffer and offset, including private storage and split allocations. |
+| `ggml_ext_metal_submit_v1` | Encode application commands on a backend-owned command buffer, ordered with native Metal work and tracked by backend synchronization. |
+
+The borrowed cuBLAS handle remains owned by the backend. Extensions must not retain
+or destroy it, or change its stream, pointer mode, math mode, or workspace. This
+CUDA-specific BLAS service is retained for existing callers; the runtime-aware
+services support CUDA, HIP, and MUSA with the same ownership rules. Extensions
+call vendor runtime and BLAS APIs directly. These services do not execute GEMM,
+quantization, or model fusion on the application's behalf.
+
+Workspace is a backend buffer. Its base is a CUDA device pointer on CUDA; callers
+must use the representation appropriate to their backend. Vulkan extensions use
+the command recording services described below.
+
+CUDA INT8 Linear, RoPE, ConvRot, RMS affine and INT8 gated residual kernels omit output
+scratch when allocation guarantees that a fresh owning output cannot overlap
+live inputs. Existing outputs and views retain overlap protection. Internal
+GEMM and quantization workspace is still reserved when required; kernels also
+handle a null workspace when the entire step needs no scratch.
+
+Compute workspace measurement preserves the storage identities and offsets of
+external tensors, including views. Separate allocations must not share a synthetic
+buffer identity: that can invent overlap, overestimate scratch, and repeatedly
+rebuild a workspace whose actual allocation is smaller than the estimate.
+
+## sd.cpp integration
+
+`src/core/ggml_extensions.cpp` and `.h` contain the shared operator schemas,
+validation, and backend registration entry point. Graph construction helpers live
+in `src/core/ggml_extend.cpp` and `.h`; they take `GGMLRunnerContext*` so backend
+selection, compatibility workarounds, and shared ConvRot nodes use the same
+context as the calling model. Per-call precision, scaling, attention, and padding
+options remain explicit. Model blocks retain weight loading and adapter
+orchestration, while the shared helpers own FP8 fallback, embedding lookup,
+full/partial RoPE, grouped convolution, and backend-dependent repeat handling.
+LoRA/LoKr construction, n-mode products, and Kronecker products stay in
+`src/model/adapter/lora_ops.cpp` and `.h`, with the same runner-context interface.
+CPU kernels and fusion rules live in `src/core/cpu/`, with registration collected
+in `cpu/registration.cpp`. CUDA-style and Vulkan implementations live in their
+respective backend directories.
+
+`src/core/metal/int8.mm` registers Metal INT8 linear and ConvRot kernels. Their
+Metal source is embedded at build time and compiled for the selected device
+during registration. The public `ggml-ext-metal.h` services expose borrowed
+objects without private backend headers. The submit callback only encodes work;
+GGML owns submission, cancellation checks, command-buffer lifetime, and
+asynchronous error tracking. Rotation scratch and any overlapping output use
+the scheduler workspace. No full floating-point weight copy is allocated.
+When runtime compilation supports Metal 4 TensorOps, INT8 multiplication uses
+`matmul2d` with INT32 cooperative results and fused scale/bias restoration.
+The SIMD-group implementation remains the fallback for older systems and
+devices where the TensorOps pipeline is unavailable.
+TensorOps multiplies at most 1024 pairs per partial result and combines those
+results with wrapping integer addition. This preserves exact INT32 semantics
+even when TensorOps uses floating-point arithmetic internally.
+
+`ggml_ext_modulate(ctx, x, shift, scale)` emits `sd.modulate.v1` for F32 inputs
+when the runner context uses CPU, CUDA/HIP/MUSA, or Vulkan.
+Its explicit inputs are `x`, `scale`, and optional `shift`; it computes
+`(x + x * scale) + shift` with separate FP32 multiply/add rounding. The output
+is contiguous and out of place. Inputs may be strided and use ggml's
+dimension-wise repetition rules. Callers prepare token, batch, or frame layouts
+before constructing the operation; normalization and residual gates remain
+separate operations. CPU, CUDA/HIP/MUSA, and Vulkan have standalone kernels.
+Other backends (including Metal, SYCL, and OpenCL), upstream ggml builds, and
+other input types retain the original MUL/ADD graph.
+
+ConvRot fusion matches the Modulate node directly, including scale-only
+modulation, instead of recognizing its former MUL/ADD/ADD decomposition.
+CUDA accepts contiguous per-channel parameters for `Modulate -> ConvRot` and
+`RMSNorm -> Mul(weight) -> Modulate -> ConvRot`. Vulkan limits the first fusion
+to widths 4096/6144/5376 and the RMSNorm fusion to width 5376.
+Other layouts execute the standalone operation followed by ConvRot. The
+gated-residual MUL/ADD fusion remains available for residual branches.
+
+`src/core/cpu/rope.cpp` registers the CPU implementation of
+`sd.rope_apply.v1` and a
+`RMSNorm -> Mul -> RoPE` fusion. The fusion accepts a one-dimensional weight
+vector. Both interleaved and split-half RoPE, partial rotation, and strided input
+rows are supported. Model construction uses the custom operator on CPU, CUDA,
+HIP, MUSA, and Vulkan. Other backends can use the scheduler's CPU fallback.
+The former `ggml_rope_apply` API, builtin opcode, and CPU implementation have
+been removed from ggml.
+
+`src/core/cuda/rope.cu` supplies standalone CUDA RoPE
+and `RMSNorm -> Mul -> RoPE [-> Scale -> FP16 Copy]` fusion. CuTe attention shares
+its RMSNorm/RoPE kernel. Partial rotation and arbitrary float-aligned input and
+position-encoding strides are supported by the standalone kernel; RMSNorm fusion
+requires contiguous position encoding and weights. Copy fusion uses workspace
+when the destination overlaps an input. These registrations do not require
+CUTLASS. The old CUDA/HIP/MUSA RoPE dispatch is removed.
+
+Shared CUDA-style kernels live in `src/core/cuda/` and are compiled separately
+for each enabled runtime. `SD_CUDA_NAMESPACE` expands to `sd_cuda`, `sd_hip`, or
+`sd_musa`, so implementations can coexist. `runtime.h` handles the small runtime
+differences, including logical 32-lane shuffles on HIP. The application defines
+and launches its kernels directly; it includes only public ggml headers.
+CMake generates the registration table from enabled backend options. The shared
+entry point uses one `SD_BACKEND_EXTENSIONS` macro; backend registrars verify the
+device kind, and disabled backends add no symbol references.
+
+`SD_USE_UPSTREAM_GGML` excludes these registrations. The stable-diffusion public C
+API is unchanged. INT8 ConvRot retains its existing packed activation layout and
+separate weight scale tensor.
+
+`src/core/cuda/cute_attention.cu` supplies CuTe Flash Attention on CUDA when
+CUTLASS headers are configured. It registers an override of `FLASH_ATTN_EXT`,
+with the existing sequence-length preference and native fallback. It also
+registers Q preparation (`RMSNorm -> Mul -> RoPE`), output scaling, and packed
+ConvRot output fusion. These kernels use only the public extension API, borrowed
+CUDA stream, and scheduler-owned workspace; they do not use ggml's CUDA pool or
+private conversion helpers. The existing CUDA fusion and CuTe disable environment
+variables are honored.
+Aligned contiguous FP16 K/V inputs are read directly without reserving conversion
+buffers. Fresh F32 outputs are written by the attention epilogue, retaining the
+original FP16 rounding before output scaling. Packed ConvRot and preallocated or
+view outputs retain the intermediate half buffer.
+
+`src/core/cuda/i8_projection.cu` supplies the INT8 GEMM projection epilogues:
+`GEMM -> Reshape -> RMSNorm -> Mul -> RoPE`, optional scaling and FP16 copy,
+and K/V prefix concatenation with FP16 output. CuTe can consume a directly
+produced FP16 Q without materializing the FP32 GEMM or RoPE intermediates.
+Independent affine operands, including broadcasted constants, are computed before
+their normalization/projection
+chain when graph dependencies allow it. The attention optimizer can then move
+the complete five-node projection when safe.
+These rules accept custom RoPE nodes. They preserve the original
+large-matrix selection thresholds, full interleaved head dimension 128, packed
+activation layout, and separate weight scales. External consumers prevent fusion;
+aliasing terminal copies use scheduler workspace. The CUDA projection kernels
+and matchers now live in sd.cpp. `GGML_CUDA_FORCE_CUBLAS` disables these epilogues.
+
+`src/core/ggml_extensions.cpp` defines `sd.linear_i8.v1` and
+`sd.quantize_i8_convrot.v1`. `src/core/cpu/int8.cpp` supplies their CPU
+implementations. The CPU kernels use the
+configured thread pool and scheduler workspace; the old CPU INT8 GEMM and ConvRot
+implementations have been removed from ggml.
+Model code keeps calling `ggml_ext_linear_i8_tensorwise`,
+`ggml_ext_mul_mat_i8_tensorwise`, and `ggml_ext_quantize_i8_convrot`. The latter two
+constructors are declared and implemented in sd.cpp and create custom nodes.
+Their former ggml declarations, implementations, ConvRot opcode, and native
+backend paths have been removed. CPU, CUDA/HIP/MUSA, and Vulkan register kernels
+and fusion rules through the extension interfaces.
+
+For already quantized activations, `ggml_ext_mul_mat_i8_tensorwise` accepts ordinary
+I8 weight and activation tensors. Its optional final `input_scale` parameter is a
+separate F32 tensor with one value per activation row. Weight scales have one
+value per output channel; weight scales and bias can be null. Existing calls
+omit `input_scale` and retain their original argument order. It computes `float(int32_dot(W, X)) * X_scale * W_scale + bias`, in that
+order. All scales are explicit graph dependencies; there is no quantized-tensor
+wrapper. Shapes use ggml's low-to-high dimension order. Call
+`sd_ggml_register_extensions` before scheduling these custom nodes.
+
+`ggml_ext_mul_mat_i8_tensorwise` also accepts F32 activations (quantized into temporary
+workspace) or the output of `ggml_ext_quantize_i8_convrot`. ConvRot keeps the existing
+packed I8 activation rows, padded to a multiple of four rows, followed by F32 row
+scales. Weight scales remain separate. CPU ConvRot groups are 1, 4, 16, 64, or 256.
+Unsupported layouts and malformed scale shapes are rejected. Separate CPU quantize
+and GEMM phases propagate cancellation, and overlapping outputs use workspace.
+
+`src/core/cuda/int8.cu` implements the same custom schemas on CUDA. It uses the
+borrowed cuBLAS handle for I8-by-I8-to-I32 GEMM and application-owned quantization,
+ConvRot, and rescaling kernels. F32 inputs are quantized into workspace; ordinary
+I8 inputs retain separate scale tensors and use a padded temporary only when
+needed by cuBLAS. Packed ConvRot inputs are consumed directly. Large ConvRot rows
+use multiple kernels when they exceed shared memory. Aliasing outputs use scratch.
+CUDA GEMM requires Turing or newer, contiguous tensors, K/N divisible by four,
+float-aligned inputs, and row counts that fit a padded signed 32-bit dimension;
+CUDA ConvRot uses groups of 256.
+Unsupported custom nodes can fall back to CPU through the scheduler.
+
+When CUTLASS is configured, `src/core/cuda/i8_gemm.cu` supplies dense and SwiGLU
+INT8 GEMM kernels on Ampere or newer. It preserves the native tile selection,
+small-row policy, Stream-K scheduling, Ada column bands, and separate scaling
+order. The same extension matches `GEMM -> SwiGLU [-> ConvRot]` and row
+concatenation for custom GEMM nodes. Weight scales remain explicit
+inputs, including the gate/up scale permutation recorded on every SwiGLU launch.
+Two independent INT8 GEMMs sharing a quantized input can also fuse with split
+SwiGLU. A paired-column iterator reads the separate gate/up weight allocations,
+and the GEMM epilogue writes only the activated output. Both intermediate
+activations are omitted through the generic fusion allocation interface; weights
+are neither copied nor concatenated. This path requires bias-free branches,
+compatible layouts and alignments, and no external intermediate consumers.
+Only interleaved weight scales and any Stream-K workspace remain temporary.
+Plans reserve bounded Stream-K workspace, cache occupancy on first execution,
+and initialize barriers on every launch, so CUDA Graph replay observes updated
+inputs and scales. External consumers retain ordinary execution; overlapping
+outputs use workspace. For fresh outputs, the CUTLASS SwiGLU epilogue writes
+an allocator-managed tensor that the subsequent ConvRot launch consumes. This
+keeps the same GPU launches while allowing its storage to be reused by other
+operations. The combined SwiGLU/ConvRot path retains its workspace intermediate
+for other cases. Shapes or alignments unsuitable for CUTLASS use cuBLAS.
+Custom I8 nodes also retain cuBLAS when the reduction could overflow I32, to
+preserve their previous accumulation behavior. Fusion disable and force-cuBLAS
+settings are honored. No CUTLASS headers are needed for the cuBLAS path.
+
+The native CUDA/HIP INT8 GEMM, standalone ConvRot, and associated fusion paths
+have been removed from ggml. The shared application implementation calls cuBLAS
+or hipBLAS directly. HIP retains the existing architecture eligibility policy;
+MUSA does not advertise INT8 BLAS support and uses scheduler fallback for GEMM.
+Standalone ConvRot kernels are compiled for all three runtimes. CUTLASS
+implementations remain CUDA-specific.
+
+`src/core/cuda/i8_gated_residual.cu` implements
+`INT8 GEMM -> [Scale] -> Mul(gate) -> Add(residual) -> [Concat]` as a registered
+CUDA fusion, for packed ConvRot inputs and ordinary I8 nodes with separate activation
+scales. The corresponding native CUDA fusion rules and CUTLASS residual epilogue
+have been removed from ggml. Its optimizer moves independent gate producers ahead
+of GEMM and optional scaling through `ggml_ext_graph_delay`, before fusion matching.
+The former native INT8 affine reorder rule has also moved into this extension.
+It preserves the original tile selection, small-row
+threshold, Stream-K scheduling, Ada column bands, and separate FP32 rounding of
+gate multiplication and residual addition. The plan bounds Stream-K workspace
+over possible occupancy values using host metadata; execution verifies the
+actual requirement against that bound. Device occupancy is cached on first
+execution; each launch rebuilds parameters for the current tensor addresses and
+initializes its Stream-K barriers. Overlapping outputs use scratch, except
+for an exact in-place residual without concatenation. External consumers and
+unsupported layouts retain ordinary operator execution. Fusion and force-cuBLAS
+disable settings are honored.
+
+`src/core/cuda/convrot.cu` registers CUDA fusion for `Scale -> ConvRot`,
+`Norm -> Mul -> ConvRot`, `Modulate -> ConvRot`,
+`RMSNorm -> Mul(weight) -> Modulate -> ConvRot`, `SwiGLU -> ConvRot`, and
+`SiLU/Sigmoid -> Mul -> ConvRot`.
+ConvRot outputs are custom nodes. SwiGLU accepts packed,
+swapped, or separate gate/up inputs. Independent modulation and up-branch
+producers are moved ahead of their consumers through the safe graph-delay API.
+Reordering keeps an owning producer together with in-place consumers, such as
+Krea2's RMSNorm weight multiplication, so the alias checks can preserve their
+storage relationship.
+The application kernel preserves the original reduction order, thread selection,
+ConvRot row padding, and separate scale tail. Prepared plans cache operation
+metadata, use the borrowed CUDA stream, and reserve scheduler workspace for
+overlapping outputs. Long rows or external intermediate consumers retain
+ordinary execution. These fusions do not require CUTLASS and honor
+`GGML_CUDA_DISABLE_FUSION`.
+
+The corresponding scale, normalization, and unary-activation fusion rules have
+been removed from ggml for CUDA and HIP. GEMM/SwiGLU/ConvRot chains are selected by the
+application INT8 extension before the standalone activation fusion.
+
+`src/core/cuda/affine.cu` registers FP32 `Mul(gate) -> Add(residual)`,
+`Concat -> [Permute] -> FP16 Copy`, and `Contiguous -> Scale -> Permute -> FP16
+Copy` fusions. Gate multiplication and residual addition retain separate FP32
+rounding. Copy layouts and alignment requirements match the original CUDA
+kernels; unsupported layouts, external intermediate consumers, self-residual
+Add inputs, and fusion disabling retain ordinary operators. Internal in-place
+Add outputs are accepted when they overwrite the complete fused Mul allocation. Exact in-place residual
+or input addresses are safe for the pointwise kernel; partial overlap, broadcast
+gate overlap, and FP16 copy overlap use scheduler workspace. These registrations
+require no CUTLASS, compile for CUDA/HIP/MUSA, and support reusable CUDA Graph
+replay. Their former native fusion implementations have been removed from ggml.
+
+CUDA kernels can opt into device replay with `ggml_ext_register_kernel_ex` and
+`GGML_EXT_KERNEL_CAPTURE_SAFE`. The original registration function retains its
+per-execution callback behavior, and the descriptor ABI is unchanged. Opting in
+means the callback enqueues replayable device work: it has no required
+per-execution host side effects or decisions based on tensor contents, and any
+private streams rejoin the borrowed stream. Lazy setup must complete during the
+first ordinary execution. Tensor metadata, attributes, and host plan parameters
+remain frozen until scheduler reallocation; tensor contents may change normally.
+
+When every selected extension opts in, the CUDA backend can capture the whole
+scheduled plan, including its native fragments, into one CUDA Graph. The first
+execution warms up kernels and library state while retaining native fragment
+caches. The next stable execution captures
+and launches the graph; later calls replay it. Changing a tensor address or the
+execution stream invalidates the capture and repeats warmup, even when the graph
+UID is unchanged. Capture failures propagate without retrying partially executed
+work. An enclosing stream capture records the plan directly instead of starting
+a nested capture.
+
+`GGML_EXT_KERNEL_REPLAY_REUSABLE` additionally permits reuse across scheduler
+plan rebuilds and requires `CAPTURE_SAFE`. Recorded work must depend only on
+complete tensor metadata, addresses, the device and immutable registration data.
+Captured resources must belong to the backend or the supplied workspace, never
+per-plan host state. Workspace contents must be initialized by recorded device
+work on each execution. Prepare/destroy still run for every rebuilt host plan.
+The backend cache retains only the executor and workspace; it compares graph
+structure, tensor metadata/addresses, selected kernels and workspace requirements
+exactly, without dereferencing old graph objects. A new metadata arena can reuse
+an equivalent capture. Each backend retains at most 16 retired plans and 256 MiB
+of their workspaces, evicting the oldest entries first. Larger workspaces retain
+plan-local capture only. Backend destruction synchronizes and releases the cache.
+This allows sd.cpp's per-step scheduler rebuilds to retain device replay.
+
+Mixed plans containing an extension without the flag retain ordinary dispatch.
+Evaluation callbacks, multiple pipeline copies, and direct backend graph calls
+also retain their previous behavior. `GGML_CUDA_DISABLE_GRAPHS` disables whole-plan
+capture as well as native CUDA Graph caching. Native operations that cannot be
+captured retain the existing fragment execution path. Whole-plan replay requires
+CUDA VMM so native scratch addresses remain valid for the backend lifetime;
+legacy scratch pools retain the existing fragment path. HIP and MUSA do not yet
+provide this plan executor. sd.cpp's CUDA registrations opt in to reusable replay.
+
+## Vulkan execution services
+
+`ggml-ext-vulkan.h` exposes `ggml_ext_vulkan_v1` through the Vulkan backend
+registry. The versioned function table reports device limits and subgroup
+capabilities, resolves tensor/workspace byte ranges, dispatches application
+SPIR-V, and copies nonoverlapping buffer ranges. Extensions do not include
+private ggml Vulkan headers or own backend command buffers and descriptors.
+
+Shaders use the `main` entry point, consecutive storage-buffer bindings starting
+at zero, and consecutive 32-bit specialization constants. Dispatch accepts actual
+workgroup counts. The backend validates ranges and dispatch limits, caches the
+complete shader/configuration per device, and releases pipelines with that
+device. Push constants and specialization values are copied; the backend does
+not retain application execution state. Pipeline creation occurs on execution,
+never during host-only plan preparation.
+
+Bindings identify either a tensor or a backend buffer, with a byte offset and
+optional byte count. `buffer_info` returns an opaque storage identity and byte
+range for alias checks. Vulkan descriptor offsets are rounded down to the
+device's required alignment; shaders must add the returned `shader_offset`.
+Workspace bindings follow the same rules. Invalid or foreign-device ranges are
+rejected before recording commands. Kernel bounds and shader capability
+requirements remain the extension's responsibility.
+
+Dispatch and copy use the backend's existing compute command context, descriptor
+pool, transfer waits, and memory barriers. They preserve ordering with native
+fragments and invalidate cached converted operands after writes. Normal backend
+synchronization completes queued work before host access or resource release.
+Vulkan copy ranges must be equal in size, nonoverlapping, and aligned to four
+bytes. There is no per-dispatch host wait or extension-owned queue.
+
+Scheduled Vulkan plans share submission counters and FLOP thresholds across
+native fragments and extension callbacks. Native scratch is initialized once
+per plan invocation, with its offsets retained across fragments. Fragment
+boundaries do not force a
+queue submission, and the final pending commands are submitted when the plan
+returns. Extension dispatches and copies count toward the device's node limit.
+Every callback still executes on every invocation; this batching neither
+captures device replay nor retains plan state across scheduler rebuilds.
+Callback errors propagate without retry. Evaluation callbacks, direct backend
+calls, synchronous devices, serialized diagnostics, and timestamp profiling
+retain fragment execution.
+
+`src/core/vulkan/rope.cpp` registers the custom RoPE implementation,
+plus `RMSNorm -> Mul -> RoPE` and `RMSNorm -> Mul -> RoPE -> Scale -> FP16 Copy`
+fusion. These kernels retain the original strides, partial-rotation behavior,
+128-channel subgroup reduction, and separate arithmetic rounding. Their SPIR-V
+is generated from application shaders with the native RoPE compilation policy;
+SPIR-V optimization is disabled because it changes the reduction results.
+Safe graph-delay rules bring independent producers ahead of fusion candidates.
+External intermediate consumers retain ordinary operators, and overlapping
+outputs use plan workspace. `GGML_VK_DISABLE_FUSION` is honored.
+
+The corresponding Vulkan shader sources and native fusion rules have been
+removed from ggml. Unsupported custom nodes retain scheduler fallback to
+another supported backend.
+
+RoPE Apply, INT8/ConvRot, CuTe attention, SageAttention, and Sol-Attn use
+application-owned implementations. This interface does not add dynamic dtypes.
+
+`src/core/vulkan/int8.cpp` owns Vulkan INT8 GEMM, ConvRot, and their fusion rules.
+The Vulkan compute metadata service exposes shared-memory limits, subgroup
+arithmetic, and cooperative-matrix properties; shape selection stays in sd.cpp.
+Shader variants preserve the former tiling and compilation policy, including
+SwiGLU, residual, row-concatenation, and modulation paths. Kernels use public
+Vulkan dispatch, buffer, copy, and workspace services without private backend
+headers. Vulkan currently handles packed group-256 activations with separate
+weight scales; unsupported custom nodes fall back to CPU.
+
+## SageAttention
+
+The `ggml_ext_sage_attn` entry point is declared and implemented in sd.cpp's
+graph construction helpers. It constructs `sd.sage_attn.v1` with explicit Q/K/V inputs
+and scale/mode attributes. ggml has no Sage-specific opcode, allocation rule, or
+backend dispatch branch.
+
+`src/core/cuda/sage_attention.cu` and its `sageattention/` device headers own the
+CUDA implementation. Quantization and temporary attention output use scheduler
+workspace, rather than an oversized output tensor. The implementation obtains
+the backend stream through the public GPU service and converts the final FP16
+output with a local CUDA kernel. It retains the SM80 compatibility path and the
+SageAttention2/2++ toolkit and architecture requirements. Capability checks use
+the application's compiled CUDA target as well as the physical device.
+
+CPU, HIP, MUSA, and Vulkan do not implement this NVIDIA-specific kernel. Existing
+model-level capability checks retain flash/default attention for unsupported
+layers; a custom Sage node itself does not silently change algorithms.
+
+## Sol-Attn
+
+The `ggml_ext_sol_attn` entry point lives in the common sd.cpp graph construction
+files and constructs `sd.sol_attn.v1`. It accepts contiguous F32 Q/K/V tensors
+with shape `[128, tokens, heads, batch]`, a positive finite scale, and finite tau.
+The output remains F32 `[128, heads, tokens, batch]`. No current sd.cpp model or
+CLI path calls this entry point; the migration preserves its low-level behavior.
+
+`src/core/cuda/sol_attention.cu` owns packing, block summaries, threshold
+selection, and the BF16 tensor-core kernel. Temporary storage is separate
+scheduler workspace, so the output allocation contains only the output tensor.
+The kernel uses the public backend stream service and direct CUDA/PTX operations.
+Registration requires CUDA 12 or newer, an SM80-or-newer device, and an application
+kernel compiled for SM80 or newer. CPU, HIP, MUSA, and Vulkan do not advertise
+support. ggml no longer contains a Sol-Attn opcode, constructor, allocation rule,
+or backend kernel.
+
+## Public graph access
+
+`ggml-graph.h` exposes leaf enumeration, append-only leaf construction, recorded
+use counts, mapped inference-graph copying, use-count copying for segments, and
+backend execution of a node range. Mapped copies retain node order and consumer
+counts from the original graph, including consumers outside a graph view. The
+caller owns tensor copies and remaps their storage and edges; invalid mappings
+leave the destination graph unchanged. Range execution uses already allocated
+tensors and retains the original graph's consumer counts.
+
+Workspace measurement, graph cuts, and evaluation callbacks use these APIs.
+Normal sd.cpp builds require only ggml's public headers. Compatibility with an
+upstream ggml lacking this header is isolated in `src/core/ggml_compat.cpp`; that
+optional build still requires matching upstream private headers. The compatibility
+layer also retains the workaround for old Metal backends wrapping failed host
+mappings. The patched Metal backend returns null directly when mapping fails.

docs/image_preprocessing.md

@@ -100,10 +100,13 @@ for individual references, identity images, or video control frames.
 --image-preprocess "target=control-frame,index=2,canny=true"
 ```
 
-Init and mask sources must have the same dimensions. The mask inherits the
-init crop, resize, and padding coordinates, while retaining its own filter,
-padding value, and Canny setting. Conflicting mask geometry is rejected. An
-omitted mask remains absent until the original pipeline creates its default mask.
+If init and mask source dimensions differ, the mask is first center-cropped to
+the init source aspect ratio and resized to the init source dimensions using
+nearest-exact interpolation. The mask then inherits the init crop, resize, and
+padding coordinates, while retaining its own filter, padding value, and Canny
+setting for that transform. Explicit mask geometry is evaluated after this
+alignment; conflicting geometry is rejected. An omitted mask remains absent
+until the original pipeline creates its default mask.
 
 ## Downstream behavior
 
@@ -128,7 +131,8 @@ To disable reference resizing before VAE encoding, use
 `--ref-image-args "resize_before_vae=false"` or the server field
 `"ref_image_args": "resize_before_vae=false"`. This is separate from
 `target=ref,mode=none`, which only skips input geometry. Model constraints
-still apply.
+still apply; MiniMax-H3 requires 32-pixel-aligned references and warns instead
+of honoring the switch.
 
 ## Server requests
 
@@ -148,12 +152,12 @@ an array of rule strings:
 OpenAI-compatible requests accept it through
 `<sd_cpp_extra_args>{...}</sd_cpp_extra_args>` in the prompt.
 Request rules replace server-default rules. Generation metadata records the
-user rules; image encodings and channel conventions are unchanged.
+user rules for replay with the same input images; image encodings and channel conventions are unchanged.
 
 ## C API
 
 Set `image_preprocess` on the existing image/video generation parameters.
-The `generate_image()` and `generate_video()` signatures are unchanged:
+Generation returns media and actual parameters together; see [generation metadata](generation_metadata.md):
 
 ```c
 sd_img_gen_params_t params;
@@ -161,7 +165,10 @@ sd_img_gen_params_init(&params);
 /* Set prompt, original-resolution input images, and generation options. */
 params.image_preprocess.rules = "target=init,mode=crop-resize,filter=lanczos;"
                                 "target=mask,filter=nearest-exact";
-bool ok = generate_image(ctx, &params, &images, &count);
+sd_image_generation_result_t* result = NULL;
+bool ok = generate_image(ctx, &params, &result);
+/* Consume result->items and their generation_info on success. */
+free_sd_image_generation_result(result);
 ```
 
 Both generation parameter initializers set `image_preprocess.rules` to `NULL`,

docs/int8_convrot.md

@@ -2,7 +2,11 @@
 
 sd.cpp can load and execute ComfyUI `int8_tensorwise` safetensors with `convrot` metadata directly. The stored INT8 weights are not converted to another weight type at load time.
 
-This requires the INT8 tensorwise/convrot extensions in the patched GGML.
+This uses sd.cpp's INT8 tensorwise/convrot operators through the patched GGML
+extension API. Schemas and graph builders live in sd.cpp, alongside the
+backend kernels and fusion rules; GGML supplies generic dispatch and workspace
+services. See [operator extensions](ggml_extensions.md) for the implementation
+and the preserved INT8 API entry points.
 Builds with `SD_USE_UPSTREAM_GGML=ON` reject these files during loading.
 
 ## Checkpoint format
@@ -89,13 +93,76 @@ The packed runtime activation tensor contains the I8 activation rows and their f
 ## Backend support
 
 - CPU provides the portable regular Hadamard, activation quantization, INT8 matrix multiplication, and scale restoration implementations.
-- NVIDIA CUDA devices with compute capability 7.5 or newer use the native accelerated path. For H256, CUDA fuses the rotation, row-wise maximum reduction, and activation quantization. It uses cuBLAS for I8 x I8 to I32 GEMM and a CUDA kernel for scale restoration and bias addition.
+- Metal supports INT8 tensorwise and convrot on Apple silicon (Apple GPU family 7 or newer). It keeps weights in INT8 and rotates and quantizes activations on the GPU. On macOS/iOS 26 or newer, it probes Metal TensorOps `matmul2d` with INT8 inputs and INT32 cooperative results. It combines 1024-product partial sums using wrapping integer addition to preserve exact accumulation on older GPUs, then applies scales and bias in registers. Older systems or unavailable TensorOps pipelines use SIMD-group matrix multiplication: INT8 tiles are converted to FP16, and FP32 partial sums cover at most 256 products before being combined as integers, preserving the I32 accumulation contract. Convrot groups 1, 4, 16, 64, and 256 are supported. These kernels require the patched GGML Metal extension services and are enabled by `SD_METAL=ON`.
+- NVIDIA CUDA devices with compute capability 7.5 or newer use the native accelerated path. For H256, CUDA fuses the rotation, row-wise maximum reduction, and activation quantization. Eligible shapes use the CUTLASS INT8 implementation when enabled; other supported shapes use cuBLAS for I8 x I8 to I32 GEMM and a CUDA kernel for scale restoration and bias addition.
 - AMD HIP devices in the CDNA, RDNA3 (including RDNA3.5), and RDNA4 families use the same INT8/H256 kernels with hipBLAS for I8 x I8 to I32 GEMM. Other AMD architectures fall back to CPU for INT8 matrix multiplication.
-- Vulkan provides native H256 activation quantization and INT8 matrix multiplication when the build and device support accelerated packed INT8 dot products. Unsupported configurations and GPU backends without these kernels use the backend scheduler to fall back to CPU.
+- MUSA shares the CUDA-style H256 activation kernels. INT8 GEMM falls back to CPU because the MUSA backend does not advertise INT8 BLAS support.
+- Vulkan provides H256 activation quantization and INT8 matrix multiplication when the build and device support accelerated packed INT8 dot products. Unsupported configurations and GPU backends without these kernels use the backend scheduler to fall back to CPU.
 
 LoRA adapters are applied at runtime without modifying the INT8 weights. The INT8 convrot path computes the base linear output, while LoRA, LoHa, LoKr, and raw weight-difference adapters compute their output corrections from the original, unrotated activation and add them to the base output. `--lora-apply-mode auto` selects this path for models containing INT8 tensorwise weights. If `immediately` is requested, sd.cpp falls back to runtime application because merging an adapter would require dequantizing and rotating its weight update, then recalculating the per-row scales and requantizing the result.
 
-The dedicated CUDA and HIP convrot activation paths currently require a group size of `256`; other supported group sizes use CPU execution.
+The shared CUDA/HIP/MUSA convrot activation paths support H64 and H256. Vulkan also supports H64 activation quantization; its INT8 matrix multiplication path currently requires H256.
+
+## Packed INT4 convrot
+
+ComfyUI `convrot_w4a4` and `asym_w4a8_int8` safetensors load directly, including
+checkpoints that mix packed INT4, INT8 tensorwise, and floating-point layers.
+Packed weights retain their checkpoint byte layout; they are not expanded or
+converted to GGUF Q4 blocks at load time.
+
+- `convrot_w4a4`: signed 4-bit weights, low nibble first, with F32 per-output
+  `weight_scale`. Activations use the same INT8 convrot quantization as above;
+  the format name does not select a native INT4 activation kernel.
+- `asym_w4a8_int8`: 4-bit indices into `weight_codebook` (16 F32 values), F32
+  `weight_s_channel`, and F8_E4M3 `weight_s_rel` per 16 input features. The
+  metadata must specify `group_size: 16`. CPU evaluates the floating-point
+  codebook; GPU kernels requantize it to INT8 with `amax / 127`, introducing a
+  small additional rounding error.
+
+Both formats imply convrot rotation even without a `convrot` flag; missing
+`convrot_groupsize` defaults to 256.
+
+Both formats reuse the convrot activation cache when the input scale is `1` and
+produce F32 outputs with optional bias. CPU provides a portable implementation.
+CUDA/HIP/MUSA use shared packed-weight kernels without requiring INT8 BLAS.
+With CUTLASS enabled, NVIDIA
+SM80 and newer devices use mixed-input S8 x S4 GEMM for eligible W4A4 shapes,
+reusing the INT8 scale epilogue without expanding the full weight tensor. W4A8
+uses CUTLASS integer MMA per 16-value group with FP32 scale restoration.
+Other CUDA shapes and builds retain the packed byte-dot path. Set
+`GGML_CUDA_W4_FORCE_DP4A=1` to compare against that path. Supported AMD targets
+use packed byte-dot instructions, with portable byte arithmetic for other targets.
+Vulkan requires integer dot product support and at least 13312 bytes of shared
+memory for W4A4 or 16452 bytes for W4A8. GPU W4 kernels require the logical input
+width to be divisible by 64; H64 and H256 are supported. When supported by the
+compiler and device, Vulkan uses NV cooperative matrix integer GEMM. W4A4
+decodes packed weights directly into cooperative matrices for aligned input
+and output widths. Complete activation tiles and the final partial tile use
+separate shader specializations. Other W4A4 shapes use double-buffered weight
+and activation tiles (20480 bytes of shared memory, or 30720 for larger tiles).
+W4A8 stages both weights and activations in shared memory in K=64 tiles,
+reusing them for each 16-value group's integer
+MMA and FP32 scale restoration. FP8 scales use a shared lookup table, and
+expanding projections visit activation tiles in groups of weight panels for
+cache reuse. Adjacent groups share a K=32 activation fragment, with
+complementary zero padding in the weights to preserve each group's scale.
+This path uses 17476 bytes of shared memory, or 22596 for larger tiles.
+Both paths use 256-thread workgroups and keep the model's
+weights packed; neither expands a full layer in device memory. Other Vulkan
+devices retain the packed dot-product path; set `GGML_VK_W4_FORCE_DP4A=1` to compare
+against it. FP32 accumulation order can differ between these paths.
+Other shapes use CPU fallback. HIP W4A8 selects wave64 WMMA by the active
+compilation target's intrinsic support, preferring the GFX12 fragment layout
+over the GFX11 layout.
+There is no GPU architecture allowlist: newer targets supporting either
+intrinsic automatically use its matching layout. Selection happens per device
+image, including multi-architecture builds; targets supporting neither intrinsic
+retain the integer-dot fallback. W4A4 retains integer dot products on HIP.
+
+LoRA application remains at runtime, using the original unrotated activations
+and logical weight dimensions. Packed weights and their codebook/scale tensors
+are excluded from weight type conversion. `SD_USE_UPSTREAM_GGML` builds reject
+these formats because they require the application extension API.
 
 ## Example
 

docs/minimax_h3.md

@@ -80,17 +80,107 @@ latents condition the diffusion transformer. Paired video and audio references
 share the same timeline. Ref2VA cannot be combined with `--init-img` or
 `--end-img` in one request.
 
-Reference images keep their aspect ratio and are only downscaled when their
-pixel area exceeds the requested generation canvas.
+Reference images keep their aspect ratio and are only downscaled when they
+exceed the configured limit; neither mode scales a reference up. The limit is
+selected by `ref_image_size`. The default, `match`, uses the pixel area of the
+requested generation canvas. Pass `--ref-image-args "ref_image_size=max"` (or
+`ref_image_size` in `sd_vid_gen_params_t.ref_image_args`) to use a 2048 px short
+edge instead, which preserves more identity detail at a higher sampling cost:
+reference latents are attended at every sampling step.
+
+Whichever limit applies, each axis is then snapped to the nearest 32-pixel
+multiple. An axis of at least 16 pixels can therefore grow by up to 16 pixels
+past the scaled size (a 48-pixel axis becomes 64), and references are never
+narrower than 32 pixels.
+`resize_before_vae=false` is not supported here; MiniMax-H3 keeps references on
+the 32-pixel grid and reports a warning.
 
 The C API exposes the same inputs through `ref_images`, `ref_videos`, and
 `ref_audios` in `sd_vid_gen_params_t`. Each `sd_ref_video_t` supplies its own
 frame rate and optional soundtrack; non-24-fps inputs are resampled internally.
 
+## Latent hires second pass (two-stage sampling)
+
+MiniMax-H3 supports `--hires` with two interchangeable second-stage upscalers:
+
+- an interpolating latent upscaler (`Latent`, `Latent (nearest)`,
+  `Latent (bicubic)`, ...): after the first pass finishes, the packed AV latent
+  is resampled in latent space to `--hires-width` x `--hires-height` and
+  refined by a second sampling pass;
+- a model-backed upscaler (`--hires-upscaler <name>` + `--hires-upscalers-dir`):
+  a H3-shaped learned latent upscaler checkpoint runs between the passes,
+  like the LTX spatial upsampler. This is the sd.cpp equivalent of the
+  two-sampler workflows used for H3 in node-based frontends (e.g. the
+  community 3D-conv latent upscaler from
+  [LBH-123-AI/Minimax_h3_latent_Upscaler](https://huggingface.co/LBH-123-AI/Minimax_h3_latent_Upscaler)).
+
+Draft at a low megapixel count (e.g. 0.4 MP) and refine at a higher one
+(e.g. 0.8 MP) instead of paying for the full resolution in every step:
+
+```sh
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "a lovely cat" --cfg-scale 1.0 -v -W 864 -H 480 --diffusion-fa --offload-to-cpu --rng cpu --video-frames 56 --hires --hires-upscaler "Latent (bicubic)" --hires-width 1216 --hires-height 672 --hires-steps 4 --hires-denoising-strength 0.4
+```
+
+or with the learned upscaler:
+
+```sh
+.\bin\Release\sd-cli.exe -M vid_gen ... --hires --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler h3-latent-upscaler-3d --hires-width 1216 --hires-height 672 --hires-steps 4 --hires-denoising-strength 0.4
+```
+
+Notes for this mode:
+
+- The learned upscaler expects a second normalization of the diffusion latent,
+  matching its training convention. The runner applies and reverses this extra
+  transform internally; callers should pass the usual H3 diffusion latents.
+- The frame count is never changed by the second stage, so the packed audio
+  latent length stays valid and the generated soundtrack is preserved.
+- Width and height still have to be multiples of 32; the hires target is
+  aligned like any other resolution. The learned upscaler additionally wants
+  an even latent size on both axes, which 32-pixel targets provide.
+- `--hires-sigmas` can replace `--hires-denoising-strength` when the second
+  stage should resume from an explicit sigma schedule, which is how most
+  two-stage H3 workflows split the sampler.
+- Keyframe conditioning (`--init-img` / `--end-img`) is not available together
+  with hires yet: those latents are encoded at the first stage resolution and
+  would have to be re-encoded for the hires canvas. Use T2VA or Ref2VA.
+- The second stage runs at the larger latent, so it is the memory-hungry half;
+  combine it with quantization, `--vae-tiling` and `--offload-to-cpu` on
+  small cards.
+- The learned upscaler is scale-conditional: the target-to-source ratio is
+  fed into the network as an embedding, 1.2-2.0x is the trained domain, and
+  downscaling is rejected.
+
 ## Shape and runtime notes
 
 - Width and height are aligned upward to a multiple of 32.
 - Frame count is aligned upward to the `17k + 5` grid, with a minimum of 5.
 - MiniMax-H3 runs at 24 fps; another requested value is overridden.
+- Attention head grouping is controlled by
+  `--model-args minimax_h3_attn_chunk_budget=2`, in GiB (default: `2`).
+  The budget estimates the size of a single FP32 Q/K tensor per group:
+  `head_dim * tokens * batch * 4 bytes * heads_per_group`. The group size adapts
+  to the sequence length, with a minimum of one head; inputs that fit use all
+  heads together. Set `0` to disable grouping, or a smaller budget such as
+  `0.25` to reduce peak activation memory for long sequences at the cost of
+  extra launches and concatenation. Each head still attends to the full sequence.
+  This is a grouping target, not a cap on total attention memory or VRAM usage;
+  other intermediates, backend workspace, and weights require additional memory.
+  `--max-vram` and `--offload-to-cpu` control weight residency, but the active
+  block's working memory must also fit within the VRAM budget.
 - The default video flow shift is 12. The audio stream is mapped internally to
   its shift of 3, so the regular samplers can operate on the packed AV latent.
+
+## HyperFlow H3 Flash
+
+[HyperFlow H3 Flash](https://huggingface.co/VideoRebirthAI/hyperflow-h3-flash)
+uses the shared components and input options described in this guide. Replace
+`--diffusion-model` with the matching HyperFlow FL2VA or Ref2VA checkpoint and
+use these sampling options:
+
+```sh
+--sampling-method euler --cfg-scale 1 --sigmas hyperflow-h3-flash-6
+```
+
+Use `hyperflow-h3-flash-4` or `hyperflow-h3-flash-8` for 4 or 8 steps. These
+presets also work with `--hires-sigmas`; their length determines the step count.
+They already include video flow shift 12, so keep the default flow shift.

docs/model_config.md

@@ -1,7 +1,7 @@
 # Model Configuration Conventions
 
-This document describes the conventions for model configuration structs and
-weight-based configuration detection.
+This document describes model configuration structs, weight-based configuration
+detection, and the definitions used to integrate models into the runtime.
 
 ## Config Types
 
@@ -60,6 +60,8 @@ static UNetConfig detect_from_weights(const String2TensorStorage& tensor_storage
 ```
 
 Use `TensorStorage` metadata, especially `n_dims` and `ne`, to infer shapes.
+Use `logical_ne0()` for Linear input widths; packed INT4 weights store half of
+the logical width in `ne[0]`. Keep raw `ne` values for storage layout calculations.
 Do not load or parse tensor data for config detection.
 
 Detection should respect `prefix`. For nested weights, construct full names from
@@ -116,3 +118,113 @@ struct.
 If a model has no weight-derived config today, it may still provide
 `detect_from_weights` for API consistency, but it should not print a config
 detection log unless it actually derives values from weights.
+
+## Model Integration Definitions
+
+Every `SDVersion` has an explicit `ModelDefinition` in
+`src/model/integrations/`. Related variants share an integration file and runner
+factory. Network blocks and their weight-derived configuration remain under
+`src/model/diffusion/`; reusable text encoder implementations remain under
+`src/model/te/`. Weight-name conversion remains in `src/name_conversion.cpp`.
+
+To add a model:
+
+1. Add its identity to `SDVersion` without renumbering existing values.
+2. Implement the network and `*Config::detect_from_weights` following the
+   conventions above, or reuse an existing runner.
+3. Add a definition in its integration file, including its component
+   factory, traits, geometry, and sampling policy.
+4. Add the family's explicit registration in `src/model_registry.cpp`. For a new
+   variant of a registered family, add its definition to that family's list.
+5. Add version detection signatures and any required weight-name mappings in
+   `src/model_loader.cpp` and `src/name_conversion.cpp` respectively. Update shared
+   conditioning or runtime code only when existing behavior cannot serve the model.
+
+Use the existing family integration as the starting point for a variant. For a
+new family, `src/model/integrations/moss_image.cpp` is a compact example of a
+conditioner/diffusion factory and its definition. Definitions collect model
+construction and defaults; they do not replace version detection, name conversion,
+or model-specific inference algorithms.
+
+The registry checks for missing, duplicate, and invalid version registrations.
+Explicit references keep integrations reachable in static-library builds. The
+existing CMake glob includes integration sources automatically. No legacy
+builder, display-name table, or prediction-default fallback needs updating.
+
+### Detection And Weight Conversion
+
+Version detection remains in `ModelLoader::get_sd_version()` in
+`src/model_loader.cpp`. Add model signatures there and preserve the order of
+overlapping checks. Detection uses tensor names and metadata, and can run before
+and after name conversion, so signatures must work at both stages. Do not load
+tensor data to identify an architecture. The registry looks up definitions by
+the detected version; it does not perform detection.
+
+Weight-name mappings and their version-based dispatch remain centralized in
+`src/name_conversion.cpp`. Add new mappings there when needed. Inference, GGUF
+export, quantization, and LoRA conversion share the `convert_tensor_name` entry
+point; preserve its prefix normalization and component conversion order.
+
+### Components And Conditioning
+
+The core factory receives `model_builders::Context`, the tokenizer configuration,
+and the output runners. Backend checks, optional IP-Adapter setup, scale
+overrides, and tokenizer usage validation remain in the common builder. A
+factory returns failure if any required additional backend cannot initialize.
+
+LLM architecture selection, tokenizer requirements, prompt preparation, and
+encoding remain centralized in `src/conditioning/conditioner.hpp`. Integration
+factories instantiate the conditioner with the model version; models requiring
+new LLM conditioning behavior still need changes in that shared implementation.
+
+`vae_kind` is the single source of truth for the VAE family; compatibility
+helpers derive their results from it. `KL`, `FLUX`, and `FLUX2` share the
+`AutoEncoderKL` implementation while retaining their distinct latent formats.
+The related traits have separate responsibilities:
+
+- `supports_video_generation` describes the model's video generation capability.
+- `video_tae` selects `TinyVideoAutoEncoder` instead of `TinyImageAutoEncoder`.
+  Image models can use a video TAE, so this is independent of video generation.
+- `fake_vae` selects FakeVAE for pixel-space models. It is separate from
+  `vae_kind`, which also controls latent-format compatibility helpers.
+- `vae_format_override` allows PiD's VAE format override.
+
+`configure_vae` handles model-specific KL adjustments such as
+PixArt's scale and SDXL's convolution scale. VAE compatibility traits also drive
+latent transformations and previews through the existing helper functions.
+
+### Sampling And Geometry
+
+`ModelSamplingPolicy` groups the defaults first: `default_prediction`,
+`default_flow_shift`, `default_sample_method`, and `default_scheduler`. Set only
+the values that differ from the struct defaults. `prediction_from_weights` and
+`flow_shift_from_weights` preserve checkpoint-dependent defaults during automatic
+prediction selection; `probe_v_prediction` enables SD2's runtime v-prediction
+probe. Explicit prediction settings take precedence over automatic selection.
+MOSS's default unconditional skip layer stays
+in the diffusion engine, with active unconditional SLG taking precedence.
+
+The resolved `prediction_t` selects the denoiser directly. Flux3, MiniMax H3 AV,
+and Ming-Image use `FLUX3_FLOW_PRED`, `H3_AV_FLOW_PRED`, and
+`MING_IMAGE_FLOW_PRED`; their CLI overrides are `flux3_flow`, `h3_av_flow`, and
+`ming_image_flow`. Explicit `sd3_flow` always selects `DiscreteFlowDenoiser`,
+and `flux_flow` always selects `FluxFlowDenoiser`, regardless of model version.
+Automatic selection retains each model's default implementation. New prediction
+values are appended before `PREDICTION_COUNT`, preserving existing valid values;
+clients using the automatic-selection sentinel must rebuild with the new header.
+
+`ModelGeometry` declares `diffusion_down_factor`, `latent_channels`, and
+`image_channels`. `ref_image_preset` defaults to `"default"`; set it only for models
+that need another reference-image preset.
+
+The common pipeline executes these policies; specialized video, multimodal,
+cache, and extension algorithms remain in their existing runtime components.
+Model-specific diffusion extra parameters are assembled in the diffusion engine.
+Video frame conversion and alignment also remain in the diffusion engine.
+`get_vae_scale_factor()` retains PiD's pixel-space special case and otherwise
+uses the VAE's scale factor.
+
+Validate migrations against the previous implementation using synthetic metadata,
+name conversions, prompt preparation, default/override selection, and focused
+builds. Real generation checks are still needed to validate numerical behavior
+when changing network or encoding algorithms.

docs/p_h3_image.md

@@ -0,0 +1,34 @@
+# How to Use
+
+P-H3-Image supports text-to-image and reference-image editing. It uses a MiniMax-H3 diffusion transformer, a single-frame VAE, and Qwen3-VL-4B with a text connector. Prompt enhancement is not included.
+
+## Download weights
+
+- Download P-H3-Image
+    - safetensors: https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/diffusion_models
+- Download P-H3-Image VAE
+    - safetensors: https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/vae
+- Download Qwen3-VL-4B with the text connector
+    - https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/text_encoders
+
+Alternatively, reuse Qwen3-VL-4B, including its vision tower, with the [separate text connector](https://huggingface.co/PrunaAI/P-H3-Image/blob/main/text_connector/diffusion_pytorch_model.safetensors). Save the connector as `p-h3-image_text_connector.safetensors` and pass it with `--embeddings-connectors`.
+
+Use the P-H3 image VAE; the MiniMax-H3 video VAE is not compatible. No audio VAE is needed.
+
+## Examples
+
+### Text-to-image
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\p-h3-image_int8_convrot.safetensors --llm ..\models\text_encoders\qwen3vl_4b_bf16.safetensors --embeddings-connectors ..\models\text_encoders\p-h3-image_text_connector.safetensors --vae ..\models\vae\p-h3-image_vae_bf16.safetensors --cfg-scale 1.0 --steps 8 --sampling-method euler --sigmas p-h3-image -p "a lovely cat holding a sign says 'p-h3-image.cpp'" -v --offload-to-cpu --rng cpu -H 2912 -W 2912
+```
+
+### Image editing
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\p-h3-image_int8_convrot.safetensors --llm ..\models\text_encoders\qwen3vl_4b_bf16.safetensors --embeddings-connectors ..\models\text_encoders\p-h3-image_text_connector.safetensors --vae ..\models\vae\p-h3-image_vae_bf16.safetensors --cfg-scale 1.0 --steps 8 --sampling-method euler --sigmas p-h3-image -r source.png -r reference.png -p "Edit <Picture 1>. Give the cat the scarf from <Picture 2>, preserving its face and the background." -v --offload-to-cpu --rng cpu -H 2912 -W 2912
+```
+
+Reference images are numbered in argument order as `<Picture 1>`, `<Picture 2>`, and so on. A plain edit instruction gets `Edit <Picture 1>. ` prepended automatically. Prompts starting with `Edit <Picture N>` or `subject_definitions:` are used directly.
+
+Use dimensions divisible by 32. The upstream default is 2912 × 2912; smaller canvases reduce memory use. Use CFG 1 and `--sigmas p-h3-image` for the upstream 8-step schedule. Keep the default VAE tiling size of 256 pixels.

docs/pe_cache.md

@@ -0,0 +1,60 @@
+# Position embedding cache
+
+`sd::PECache` stores CPU-side `sd::Tensor<float>` data. Each model runner owns
+its cache and calls the existing `make_input()` on every graph build. Cache
+hits skip PE generation; allocation and upload remain ordinary graph-input
+operations. `GGMLRunner` has no PE-specific code or resource hooks.
+
+The cache keeps the most recent key and tensor for each named PE input. A
+matching key returns the existing tensor by reference. Changing its parameters
+replaces that input's cached result. Different named inputs have independent
+storage. Alternating two layouts under the same name recomputes their PE;
+there is no LRU, device-buffer cache, memory budget, or graph lifecycle callback.
+CPU data lives with the model runner, including across `runner_end()` calls,
+and is released when its named entry is replaced or the cache is destroyed.
+
+## Adding a model
+
+Include `core/pe_cache.h` and declare a member:
+
+```cpp
+sd::PECache pe_cache_;
+```
+
+Construct the key before generating PE and place the generation code in the
+lazy callback:
+
+```cpp
+sd::PECacheKey key("model.image_pe");
+key.add(height).add(width).add(head_dim).add(theta)
+    .add(circular_x_enabled).add(circular_y_enabled);
+auto pe = make_input(pe_cache_.get_or_create(key, [&]() {
+    auto values = generate_position_embedding(/* existing arguments */);
+    return sd::Tensor<float>({2, 2, head_dim / 2, token_count}, std::move(values));
+}));
+```
+
+Keys must cover every variable affecting PE values or shape: geometry, text
+length, frequency settings, reference-image ordering and offsets, circular
+settings, and video/audio positions or frame rates as applicable.
+`add_shape()` records all four ggml dimensions, including trailing singleton
+dimensions. `add_shapes()` records an ordered list of reference shapes.
+`add(sd::Tensor<T>)` records both shape and values for explicit position inputs.
+Keys compare their complete encoded fields rather than a data pointer or hash.
+
+Use distinct names for PE inputs that coexist in a graph, such as Anima's
+adapter query and key PE. `make_input()` defers reading the CPU data until graph
+execution: do not replace a named entry while an unfinished graph references
+it. Adding or updating other named entries does not invalidate its storage.
+Return tensor shapes with the contiguous dimension first, and preserve each
+model's existing F32 values, layout, and final transformations.
+
+## Current integrations
+
+The cache is used by MiniT2I, the Flux runner and its variants, Flux3, Qwen Image,
+Qwen Image 2.1, Z-Image and L2P, Anima, Boogu, ERNIE Image, Fibo2, Ideogram4,
+Iris, Krea2, Lens, LLaDA Image, MAGE Flow, Ming Image, MOSS Image, PiD, PixArt,
+Wan, Hunyuan Video, LingBot Video, and LTX audio/video.
+
+Position embeddings stored as model weights and graph-side RoPE operations
+remain on their existing paths, including MiniMax H3 and SenseNova U1.

docs/performance.md

@@ -159,6 +159,25 @@ allocator/scheduler capacity. Compatible gallocr reservations are reused across
 graphs; idle workspaces can be reclaimed under pressure and are freed at runner
 completion. Cross-graph caches survive individual graphs, but cut buffers do not.
 
+On discrete Vulkan devices, runners reuse released device buffers of the same
+size to reduce allocation overhead during segmented execution. This idle buffer
+cache is limited to 1 GiB per device and is reclaimed before allocating a new
+size or at runner completion. It is outside the managed `--max-vram` budget,
+so physical VRAM usage can be higher. Set `GGML_VK_REUSE_BUFFERS=0` to disable
+it. Unified-memory and host-memory modes do not use this cache.
+
+Discrete NVIDIA and non-GCN AMD Vulkan devices use a separate transfer queue by
+default when available and graphics queue use is disabled, so weight prefetch can overlap compute. Set
+`GGML_VK_ASYNC_USE_TRANSFER_QUEUE=0` to disable it.
+
+On discrete NVIDIA devices, allocation chunks can grow up to 4 GiB, bounded by the device's buffer and allocation
+limits, to reduce fragmentation within large graph workspaces. Other devices
+retain the 1 GiB default. `GGML_VK_SUBALLOCATION_BLOCK_SIZE` overrides this limit
+in bytes on any vendor; for example, `1073741824` selects 1 GiB and `4294967296`
+selects 4 GiB, subject to device limits. This is a soft chunk limit, not a reservation
+or a change to `--max-vram`: a single oversized tensor or exhausting the allocator's
+chunk count can require a larger allocation, which may fail.
+
 The recommended shape for "biggest model my card can host" is:
 
 ```shell

docs/relay_vsr.md

@@ -0,0 +1,50 @@
+# RelayVSR image and video super-resolution
+
+[RelayVSR](https://github.com/kopperx/RelayVSR) upscales images and videos by
+2x or 4x with `-M vid_upscale`.
+
+## Weights
+
+[ComfyUI safetensors weights](https://huggingface.co/Comfy-Org/RelayVSR) load
+directly. Select the matching pair for the requested scale:
+
+| `--upscale-factor` | `--diffusion-model` | `--vae` |
+| --- | --- | --- |
+| 2 | `relayvsr_x2_lightx2v_int8_convrot.safetensors` | `relayvsr_flash_vae_x2_fp32.safetensors` |
+| 4 | `relayvsr_x4_int8_convrot.safetensors` | `relayvsr_flash_vae_x4_fp32.safetensors` |
+
+No separate text encoder or ordinary Wan VAE is needed.
+
+## Usage
+
+For a video, extract numbered frames into a directory, then run:
+
+```sh
+sd-cli -M vid_upscale \
+  --diffusion-model models/diffusion_models/relayvsr_x2_lightx2v_int8_convrot.safetensors \
+  --vae models/vae/relayvsr_flash_vae_x2_fp32.safetensors \
+  --input-frames input_frames --upscale-factor 2 --fps 24 \
+  --vsr-window-size 16 -s 42 -o output.avi
+```
+
+Frames are read in filename order. Set `--fps` to the source video's frame rate.
+Video outputs support AVI, WebM, and animated WebP. To save individual frames,
+use an output pattern such as `-o output/%04d.png`.
+
+For a single image, replace `--input-frames input_frames` with `-i input.png`
+and use `-o output.png` (or JPEG/WebP). No FPS setting is needed for an image.
+`-i` / `--init-img` and `--input-frames` are mutually exclusive.
+
+## Settings and limits
+
+- `--upscale-factor` is required. Use 2 or 4 with the corresponding weights.
+  Output width and height are exactly the input dimensions times the scale.
+- `--vsr-window-size` sets the reference-frame window, including both endpoints.
+  The default 16 selects reference frames 0, 15, 30, and so on, and always
+  includes the final frame. Every input frame appears once in the output.
+- `--flash-attn auto` and `--conv-direct auto` are enabled by default. See
+  [backend selection](backend.md) to choose a device or weight placement.
+- Sampler, scheduler, and step settings do not configure RelayVSR.
+  `--vsr-temporal-chunk-size` is a SeedVR2 option.
+- Long videos require enough system memory to hold the input and output frames.
+  Audio is not copied to the output.

docs/sage_attention.md

@@ -4,8 +4,9 @@
 the high-noise diffusion model when present. Python, PyTorch, and Triton are not
 required at build time or runtime.
 
-The CUDA backend automatically selects a kernel supported by both the GPU and
-the compiled CUDA toolkit:
+The sd.cpp CUDA extension registers SageAttention through GGML's generic
+operator API. It selects a kernel supported by the GPU, compiled CUDA target,
+and toolkit:
 
 | GPU / toolkit | Implementation |
 | --- | --- |

docs/seedvr2.md

@@ -0,0 +1,69 @@
+# SeedVR2 image and video super-resolution
+
+SeedVR2 upscales images and videos with `-M vid_upscale`. Support is experimental.
+
+## Weights
+
+These [ComfyUI safetensors weights](https://huggingface.co/Comfy-Org/SeedVR2)
+load directly, without conversion or a separate text encoder:
+
+| Model | `--diffusion-model` | `--vae` |
+| --- | --- | --- |
+| 3B | `seedvr2_3b_int8_convrot.safetensors` | `seedvr2_ema_vae_fp16.safetensors` |
+| 7B Sharp | `seedvr2_7b_sharp_int8_convrot.safetensors` | `seedvr2_ema_vae_fp16.safetensors` |
+
+Use the matching SeedVR2 VAE for either model. The 7B model requires more memory.
+
+## Upscale an image
+
+```sh
+sd-cli -M vid_upscale \
+  --diffusion-model models/diffusion_models/seedvr2_3b_int8_convrot.safetensors \
+  --vae models/vae/seedvr2_ema_vae_fp16.safetensors \
+  -i input.png --upscale-factor 2 -s 42 -o restored.png
+```
+
+`-i` / `--init-img` accepts a single image. Outputs can be PNG, JPEG, or WebP.
+No FPS or temporal chunk setting is needed for an image.
+
+## Upscale a video
+
+Extract the video into numbered frames, then pass the frame directory:
+
+```sh
+mkdir frames
+ffmpeg -i input.mp4 -fps_mode passthrough frames/%06d.png
+sd-cli -M vid_upscale \
+  --diffusion-model models/diffusion_models/seedvr2_3b_int8_convrot.safetensors \
+  --vae models/vae/seedvr2_ema_vae_fp16.safetensors \
+  --input-frames frames --upscale-factor 2 --fps 24 \
+  --vsr-temporal-chunk-size 1 -s 42 -o restored.avi
+```
+
+Set `--fps` to the source video's frame rate; 24 is only the example above.
+Frames are read in filename order. Use either `-i` or `--input-frames`, not both.
+
+Video outputs support AVI, WebM, and animated WebP. To save individual frames,
+use an output pattern such as `-o restored/%06d.png`.
+
+## Settings and limits
+
+- `--upscale-factor` is required and must be a positive integer. Output width
+  and height are exactly the input dimensions multiplied by this value.
+- `--vsr-temporal-chunk-size` controls how many video frames are processed
+  together. Valid values are 1, 5, 9, 13, and so on; the default is 5. A value of
+  1 processes each frame independently and uses less memory. Larger values use
+  neighboring frames but require more memory. Chunk boundaries may be visible.
+- Frame count and order are preserved, including a shorter final chunk. Audio
+  is not copied to the output.
+- If you run out of GPU memory, try `--vsr-temporal-chunk-size 1`. A five-frame
+  chunk at 1728x960 output resolution exceeded 24 GB of GPU memory in testing.
+- `--flash-attn auto` and `--conv-direct auto` are enabled by default. See
+  [backend selection](backend.md) to choose a device or weight placement.
+- SeedVR2 does not use sampler, scheduler, or step settings.
+  `--vsr-window-size` is a RelayVSR option and does not affect SeedVR2.
+
+3B has been tested on CUDA and CPU; 7B has been tested on CUDA with small inputs.
+Other backends have not been tested with this integration.
+
+Model reference: [ByteDance-Seed/SeedVR](https://github.com/ByteDance-Seed/SeedVR).

docs/wan.md

@@ -184,6 +184,8 @@ Notes:
 
 <video src=../assets/wan/Wan2.2_5B_i2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
 
+TI2V 5B can decode multiple latent frames together when temporal tiling is enabled. For example, `--temporal-tiling --vae-tiling --vae-tile-size 256x256 --extra-tiling-args "temporal_tile_frames=4,temporal_decode_frames=2"` keeps up to four latent frames in each temporal tile and decodes up to two at once, with spatial tiles of 256×256 pixels. `temporal_decode_frames` defaults to `1`, preserving frame-by-frame decoding; other model versions ignore it. Increasing it requires more memory. Adjust the spatial tile size to fit the available memory.
+
 ### Wan2.1 FLF2V 14B
 
 ```

examples/cli/CMakeLists.txt

@@ -2,8 +2,10 @@ set(TARGET sd-cli)
 
 add_executable(${TARGET}
     ../common/common.cpp
+    ../common/generation_metadata.cpp
     ../common/log.cpp
     ../common/media_io.cpp
+    ../common/media_metadata.cpp
     image_metadata.cpp
     main.cpp
 )

examples/cli/README.md

@@ -31,14 +31,21 @@ rule for edge detection. See
 [Image preprocessing](../../docs/image_preprocessing.md) for input selectors,
 input defaults, downstream model processing, mask alignment, and examples.
 
-Metadata mode inspects PNG/JPEG container metadata without loading any model:
+Metadata mode inspects PNG, JPEG, WebP (including animated WebP), AVI, WebM, and Matroska
+container metadata without loading any model. Use `--image` for both images and videos:
 
 ```bash
 ./bin/sd-cli -M metadata --image ./output.png
 ./bin/sd-cli -M metadata --image ./output.jpg --metadata-format json
+./bin/sd-cli -M metadata --image ./output.webm
+./bin/sd-cli -M metadata --image ./output.avi --metadata-format json
 ./bin/sd-cli -M metadata --image ./output.png --metadata-raw
 ./bin/sd-cli -M metadata --image ./output.png --metadata-all
 ```
 
 For completely black or white images or videos, NaNs, and the `--linear-scale` /
 `--attn-scale` workaround, see [Troubleshooting](../../docs/troubleshooting.md).
+
+Generation outputs embed the core [generation record](../../docs/generation_metadata.md).
+Images retain the native JSON and an additional WebUI-compatible infotext.
+`--disable-image-metadata` omits both from files.

examples/cli/image_metadata.cpp

@@ -2,12 +2,14 @@
 
 #include <algorithm>
 #include <cctype>
+#include <codecvt>
 #include <cstdint>
 #include <cstring>
 #include <filesystem>
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <locale>
 #include <map>
 #include <set>
 #include <sstream>
@@ -391,8 +393,39 @@ namespace {
                                json& report,
                                std::string& error);
 
+    json parse_exif_user_comment(const uint8_t* data, size_t size) {
+        if (size >= 8 && memcmp(data, "ASCII\0\0\0", 8) == 0) {
+            return trim_trailing_nuls(bytes_to_string(data + 8, data + size));
+        }
+        if (size < 8 || memcmp(data, "UNICODE\0", 8) != 0) {
+            return json{{"raw_hex_preview", hex_preview(data, size)}};
+        }
+        size_t offset = 8;
+        // WebUI's piexif writer uses big-endian UTF-16 without a BOM.
+        bool little_endian = false;
+        if (size >= 10 && ((data[8] == 0xfe && data[9] == 0xff) || (data[8] == 0xff && data[9] == 0xfe))) {
+            little_endian = data[8] == 0xff;
+            offset        = 10;
+        }
+        if ((size - offset) % 2 != 0) {
+            return json{{"error", "truncated UTF-16 UserComment"}, {"raw_hex_preview", hex_preview(data, size)}};
+        }
+        std::u16string text;
+        for (; offset < size; offset += 2) {
+            text.push_back(static_cast<char16_t>(little_endian ? data[offset] | (uint16_t(data[offset + 1]) << 8)
+                                                               : (uint16_t(data[offset]) << 8) | data[offset + 1]));
+        }
+        try {
+            std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
+            return trim_trailing_nuls(converter.to_bytes(text));
+        } catch (const std::range_error&) {
+            return json{{"error", "invalid UTF-16 UserComment"}, {"raw_hex_preview", hex_preview(data, size)}};
+        }
+    }
+
     json parse_exif_value(const std::vector<uint8_t>& data,
                           size_t value_offset,
+                          uint16_t tag,
                           uint16_t type,
                           uint32_t count,
                           bool little_endian,
@@ -466,8 +499,9 @@ namespace {
                 break;
             }
             case 7:
-                value = bytes_to_string(data.data() + value_offset,
-                                        data.data() + value_offset + count);
+                value = tag == 0x9286 ? parse_exif_user_comment(data.data() + value_offset, count)
+                                      : json(bytes_to_string(data.data() + value_offset,
+                                                             data.data() + value_offset + count));
                 break;
             case 9: {
                 if (count == 1) {
@@ -591,6 +625,7 @@ namespace {
 
             tag_json["value"] = parse_exif_value(data,
                                                  value_offset,
+                                                 tag,
                                                  type,
                                                  value_count,
                                                  little_endian,
@@ -1098,6 +1133,178 @@ namespace {
         return true;
     }
 
+    static bool parse_avi_chunks(const std::vector<uint8_t>& data, size_t offset, size_t end, bool info, bool include_raw, json& result, std::string& error) {
+        while (offset < end) {
+            if (end - offset < 8) {
+                error = "truncated AVI chunk header";
+                return false;
+            }
+            const auto name   = bytes_to_string(data.data() + offset, data.data() + offset + 4);
+            const size_t size = read_u32_le(data, offset + 4);
+            offset += 8;
+            if (size > end - offset || (size & 1u) > end - offset - size) {
+                error = "AVI chunk exceeds its container";
+                return false;
+            }
+            json entry = {{"entry_type", "chunk"}, {"name", name}, {"length", size}, {"metadata_like", info}};
+            if (name == "LIST") {
+                if (size < 4) {
+                    error = "truncated AVI LIST type";
+                    return false;
+                }
+                const auto type            = bytes_to_string(data.data() + offset, data.data() + offset + 4);
+                entry["data"]["list_type"] = type;
+                if (type == "INFO" && !info &&
+                    !parse_avi_chunks(data, offset + 4, offset + size, true, include_raw, result, error))
+                    return false;
+            } else if (info) {
+                entry["data"]["text"] = trim_trailing_nuls(bytes_to_string(data.data() + offset, data.data() + offset + size));
+            } else {
+                append_raw_preview(entry, data.data() + offset, size, include_raw);
+            }
+            result["entries"].push_back(std::move(entry));
+            offset += size + (size & 1u);
+        }
+        return true;
+    }
+
+    static bool parse_avi(const std::vector<uint8_t>& data, bool include_raw, json& result, std::string& error) {
+        result        = {{"format", "AVI"}, {"entries", json::array()}};
+        size_t offset = 0;
+        while (offset < data.size()) {
+            if (data.size() - offset < 12 || memcmp(data.data() + offset, "RIFF", 4) != 0 ||
+                memcmp(data.data() + offset + 8, offset == 0 ? "AVI " : "AVIX", 4) != 0) {
+                error = "invalid AVI RIFF header";
+                return false;
+            }
+            const size_t size = read_u32_le(data, offset + 4);
+            if (size < 4 || size > data.size() - offset - 8 ||
+                (size & 1u) > data.size() - offset - 8 - size) {
+                error = "AVI RIFF size exceeds file size";
+                return false;
+            }
+            if (!parse_avi_chunks(data, offset + 12, offset + 8 + size, false, include_raw, result, error))
+                return false;
+            offset += 8 + size + (size & 1u);
+        }
+        return true;
+    }
+
+    static bool read_ebml_uint(const std::vector<uint8_t>& data, size_t& offset, size_t end, bool id, uint64_t& value, bool& unknown) {
+        if (offset == end || data[offset] == 0)
+            return false;
+        uint8_t mask  = 0x80;
+        size_t length = 1;
+        while (!(data[offset] & mask)) {
+            mask >>= 1;
+            ++length;
+        }
+        if (length > (id ? 4u : 8u) || length > end - offset)
+            return false;
+        value = data[offset++] & (id ? 0xff : mask - 1);
+        for (size_t i = 1; i < length; ++i)
+            value = (value << 8) | data[offset++];
+        unknown = !id && value == (uint64_t(1) << (7 * length)) - 1;
+        return true;
+    }
+
+    static bool is_webm_segment_child(uint64_t id) {
+        return id == 0x114D9B74 || id == 0x1549A966 || id == 0x1654AE6B ||
+               id == 0x1F43B675 || id == 0x1C53BB6B || id == 0x1941A469 ||
+               id == 0x1043A770 || id == 0x1254C367;
+    }
+
+    static bool parse_webm_elements(const std::vector<uint8_t>& data, size_t& offset, size_t end, uint64_t parent, bool unknown_cluster, int depth, bool include_raw, json& result, std::string& error) {
+        if (depth > 16) {
+            error = "WebM tag nesting exceeds limit";
+            return false;
+        }
+        std::string tag_name;
+        std::string tag_text;
+        bool has_text = false;
+        while (offset < end) {
+            const auto start = offset;
+            uint64_t id, size;
+            bool unknown;
+            if (!read_ebml_uint(data, offset, end, true, id, unknown)) {
+                error = "invalid or truncated WebM element ID";
+                return false;
+            }
+            // An unknown-size Cluster ends at the next Segment-level element.
+            if (unknown_cluster && is_webm_segment_child(id)) {
+                offset = start;
+                return true;
+            }
+            if (!read_ebml_uint(data, offset, end, false, size, unknown) ||
+                (!unknown && size > end - offset)) {
+                error = "invalid or truncated WebM element size";
+                return false;
+            }
+            if (unknown && id != 0x18538067 && id != 0x1F43B675) {
+                error = "unsupported unknown-size WebM element";
+                return false;
+            }
+            const size_t stop    = unknown ? end : offset + static_cast<size_t>(size);
+            const bool container = (parent == 0 && (id == 0x1A45DFA3 || id == 0x18538067)) ||
+                                   (parent == 0x18538067 && (id == 0x1254C367 || (id == 0x1F43B675 && unknown))) ||
+                                   (parent == 0x1254C367 && id == 0x7373) ||
+                                   ((parent == 0x7373 || parent == 0x67C8) && id == 0x67C8);
+            if (parent == 0 && id == 0x18538067)
+                result["has_segment"] = true;
+            if (parent == 0x1A45DFA3 && id == 0x4282) {
+                const auto type = bytes_to_string(data.data() + offset, data.data() + stop);
+                if (type != "webm" && type != "matroska") {
+                    error = "EBML document is not WebM or Matroska";
+                    return false;
+                }
+                result["doc_type"] = type;
+                result["format"]   = type == "webm" ? "WEBM" : "MATROSKA";
+            }
+            if (parent == 0x67C8 && (id == 0x45A3 || id == 0x4487)) {
+                auto value = bytes_to_string(data.data() + offset, data.data() + stop);
+                if (id == 0x45A3)
+                    tag_name = std::move(value);
+                else {
+                    tag_text = std::move(value);
+                    has_text = true;
+                }
+            }
+            if (parent == 0 || parent == 0x18538067) {
+                std::ostringstream name;
+                name << "0x" << std::hex << std::uppercase << id;
+                json entry = {{"entry_type", "element"}, {"name", name.str()}, {"length", stop - offset}, {"metadata_like", false}};
+                if (unknown)
+                    entry["unknown_size"] = true;
+                append_raw_preview(entry, data.data() + offset, stop - offset, include_raw);
+                result["entries"].push_back(std::move(entry));
+            }
+            if (container) {
+                if (!parse_webm_elements(data, offset, stop, id, id == 0x1F43B675 && unknown,
+                                         depth + 1, include_raw, result, error))
+                    return false;
+            } else {
+                offset = stop;
+            }
+        }
+        if (parent == 0x67C8 && has_text) {
+            result["entries"].push_back({{"entry_type", "tag"}, {"name", tag_name}, {"metadata_like", true}, {"data", {{"text", tag_text}}}});
+        }
+        return true;
+    }
+
+    static bool parse_webm(const std::vector<uint8_t>& data, bool include_raw, json& result, std::string& error) {
+        result        = {{"format", "WEBM"}, {"entries", json::array()}};
+        size_t offset = 0;
+        if (!parse_webm_elements(data, offset, data.size(), 0, false, 0, include_raw, result, error))
+            return false;
+        if (!result.contains("doc_type") || !result.value("has_segment", false)) {
+            error = "missing WebM document type or Segment";
+            return false;
+        }
+        result.erase("has_segment");
+        return true;
+    }
+
     std::string abbreviate(const std::string& value, bool brief) {
         if (!brief || value.size() <= 240) {
             return value;
@@ -1116,7 +1323,7 @@ namespace {
             return;
         }
         if (value.is_primitive()) {
-            out << prefix << key << ": " << value.dump() << "\n";
+            out << prefix << key << ": " << value.dump(-1, ' ', false, json::error_handler_t::replace) << "\n";
             return;
         }
         if (value.is_array()) {
@@ -1125,7 +1332,7 @@ namespace {
                 if (item.is_string()) {
                     out << prefix << "  - " << abbreviate(item.get<std::string>(), brief) << "\n";
                 } else if (item.is_primitive()) {
-                    out << prefix << "  - " << item.dump() << "\n";
+                    out << prefix << "  - " << item.dump(-1, ' ', false, json::error_handler_t::replace) << "\n";
                 } else {
                     out << prefix << "  -\n";
                     for (auto it = item.begin(); it != item.end(); ++it) {
@@ -1158,10 +1365,12 @@ namespace {
         }
 
         for (const auto& entry : entries) {
-            const bool is_chunk    = entry.value("entry_type", "") == "chunk";
-            const std::string name = is_chunk ? entry.value("name", "unknown")
-                                              : entry.value("marker", "unknown");
-            out << (is_chunk ? "Chunk: " : "Segment: ") << name << "\n";
+            const auto type   = entry.value("entry_type", "");
+            const auto name   = entry.value("name", entry.value("marker", "unknown"));
+            const char* label = type == "chunk" ? "Chunk" : type == "tag"   ? "Tag"
+                                                        : type == "element" ? "Element"
+                                                                            : "Segment";
+            out << label << ": " << name << "\n";
             for (auto it = entry.begin(); it != entry.end(); ++it) {
                 if (it.key() == "entry_type" || it.key() == "name" || it.key() == "marker" ||
                     it.key() == "metadata_like") {
@@ -1211,7 +1420,13 @@ namespace {
             return parse_webp(data, include_raw, report, error);
         }
 
-        error = "unsupported image format; only PNG, JPEG, and WebP are supported";
+        if (data.size() >= 12 && memcmp(data.data(), "RIFF", 4) == 0 &&
+            memcmp(data.data() + 8, "AVI ", 4) == 0)
+            return parse_avi(data, include_raw, report, error);
+        if (data.size() >= 4 && read_u32_be(data, 0) == 0x1A45DFA3)
+            return parse_webm(data, include_raw, report, error);
+
+        error = "unsupported media format; only PNG, JPEG, WebP, AVI, WebM, and Matroska are supported";
         return false;
     }
 
@@ -1230,7 +1445,7 @@ bool print_image_metadata(const std::string& image_path,
 
     if (options.output_format == MetadataOutputFormat::JSON) {
         visible_report["file"] = image_path;
-        out << visible_report.dump(2) << "\n";
+        out << visible_report.dump(2, ' ', false, json::error_handler_t::replace) << "\n";
     } else {
         print_text_report(image_path, visible_report, options.include_structural, options.brief, out);
     }

examples/cli/main.cpp

@@ -3,6 +3,7 @@
 #include <time.h>
 #include <algorithm>
 #include <cctype>
+#include <climits>
 #include <filesystem>
 #include <functional>
 #include <iostream>
@@ -33,10 +34,14 @@ const char* previews_str[] = {
 std::regex format_specifier_regex("(?:[^%]|^)(?:%%)*(%\\d{0,3}d)");
 
 struct SDCliParams {
-    SDMode mode             = IMG_GEN;
-    std::string output_path = "output.png";
-    int output_begin_idx    = -1;
-    int compression_quality = 90;
+    std::string input_frames;
+    int video_upscale_factor    = 0;
+    int vsr_window_size         = 16;
+    int vsr_temporal_chunk_size = 5;
+    SDMode mode                 = IMG_GEN;
+    std::string output_path     = "output.png";
+    int output_begin_idx        = -1;
+    int compression_quality     = 90;
     std::string image_path;
     std::string metadata_format = "text";
 
@@ -62,6 +67,7 @@ struct SDCliParams {
         ArgOptions options;
 
         options.string_options = {
+            {"", "--input-frames", "directory of ordered RGB frames for vid_upscale (use -i for a single image)", 0, &input_frames},
             {"-o",
              "--output",
              "path to write result image to. you can use printf-style %d format specifiers for image sequences (default: ./output.png) (eg. output_%03d.png). Single-file video outputs support .avi, .webm, and animated .webp",
@@ -69,7 +75,7 @@ struct SDCliParams {
              &output_path},
             {"",
              "--image",
-             "path to the image to inspect (for metadata mode)",
+             "path to the image or video to inspect (for metadata mode)",
              0,
              &image_path},
             {"",
@@ -90,6 +96,9 @@ struct SDCliParams {
         };
 
         options.int_options = {
+            {"", "--upscale-factor", "required positive video upscale factor (RelayVSR: 2 or 4; SeedVR2: integer)", &video_upscale_factor},
+            {"", "--vsr-window-size", "RelayVSR window including both endpoints (default: 16)", &vsr_window_size},
+            {"", "--vsr-temporal-chunk-size", "SeedVR2 input frames per independent chunk: 1 or 4n+1 (default: 5)", &vsr_temporal_chunk_size},
             {"",
              "--preview-interval",
              "preview interval: in each sampling pass, positive N updates every Nth denoiser step and -N previews only completed logical step N; 0 previews the final completed step of the first pass (base-resolution or high-noise). Default: 1",
@@ -190,7 +199,7 @@ struct SDCliParams {
         options.manual_options = {
             {"-M",
              "--mode",
-             "run mode, one of [img_gen, adetailer, vid_gen, upscale, convert, metadata], default: img_gen",
+             "run mode, one of [img_gen, adetailer, vid_gen, vid_upscale, upscale, convert, metadata], default: img_gen",
              on_mode_arg},
             {"",
              "--preview",
@@ -220,6 +229,10 @@ struct SDCliParams {
     }
 
     bool validate() {
+        if (mode == VID_UPSCALE && video_upscale_factor <= 0) {
+            LOG_ERROR("vid_upscale requires a positive --upscale-factor");
+            return false;
+        }
         if (mode != METADATA) {
             if (output_path.length() == 0) {
                 LOG_ERROR("error: the following arguments are required: output_path");
@@ -227,7 +240,7 @@ struct SDCliParams {
             }
         } else {
             if (image_path.empty()) {
-                LOG_ERROR("error: metadata mode needs an image path (--image)");
+                LOG_ERROR("error: metadata mode needs an image or video path (--image)");
                 return false;
             }
             if (metadata_format != "text" && metadata_format != "json") {
@@ -441,10 +454,11 @@ static fs::path get_video_audio_sidecar_path(const SDCliParams& cli_params) {
 bool save_results(const SDCliParams& cli_params,
                   const SDContextParams& ctx_params,
                   const SDGenerationParams& gen_params,
-                  sd_image_t* results,
+                  SDImageVec& results,
                   int num_results,
-                  const sd_audio_t* generated_audio = nullptr) {
-    if (results == nullptr || num_results <= 0) {
+                  const sd_audio_t* generated_audio,
+                  const std::string& video_info) {
+    if (results.empty() || num_results <= 0) {
         return false;
     }
 
@@ -487,12 +501,10 @@ bool save_results(const SDCliParams& cli_params,
         if (!img.data)
             return false;
 
-        int images_per_batch        = gen_params.batch_count > 0 ? std::max(1, num_results / gen_params.batch_count) : 1;
-        const int64_t metadata_seed = cli_params.mode == VID_GEN ? gen_params.seed : gen_params.seed + idx / images_per_batch;
-        std::string params          = gen_params.embed_image_metadata
-                                          ? get_image_params(ctx_params, gen_params, metadata_seed, cli_params.mode)
-                                          : "";
-        const bool ok               = write_image_to_file(path.u8string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality);
+        std::string params = gen_params.embed_image_metadata
+                                 ? results.generation_info(idx)
+                                 : "";
+        const bool ok      = write_image_to_file(path.u8string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality);
         LOG_INFO("save result image %d to '%s' (%s)", idx, path.u8string().c_str(), ok ? "success" : "failure");
         return ok;
     };
@@ -530,11 +542,13 @@ bool save_results(const SDCliParams& cli_params,
         return sucessful_reults != 0;
     }
 
-    if (cli_params.mode == VID_GEN && num_results > 1) {
+    const bool video_upscale_output = cli_params.mode == VID_UPSCALE &&
+                                      (gen_params.init_image_path.empty() || ext_lower == ".avi" || ext_lower == ".webm");
+    if ((cli_params.mode == VID_GEN && num_results > 1) || video_upscale_output) {
         if (ext_lower != ".avi" && ext_lower != ".webp" && ext_lower != ".webm")
             ext = ".avi";
         std::string params = gen_params.embed_image_metadata
-                                 ? get_image_params(ctx_params, gen_params, gen_params.seed, cli_params.mode)
+                                 ? video_info
                                  : "";
 
         fs::path video_path = base_path;
@@ -542,7 +556,7 @@ bool save_results(const SDCliParams& cli_params,
         std::string final_ext_lower = ext.u8string();
         std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower);
         const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm");
-        if (create_video_from_sd_images(video_path.u8string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) {
+        if (create_video_from_sd_images(video_path.u8string().c_str(), results.data(), num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) {
             LOG_INFO("save result video to '%s'", video_path.u8string().c_str());
             if (generated_audio != nullptr && !mux_audio) {
                 fs::path wav_path = video_path;
@@ -607,30 +621,29 @@ static bool apply_adetailer(sd_ctx_t* sd_ctx,
             continue;
         }
         sd_img_gen_params_t ad_generation_params = img_gen_params;
-        ad_generation_params.seed                = img_gen_params.seed + i;
+        ad_generation_params.seed                = generation_seed(results.generation_info(i), img_gen_params.seed);
         if (mode == IMG_GEN) {
             ad_generation_params.width    = 512;
             ad_generation_params.height   = 512;
             ad_generation_params.strength = 0.4f;
         }
-        sd_image_t* detailed_images = nullptr;
-        int detailed_count          = 0;
+        sd_image_generation_result_t* detailed = nullptr;
         if (!adetail_image(ad_ctx.get(),
                            sd_ctx,
                            results[i],
                            &ad_params,
                            &ad_generation_params,
-                           &detailed_images,
-                           &detailed_count) ||
-            detailed_count <= 0 || detailed_images == nullptr || detailed_images[0].data == nullptr) {
-            free_sd_images(detailed_images, detailed_count);
+                           &detailed) ||
+            detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) {
+            free_sd_image_generation_result(detailed);
             LOG_ERROR("ADetailer failed for image %d", i + 1);
             return false;
         }
+        results.append_generation_info(i, detailed->items[0].generation_info, ctx_params.lora_model_dir);
         free(results[i].data);
-        results[i]         = detailed_images[0];
-        detailed_images[0] = {0, 0, 0, nullptr};
-        free_sd_images(detailed_images, detailed_count);
+        results[i]               = detailed->items[0].image;
+        detailed->items[0].image = {};
+        free_sd_image_generation_result(detailed);
     }
     return true;
 }
@@ -666,6 +679,72 @@ int main(int argc, const char* argv[]) {
         return 0;
     }
 
+    if (cli_params.mode == VID_UPSCALE) {
+        if (cli_params.input_frames.empty() == gen_params.init_image_path.empty()) {
+            LOG_ERROR("vid_upscale requires exactly one input: -i/--init-img or --input-frames");
+            return 1;
+        }
+        ctx_params.prepare_backend_assignments();
+        if (ctx_params.diffusion_model_path.empty() || ctx_params.vae_path.empty() || gen_params.fps <= 0) {
+            LOG_ERROR("vid_upscale requires --diffusion-model, --vae and positive --fps");
+            return 1;
+        }
+        std::vector<SDImageOwner> owned_frames;
+        if (!gen_params.init_image_path.empty()) {
+            owned_frames.emplace_back();
+            if (!load_sd_image_from_file(owned_frames.back().put(), gen_params.init_image_path.c_str(), 0, 0, 3)) {
+                LOG_ERROR("load image from '%s' failed", gen_params.init_image_path.c_str());
+                return 1;
+            }
+        } else if (!load_images_from_dir(cli_params.input_frames, owned_frames) || owned_frames.empty()) {
+            LOG_ERROR("vid_upscale input contains no readable frames");
+            return 1;
+        }
+        if (owned_frames.size() > size_t(INT_MAX))
+            return 1;
+        std::vector<sd_image_t> frames;
+        for (const auto& frame : owned_frames)
+            frames.push_back(frame.get());
+        sd_video_upscaler_ctx_params_t params;
+        sd_video_upscaler_ctx_params_init(&params);
+        params.diffusion_model_path  = ctx_params.diffusion_model_path.c_str();
+        params.vae_path              = ctx_params.vae_path.c_str();
+        params.upscale_factor        = cli_params.video_upscale_factor;
+        params.n_threads             = ctx_params.n_threads;
+        params.backend               = ctx_params.effective_backend.c_str();
+        params.params_backend        = ctx_params.effective_params_backend.c_str();
+        params.flash_attn_mode       = ctx_params.flash_attn_mode.c_str();
+        std::string conv_direct_mode = ctx_params.conv_direct_mode;
+        if (conv_direct_mode.empty()) {
+            conv_direct_mode = "auto";
+            if (ctx_params.diffusion_conv_direct)
+                conv_direct_mode += ",diffusion=on";
+            if (ctx_params.vae_conv_direct)
+                conv_direct_mode += ",vae=on";
+        }
+        params.conv_direct_mode = conv_direct_mode.c_str();
+        std::unique_ptr<video_upscaler_ctx_t, decltype(&free_video_upscaler_ctx)> upscaler(
+            new_video_upscaler_ctx(&params),
+            free_video_upscaler_ctx);
+        if (!upscaler)
+            return 1;
+        sd_video_generation_result_t* output = nullptr;
+        sd_video_upscale_params_t upscale_params;
+        sd_video_upscale_params_init(&upscale_params);
+        upscale_params.frames              = frames.data();
+        upscale_params.frame_count         = int(frames.size());
+        upscale_params.fps                 = gen_params.fps;
+        upscale_params.window_size         = cli_params.vsr_window_size;
+        upscale_params.temporal_chunk_size = cli_params.vsr_temporal_chunk_size;
+        upscale_params.seed                = gen_params.seed;
+        if (!upscale_video(upscaler.get(), &upscale_params, &output))
+            return 1;
+        std::string video_info = output->generation_info;
+        SDImageVec results;
+        results.adopt_video(output);
+        return save_results(cli_params, ctx_params, gen_params, results, int(results.size()), nullptr, video_info) ? 0 : 1;
+    }
+
     if (!gen_params.ad_model_path.empty() && cli_params.mode != IMG_GEN && cli_params.mode != ADETAILER) {
         LOG_ERROR("--ad-model is only supported in image generation and adetailer modes");
         return 1;
@@ -884,6 +963,7 @@ int main(int argc, const char* argv[]) {
     SDImageVec results;
     int num_results             = 0;
     sd_audio_t* generated_audio = nullptr;
+    std::string video_info;
 
     if (cli_params.mode == UPSCALE) {
         num_results = 1;
@@ -896,18 +976,6 @@ int main(int argc, const char* argv[]) {
             return 1;
         }
 
-        if (gen_params.sample_params.sample_method == SAMPLE_METHOD_COUNT) {
-            gen_params.sample_params.sample_method = sd_get_default_sample_method(sd_ctx.get());
-        }
-
-        if (gen_params.high_noise_sample_params.sample_method == SAMPLE_METHOD_COUNT) {
-            gen_params.high_noise_sample_params.sample_method = sd_get_default_sample_method(sd_ctx.get());
-        }
-
-        if (gen_params.sample_params.scheduler == SCHEDULER_COUNT) {
-            gen_params.sample_params.scheduler = sd_get_default_scheduler(sd_ctx.get(), gen_params.sample_params.sample_method);
-        }
-
         sd_img_gen_params_t img_gen_params{};
         const bool use_img_gen_params = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER;
         if (use_img_gen_params) {
@@ -915,23 +983,24 @@ int main(int argc, const char* argv[]) {
         }
 
         if (cli_params.mode == IMG_GEN) {
-            sd_image_t* generated_images = nullptr;
-            if (!generate_image(sd_ctx.get(), &img_gen_params, &generated_images, &num_results)) {
-                generated_images = nullptr;
-                num_results      = 0;
-            }
-            results.adopt(generated_images, num_results);
+            sd_image_generation_result_t* generated = nullptr;
+            generate_image(sd_ctx.get(), &img_gen_params, &generated);
+            results.adopt(generated, ctx_params.lora_model_dir);
+            num_results = results.count();
         } else if (cli_params.mode == ADETAILER) {
             num_results = 1;
             results.push_back(gen_params.init_image.release());
         } else if (cli_params.mode == VID_GEN) {
-            sd_vid_gen_params_t vid_gen_params = gen_params.to_sd_vid_gen_params_t();
-            sd_image_t* generated_video        = nullptr;
-            if (!generate_video(sd_ctx.get(), &vid_gen_params, &generated_video, &num_results, &generated_audio, &cli_params.preview_fps)) {
-                generated_video = nullptr;
+            sd_vid_gen_params_t vid_gen_params      = gen_params.to_sd_vid_gen_params_t();
+            sd_video_generation_result_t* generated = nullptr;
+            if (generate_video(sd_ctx.get(), &vid_gen_params, &generated)) {
+                gen_params.fps = cli_params.preview_fps = generated->fps;
+                generated_audio                         = generated->audio;
+                generated->audio                        = nullptr;
+                num_results                             = generated->frame_count;
+                results.adopt_video(generated, ctx_params.lora_model_dir);
+                video_info = results.generation_info(0);
             }
-            gen_params.fps = cli_params.preview_fps;
-            results.adopt(generated_video, num_results);
         }
 
         if (!results) {
@@ -971,21 +1040,20 @@ int main(int argc, const char* argv[]) {
                 SDImageOwner current_image(results[i]);
                 results[i] = {0, 0, 0, nullptr};
                 for (int u = 0; u < gen_params.upscale_repeats; ++u) {
-                    sd_image_t* upscaled_images = nullptr;
-                    int upscaled_count          = 0;
-                    bool upscale_ok             = upscale(upscaler_ctx.get(),
-                                                          current_image.get(),
-                                                          upscale_factor,
-                                                          &upscaled_images,
-                                                          &upscaled_count);
-                    if (!upscale_ok || upscaled_count <= 0 || upscaled_images[0].data == nullptr) {
-                        free_sd_images(upscaled_images, upscaled_count);
+                    sd_image_generation_result_t* upscaled = nullptr;
+                    bool upscale_ok                        = upscale(upscaler_ctx.get(),
+                                                                     current_image.get(),
+                                                                     upscale_factor,
+                                                                     &upscaled);
+                    if (!upscale_ok || !upscaled || upscaled->count <= 0 || upscaled->items[0].image.data == nullptr) {
+                        free_sd_image_generation_result(upscaled);
                         LOG_ERROR("upscale failed");
                         break;
                     }
-                    sd_image_t upscaled_image = upscaled_images[0];
-                    upscaled_images[0]        = {0, 0, 0, nullptr};
-                    free_sd_images(upscaled_images, upscaled_count);
+                    results.append_generation_info(i, upscaled->items[0].generation_info);
+                    sd_image_t upscaled_image = upscaled->items[0].image;
+                    upscaled->items[0].image  = {};
+                    free_sd_image_generation_result(upscaled);
                     current_image.reset(upscaled_image);
                 }
                 results[i] = current_image.release();  // Set the final upscaled image as the result
@@ -993,7 +1061,9 @@ int main(int argc, const char* argv[]) {
         }
     }
 
-    if (!save_results(cli_params, ctx_params, gen_params, results.data(), num_results, generated_audio)) {
+    if (!video_info.empty() && results.count() > 0)
+        video_info = results.generation_info(0);
+    if (!save_results(cli_params, ctx_params, gen_params, results, num_results, generated_audio, video_info)) {
         free_sd_audio(generated_audio);
         return 1;
     }

examples/common/common.cpp

@@ -35,6 +35,7 @@ const char* const modes_str[] = {
     "convert",
     "upscale",
     "metadata",
+    "vid_upscale",
 };
 
 static sd_vae_format_t str_to_vae_format(const std::string& value) {
@@ -464,7 +465,7 @@ ArgOptions SDContextParams::get_options() {
          &uncond_diffusion_model_path},
         {"",
          "--embeddings-connectors",
-         "path to LTXAV embeddings connectors",
+         "path to embeddings connectors (LTXAV or P-H3-Image)",
          0,
          &embeddings_connectors_path},
         {"",
@@ -818,7 +819,7 @@ ArgOptions SDContextParams::get_options() {
          on_sampler_rng_arg},
         {"",
          "--prediction",
-         "prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow, sefi_flow]",
+         "prediction type override, one of [eps, v, edm_v, sd3_flow, flux_flow, sefi_flow, minit2i_flow, sensenova_u1_flow, flux3_flow, h3_av_flow, ming_image_flow]",
          on_prediction_arg},
         {"",
          "--lora-apply-mode",
@@ -1135,7 +1136,7 @@ ArgOptions SDGenerationParams::get_options() {
          &extra_ad_args},
         {"-i",
          "--init-img",
-         "path to the init image",
+         "path to the init image, or a single input image for vid_upscale",
          0,
          &init_image_path},
         {"",
@@ -1187,6 +1188,13 @@ ArgOptions SDGenerationParams::get_options() {
          "under --hires-upscalers-dir (default: Latent)",
          0,
          &hires_upscaler},
+        {"",
+         "--control-type",
+         "kind of the control image, one of [openpose, depth, scribble, canny, normal, segment, tile, repaint]. "
+         "Required by an SDXL Control Net Union model (tile and repaint only exist in its ProMax variant), ignored by any other Control Net. "
+         "Also accepted: hed, softedge, pidi, ted (scribble); lineart, anime_lineart, mlsd (canny); seg (segment); inpaint, outpaint (repaint)",
+         0,
+         &control_type},
         {"",
          "--extra-sample-args",
          "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware; llada_image supports uniform; lms supports lms_max_order, lms_shift, lms_divisions; noise-injecting samplers support noise_sampler with value iid (default except for dpm++2m_sde_bt) or brownian_tree; brownian_tree_rng supports cpu (default), cuda, std_default or sampler_rng",
@@ -1194,7 +1202,7 @@ ArgOptions SDGenerationParams::get_options() {
          &extra_sample_args},
         {"",
          "--extra-tiling-args",
-         "extra VAE tiling args, key=value list. Supported video VAEs accept temporal_tile_frames/temporal_tile_size (default: 4), temporal_tile_overlap (default: 1)",
+         "extra VAE tiling args, key=value list. Supported video VAEs accept temporal_tile_frames/temporal_tile_size (default: 4), temporal_tile_overlap (default: 1); Wan2.2 TI2V 5B accepts temporal_decode_frames (default: 1)",
          (int)',',
          &extra_tiling_args},
         {"",
@@ -1530,6 +1538,14 @@ ArgOptions SDGenerationParams::get_options() {
         std::string sigmas_str = value;
         if (sigmas_str == "qwen-image-2.1-turbo") {
             sigmas_str = "1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0";
+        } else if (sigmas_str == "p-h3-image") {
+            sigmas_str = "1.0,0.984753,0.967905,0.951688,0.925372,0.881923,0.810083,0.528718,0.0";
+        } else if (sigmas_str == "hyperflow-h3-flash-4") {
+            sigmas_str = "1.0,0.962575853,0.755781889,0.312497854,0.0";
+        } else if (sigmas_str == "hyperflow-h3-flash-6") {
+            sigmas_str = "1.0,0.982655942,0.914642572,0.674755692,0.385223746,0.159055337,0.0";
+        } else if (sigmas_str == "hyperflow-h3-flash-8") {
+            sigmas_str = "1.0,0.993909776,0.9842875,0.966063678,0.923076928,0.834942341,0.696852028,0.468753338,0.0";
         }
         if (!sigmas_str.empty() && sigmas_str.front() == '[') {
             sigmas_str.erase(0, 1);
@@ -1777,11 +1793,11 @@ ArgOptions SDGenerationParams::get_options() {
          on_scheduler_arg},
         {"",
          "--sigmas",
-         "custom sigma values for the sampler, comma-separated (e.g., \"14.61,7.8,3.5,0.0\") or preset: qwen-image-2.1-turbo.",
+         "custom sigma values for the sampler, comma-separated (e.g., \"14.61,7.8,3.5,0.0\") or preset: qwen-image-2.1-turbo, p-h3-image, hyperflow-h3-flash-4, hyperflow-h3-flash-6, hyperflow-h3-flash-8.",
          on_sigmas_arg},
         {"",
          "--hires-sigmas",
-         "custom sigma values for the highres fix second pass, comma-separated (e.g., \"0.85,0.725,0.421875,0.0\") or preset: qwen-image-2.1-turbo.",
+         "custom sigma values for the highres fix second pass, comma-separated (e.g., \"0.85,0.725,0.421875,0.0\") or preset: qwen-image-2.1-turbo, p-h3-image, hyperflow-h3-flash-4, hyperflow-h3-flash-6, hyperflow-h3-flash-8.",
          on_hires_sigmas_arg},
         {"",
          "--skip-layers",
@@ -1959,6 +1975,7 @@ static bool parse_image_json_field(const json& parent,
     return decode_base64_image(parent.at(key).get<std::string>(), channels, 0, 0, out_image);
 }
 
+/** Parses a JSON array of base64 image strings into owned images. */
 static bool parse_image_array_json_field(const json& parent,
                                          const char* key,
                                          int channels,
@@ -1988,6 +2005,84 @@ static bool parse_image_array_json_field(const json& parent,
     return true;
 }
 
+/** Parses a JSON array of base64 WAV strings into owned audio clips. */
+static bool parse_audio_array_json_field(const json& parent,
+                                         const char* key,
+                                         std::vector<SDAudioOwner>& out_audios) {
+    if (!parent.contains(key)) {
+        return true;
+    }
+    if (parent.at(key).is_null()) {
+        out_audios.clear();
+        return true;
+    }
+    if (!parent.at(key).is_array()) {
+        return false;
+    }
+
+    out_audios.clear();
+    for (const auto& item : parent.at(key)) {
+        if (!item.is_string()) {
+            return false;
+        }
+        std::string encoded = item.get<std::string>();
+        auto comma_pos      = encoded.find(',');
+        if (comma_pos != std::string::npos) {
+            encoded = encoded.substr(comma_pos + 1);
+        }
+        std::vector<uint8_t> audio_bytes = decode_base64_bytes(encoded);
+        if (audio_bytes.empty()) {
+            return false;
+        }
+        std::vector<float> samples;
+        uint32_t sample_rate = 0;
+        uint32_t channels    = 0;
+        if (!load_wav_from_memory(audio_bytes.data(), audio_bytes.size(), samples, sample_rate, channels)) {
+            return false;
+        }
+        SDAudioOwner audio;
+        audio.reset(std::move(samples), sample_rate, channels);
+        out_audios.push_back(std::move(audio));
+    }
+    return true;
+}
+
+/** Parses a JSON array of per-video base64 frame arrays into owned videos. */
+static bool parse_video_array_json_field(const json& parent,
+                                         const char* key,
+                                         std::vector<std::vector<SDImageOwner>>& out_videos) {
+    if (!parent.contains(key)) {
+        return true;
+    }
+    if (parent.at(key).is_null()) {
+        out_videos.clear();
+        return true;
+    }
+    if (!parent.at(key).is_array()) {
+        return false;
+    }
+
+    out_videos.clear();
+    for (const auto& video : parent.at(key)) {
+        if (!video.is_array() || video.empty()) {
+            return false;
+        }
+        std::vector<SDImageOwner> frames;
+        for (const auto& frame : video) {
+            if (!frame.is_string()) {
+                return false;
+            }
+            SDImageOwner image;
+            if (!decode_base64_image(frame.get<std::string>(), 0, 0, 0, image)) {
+                return false;
+            }
+            frames.push_back(std::move(image));
+        }
+        out_videos.push_back(std::move(frames));
+    }
+    return true;
+}
+
 static bool parse_lora_json_field(const json& parent,
                                   const std::function<std::string(const std::string&)>& lora_path_resolver,
                                   std::map<std::string, float>& lora_map,
@@ -2102,6 +2197,7 @@ bool SDGenerationParams::parse_image_preprocess_json(const std::string& json_str
     return true;
 }
 
+/** Populates generation params from a JSON payload (server APIs); returns false on any malformed field. */
 bool SDGenerationParams::from_json_str(
     const std::string& json_str,
     const std::function<std::string(const std::string&)>& lora_path_resolver) {
@@ -2167,6 +2263,7 @@ bool SDGenerationParams::from_json_str(
 
     load_if_exists("strength", strength);
     load_if_exists("control_strength", control_strength);
+    load_if_exists("control_type", control_type);
     load_if_exists("ip_adapter_strength", ip_adapter_strength);
     load_if_exists("moe_boundary", moe_boundary);
     load_if_exists("vace_strength", vace_strength);
@@ -2330,8 +2427,11 @@ bool SDGenerationParams::from_json_str(
         return false;
     }
     if (!parse_image_array_json_field(j, "ref_images", 0, ref_images) ||
-        !parse_image_array_json_field(j, "control_frames", 3, control_frames)) {
-        LOG_ERROR("invalid input image array");
+        !parse_image_array_json_field(j, "control_frames", 3, control_frames) ||
+        !parse_audio_array_json_field(j, "ref_audios", ref_audios) ||
+        !parse_audio_array_json_field(j, "ref_video_audios", ref_video_audios) ||
+        !parse_video_array_json_field(j, "ref_videos", ref_videos)) {
+        LOG_ERROR("invalid input image/audio/video array");
         return false;
     }
 
@@ -2548,16 +2648,20 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s
         return false;
     }
 
-    if (seed < 0) {
-        srand((int)time(nullptr));
-        seed = rand();
-    }
-
     if (strict) {
         batch_count                = std::clamp(batch_count, 1, 8);
         sample_params.sample_steps = std::clamp(sample_params.sample_steps, 1, 100);
     }
 
+    resolved_control_type = SD_CONTROL_TYPE_NONE;
+    if (!control_type.empty()) {
+        resolved_control_type = str_to_sd_control_type(control_type.c_str());
+        if (resolved_control_type == SD_CONTROL_TYPE_COUNT) {
+            LOG_ERROR("invalid control type '%s' (openpose, depth, scribble, canny, normal, segment, tile, repaint)", control_type.c_str());
+            return false;
+        }
+    }
+
     hires_upscaler_model_path.clear();
     if (hires_enabled) {
         if (hires_upscaler.empty()) {
@@ -2588,6 +2692,7 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s
     return true;
 }
 
+/** Validates params for a generation mode; enforces mode-specific input rules. */
 bool SDGenerationParams::validate(SDMode mode) {
     if (!image_preprocess.empty() && mode != IMG_GEN && mode != VID_GEN) {
         LOG_ERROR("--image-preprocess requires img_gen or vid_gen mode");
@@ -2641,12 +2746,13 @@ bool SDGenerationParams::validate(SDMode mode) {
         return false;
     }
 
-    if (ref_video_audio_paths.size() > ref_video_paths.size()) {
+    if (ref_video_audio_paths.size() + ref_video_audios.size() > ref_video_paths.size() + ref_videos.size()) {
         LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video");
         return false;
     }
 
-    if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty())) {
+    if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty() ||
+                            !ref_videos.empty() || !ref_video_audios.empty() || !ref_audios.empty())) {
         LOG_ERROR("error: reference video and audio inputs require vid_gen mode");
         return false;
     }
@@ -2795,6 +2901,7 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() {
     params.qwen_image_layers   = qwen_image_layers;
     params.control_image       = control_image.get();
     params.control_strength    = control_strength;
+    params.control_type        = resolved_control_type;
     params.ip_adapter_image    = ip_adapter_image.get();
     params.ip_adapter_strength = ip_adapter_strength;
     params.pm_params           = pm_params;
@@ -2919,6 +3026,7 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
     params.circular_x                = circular || circular_x;
     params.circular_y                = circular || circular_y;
     params.image_preprocess          = {image_preprocess.c_str()};
+    params.ref_image_args            = ref_image_args.c_str();
     return params;
 }
 
@@ -2999,6 +3107,7 @@ std::string SDGenerationParams::to_string() const {
         << "  vace_strength: " << vace_strength << ",\n"
         << "  strength: " << strength << ",\n"
         << "  control_strength: " << control_strength << ",\n"
+        << "  control_type: \"" << control_type << "\",\n"
         << "  seed: " << seed << ",\n"
         << "  upscale_repeats: " << upscale_repeats << ",\n"
         << "  upscale_tile_size: " << upscale_tile_size << ",\n"
@@ -3028,301 +3137,3 @@ std::string SDGenerationParams::to_string() const {
 std::string version_string() {
     return std::string("stable-diffusion.cpp version ") + sd_version() + ", commit " + sd_commit();
 }
-
-static std::string safe_json_string(const char* value) {
-    return value ? value : "";
-}
-
-static void set_json_basename_if_not_empty(json& target, const char* key, const std::string& path) {
-    if (!path.empty()) {
-        target[key] = sd_basename(path);
-    }
-}
-
-static json build_sampling_metadata_json(const sd_sample_params_t& sample_params,
-                                         const std::vector<int>& skip_layers,
-                                         const std::vector<float>* custom_sigmas = nullptr) {
-    json sampling = {
-        {"steps", sample_params.sample_steps},
-        {"eta", sample_params.eta},
-        {"shifted_timestep", sample_params.shifted_timestep},
-        {"flow_shift", sample_params.flow_shift},
-        {"extra_sample_args", safe_json_string(sample_params.extra_sample_args)},
-        {"guidance",
-         {
-             {"txt_cfg", sample_params.guidance.txt_cfg},
-             {"img_cfg", sample_params.guidance.img_cfg},
-             {"distilled_guidance", sample_params.guidance.distilled_guidance},
-             {"slg",
-              {
-                  {"scale", sample_params.guidance.slg.scale},
-                  {"layers", skip_layers},
-                  {"start", sample_params.guidance.slg.layer_start},
-                  {"end", sample_params.guidance.slg.layer_end},
-              }},
-         }},
-    };
-    if (sample_params.sample_method != SAMPLE_METHOD_COUNT) {
-        sampling["method"] = safe_json_string(sd_sample_method_name(sample_params.sample_method));
-    }
-    if (sample_params.scheduler != SCHEDULER_COUNT) {
-        sampling["scheduler"] = safe_json_string(sd_scheduler_name(sample_params.scheduler));
-    }
-    if (custom_sigmas != nullptr) {
-        sampling["custom_sigmas"] = *custom_sigmas;
-    }
-    return sampling;
-}
-
-std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
-                                            const SDGenerationParams& gen_params,
-                                            int64_t seed,
-                                            SDMode mode) {
-    json root;
-    root["schema"] = "sdcpp.image.params/v1";
-    root["mode"]   = "img_gen";
-    if (mode == VID_GEN) {
-        root["mode"] = "vid_gen";
-    } else if (mode == ADETAILER) {
-        root["mode"] = "adetailer";
-    }
-    root["generator"] = {
-        {"name", "stable-diffusion.cpp"},
-        {"version", safe_json_string(sd_version())},
-        {"commit", safe_json_string(sd_commit())},
-    };
-    root["seed"]   = seed;
-    root["width"]  = gen_params.get_resolved_width();
-    root["height"] = gen_params.get_resolved_height();
-
-    root["prompt"] = {
-        {"positive", gen_params.prompt},
-        {"negative", gen_params.negative_prompt},
-    };
-    if (!gen_params.ad_model_path.empty()) {
-        root["adetailer"] = {
-            {"model", sd_basename(gen_params.ad_model_path)},
-            {"prompt", gen_params.ad_prompt},
-            {"negative_prompt", gen_params.ad_negative_prompt},
-            {"extra_args", gen_params.extra_ad_args},
-        };
-    }
-    root["sampling"] = build_sampling_metadata_json(gen_params.sample_params,
-                                                    gen_params.skip_layers,
-                                                    &gen_params.custom_sigmas);
-
-    json models;
-    set_json_basename_if_not_empty(models, "model", ctx_params.model_path);
-    set_json_basename_if_not_empty(models, "clip_l", ctx_params.clip_l_path);
-    set_json_basename_if_not_empty(models, "clip_g", ctx_params.clip_g_path);
-    set_json_basename_if_not_empty(models, "clip_vision", ctx_params.clip_vision_path);
-    set_json_basename_if_not_empty(models, "t5xxl", ctx_params.t5xxl_path);
-    set_json_basename_if_not_empty(models, "llm", ctx_params.llm_path);
-    set_json_basename_if_not_empty(models, "llm_vision", ctx_params.llm_vision_path);
-    set_json_basename_if_not_empty(models, "diffusion_model", ctx_params.diffusion_model_path);
-    set_json_basename_if_not_empty(models, "high_noise_diffusion_model", ctx_params.high_noise_diffusion_model_path);
-    set_json_basename_if_not_empty(models, "uncond_diffusion_model", ctx_params.uncond_diffusion_model_path);
-    set_json_basename_if_not_empty(models, "vae", ctx_params.vae_path);
-    set_json_basename_if_not_empty(models, "taesd", ctx_params.taesd_path);
-    set_json_basename_if_not_empty(models, "control_net", ctx_params.control_net_path);
-    root["models"] = std::move(models);
-
-    root["clip_skip"]           = gen_params.clip_skip;
-    root["strength"]            = gen_params.strength;
-    root["control_strength"]    = gen_params.control_strength;
-    root["ip_adapter_strength"] = gen_params.ip_adapter_strength;
-    root["ref_image_args"]      = gen_params.ref_image_args;
-    root["image_preprocess"]    = gen_params.image_preprocess;
-    root["increase_ref_index"]  = gen_params.increase_ref_index;
-    if (mode == VID_GEN) {
-        root["video"] = {
-            {"frame_count", gen_params.video_frames},
-            {"fps", gen_params.fps},
-        };
-        root["moe_boundary"]        = gen_params.moe_boundary;
-        root["vace_strength"]       = gen_params.vace_strength;
-        root["high_noise_sampling"] = build_sampling_metadata_json(gen_params.high_noise_sample_params,
-                                                                   gen_params.high_noise_skip_layers);
-    }
-
-    root["rng"] = safe_json_string(sd_rng_type_name(ctx_params.rng_type));
-    if (ctx_params.sampler_rng_type != RNG_TYPE_COUNT) {
-        root["sampler_rng"] = safe_json_string(sd_rng_type_name(ctx_params.sampler_rng_type));
-    }
-
-    json loras = json::array();
-    for (const auto& entry : gen_params.lora_map) {
-        loras.push_back({
-            {"name", sd_basename(entry.first)},
-            {"multiplier", entry.second},
-            {"is_high_noise", false},
-        });
-    }
-    for (const auto& entry : gen_params.high_noise_lora_map) {
-        loras.push_back({
-            {"name", sd_basename(entry.first)},
-            {"multiplier", entry.second},
-            {"is_high_noise", true},
-        });
-    }
-    if (!loras.empty()) {
-        root["loras"] = std::move(loras);
-    }
-
-    if (gen_params.hires_enabled) {
-        root["hires"] = {
-            {"enabled", gen_params.hires_enabled},
-            {"upscaler", gen_params.hires_upscaler},
-            {"model", gen_params.hires_upscaler_model_path.empty() ? "" : sd_basename(gen_params.hires_upscaler_model_path)},
-            {"scale", gen_params.hires_scale},
-            {"target_width", gen_params.hires_width},
-            {"target_height", gen_params.hires_height},
-            {"steps", gen_params.hires_steps},
-            {"denoising_strength", gen_params.hires_denoising_strength},
-            {"custom_sigmas", gen_params.hires_custom_sigmas},
-            {"upscale_tile_size", gen_params.hires_upscale_tile_size},
-        };
-    }
-
-    if (gen_params.cache_params.mode != SD_CACHE_DISABLED) {
-        root["cache"] = {
-            {"requested_mode", gen_params.cache_mode},
-            {"requested_option", gen_params.cache_option},
-            {"mode", gen_params.cache_params.mode},
-            {"scm_mask", gen_params.scm_mask},
-            {"scm_policy_dynamic", gen_params.scm_policy_dynamic},
-            {"reuse_threshold", gen_params.cache_params.reuse_threshold},
-            {"start_percent", gen_params.cache_params.start_percent},
-            {"end_percent", gen_params.cache_params.end_percent},
-            {"error_decay_rate", gen_params.cache_params.error_decay_rate},
-            {"use_relative_threshold", gen_params.cache_params.use_relative_threshold},
-            {"reset_error_on_compute", gen_params.cache_params.reset_error_on_compute},
-            {"Fn_compute_blocks", gen_params.cache_params.Fn_compute_blocks},
-            {"Bn_compute_blocks", gen_params.cache_params.Bn_compute_blocks},
-            {"residual_diff_threshold", gen_params.cache_params.residual_diff_threshold},
-            {"max_warmup_steps", gen_params.cache_params.max_warmup_steps},
-            {"max_cached_steps", gen_params.cache_params.max_cached_steps},
-            {"max_continuous_cached_steps", gen_params.cache_params.max_continuous_cached_steps},
-            {"taylorseer_n_derivatives", gen_params.cache_params.taylorseer_n_derivatives},
-            {"taylorseer_skip_interval", gen_params.cache_params.taylorseer_skip_interval},
-            {"spectrum_w", gen_params.cache_params.spectrum_w},
-            {"spectrum_m", gen_params.cache_params.spectrum_m},
-            {"spectrum_lam", gen_params.cache_params.spectrum_lam},
-            {"spectrum_window_size", gen_params.cache_params.spectrum_window_size},
-            {"spectrum_flex_window", gen_params.cache_params.spectrum_flex_window},
-            {"spectrum_warmup_steps", gen_params.cache_params.spectrum_warmup_steps},
-            {"spectrum_stop_percent", gen_params.cache_params.spectrum_stop_percent},
-        };
-    }
-
-    if (gen_params.vae_tiling_params.enabled ||
-        gen_params.vae_tiling_params.temporal_tiling ||
-        !gen_params.extra_tiling_args.empty()) {
-        root["vae_tiling"] = {
-            {"enabled", gen_params.vae_tiling_params.enabled},
-            {"temporal_tiling", gen_params.vae_tiling_params.temporal_tiling},
-            {"tile_size_w", gen_params.vae_tiling_params.tile_size_w},
-            {"tile_size_h", gen_params.vae_tiling_params.tile_size_h},
-            {"target_overlap", gen_params.vae_tiling_params.target_overlap},
-            {"rel_size_w", gen_params.vae_tiling_params.rel_size_w},
-            {"rel_size_h", gen_params.vae_tiling_params.rel_size_h},
-            {"extra_tiling_args", gen_params.extra_tiling_args},
-        };
-    }
-
-    return root.dump();
-}
-
-std::string get_image_params(const SDContextParams& ctx_params,
-                             const SDGenerationParams& gen_params,
-                             int64_t seed,
-                             SDMode mode) {
-    std::string parameter_string;
-    if (gen_params.prompt_with_lora.size() != 0) {
-        parameter_string += gen_params.prompt_with_lora + "\n";
-    } else {
-        parameter_string += gen_params.prompt + "\n";
-    }
-    if (gen_params.negative_prompt.size() != 0) {
-        parameter_string += "Negative prompt: " + gen_params.negative_prompt + "\n";
-    }
-    parameter_string += "Steps: " + std::to_string(gen_params.sample_params.sample_steps) + ", ";
-    parameter_string += "CFG scale: " + std::to_string(gen_params.sample_params.guidance.txt_cfg) + ", ";
-    if (gen_params.sample_params.guidance.slg.scale != 0 && gen_params.skip_layers.size() != 0) {
-        parameter_string += "SLG scale: " + std::to_string(gen_params.sample_params.guidance.slg.scale) + ", ";
-        parameter_string += "Skip layers: [";
-        for (const auto& layer : gen_params.skip_layers) {
-            parameter_string += std::to_string(layer) + ", ";
-        }
-        parameter_string += "], ";
-        parameter_string += "Skip layer start: " + std::to_string(gen_params.sample_params.guidance.slg.layer_start) + ", ";
-        parameter_string += "Skip layer end: " + std::to_string(gen_params.sample_params.guidance.slg.layer_end) + ", ";
-    }
-    parameter_string += "Guidance: " + std::to_string(gen_params.sample_params.guidance.distilled_guidance) + ", ";
-    parameter_string += "Eta: " + std::to_string(gen_params.sample_params.eta) + ", ";
-    if (!gen_params.extra_sample_args.empty()) {
-        parameter_string += "Extra sample args: " + gen_params.extra_sample_args + ", ";
-    }
-    if (!gen_params.ad_model_path.empty()) {
-        parameter_string += "ADetailer model: " + sd_basename(gen_params.ad_model_path) + ", ";
-        if (!gen_params.ad_prompt.empty()) {
-            parameter_string += "ADetailer prompt: " + gen_params.ad_prompt + ", ";
-        }
-        if (!gen_params.ad_negative_prompt.empty()) {
-            parameter_string += "ADetailer negative prompt: " + gen_params.ad_negative_prompt + ", ";
-        }
-        if (!gen_params.extra_ad_args.empty()) {
-            parameter_string += "ADetailer args: " + gen_params.extra_ad_args + ", ";
-        }
-    }
-    parameter_string += "Seed: " + std::to_string(seed) + ", ";
-    parameter_string += "Size: " + std::to_string(gen_params.get_resolved_width()) + "x" + std::to_string(gen_params.get_resolved_height()) + ", ";
-    parameter_string += "Model: " + sd_basename(ctx_params.model_path) + ", ";
-    parameter_string += "RNG: " + std::string(sd_rng_type_name(ctx_params.rng_type)) + ", ";
-    if (ctx_params.sampler_rng_type != RNG_TYPE_COUNT) {
-        parameter_string += "Sampler RNG: " + std::string(sd_rng_type_name(ctx_params.sampler_rng_type)) + ", ";
-    }
-    parameter_string += "Sampler: " + std::string(sd_sample_method_name(gen_params.sample_params.sample_method));
-    if (!gen_params.custom_sigmas.empty()) {
-        parameter_string += ", Custom Sigmas: [";
-        for (size_t i = 0; i < gen_params.custom_sigmas.size(); ++i) {
-            std::ostringstream oss;
-            oss << std::fixed << std::setprecision(4) << gen_params.custom_sigmas[i];
-            parameter_string += oss.str() + (i == gen_params.custom_sigmas.size() - 1 ? "" : ", ");
-        }
-        parameter_string += "]";
-    } else if (gen_params.sample_params.scheduler != SCHEDULER_COUNT) {  // Only show schedule if not using custom sigmas
-        parameter_string += " " + std::string(sd_scheduler_name(gen_params.sample_params.scheduler));
-    }
-    parameter_string += ", ";
-    for (const auto& te : {ctx_params.clip_l_path, ctx_params.clip_g_path, ctx_params.t5xxl_path, ctx_params.llm_path, ctx_params.llm_vision_path}) {
-        if (!te.empty()) {
-            parameter_string += "TE: " + sd_basename(te) + ", ";
-        }
-    }
-    if (!ctx_params.diffusion_model_path.empty()) {
-        parameter_string += "Unet: " + sd_basename(ctx_params.diffusion_model_path) + ", ";
-    }
-    if (!ctx_params.uncond_diffusion_model_path.empty()) {
-        parameter_string += "Uncond Unet: " + sd_basename(ctx_params.uncond_diffusion_model_path) + ", ";
-    }
-    if (!ctx_params.vae_path.empty()) {
-        parameter_string += "VAE: " + sd_basename(ctx_params.vae_path) + ", ";
-    }
-    if (gen_params.clip_skip != -1) {
-        parameter_string += "Clip skip: " + std::to_string(gen_params.clip_skip) + ", ";
-    }
-    if (gen_params.hires_enabled) {
-        parameter_string += "Hires upscale: " + gen_params.hires_upscaler + ", ";
-        parameter_string += "Hires scale: " + std::to_string(gen_params.hires_scale) + ", ";
-        parameter_string += "Hires resize: " + std::to_string(gen_params.hires_width) + "x" + std::to_string(gen_params.hires_height) + ", ";
-        parameter_string += "Hires steps: " + std::to_string(gen_params.hires_steps) + ", ";
-        parameter_string += "Denoising strength: " + std::to_string(gen_params.hires_denoising_strength) + ", ";
-        if (!gen_params.hires_custom_sigmas.empty()) {
-            parameter_string += "Hires custom sigmas: " + vec_to_string(gen_params.hires_custom_sigmas) + ", ";
-        }
-    }
-    parameter_string += "Version: stable-diffusion.cpp";
-    parameter_string += ", SDCPP: " + build_sdcpp_image_metadata_json(ctx_params, gen_params, seed, mode);
-    return parameter_string;
-}

examples/common/common.h

@@ -1,6 +1,8 @@
 #ifndef __EXAMPLES_COMMON_COMMON_H__
 #define __EXAMPLES_COMMON_COMMON_H__
 
+#include "generation_metadata.h"
+
 #include <cmath>
 #include <cstdint>
 #include <functional>
@@ -20,7 +22,7 @@ std::string argv_to_utf8(int index, const char** argv);
 #define BOOL_STR(b) ((b) ? "true" : "false")
 
 extern const char* const modes_str[];
-#define SD_ALL_MODES_STR "img_gen, adetailer, vid_gen, convert, upscale, metadata"
+#define SD_ALL_MODES_STR "img_gen, adetailer, vid_gen, convert, upscale, metadata, vid_upscale"
 
 enum SDMode {
     IMG_GEN,
@@ -29,6 +31,7 @@ enum SDMode {
     CONVERT,
     UPSCALE,
     METADATA,
+    VID_UPSCALE,
     MODE_COUNT
 };
 
@@ -225,6 +228,7 @@ struct SDGenerationParams {
     std::string end_image_path;
     std::string mask_image_path;
     std::string control_image_path;
+    std::string control_type;
     std::string ip_adapter_image_path;
     std::vector<std::string> ref_image_paths;
     std::vector<std::string> ref_video_paths;
@@ -289,6 +293,7 @@ struct SDGenerationParams {
     std::string prompt_with_lora;  // for metadata record only
     std::vector<sd_lora_t> lora_vec;
     sd_hires_upscaler_t resolved_hires_upscaler;
+    sd_control_type_t resolved_control_type = SD_CONTROL_TYPE_NONE;
 
     // Owned execution payload.
     SDImageOwner init_image;
@@ -338,13 +343,4 @@ struct SDGenerationParams {
 };
 
 std::string version_string();
-std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
-                                            const SDGenerationParams& gen_params,
-                                            int64_t seed,
-                                            SDMode mode = IMG_GEN);
-std::string get_image_params(const SDContextParams& ctx_params,
-                             const SDGenerationParams& gen_params,
-                             int64_t seed,
-                             SDMode mode = IMG_GEN);
-
 #endif  // __EXAMPLES_COMMON_COMMON_H__

examples/common/generation_metadata.cpp

@@ -0,0 +1,164 @@
+#include "generation_metadata.h"
+
+#include <cctype>
+#include <filesystem>
+#include <map>
+
+#include "json.hpp"
+
+static void relativize_lora_paths(nlohmann::json& info, const std::filesystem::path& root) {
+    if (info.contains("loras")) {
+        for (auto& lora : info["loras"]) {
+            const auto path = lora.value("path", "");
+            if (path.empty())
+                continue;
+            std::error_code error;
+            const auto absolute = std::filesystem::absolute(std::filesystem::u8path(path), error);
+            if (error)
+                continue;
+            const auto relative = absolute.lexically_normal().lexically_relative(root);
+            if (!relative.empty())
+                lora["path"] = relative.generic_u8string();
+        }
+    }
+    if (info.contains("inpaint_params"))
+        relativize_lora_paths(info["inpaint_params"], root);
+    if (info.contains("postprocess")) {
+        for (auto& operation : info["postprocess"])
+            relativize_lora_paths(operation, root);
+    }
+}
+
+std::string generation_info_with_relative_lora_paths(const std::string& info, const std::string& lora_model_dir) {
+    if (info.empty() || lora_model_dir.empty())
+        return info;
+    std::error_code error;
+    const auto root = std::filesystem::absolute(std::filesystem::u8path(lora_model_dir), error);
+    if (error)
+        return info;
+    auto result = nlohmann::json::parse(info);
+    relativize_lora_paths(result, root.lexically_normal());
+    return result.dump();
+}
+
+std::string merge_generation_info(const std::string& source, const std::string& operation) {
+    if (source.empty())
+        return operation;
+    auto result     = nlohmann::json::parse(source);
+    const auto next = nlohmann::json::parse(operation);
+    result["postprocess"].push_back(next);
+    return result.dump();
+}
+
+int64_t generation_seed(const std::string& info, int64_t fallback) {
+    if (info.empty())
+        return fallback;
+    const auto value   = nlohmann::json::parse(info);
+    const auto& params = value.contains("inpaint_params") ? value.at("inpaint_params") : value;
+    return params.contains("seed") ? std::stoll(params.at("seed").get<std::string>()) : fallback;
+}
+
+std::string generation_info_to_webui(const std::string& text) {
+    using nlohmann::json;
+    if (text.empty())
+        return "";
+    const auto info = json::parse(text);
+    if (info.value("schema", "") != "sdcpp.generation/v1")
+        return "";
+    // Standalone postprocessing has no source diffusion settings to import.
+    if (!info.contains("sample_params"))
+        return "";
+    const auto& sampling = info.at("sample_params");
+    std::string result   = info.value("prompt", "");
+    if (info.contains("loras")) {
+        bool prompt_trimmed = false;
+        for (const auto& lora : info.at("loras")) {
+            // WebUI tags cannot restrict a LoRA to the high-noise model.
+            if (lora.value("is_high_noise", false))
+                continue;
+            auto name            = lora.at("path").get<std::string>();
+            const auto separator = name.find_last_of("/\\");
+            if (separator != std::string::npos)
+                name.erase(0, separator + 1);
+            const auto extension = name.find_last_of('.');
+            if (extension != std::string::npos && extension > 0)
+                name.erase(extension);
+            if (name.empty() || name.find_first_of(":<>\r\n") != std::string::npos)
+                continue;
+            if (!result.empty()) {
+                if (!prompt_trimmed) {
+                    while (!result.empty() && std::isspace(static_cast<unsigned char>(result.back())))
+                        result.pop_back();
+                    prompt_trimmed = true;
+                }
+                if (!result.empty())
+                    result += " ";
+            }
+            result += "<lora:" + name + ":" + lora.at("multiplier").dump() + ">";
+        }
+    }
+    const auto negative = info.value("negative_prompt", "");
+    if (!negative.empty())
+        result += "\nNegative prompt: " + negative;
+    result += "\n";
+    bool first       = true;
+    const auto field = [&](const char* key, const json& value) {
+        std::string v = value.is_string() ? value.get<std::string>() : value.dump();
+        if (v.find_first_of(",\n\r:\"") != std::string::npos)
+            v = json(v).dump(-1, ' ', false);
+        if (!first)
+            result += ", ";
+        first = false;
+        result += std::string(key) + ": " + v;
+    };
+    field("Steps", sampling.contains("custom_sigmas") ? json(sampling.at("custom_sigmas").size() - 1) : sampling.at("sample_steps"));
+    static const std::map<std::string, std::string> samplers = {
+        {"euler", "Euler"}, {"euler_a", "Euler a"}, {"heun", "Heun"}, {"dpm2", "DPM2"}, {"dpm++2s_a", "DPM++ 2S a"}, {"dpm++2m", "DPM++ 2M"}, {"dpm++2m_sde", "DPM++ 2M SDE"}, {"lcm", "LCM"}};
+    const auto method = sampling.at("sample_method").get<std::string>();
+    auto mapped       = samplers.find(method);
+    field(mapped != samplers.end() ? "Sampler" : "SDCPP Sampler", mapped != samplers.end() ? mapped->second : method);
+    static const std::map<std::string, std::string> schedules = {
+        {"discrete", "Normal"}, {"karras", "Karras"}, {"exponential", "Exponential"}, {"sgm_uniform", "SGM Uniform"}, {"simple", "Simple"}, {"ddim_uniform", "DDIM"}, {"beta", "Beta"}};
+    if (sampling.contains("scheduler")) {
+        const auto scheduler = sampling.at("scheduler").get<std::string>();
+        auto mapped_schedule = schedules.find(scheduler);
+        field(mapped_schedule != schedules.end() ? "Schedule type" : "SDCPP Schedule", mapped_schedule != schedules.end() ? mapped_schedule->second : scheduler);
+    }
+    field("CFG scale", sampling.at("guidance").at("txt_cfg"));
+    field("Seed", info.at("seed"));
+    field("Size", std::to_string(info.at("width").get<int>()) + "x" + std::to_string(info.at("height").get<int>()));
+    if (info.contains("models")) {
+        const auto& models   = info.at("models");
+        auto model           = models.value("diffusion_model", models.value("model", ""));
+        const auto extension = model.find_last_of('.');
+        if (extension != std::string::npos && extension > 0)
+            model.erase(extension);
+        if (!model.empty())
+            field("Model", model);
+        if (models.contains("vae"))
+            field("VAE", models.at("vae"));
+    }
+    if (sampling.contains("eta"))
+        field("Eta", sampling.at("eta"));
+    if (info.value("clip_skip", 0) > 0)
+        field("Clip skip", info.at("clip_skip"));
+    if (info.contains("strength"))
+        field("Denoising strength", info.at("strength"));
+    if (info.contains("hires") && info.at("hires").value("enabled", false)) {
+        const auto& hires = info.at("hires");
+        field("Hires resize", std::to_string(hires.at("target_width").get<int>()) + "x" + std::to_string(hires.at("target_height").get<int>()));
+        if (hires.value("steps", 0) > 0)
+            field("Hires steps", hires.at("steps"));
+        field("Denoising strength", hires.at("denoising_strength"));
+        field("Hires upscaler", hires.at("upscaler"));
+    }
+    field("Version", "stable-diffusion.cpp");
+    return result;
+}
+
+std::string generation_info_to_image_metadata(const std::string& info) {
+    if (info.empty())
+        return "";
+    const auto parameters = generation_info_to_webui(info);
+    return parameters.empty() ? info : parameters + ", SDCPP: " + info;
+}

examples/common/generation_metadata.h

@@ -0,0 +1,13 @@
+#ifndef __EXAMPLES_COMMON_GENERATION_METADATA_H__
+#define __EXAMPLES_COMMON_GENERATION_METADATA_H__
+
+#include <cstdint>
+#include <string>
+
+std::string merge_generation_info(const std::string& source, const std::string& operation);
+std::string generation_info_with_relative_lora_paths(const std::string& info, const std::string& lora_model_dir);
+int64_t generation_seed(const std::string& info, int64_t fallback = -1);
+std::string generation_info_to_webui(const std::string& text);
+std::string generation_info_to_image_metadata(const std::string& info);
+
+#endif  // __EXAMPLES_COMMON_GENERATION_METADATA_H__

examples/common/media_io.cpp

@@ -1,5 +1,7 @@
 #include "media_io.h"
+#include "generation_metadata.h"
 #include "log.h"
+#include "media_metadata.h"
 #include "resource_owners.hpp"
 
 #include <algorithm>
@@ -316,23 +318,22 @@ uint8_t* decode_webp_image_to_buffer(const uint8_t* data,
     return output.release();
 }
 
-std::string build_webp_xmp_packet(const std::string& parameters) {
-    if (parameters.empty()) {
-        return "";
-    }
-
-    const std::string escaped_parameters = xml_escape(parameters);
-    return "<?xpacket begin=\"\" id=\"W5M0MpCehiHzreSzNTczkc9d\"?>\n"
-           "<x:xmpmeta xmlns:x=\"adobe:ns:meta/\">\n"
-           "  <rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n"
-           "    <rdf:Description xmlns:sdcpp=\"https://github.com/leejet/stable-diffusion.cpp/ns/1.0/\">\n"
-           "      <sdcpp:parameters>" +
-           escaped_parameters +
-           "</sdcpp:parameters>\n"
-           "    </rdf:Description>\n"
-           "  </rdf:RDF>\n"
-           "</x:xmpmeta>\n"
-           "<?xpacket end=\"w\"?>";
+static bool embed_webp_metadata(std::vector<uint8_t>& image, const std::string& parameters) {
+    if (parameters.empty())
+        return true;
+    WebPData data{image.data(), image.size()};
+    WebPMuxPtr mux(WebPMuxCreate(&data, 1));
+    if (!mux)
+        return false;
+    const auto xmp = generation_xmp(parameters);
+    WebPData xmp_data{reinterpret_cast<const uint8_t*>(xmp.data()), xmp.size()};
+    if (WebPMuxSetChunk(mux.get(), "XMP ", &xmp_data, 1) != WEBP_MUX_OK)
+        return false;
+    WebPDataGuard assembled;
+    if (WebPMuxAssemble(mux.get(), &assembled.data) != WEBP_MUX_OK)
+        return false;
+    image.assign(assembled.data.bytes, assembled.data.bytes + assembled.data.size);
+    return true;
 }
 
 bool encode_webp_image_to_vector(const uint8_t* image,
@@ -376,37 +377,7 @@ bool encode_webp_image_to_vector(const uint8_t* image,
 
     out.assign(encoded.get(), encoded.get() + encoded_size);
 
-    if (parameters.empty()) {
-        return true;
-    }
-
-    WebPData image_data;
-    WebPDataInit(&image_data);
-    WebPDataGuard assembled_data;
-
-    image_data.bytes = out.data();
-    image_data.size  = out.size();
-
-    WebPMuxPtr mux(WebPMuxNew());
-    if (mux == nullptr) {
-        return false;
-    }
-
-    const std::string xmp_packet = build_webp_xmp_packet(parameters);
-    WebPData xmp_data;
-    WebPDataInit(&xmp_data);
-    xmp_data.bytes = reinterpret_cast<const uint8_t*>(xmp_packet.data());
-    xmp_data.size  = xmp_packet.size();
-
-    const bool ok = WebPMuxSetImage(mux.get(), &image_data, 1) == WEBP_MUX_OK &&
-                    WebPMuxSetChunk(mux.get(), "XMP ", &xmp_data, 1) == WEBP_MUX_OK &&
-                    WebPMuxAssemble(mux.get(), &assembled_data.data) == WEBP_MUX_OK;
-
-    if (ok) {
-        out.assign(assembled_data.data.bytes, assembled_data.data.bytes + assembled_data.data.size);
-    }
-
-    return ok;
+    return embed_webp_metadata(out, generation_info_to_image_metadata(parameters));
 }
 
 #ifdef SD_USE_WEBM
@@ -739,6 +710,7 @@ std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
                                             const std::string& parameters,
                                             int quality) {
     std::vector<uint8_t> buffer;
+    const auto metadata = format == EncodedImageFormat::WEBP ? std::string() : generation_info_to_image_metadata(parameters);
 
     auto write_func = [&buffer](void* context, void* data, int size) {
         (void)context;
@@ -761,7 +733,7 @@ std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
             result = stbi_write_jpg_to_func(c_func, &ctx, width, height, channels, image, quality);
             break;
         case EncodedImageFormat::PNG:
-            result = stbi_ext_write_png_to_func(c_func, &ctx, width, height, channels, image, width * channels, parameters.empty() ? nullptr : parameters.c_str());
+            result = stbi_ext_write_png_to_func(c_func, &ctx, width, height, channels, image, width * channels, metadata.empty() ? nullptr : metadata.c_str());
             break;
         case EncodedImageFormat::WEBP:
 #ifdef SD_USE_WEBP
@@ -779,9 +751,12 @@ std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
             break;
     }
 
-    if (!result) {
-        buffer.clear();
+    if (result && format == EncodedImageFormat::JPEG && !embed_jpeg_metadata(buffer, metadata)) {
+        LOG_ERROR("generation metadata exceeds the container capacity or the encoded image is invalid");
+        result = 0;
     }
+    if (!result)
+        buffer.clear();
     return buffer;
 }
 
@@ -794,18 +769,8 @@ bool write_image_to_file(const std::string& path,
                          int quality) {
     const EncodedImageFormat format = encoded_image_format_from_path(path);
 
-    switch (format) {
-        case EncodedImageFormat::JPEG:
-            return stbi_write_jpg(path.c_str(), width, height, channels, image, quality, parameters.empty() ? nullptr : parameters.c_str()) != 0;
-        case EncodedImageFormat::PNG:
-            return stbi_write_png(path.c_str(), width, height, channels, image, 0, parameters.empty() ? nullptr : parameters.c_str()) != 0;
-        case EncodedImageFormat::WEBP: {
-            const std::vector<uint8_t> encoded = encode_image_to_vector(format, image, width, height, channels, parameters, quality);
-            return !encoded.empty() && write_binary_file_bytes(path, encoded);
-        }
-        default:
-            return false;
-    }
+    const auto encoded = encode_image_to_vector(format, image, width, height, channels, parameters, quality);
+    return !encoded.empty() && write_binary_file_bytes(path, encoded);
 }
 
 uint8_t* load_image_from_file(const char* image_path,
@@ -1384,7 +1349,10 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou
 
 #ifdef SD_USE_WEBP
     if (format == "webp") {
-        return create_animated_webp_from_sd_images_to_vector(images, num_images, fps, quality);
+        auto data = create_animated_webp_from_sd_images_to_vector(images, num_images, fps, quality);
+        if (!data.empty() && !embed_webp_metadata(data, parameters))
+            return {};
+        return data;
     }
 #endif
 
@@ -1469,70 +1437,94 @@ static uint32_t read_le32(const uint8_t* data) {
            (static_cast<uint32_t>(data[3]) << 24);
 }
 
+/** Loads a WAV file from disk and delegates decoding to load_wav_from_memory(). */
 bool load_wav_from_file(const std::string& path,
                         std::vector<float>& interleaved_samples,
                         uint32_t& sample_rate,
                         uint32_t& channels) {
+    std::ifstream file(fs::u8path(path), std::ios::binary);
+    if (!file) {
+        interleaved_samples.clear();
+        sample_rate = 0;
+        channels    = 0;
+        return false;
+    }
+    file.seekg(0, std::ios::end);
+    const std::streamoff file_size = file.tellg();
+    file.seekg(0, std::ios::beg);
+    if (file_size < 0) {
+        interleaved_samples.clear();
+        sample_rate = 0;
+        channels    = 0;
+        return false;
+    }
+    std::vector<uint8_t> buffer(static_cast<size_t>(file_size));
+    if (file_size > 0 && !file.read(reinterpret_cast<char*>(buffer.data()), file_size)) {
+        interleaved_samples.clear();
+        sample_rate = 0;
+        channels    = 0;
+        return false;
+    }
+    return load_wav_from_memory(buffer.data(), buffer.size(), interleaved_samples, sample_rate, channels);
+}
+
+/** Parses a RIFF/WAVE buffer in place; compares chunk sizes against the remaining
+ * bytes so payload-controlled sizes cannot wrap size_t on 32-bit builds. */
+bool load_wav_from_memory(const uint8_t* data,
+                          size_t size,
+                          std::vector<float>& interleaved_samples,
+                          uint32_t& sample_rate,
+                          uint32_t& channels) {
     interleaved_samples.clear();
     sample_rate = 0;
     channels    = 0;
 
-    std::ifstream file(fs::u8path(path), std::ios::binary);
-    uint8_t riff_header[12];
-    if (!file.read(reinterpret_cast<char*>(riff_header), sizeof(riff_header)) ||
-        std::memcmp(riff_header, "RIFF", 4) != 0 ||
-        std::memcmp(riff_header + 8, "WAVE", 4) != 0) {
+    if (data == nullptr ||
+        size < 12 ||
+        std::memcmp(data, "RIFF", 4) != 0 ||
+        std::memcmp(data + 8, "WAVE", 4) != 0) {
         return false;
     }
 
     uint16_t audio_format    = 0;
     uint16_t bits_per_sample = 0;
     uint16_t block_align     = 0;
-    std::streampos data_pos  = std::streampos(-1);
+    size_t data_pos          = SIZE_MAX;
     uint32_t data_size       = 0;
 
-    while (file.good()) {
-        uint8_t chunk_header[8];
-        if (!file.read(reinterpret_cast<char*>(chunk_header), sizeof(chunk_header))) {
-            break;
-        }
-        uint32_t chunk_size           = read_le32(chunk_header + 4);
-        std::streampos chunk_data_pos = file.tellg();
+    size_t pos = 12;
+    while (pos + 8 <= size) {
+        uint32_t chunk_size    = read_le32(data + pos + 4);
+        size_t chunk_data_pos  = pos + 8;
+        const size_t remaining = size - chunk_data_pos;  // safe: chunk_data_pos <= size
+        // Chunk sizes come from the payload, so compare against the remaining bytes
+        // instead of adding them to offsets (avoids size_t wrap on 32-bit builds).
+        const bool data_fits = static_cast<size_t>(chunk_size) <= remaining;
 
-        if (std::memcmp(chunk_header, "fmt ", 4) == 0) {
-            if (chunk_size < 16) {
+        if (std::memcmp(data + pos, "fmt ", 4) == 0) {
+            if (chunk_size < 16 || !data_fits) {
                 return false;
             }
-            std::vector<uint8_t> fmt(chunk_size);
-            if (!file.read(reinterpret_cast<char*>(fmt.data()), chunk_size)) {
-                return false;
-            }
-            audio_format    = read_le16(fmt.data());
-            channels        = read_le16(fmt.data() + 2);
-            sample_rate     = read_le32(fmt.data() + 4);
-            block_align     = read_le16(fmt.data() + 12);
-            bits_per_sample = read_le16(fmt.data() + 14);
+            audio_format    = read_le16(data + chunk_data_pos);
+            channels        = read_le16(data + chunk_data_pos + 2);
+            sample_rate     = read_le32(data + chunk_data_pos + 4);
+            block_align     = read_le16(data + chunk_data_pos + 12);
+            bits_per_sample = read_le16(data + chunk_data_pos + 14);
             if (audio_format == 0xfffe && chunk_size >= 40) {
-                audio_format = read_le16(fmt.data() + 24);
+                audio_format = read_le16(data + chunk_data_pos + 24);
             }
-        } else if (std::memcmp(chunk_header, "data", 4) == 0) {
+        } else if (std::memcmp(data + pos, "data", 4) == 0) {
             data_pos  = chunk_data_pos;
             data_size = chunk_size;
-            file.seekg(chunk_size, std::ios::cur);
-        } else {
-            file.seekg(chunk_size, std::ios::cur);
-        }
-
-        if (!file.good()) {
-            break;
-        }
-        if ((chunk_size & 1) != 0) {
-            file.seekg(1, std::ios::cur);
         }
+        // Only advance past a chunk whose size and pad byte fully fit (keeps the sum
+        // overflow-free); a chunk truncated at the buffer end terminates the walk.
+        const bool chunk_fits = data_fits && ((chunk_size & 1) == 0 || static_cast<size_t>(chunk_size) < remaining);
+        pos                   = chunk_fits ? chunk_data_pos + static_cast<size_t>(chunk_size) + (chunk_size & 1) : size;
     }
 
     const uint32_t bytes_per_sample = (bits_per_sample + 7) / 8;
-    if (data_pos == std::streampos(-1) || data_size == 0 || channels == 0 || sample_rate == 0 ||
+    if (data_pos == SIZE_MAX || data_size == 0 || channels == 0 || sample_rate == 0 ||
         block_align == 0 || bytes_per_sample == 0 || block_align < channels * bytes_per_sample ||
         (audio_format != 1 && audio_format != 3)) {
         return false;
@@ -1542,16 +1534,14 @@ bool load_wav_from_file(const std::string& path,
     if (frame_count == 0 || frame_count > SIZE_MAX / channels) {
         return false;
     }
-    std::vector<uint8_t> pcm(data_size);
-    file.clear();
-    file.seekg(data_pos);
-    if (!file.read(reinterpret_cast<char*>(pcm.data()), data_size)) {
+    if (static_cast<size_t>(data_size) > size - data_pos) {
         return false;
     }
+    const uint8_t* pcm = data + data_pos;
 
     interleaved_samples.resize(static_cast<size_t>(frame_count * channels));
     for (uint64_t frame = 0; frame < frame_count; ++frame) {
-        const uint8_t* frame_data = pcm.data() + frame * block_align;
+        const uint8_t* frame_data = pcm + frame * block_align;
         for (uint32_t channel = 0; channel < channels; ++channel) {
             const uint8_t* sample_data = frame_data + channel * bytes_per_sample;
             float sample               = 0.0f;

examples/common/media_io.h

@@ -16,6 +16,7 @@ enum class EncodedImageFormat {
 
 EncodedImageFormat encoded_image_format_from_path(const std::string& path);
 
+// parameters is the core sdcpp.generation/v1 JSON; empty disables both metadata formats.
 std::vector<uint8_t> encode_image_to_vector(EncodedImageFormat format,
                                             const uint8_t* image,
                                             int width,
@@ -125,4 +126,11 @@ bool load_wav_from_file(const std::string& path,
                         uint32_t& sample_rate,
                         uint32_t& channels);
 
+/** Decodes a WAV buffer (PCM 8/16/24/32 or IEEE float 32/64) into interleaved samples. */
+bool load_wav_from_memory(const uint8_t* data,
+                          size_t size,
+                          std::vector<float>& interleaved_samples,
+                          uint32_t& sample_rate,
+                          uint32_t& channels);
+
 #endif  // __MEDIA_IO_H__

examples/common/media_metadata.cpp

@@ -0,0 +1,47 @@
+#include "media_metadata.h"
+
+std::string generation_xmp(const std::string& info) {
+    std::string escaped;
+    for (char c : info) {
+        switch (c) {
+            case '&':
+                escaped += "&";
+                break;
+            case '<':
+                escaped += "&lt;";
+                break;
+            case '>':
+                escaped += "&gt;";
+                break;
+            case '"':
+                escaped += "&quot;";
+                break;
+            case '\'':
+                escaped += "&apos;";
+                break;
+            default:
+                escaped += c;
+                break;
+        }
+    }
+    return "<x:xmpmeta xmlns:x=\"adobe:ns:meta/\"><rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">"
+           "<rdf:Description rdf:about=\"\" xmlns:sdcpp=\"https://github.com/leejet/stable-diffusion.cpp/ns/1.0/\">"
+           "<sdcpp:parameters>" +
+           escaped + "</sdcpp:parameters></rdf:Description></rdf:RDF></x:xmpmeta>";
+}
+
+bool embed_jpeg_metadata(std::vector<uint8_t>& image, const std::string& parameters) {
+    if (parameters.empty())
+        return true;
+    const char label[]    = "parameters";
+    const size_t overhead = sizeof(label) + 1;
+    if (image.size() < 2 || parameters.size() > 65533 - overhead)
+        return false;
+    const size_t length          = overhead + parameters.size() + 2;
+    std::vector<uint8_t> segment = {0xff, 0xfe, static_cast<uint8_t>(length >> 8), static_cast<uint8_t>(length)};
+    segment.insert(segment.end(), label, label + sizeof(label));
+    segment.insert(segment.end(), parameters.begin(), parameters.end());
+    segment.push_back(0);
+    image.insert(image.begin() + 2, segment.begin(), segment.end());
+    return true;
+}

examples/common/media_metadata.h

@@ -0,0 +1,11 @@
+#ifndef __EXAMPLES_COMMON_MEDIA_METADATA_H__
+#define __EXAMPLES_COMMON_MEDIA_METADATA_H__
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+std::string generation_xmp(const std::string& parameters);
+bool embed_jpeg_metadata(std::vector<uint8_t>& image, const std::string& parameters);
+
+#endif  // __EXAMPLES_COMMON_MEDIA_METADATA_H__

examples/common/resource_owners.hpp

@@ -8,6 +8,7 @@
 #include <utility>
 #include <vector>
 
+#include "generation_metadata.h"
 #include "stable-diffusion.h"
 
 struct FreeDeleter {
@@ -175,6 +176,7 @@ public:
 class SDImageVec {
 private:
     std::vector<sd_image_t> images_;
+    std::vector<std::string> generation_infos_;
 
 public:
     SDImageVec() = default;
@@ -183,13 +185,14 @@ public:
     SDImageVec& operator=(const SDImageVec&) = delete;
 
     SDImageVec(SDImageVec&& other) noexcept
-        : images_(std::move(other.images_)) {
+        : images_(std::move(other.images_)), generation_infos_(std::move(other.generation_infos_)) {
     }
 
     SDImageVec& operator=(SDImageVec&& other) noexcept {
         if (this != &other) {
             clear();
-            images_ = std::move(other.images_);
+            images_           = std::move(other.images_);
+            generation_infos_ = std::move(other.generation_infos_);
         }
         return *this;
     }
@@ -210,6 +213,41 @@ public:
         images_.reserve(count);
     }
 
+    const std::string& generation_info(size_t index) const {
+        static const std::string empty;
+        return index < generation_infos_.size() ? generation_infos_[index] : empty;
+    }
+
+    void append_generation_info(size_t index, const std::string& info, const std::string& lora_model_dir = {}) {
+        generation_infos_.resize(images_.size());
+        generation_infos_.at(index) = merge_generation_info(generation_infos_.at(index), generation_info_with_relative_lora_paths(info, lora_model_dir));
+    }
+
+    void adopt(sd_image_generation_result_t* result, const std::string& lora_model_dir = {}) {
+        clear();
+        if (!result)
+            return;
+        for (int i = 0; i < result->count; ++i) {
+            images_.push_back(result->items[i].image);
+            result->items[i].image = {};
+            generation_infos_.push_back(generation_info_with_relative_lora_paths(result->items[i].generation_info, lora_model_dir));
+        }
+        free_sd_image_generation_result(result);
+    }
+
+    void adopt_video(sd_video_generation_result_t* result, const std::string& lora_model_dir = {}) {
+        clear();
+        if (!result)
+            return;
+        const auto info = generation_info_with_relative_lora_paths(result->generation_info, lora_model_dir);
+        for (int i = 0; i < result->frame_count; ++i) {
+            images_.push_back(result->frames[i]);
+            result->frames[i] = {};
+            generation_infos_.push_back(info);
+        }
+        free_sd_video_generation_result(result);
+    }
+
     void adopt(sd_image_t* images, int count) {
         clear();
         if (images == nullptr || count <= 0) {
@@ -270,6 +308,7 @@ public:
             image.data = nullptr;
         }
         images_.clear();
+        generation_infos_.clear();
     }
 };
 

examples/server/CMakeLists.txt

@@ -64,8 +64,10 @@ endif()
 
 add_executable(${TARGET}
     ../common/common.cpp
+    ../common/generation_metadata.cpp
     ../common/log.cpp
     ../common/media_io.cpp
+    ../common/media_metadata.cpp
     main.cpp
     runtime.cpp
     model_catalog.cpp

examples/server/README.md

@@ -76,6 +76,16 @@ Recipes declaring `video_frames` also switch the form to Video Generation;
 other recipes switch to Image Generation. Generation is enabled after the
 selected recipe is loaded. Changing a selection does not reload the model
 until you click Load.
+
+The generation status bar shows the current stage and elapsed time even when
+Preview Mode is `none`. During sampling it shows completed steps for the current
+pass; prompt encoding and VAE decoding show an activity indicator when their
+total work is unknown. Sampling reaching 100% does not mean the job is finished:
+decoding, ADetailer, and output encoding may still follow.
+The canvas keeps the submitted output aspect ratio, and stage changes keep the
+same layout. Editing the next request's resolution does not resize the current
+preview or result.
+
 The server can also start with the existing model arguments shown below;
 existing API clients then work without selecting a recipe. The UI automatically
 selects **Command-line model (all features)** and uses the startup generation
@@ -89,8 +99,8 @@ FLUX 1/2/3, Chroma, Qwen Image and editing variants; Z-Image, Anima, ERNIE,
 LENS, PiD and other supported image families; AnimateDiff, Wan, LTX,
 HunyuanVideo, LingBot Video and MiniMax-H3. Model-specific text encoder
 labels identify the expected encoder family and size. Loading a video model enables **Video Generation**; I2V also requires a
-start image. The Models panel can be collapsed and remembers its open state.
-Its header shows elapsed loading time from the status API and keeps the final
+start image. The model button below the prompt opens model and recipe selection.
+Its status summary shows elapsed loading time from the status API and keeps the final
 time after success or failure, including after a page refresh.
 Built-ins are embedded in the
 binary from [recipes/](recipes/), with one JSON file per recipe. Adding or

examples/server/api.md

@@ -85,7 +85,7 @@ available during loading. Model lists are empty when no model is loaded;
 | GET | `/sdcpp/v1/model-files` | `{files: [{id, name, category, root, relative_path}], errors: [...]}` |
 | POST | `/sdcpp/v1/models/refresh` | Rescan recipes and files; return `{errors: [...]}` |
 | GET | `/sdcpp/v1/models/status` | Current state, recipe, selection, error, loading time and optional revision |
-| POST | `/sdcpp/v1/models/load` | Load the submitted recipe and selection; return status after completion |
+| POST | `/sdcpp/v1/models/load` | Load the submitted recipe and selection; return status after completion. A load is a no-op returning the current status with `already_loaded: true` only when the state is ready and the fully resolved selection, the complete recipe object, and the resolved component paths all match the active model; a catalog refresh with modified recipe metadata or a retargeted model path therefore reloads even with an unchanged recipe ID and selection |
 | POST | `/sdcpp/v1/models/unload` | Release the active context; return empty status |
 
 Load request:
@@ -183,6 +183,9 @@ Default-enabled features are `img2img`, `mask_image`, `lora`, `vae_tiling`,
 | `clip_skip` | CLIP Skip |
 | `img2img`, `mask_image` | Image initialization/strength and masks |
 | `ref_images` | Reference image inputs |
+| `ref_videos` | Reference videos as frame arrays |
+| `ref_audios` | Standalone reference audio; runtime requires an Audio VAE or Audio Encoder |
+| `ref_video_audios` | Soundtracks paired with reference videos; runtime requires an Audio VAE |
 | `controlnet` | Control image and ControlNet strength |
 | `control_image` | Control image without a separate ControlNet (FLUX Canny/Depth) |
 | `ip_adapter` | IP-Adapter image and strength |
@@ -241,6 +244,7 @@ the same one-recipe-per-file schema:
   "id": "my-flux",
   "name": "My FLUX",
   "description": "FLUX.1 dev",
+  "size_multiple": 16,
   "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-dev",
   "slots": [
     {"id": "diffusion_model", "required": true},
@@ -256,7 +260,8 @@ the same one-recipe-per-file schema:
 
 IDs contain 1–128 ASCII letters, digits, dots, underscores or hyphens.
 `name`, `schema_version: 1` and `slots` are required; `description` and
-`model_url`, `required_any`, `defaults`, `context`, `features` and `disabled_features` are optional. Each slot requires only `id` and boolean `required`.
+`model_url`, `required_any`, `defaults`, `context`, `features`, `disabled_features`
+and `size_multiple`, `video_constraints` are optional. Each slot requires only `id` and boolean `required`.
 The server supplies the default label and search directories for each slot.
 An optional nonempty `label` overrides the display name, for example
 `{"id": "diffusion_model", "label": "Low-noise diffusion model", "required": true}`.
@@ -264,6 +269,53 @@ Legacy `category` fields remain accepted, but must match one of the slot's
 supported directories; they cannot redefine or restrict its search directories.
 At least one required `model` or `diffusion_model` slot is necessary.
 
+An optional top-level `models` object maps slot IDs to file IDs, providing
+recipe-local defaults for component selection. A load request may omit
+`models` entirely and load with the recipe's defaults. Selections sent in the
+request always override recipe defaults, slot by slot. An empty string
+disables a slot's recipe default (omitting the slot selects it); the empty
+key is then normalized out of the reported selection, so for slots without a
+recipe default an empty selection loads identically to an omitted one.
+
+`size_multiple` is a positive integer from 1 to 4096 that declares the width and
+height increment used by the web UI. It defaults to `8` when omitted; the UI also
+uses `8` before a recipe is selected. For example, `"size_multiple": 16` makes
+generated size presets multiples of 16. Exact dimensions that do not match the
+increment require correction before submission. This is recipe metadata, not a
+generation parameter, and does not change the core model's alignment behavior.
+
+`video_constraints` is optional metadata for the web UI's Frames / Duration
+controls. It does not override the runtime's model-specific rules. For example:
+
+```json
+"video_constraints": {
+  "frame_step": 4,
+  "frame_offset": 1,
+  "frame_rounding": "floor",
+  "min_frames": 1
+}
+```
+
+Legal frame counts are `frame_step * n + frame_offset`, with integer `n >= 0`,
+within the inclusive `min_frames` / `max_frames` range. All fields are optional:
+
+| Field | Default | Accepted values |
+| --- | --- | --- |
+| `frame_step` | `1` | Integer 1–4096 |
+| `frame_offset` | `0` | Integer 0 to `frame_step - 1` |
+| `frame_rounding` | `"floor"` | `"floor"` or `"ceil"` |
+| `min_frames` | `1` | Integer 1–4096 |
+| `max_frames` | `4096` | Integer 1–4096, at least `min_frames` |
+| `fixed_fps` | Unset | Integer 1–240; locks the FPS control |
+
+The range must contain at least one legal frame count. Unknown keys are rejected.
+Omitting the object adds no frame alignment in the UI. `defaults.fps` only sets
+the initial FPS; it does not lock it. The UI multiplies requested duration by FPS,
+aligns using the declared rounding direction, then clamps to the legal range.
+Duration stays in the UI; generation requests still send only `video_frames` and
+`fps`. Switching input modes preserves the aligned length. Changing FPS keeps
+frames in Frames mode, and keeps the requested seconds in Duration mode.
+
 `model_url` is an optional HTTPS Hugging Face model repository link, shown beside
 the recipe description. Built-ins link to the model's author/publisher, not repacks.
 `required_any` contains groups of declared slot IDs. At least one model in each
@@ -447,10 +499,6 @@ image establishes the generation dimensions if `size` is omitted. Input
 geometry follows `image_preprocess`: references preserve their dimensions by
 default, while init and mask use the generation canvas preset.
 
-When the request starts without preconfigured reference images, the first uploaded image becomes the img2img init image if the loaded model does not support reference-image conditioning, the final reference list is empty, a mask is present, or an explicit numeric `strength` override is provided. Otherwise, only the final reference list is used for image conditioning, without initializing the img2img latent. Requests with preconfigured references retain their existing init-image selection.
-
-Routing uses the reference list and mask after applying `sd_cpp_extra_args`. Both `ref_images: []` and `ref_images: null` clear the reference list; `mask_image: null` clears an uploaded mask. An explicit `init_image` in `sd_cpp_extra_args`, including `null`, takes precedence over the implicit init-image selection.
-
 Reference encoding then follows model presets and `ref_image_args`. To skip
 input geometry for references and disable resizing before VAE encoding, include
 this in `prompt`:
@@ -727,6 +775,7 @@ Field types:
 | `queue_position` | `integer` |
 | `result` | `object \| null` |
 | `error` | `object \| null` |
+| `progress` | `object` |
 | `preview` | `object \| null` |
 
 `created`, `started`, and `completed` are integer Unix timestamps in
@@ -734,6 +783,57 @@ Field types:
 the elapsed generation time in seconds, including output encoding and excluding
 queue time.
 
+`progress` is available independently of the preview setting, for both image
+and video jobs. Continue polling until `status` is terminal: a sampling pass
+can finish before decoding, ADetailer, or output encoding finishes.
+
+| Field | Type | Notes |
+| --- | --- | --- |
+| `progress.stage` | `string` | Current execution stage; see below |
+| `progress.detail` | `string` | Stage detail, or an empty string |
+| `progress.pass` | `integer` | Sampling pass number within the job; zero before sampling |
+| `progress.completed` | `integer \| null` | Completed units in the current stage/pass |
+| `progress.total` | `integer \| null` | Actual total units, or null when unknown |
+| `progress.unit` | `string \| null` | `steps` for sampling, `images` for image output encoding; null when uncounted |
+| `progress.stage_elapsed_ms` | `integer` | Elapsed time in the current stage/detail/pass |
+| `progress.elapsed_ms` | `integer` | Job execution time, excluding queue time; zero before execution |
+
+Stages include `queued`, `waiting` (waiting for the execution lock), `preparing`,
+`conditioning`, `sampling`, `vae_encode`, `vae_decode`, `audio_decode`,
+`upscaling`, `adetailer`, and `encoding`. Stages may repeat or be skipped.
+Details include `positive_prompt` / `negative_prompt` / `image_condition` for
+conditioning, `high_noise` / `low_noise` for two-stage sampling, and `image` /
+`video` for output encoding. During ADetailer, `detail` identifies the inner
+stage, such as `conditioning` or `sampling`. Clients should tolerate unknown
+stage, detail, and unit values.
+
+Sampling counts completed logical steps using the actual schedule, including
+strength adjustments. Each new sampling pass increments `pass` and starts at
+zero, including batch items, highres passes, high/low-noise passes, and ADetailer
+passes. The ratio is **current-pass progress**, not overall job completion.
+The progress snapshot is published atomically. Preview metadata still describes
+the preview frame, which may be older than the current progress.
+
+Unknown totals are represented by null counts, not an estimated percentage.
+For example, prompt encoding can report:
+
+```json
+"progress": {
+  "stage": "conditioning",
+  "detail": "positive_prompt",
+  "pass": 0,
+  "completed": null,
+  "total": null,
+  "unit": null,
+  "stage_elapsed_ms": 3200,
+  "elapsed_ms": 4100
+}
+```
+
+Elapsed times freeze when a job completes, fails, or is cancelled; the last
+stage is retained for inspection. Use `status` to determine the final outcome.
+Elapsed time alone does not indicate that computation is advancing.
+
 `preview` sub-fields:
 
 | Field | Type | Notes |
@@ -877,7 +977,7 @@ Shared default fields used by both `img_gen` and `vid_gen`:
 
 `vae_tiling_params.tile_size_w` and `tile_size_h` are in **image pixels**, with `0` selecting the 256-pixel default. Both encode and decode use these sizes without an encoding multiplier. Positive `rel_size_w`/`rel_size_h` values override the corresponding absolute size: values up to 1 are dimension fractions, and values greater than 1 are target tile counts. Set `enabled` to use spatial tiling. Sizes are aligned down to the VAE scale factor and capped at the input dimensions; explicit sizes below the minimum supported tile size are rejected. These fields previously used latent units; see [VAE tiling](../../docs/performance.md#use-vae-tiling-to-reduce-encode-and-decode-memory-usage) for migration and OOM retry behavior.
 
-`vae_tiling_params.extra_tiling_args` accepts a key=value list. Supported video VAEs accept `temporal_tile_frames` (alias `temporal_tile_size`, default `4`) and `temporal_tile_overlap` (default `1`).
+`vae_tiling_params.extra_tiling_args` accepts a key=value list. Supported video VAEs accept `temporal_tile_frames` (alias `temporal_tile_size`, default `4`) and `temporal_tile_overlap` (default `1`). Wan2.2 TI2V 5B uses `temporal_decode_frames` (default `1`) to decode multiple latent frames per forward pass, trading additional memory for throughput; other model versions ignore this option.
 LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEHV use overlap blending. MiniMax H3 keeps its model-specific fixed temporal windows because its latent-to-frame mapping is non-linear.
 
 `img_gen`-specific default fields:
@@ -888,6 +988,7 @@ LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEH
 | `ref_image_args` | `string` |
 | `increase_ref_index` | `boolean` |
 | `control_strength` | `number` |
+| `control_type` | `string` |
 | `ip_adapter_strength` | `number` |
 | `hires` | `object` |
 | `hires.enabled` | `boolean` |
@@ -906,6 +1007,7 @@ LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEH
 | --- | --- |
 | `video_frames` | `integer` |
 | `fps` | `integer` |
+| `ref_image_args` | `string` |
 | `moe_boundary` | `number` |
 | `vace_strength` | `number` |
 | `high_noise_sample_params` | `object` |
@@ -1067,6 +1169,7 @@ Example:
   "ref_image_args": "",
   "increase_ref_index": false,
   "control_strength": 0.9,
+  "control_type": "",
   "ip_adapter_strength": 1.0,
   "embed_image_metadata": true,
 
@@ -1192,6 +1295,7 @@ Top-level scalar fields:
 | `image_preprocess` | `string \| array<string>` |
 | `increase_ref_index` | `boolean` |
 | `control_strength` | `number` |
+| `control_type` | `string` |
 | `ip_adapter_strength` | `number` |
 | `embed_image_metadata` | `boolean` |
 | `preview` | `string` |
@@ -1403,7 +1507,7 @@ Compared with `img_gen`, the `vid_gen` request body:
 - `vid_gen` is a single video sequence job, so `batch_count` is not part of the request schema
 - `mask_image`, `control_image`, `control_strength`, `ip_adapter_image`, `ip_adapter_strength`, and `embed_image_metadata` are not part of the request schema
 - `ref_images` is accepted for MiniMax-H3 Ref2VA conditioning; other video model families currently ignore it
-- `vid_gen` adds `end_image`, `control_frames`, `high_noise_sample_params`, `video_frames`, `fps`, `moe_boundary`, and `vace_strength`
+- `vid_gen` adds `end_image`, `control_frames`, `high_noise_sample_params`, `video_frames`, `fps`, `ref_image_args`, `moe_boundary`, and `vace_strength`
 
 Example:
 
@@ -1418,6 +1522,7 @@ Example:
   "seed": -1,
   "video_frames": 33,
   "fps": 16,
+  "ref_image_args": "",
   "moe_boundary": 0.875,
   "vace_strength": 1.0,
 
@@ -1499,6 +1604,7 @@ Example:
 - Images retain request order and correspond to `<Picture 1>`, `<Picture 2>`, and so on in the prompt.
 - For MiniMax-H3, non-empty `ref_images` cannot be combined with `init_image` or `end_image`. A conflicting API request fails during generation; the WebUI checks this before submission.
 - MiniMax-H3 does not support `control_frames`; leave that array empty.
+- `ref_image_args` applies to `vid_gen`; MiniMax-H3 honors `ref_image_size` there (`match` default, `max` for a 2048 px short edge). Only references above the limit are downscaled; each axis is then snapped to the nearest 32 px multiple.
 - Other video model families currently ignore `ref_images`.
 - `features_by_mode.vid_gen` does not currently advertise `ref_images`; its absence is not an indication that MiniMax-H3 reference images are unsupported.
 
@@ -1549,6 +1655,7 @@ Top-level scalar fields:
 | `seed` | `integer` |
 | `video_frames` | `integer` |
 | `fps` | `integer` |
+| `ref_image_args` | `string` |
 | `moe_boundary` | `number` |
 | `vace_strength` | `number` |
 | `preview` | `string` |
@@ -1690,15 +1797,26 @@ When omitted, backend defaults apply to these fields:
 
 ### Frame Count Semantics
 
-`video_frames` is the requested target length, but the current core video path internally normalizes the effective frame count to the largest `4n + 1` value that does not exceed the requested count.
+`video_frames` is the requested target length. The core normalizes it according
+to the loaded model:
 
-Examples:
+| Model family | Frame alignment | Fixed FPS |
+| --- | --- | --- |
+| Wan, HunyuanVideo, LingBot | `4n + 1`, round down | Wan 2.2 S2V: 16 |
+| LTX | `8n + 1`, round down | None |
+| MiniMax-H3 | `17n + 5`, round up, minimum 5 | 24 |
+| AnimateDiff | Whole frames, maximum 32 | None |
+
+Wan examples:
 
 - `video_frames = 33` stays `33`
 - `video_frames = 34` becomes `33`
 - `video_frames = 32` becomes `29`
 
-The completed job payload includes the actual decoded `frame_count`.
+At 16 FPS, a 5-second target requests 80 frames and aligns to 77 frames for Wan,
+or approximately 4.813 seconds. Audio-driven generation and temporal upscaling
+can change the final length. Use the completed job's actual decoded `frame_count`
+and `fps` for the final playback duration.
 
 ### Completion Result
 
@@ -1776,3 +1894,14 @@ Example cancelled job:
 - `400 Bad Request` for an empty body, unsupported model mode, invalid JSON, invalid generation parameters, or an unsupported output format
 - `429 Too Many Requests` when the job queue is full
 - `500 Internal Server Error` for unexpected server exceptions during submission
+
+## Actual generation parameters
+
+Generation responses expose the core `sdcpp.generation/v1` record as
+`generation_info`, independently of `embed_image_metadata`. Native async image
+result items and OpenAI image items contain the parameter snapshot with each
+image's actual seed and without `batch_count`. Native video
+results contain the video request snapshot. SD API `info.generation_info` is an array
+aligned with `info.infotexts` and `info.all_seeds`; the outer `parameters` field
+remains the request echo. See [generation metadata](../../docs/generation_metadata.md)
+for field semantics, postprocessing and container storage.

examples/server/async_jobs.cpp

@@ -92,6 +92,63 @@ static void set_preview_callback(AsyncGenerationJob& job) {
     sd_set_preview_callback(preview_callback, mode, interval, true, false, &job);
 }
 
+static void update_job_progress(AsyncGenerationJob& job, const char* stage, const char* detail = "", int sample_pass = -1, int64_t completed = 0, int64_t total = 0, const char* unit = "") {
+    std::lock_guard<std::mutex> lock(job.progress_mutex);
+    auto& progress = job.progress;
+    if (sample_pass < 0) {
+        sample_pass = progress.sample_pass;
+    }
+    if (progress.stage != stage || progress.detail != detail || progress.sample_pass != sample_pass) {
+        progress.stage_started = std::chrono::steady_clock::now();
+    }
+    progress.stage       = stage;
+    progress.detail      = detail;
+    progress.sample_pass = sample_pass;
+    progress.completed   = completed;
+    progress.total       = total;
+    progress.unit        = unit;
+}
+
+static void finish_job_progress(AsyncGenerationJob& job) {
+    std::lock_guard<std::mutex> lock(job.progress_mutex);
+    job.progress.finished = std::chrono::steady_clock::now();
+}
+
+class JobCallbacksGuard {
+    sd_ctx_t* ctx_;
+    AsyncGenerationJob& job_;
+    bool detailing_ = false;
+
+    static void progress_callback(const sd_generation_progress_t* progress, void* data) {
+        auto& guard = *static_cast<JobCallbacksGuard*>(data);
+        update_job_progress(guard.job_, guard.detailing_ ? "adetailer" : progress->stage,
+                            guard.detailing_ ? progress->stage : progress->detail,
+                            progress->sample_pass, progress->completed, progress->total, progress->unit);
+    }
+
+public:
+    JobCallbacksGuard(sd_ctx_t* ctx, AsyncGenerationJob& job)
+        : ctx_(ctx), job_(job) {
+        clear_preview_callback();
+        set_preview_callback(job);
+        sd_set_generation_progress_callback(ctx_, progress_callback, this);
+    }
+
+    ~JobCallbacksGuard() {
+        sd_set_generation_progress_callback(ctx_, nullptr, nullptr);
+        clear_preview_callback();
+    }
+
+    void begin_detailing() {
+        clear_preview_callback();
+        detailing_ = true;
+        update_job_progress(job_, "adetailer");
+    }
+
+    JobCallbacksGuard(const JobCallbacksGuard&)            = delete;
+    JobCallbacksGuard& operator=(const JobCallbacksGuard&) = delete;
+};
+
 const char* async_job_kind_name(AsyncJobKind kind) {
     switch (kind) {
         case AsyncJobKind::ImgGen:
@@ -179,12 +236,14 @@ bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job) {
     job.status       = AsyncJobStatus::Cancelled;
     job.completed_at = unix_timestamp_now_ms();
     job.result_images_b64.clear();
+    job.result_generation_info = nullptr;
     job.result_media_b64.clear();
     job.result_media_mime_type.clear();
     job.result_frame_count = 0;
     job.result_fps         = 0;
     job.error_code         = "cancelled";
     job.error_message      = "job cancelled by client";
+    finish_job_progress(job);
     return true;
 }
 
@@ -199,6 +258,24 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
     result["completed"]        = job.completed_at == 0 ? json(nullptr) : json(job.completed_at);
     result["queue_position"]   = 0;
 
+    {
+        std::lock_guard<std::mutex> lock(job.progress_mutex);
+        const auto& progress = job.progress;
+        const auto end       = progress.finished == std::chrono::steady_clock::time_point{}
+                                   ? std::chrono::steady_clock::now()
+                                   : progress.finished;
+        result["progress"]   = {
+            {"stage", progress.stage},
+            {"detail", progress.detail},
+            {"pass", progress.sample_pass},
+            {"completed", progress.total > 0 ? json(progress.completed) : json(nullptr)},
+            {"total", progress.total > 0 ? json(progress.total) : json(nullptr)},
+            {"unit", progress.unit.empty() ? json(nullptr) : json(progress.unit)},
+            {"stage_elapsed_ms", std::chrono::duration_cast<std::chrono::milliseconds>(end - progress.stage_started).count()},
+            {"elapsed_ms", job.started_at == 0 ? 0 : std::max<int64_t>(0, (job.completed_at == 0 ? unix_timestamp_now_ms() : job.completed_at) - job.started_at)},
+        };
+    }
+
     if (job.status == AsyncJobStatus::Queued) {
         size_t position = 1;
         for (const auto& queued_id : manager.queue) {
@@ -218,11 +295,12 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
                 {"fps", job.result_fps},
                 {"frame_count", job.result_frame_count},
                 {"b64_json", job.result_media_b64},
+                {"generation_info", job.result_generation_info},
             };
         } else {
             json images = json::array();
             for (size_t i = 0; i < job.result_images_b64.size(); ++i) {
-                images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}});
+                images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}, {"generation_info", job.result_generation_info.at(i)}});
             }
             result["result"] = {
                 {"output_format", job.img_gen.output_format},
@@ -234,11 +312,11 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
                job.status == AsyncJobStatus::Cancelled) {
         result["result"] = nullptr;
         result["error"]  = {
-             {"code",
+            {"code",
              job.error_code.empty()
-                  ? (job.status == AsyncJobStatus::Cancelled ? "cancelled" : "generation_failed")
-                  : job.error_code},
-             {"message", job.error_message},
+                 ? (job.status == AsyncJobStatus::Cancelled ? "cancelled" : "generation_failed")
+                 : job.error_code},
+            {"message", job.error_message},
         };
     } else {
         result["result"] = nullptr;
@@ -263,7 +341,9 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
 bool execute_img_gen_job(ServerRuntime& runtime,
                          AsyncGenerationJob& job,
                          std::vector<std::string>& output_images,
+                         json& output_info,
                          std::string& error_message) {
+    output_info                = json::array();
     sd_img_gen_params_t params = job.img_gen.to_sd_img_gen_params_t();
 
     SDImageVec results;
@@ -274,21 +354,19 @@ bool execute_img_gen_job(ServerRuntime& runtime,
             return false;
         }
         JobCancellationMonitor cancellation(runtime.sd_ctx, job);
-        set_preview_callback(job);
-        sd_image_t* raw_results = nullptr;
-        int num_results         = 0;
-        if (!generate_image(runtime.sd_ctx, &params, &raw_results, &num_results)) {
-            raw_results = nullptr;
-            num_results = 0;
-        }
-        clear_preview_callback();
-        results.adopt(raw_results, num_results);
+        JobCallbacksGuard callbacks(runtime.sd_ctx, job);
+        sd_image_generation_result_t* generated = nullptr;
+        generate_image(runtime.sd_ctx, &params, &generated);
+        results.adopt(generated, runtime.ctx_params->lora_model_dir);
         if (job.cancel_requested.load()) {
             return false;
         }
-        if (!results.empty() && !apply_server_adetailer(runtime, job.img_gen, params, results)) {
-            error_message = "ADetailer failed";
-            return false;
+        if (!results.empty() && job.img_gen.adetailer_enabled && !runtime.default_gen_params->ad_model_path.empty()) {
+            callbacks.begin_detailing();
+            if (!apply_server_adetailer(runtime, job.img_gen, params, results)) {
+                error_message = "ADetailer failed";
+                return false;
+            }
         }
     }
 
@@ -305,17 +383,14 @@ bool execute_img_gen_job(ServerRuntime& runtime,
         encoded_format = EncodedImageFormat::WEBP;
     }
 
-    int batch_count      = job.img_gen.gen_params.batch_count;
-    int images_per_batch = batch_count > 0 ? std::max(1, num_results / batch_count) : 1;
+    update_job_progress(job, "encoding", "image", -1, 0, num_results, "images");
     for (int i = 0; i < num_results; ++i) {
         if (results[i].data == nullptr) {
             continue;
         }
 
         const std::string metadata = job.img_gen.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime.ctx_params,
-                                                            job.img_gen.gen_params,
-                                                            job.img_gen.gen_params.seed + i / images_per_batch)
+                                         ? results.generation_info(i)
                                          : "";
         auto image_bytes           = encode_image_to_vector(encoded_format,
                                                             results[i].data,
@@ -328,6 +403,8 @@ bool execute_img_gen_job(ServerRuntime& runtime,
             continue;
         }
         output_images.push_back(base64_encode(image_bytes));
+        output_info.push_back(json::parse(results.generation_info(i)));
+        update_job_progress(job, "encoding", "image", -1, i + 1, num_results, "images");
     }
 
     if (output_images.empty()) {
@@ -344,11 +421,10 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
                          std::string& output_media_mime_type,
                          int& output_frame_count,
                          int& output_fps,
+                         json& output_info,
                          std::string& error_message) {
     sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t();
-    std::string str_params     = job.vid_gen.gen_params.embed_image_metadata
-                                     ? get_image_params(*runtime.ctx_params, job.vid_gen.gen_params, job.vid_gen.gen_params.seed, VID_GEN)
-                                     : "";
+    std::string str_params;
 
     SDImageVec results;
     int num_results             = 0;
@@ -360,13 +436,17 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
             return false;
         }
         JobCancellationMonitor cancellation(runtime.sd_ctx, job);
-        set_preview_callback(job);
-        sd_image_t* raw_results = nullptr;
-        if (!generate_video(runtime.sd_ctx, &params, &raw_results, &num_results, &generated_audio, &output_fps)) {
-            raw_results = nullptr;
+        JobCallbacksGuard callbacks(runtime.sd_ctx, job);
+        sd_video_generation_result_t* generated = nullptr;
+        if (generate_video(runtime.sd_ctx, &params, &generated)) {
+            output_fps       = generated->fps;
+            generated_audio  = generated->audio;
+            generated->audio = nullptr;
+            results.adopt_video(generated, runtime.ctx_params->lora_model_dir);
+            output_info = json::parse(results.generation_info(0));
+            if (job.vid_gen.gen_params.embed_image_metadata)
+                str_params = results.generation_info(0);
         }
-        clear_preview_callback();
-        results.adopt(raw_results, num_results);
     }
 
     num_results = results.count();
@@ -376,6 +456,7 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
         return false;
     }
 
+    update_job_progress(job, "encoding", "video");
     std::vector<uint8_t> video_bytes = create_video_from_sd_images_to_vector(job.vid_gen.output_format,
                                                                              results.data(),
                                                                              num_results,
@@ -424,9 +505,11 @@ void async_job_worker(ServerRuntime& runtime) {
             job             = it->second;
             job->status     = AsyncJobStatus::Generating;
             job->started_at = unix_timestamp_now_ms();
+            update_job_progress(*job, "waiting");
         }
 
         std::vector<std::string> output_images;
+        json output_info;
         std::string output_media_b64;
         std::string output_media_mime_type;
         int output_frame_count = 0;
@@ -435,7 +518,7 @@ void async_job_worker(ServerRuntime& runtime) {
         bool ok = false;
 
         if (job->kind == AsyncJobKind::ImgGen) {
-            ok = execute_img_gen_job(runtime, *job, output_images, error_message);
+            ok = execute_img_gen_job(runtime, *job, output_images, output_info, error_message);
         } else if (job->kind == AsyncJobKind::VidGen) {
             ok = execute_vid_gen_job(runtime,
                                      *job,
@@ -443,6 +526,7 @@ void async_job_worker(ServerRuntime& runtime) {
                                      output_media_mime_type,
                                      output_frame_count,
                                      output_fps,
+                                     output_info,
                                      error_message);
         } else {
             error_message = "unsupported job kind";
@@ -456,6 +540,7 @@ void async_job_worker(ServerRuntime& runtime) {
             }
 
             job->completed_at = unix_timestamp_now_ms();
+            finish_job_progress(*job);
             if (job->cancel_requested.load()) {
                 job->status        = AsyncJobStatus::Cancelled;
                 job->error_code    = "cancelled";
@@ -463,6 +548,7 @@ void async_job_worker(ServerRuntime& runtime) {
             } else if (ok) {
                 job->status                 = AsyncJobStatus::Completed;
                 job->result_images_b64      = std::move(output_images);
+                job->result_generation_info = std::move(output_info);
                 job->result_media_b64       = std::move(output_media_b64);
                 job->result_media_mime_type = std::move(output_media_mime_type);
                 job->result_frame_count     = output_frame_count;
@@ -474,6 +560,7 @@ void async_job_worker(ServerRuntime& runtime) {
                 job->error_code    = "generation_failed";
                 job->error_message = error_message.empty() ? "unknown generation error" : error_message;
                 job->result_images_b64.clear();
+                job->result_generation_info = nullptr;
                 job->result_media_b64.clear();
                 job->result_media_mime_type.clear();
                 job->result_frame_count = 0;

examples/server/async_jobs.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <atomic>
+#include <chrono>
 #include <condition_variable>
 #include <cstdint>
 #include <deque>
@@ -29,6 +30,17 @@ enum class AsyncJobStatus {
 const char* async_job_kind_name(AsyncJobKind kind);
 const char* async_job_status_name(AsyncJobStatus status);
 
+struct AsyncJobProgress {
+    std::string stage = "queued";
+    std::string detail;
+    std::string unit;
+    int sample_pass                                     = 0;
+    int64_t completed                                   = 0;
+    int64_t total                                       = 0;
+    std::chrono::steady_clock::time_point stage_started = std::chrono::steady_clock::now();
+    std::chrono::steady_clock::time_point finished{};
+};
+
 struct AsyncGenerationJob {
     std::string id;
     AsyncJobKind kind     = AsyncJobKind::ImgGen;
@@ -40,12 +52,15 @@ struct AsyncGenerationJob {
     ImgGenJobRequest img_gen;
     VidGenJobRequest vid_gen;
     std::vector<std::string> result_images_b64;
+    json result_generation_info;
     std::string result_media_b64;
     std::string result_media_mime_type;
     int result_frame_count = 0;
     int result_fps         = 0;
     std::string error_code;
     std::string error_message;
+    mutable std::mutex progress_mutex;
+    AsyncJobProgress progress;
     mutable std::mutex preview_mutex;
     std::string preview_b64;
     int preview_pass          = 0;
@@ -74,6 +89,7 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
 bool execute_img_gen_job(ServerRuntime& runtime,
                          AsyncGenerationJob& job,
                          std::vector<std::string>& output_images,
+                         json& output_info,
                          std::string& error_message);
 bool execute_vid_gen_job(ServerRuntime& runtime,
                          AsyncGenerationJob& job,
@@ -81,5 +97,6 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
                          std::string& output_media_mime_type,
                          int& output_frame_count,
                          int& output_fps,
+                         json& output_info,
                          std::string& error_message);
 void async_job_worker(ServerRuntime& runtime);

examples/server/model_catalog.cpp

@@ -19,7 +19,7 @@ struct ModelSlot {
     const char* id;
     const char* label;
     std::vector<std::string> categories;
-    std::string SDContextParams::*field;
+    std::string SDContextParams::* field;
 };
 
 static const ModelSlot k_model_slots[] = {
@@ -66,6 +66,14 @@ static void clear_model_paths(SDContextParams& params) {
     }
 }
 
+bool component_paths_match(const SDContextParams& a, const SDContextParams& b) {
+    for (const auto& slot : k_model_slots) {
+        if ((a.*slot.field) != (b.*slot.field))
+            return false;
+    }
+    return true;
+}
+
 std::string model_path_key(const fs::path& path) {
 #ifdef _WIN32
     const auto value = path.generic_wstring();
@@ -291,6 +299,7 @@ static void apply_recipe_context(const json& settings, SDContextParams& params)
     }
 }
 
+// Known recipe feature names with their defaults; recipes may flip any of these.
 static const std::map<std::string, bool> k_recipe_features = {
     {"img2img", true},
     {"mask_image", true},
@@ -312,6 +321,9 @@ static const std::map<std::string, bool> k_recipe_features = {
     {"init_image", false},
     {"end_image", false},
     {"control_frames", false},
+    {"ref_videos", false},
+    {"ref_audios", false},
+    {"ref_video_audios", false},
     {"vace", false},
     {"high_noise", false},
     {"cache_unet", false},
@@ -365,17 +377,44 @@ json model_generation_features(const ServerRuntime& runtime) {
     if (runtime.models->active_recipe.value("source", "") == "command_line")
         return features;
     // Optional components only become usable once selected and loaded.
-    features["ip_adapter"]  = features.value("ip_adapter", false) && !runtime.ctx_params->ip_adapter_path.empty();
-    features["controlnet"]  = features.value("controlnet", false) && !runtime.ctx_params->control_net_path.empty();
-    features["high_noise"]  = features.value("high_noise", false) && !runtime.ctx_params->high_noise_diffusion_model_path.empty();
-    features["preview_tae"] = features.value("preview_tae", false) && !runtime.ctx_params->taesd_path.empty();
+    features["ip_adapter"]       = features.value("ip_adapter", false) && !runtime.ctx_params->ip_adapter_path.empty();
+    features["controlnet"]       = features.value("controlnet", false) && !runtime.ctx_params->control_net_path.empty();
+    features["high_noise"]       = features.value("high_noise", false) && !runtime.ctx_params->high_noise_diffusion_model_path.empty();
+    features["preview_tae"]      = features.value("preview_tae", false) && !runtime.ctx_params->taesd_path.empty();
+    features["ref_audios"]       = features.value("ref_audios", false) &&
+                                   (!runtime.ctx_params->audio_vae_path.empty() || !runtime.ctx_params->audio_encoder_path.empty());
+    features["ref_video_audios"] = features.value("ref_video_audios", false) && !runtime.ctx_params->audio_vae_path.empty();
     if (!runtime.ctx_params->pulid_weights_path.empty())
         features["slg"] = false;
     return features;
 }
 
+static void validate_video_constraints(const json& constraints) {
+    check_keys(constraints, {"frame_step", "frame_offset", "frame_rounding", "min_frames", "max_frames", "fixed_fps"});
+    for (const auto* key : {"frame_step", "frame_offset", "min_frames", "max_frames", "fixed_fps"}) {
+        const int minimum = std::string(key) == "frame_offset" ? 0 : 1;
+        const int maximum = std::string(key) == "fixed_fps" ? 240 : 4096;
+        if (constraints.contains(key) &&
+            (!constraints[key].is_number_integer() || constraints[key] < minimum || constraints[key] > maximum)) {
+            throw std::runtime_error(std::string("video_constraints.") + key + " is outside its integer range");
+        }
+    }
+    const int step    = constraints.value("frame_step", 1);
+    const int offset  = constraints.value("frame_offset", 0);
+    const int minimum = constraints.value("min_frames", 1);
+    const int maximum = constraints.value("max_frames", 4096);
+    if (offset >= step || minimum > maximum ||
+        offset + std::max(0, (minimum - offset + step - 1) / step) * step > maximum) {
+        throw std::runtime_error("video_constraints must describe a nonempty frame range with frame_offset < frame_step");
+    }
+    if (constraints.contains("frame_rounding") &&
+        constraints["frame_rounding"] != "floor" && constraints["frame_rounding"] != "ceil") {
+        throw std::runtime_error("video_constraints.frame_rounding must be floor or ceil");
+    }
+}
+
 static void validate_recipe(const json& recipe) {
-    check_keys(recipe, {"schema_version", "id", "name", "description", "model_url", "slots", "required_any", "defaults", "context", "features", "disabled_features"});
+    check_keys(recipe, {"schema_version", "id", "name", "description", "model_url", "slots", "required_any", "models", "defaults", "context", "features", "disabled_features", "size_multiple", "video_constraints"});
     if (!recipe.at("schema_version").is_number_integer() || recipe.at("schema_version") != 1 ||
         !valid_identifier(recipe.at("id").get<std::string>()) || recipe.at("name").get<std::string>().empty()) {
         throw std::runtime_error("invalid schema_version, id or name");
@@ -383,6 +422,13 @@ static void validate_recipe(const json& recipe) {
     if (recipe.contains("description") && !recipe["description"].is_string()) {
         throw std::runtime_error("description must be a string");
     }
+    if (recipe.contains("size_multiple") &&
+        (!recipe["size_multiple"].is_number_integer() || recipe["size_multiple"] < 1 || recipe["size_multiple"] > 4096)) {
+        throw std::runtime_error("size_multiple must be an integer from 1 to 4096");
+    }
+    if (recipe.contains("video_constraints")) {
+        validate_video_constraints(recipe["video_constraints"]);
+    }
     if (recipe.contains("model_url")) {
         const auto url           = recipe.at("model_url").get<std::string>();
         const std::string prefix = "https://huggingface.co/";
@@ -411,6 +457,19 @@ static void validate_recipe(const json& recipe) {
     if (!has_required_model) {
         throw std::runtime_error("a required model or diffusion_model slot is needed");
     }
+    // Optional recipe-local model defaults: slot ID to file ID mappings that
+    // prefill slots a load request does not select itself.
+    if (recipe.contains("models")) {
+        const auto& recipe_models = recipe.at("models");
+        if (!recipe_models.is_object() || recipe_models.size() > 32)
+            throw std::runtime_error("models must be an object of at most 32 slot entries");
+        std::set<std::string> default_ids;
+        for (auto it = recipe_models.begin(); it != recipe_models.end(); ++it) {
+            if (!ids.count(it.key() == "taesd" ? "tae" : it.key()) || !it.value().is_string() || it.value().get<std::string>().empty() ||
+                !default_ids.insert(it.key() == "taesd" ? "tae" : it.key()).second)
+                throw std::runtime_error("unknown or invalid default model: " + it.key());
+        }
+    }
     if (recipe.contains("required_any")) {
         const auto& groups = recipe.at("required_any");
         if (!groups.is_array() || groups.size() > 32)
@@ -435,6 +494,7 @@ static void validate_recipe(const json& recipe) {
 
 static json normalize_recipe(json recipe) {
     validate_recipe(recipe);
+    recipe["size_multiple"]      = recipe.value("size_multiple", 8);
     recipe["effective_features"] = resolve_recipe_features(recipe);
     for (auto& slot : recipe.at("slots")) {
         const auto& spec   = *find_model_slot(slot.at("id").get<std::string>());
@@ -598,6 +658,7 @@ void initialize_server_models(ServerRuntime& runtime) {
             {"description", "Uses the loaded command-line model and its generation defaults."},
             {"slots", json::array()},
             {"defaults", json::object()},
+            {"size_multiple", 8},
             {"features", features},
         };
         models.active_recipe["effective_features"] = resolve_recipe_features(models.active_recipe);
@@ -606,10 +667,11 @@ void initialize_server_models(ServerRuntime& runtime) {
     refresh_model_catalog(runtime);
 }
 
-void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe) {
+void prepare_model_load(ServerRuntime& runtime, json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe) {
     check_keys(body, {"recipe", "models"});
     const auto recipe_id = body.at("recipe").get<std::string>();
-    auto selection       = body.at("models");
+    // `models` is optional; unselected slots may fall back to recipe defaults below.
+    auto selection = body.value("models", json::object());
     if (!selection.is_object())
         throw std::runtime_error("models must be an object");
     auto& models = *runtime.models;
@@ -624,6 +686,20 @@ void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParam
     defaults = models.base_defaults;
     apply_recipe_context(recipe.value("context", json::object()), context);
     apply_recipe_defaults(recipe.value("defaults", json::object()), defaults);
+    // Recipe-provided default models fill any slot the request did not select.
+    // An explicit selection blocks the default for that slot, including an
+    // empty string (a deliberate opt-out of the default). The tae/taesd pair
+    // is one slot: an explicit request selection of either spelling overrides
+    // the recipe default for that slot.
+    if (recipe.contains("models")) {
+        for (auto it = recipe.at("models").begin(); it != recipe.at("models").end(); ++it) {
+            if (selection.contains(it.key()))
+                continue;
+            if ((it.key() == "tae" || it.key() == "taesd") && (selection.contains("tae") || selection.contains("taesd")))
+                continue;
+            selection[it.key()] = it.value();
+        }
+    }
     std::set<std::string> allowed;
     for (const auto& slot : recipe.at("slots")) {
         const auto id = slot.at("id").get<std::string>();
@@ -635,6 +711,9 @@ void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParam
                 if (selection.contains(id))
                     throw std::runtime_error("specify only one of tae and taesd");
                 selection[id] = selection.at(alias);
+                // Keep only the declared spelling: equivalent loads must resolve
+                // to an identical selection for the idempotent-load comparison.
+                selection.erase(alias);
             }
         }
         const std::string file_id = selection.value(id, std::string());
@@ -658,6 +737,20 @@ void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParam
         if (!allowed.count(it.key()))
             throw std::runtime_error("unknown model slot: " + it.key());
     }
+    // Omitting an optional slot and sending it as an empty string resolve to
+    // the same context; drop the empty key so equivalent loads compare
+    // identically instead of rebuilding the context. The explicit empty
+    // string already overrode any recipe default during the merge, so its
+    // intent is preserved.
+    for (auto it = selection.begin(); it != selection.end();) {
+        if (!it->is_string() || !it->get<std::string>().empty()) {
+            ++it;
+            continue;
+        }
+        it = selection.erase(it);
+    }
+    // Echo the fully resolved selection (recipe defaults merged in) for status reporting.
+    body["models"] = selection;
     for (const auto& group : recipe.value("required_any", json::array())) {
         if (std::none_of(group.begin(), group.end(), [&](const json& value) {
                 return !selection.value(value.get<std::string>(), std::string()).empty();

examples/server/model_catalog.h

@@ -51,6 +51,14 @@ void initialize_server_models(ServerRuntime& runtime);
 json resolve_recipe_features(const json& recipe);
 json model_generation_features(const ServerRuntime& runtime);
 
-void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe = nullptr);
+// Resolves the load request into context/generation params. `body` is updated
+// so `models` reflects the fully resolved selection (recipe defaults merged in),
+// letting the caller report exactly which components got loaded.
+void prepare_model_load(ServerRuntime& runtime, json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe = nullptr);
+
+// True when both parameter sets resolve to the same component files. File IDs
+// stay stable if a model directory symlink is retargeted inside the configured
+// root, so the idempotent-load check compares the resolved paths too.
+bool component_paths_match(const SDContextParams& a, const SDContextParams& b);
 
 #endif

examples/server/openai_edit_policy.h

@@ -1,44 +0,0 @@
-#ifndef __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__
-#define __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__
-
-#include "common/resource_owners.hpp"
-
-enum class OpenAIEditInitImageAction {
-    KeepExisting,
-    UseUploaded,
-    Clear,
-};
-
-inline bool openai_edit_uses_uploaded_image_as_init(bool supports_ref_image_conditioning,
-                                                    bool has_ref_images,
-                                                    bool has_mask,
-                                                    bool has_explicit_strength_override) {
-    return !supports_ref_image_conditioning || !has_ref_images || has_mask || has_explicit_strength_override;
-}
-
-inline OpenAIEditInitImageAction apply_openai_edit_init_image_policy(
-    SDImageOwner& init_image,
-    SDImageOwner& uploaded_image,
-    bool uploaded_image_was_first_reference,
-    bool supports_ref_image_conditioning,
-    bool has_ref_images,
-    bool has_mask,
-    bool has_extra_init_image_override,
-    bool has_explicit_strength_override) {
-    if (!uploaded_image_was_first_reference || has_extra_init_image_override) {
-        return OpenAIEditInitImageAction::KeepExisting;
-    }
-
-    if (openai_edit_uses_uploaded_image_as_init(supports_ref_image_conditioning,
-                                                has_ref_images,
-                                                has_mask,
-                                                has_explicit_strength_override)) {
-        init_image = std::move(uploaded_image);
-        return OpenAIEditInitImageAction::UseUploaded;
-    }
-
-    init_image.reset();
-    return OpenAIEditInitImageAction::Clear;
-}
-
-#endif  // __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__

examples/server/recipes/README.md

@@ -77,3 +77,35 @@ These are built-in catalog rules. Users may keep personal presets in
 `<model-dir>/recipes/`; they need not add those presets to the built-in list.
 See [the recipe schema and feature reference](../api.md#user-recipe-format)
 for supported fields.
+
+## Output dimension alignment
+
+Set `size_multiple` when the runtime's output dimension alignment differs from
+the default of `8`. Match `GenerationRequest::align_image_size` in
+[`src/pipeline/request.cpp`](../../../src/pipeline/request.cpp): the VAE scale
+factor multiplied by the model definition's `diffusion_down_factor`. The scale
+comes from [`VAE::get_scale_factor`](../../../src/model/vae/vae.hpp), with the
+PiD override in [`diffusion_engine.cpp`](../../../src/pipeline/diffusion_engine.cpp);
+model geometry is defined under [`src/model/integrations`](../../../src/model/integrations/).
+
+Use the runtime's output alignment, not the nominal patch size: some models pad
+patches internally and crop their output back to the requested size. Pixel-space
+models with an alignment of `1` explicitly use `size_multiple: 1`. MiniT2I also
+has this alignment, but its separate fixed `512 × 512` output restriction still
+applies. Reference-image preprocessing constraints are independent of this field.
+
+## Video length constraints
+
+Video recipes declare `video_constraints` to match `align_video_frames` in
+[`diffusion_engine.cpp`](../../../src/pipeline/diffusion_engine.cpp) and the FPS
+and AnimateDiff overrides in
+[`request.cpp`](../../../src/pipeline/request.cpp). Wan, HunyuanVideo and LingBot
+round down to `4n + 1`; LTX rounds down to `8n + 1`. MiniMax-H3 rounds up to
+`17n + 5`, with at least 5 frames and fixed 24 FPS. AnimateDiff accepts whole
+frames up to 32. Wan 2.2 S2V fixes FPS at 16.
+
+Declare fixed FPS only where the runtime enforces it; recommended FPS belongs
+in `defaults.fps`. These constraints guide UI conversion and estimates, not
+runtime behavior. Audio-driven output and temporal upscaling may still change
+the final frame count. See the [recipe schema](../api.md#user-recipe-format)
+for field defaults and validation rules.

examples/server/recipes/animatediff-sd15.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "animatediff-sd15",
   "name": "AnimateDiff SD 1.5",
+  "size_multiple": 64,
   "description": "Use Video Generation with an SD 1.5 checkpoint and compatible motion module.",
   "model_url": "https://huggingface.co/guoyww/animatediff",
   "slots": [
@@ -22,6 +23,13 @@
       "required": false
     }
   ],
+  "video_constraints": {
+    "frame_step": 1,
+    "frame_offset": 0,
+    "frame_rounding": "floor",
+    "min_frames": 1,
+    "max_frames": 32
+  },
   "defaults": {
     "sampling_method": "euler",
     "video_frames": 8,

examples/server/recipes/chroma-radiance.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "chroma-radiance",
   "name": "Chroma Radiance",
+  "size_multiple": 1,
   "description": "Pixel-space generation without a VAE. Also accepts the x0 and experimental patch-32 variants.",
   "model_url": "https://huggingface.co/lodestones/Chroma1-Radiance",
   "slots": [

examples/server/recipes/cosxl-edit.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "cosxl-edit",
   "name": "CosXL Edit",
+  "size_multiple": 64,
   "description": "Supply the image to edit as a reference image.",
   "model_url": "https://huggingface.co/stabilityai/cosxl",
   "slots": [

examples/server/recipes/defaults.md

@@ -5,6 +5,9 @@ Reviewed against model-publisher cards and reference implementations on 2026-10-
 These are starting values, not limits on supported resolutions or generation settings.
 Built-in step defaults above 20 are reduced to 20; the cited sources may recommend more steps.
 Users can still choose a higher step count.
+Except for PiD recipes, built-in dimensions above 1,048,576 pixels (1024 x 1024) are reduced to 1024 x 1024, even when the cited source uses a larger size.
+PiD recipes retain their 2048 x 2048 defaults.
+Users can still choose a higher resolution.
 When several aspect ratios or ranges are published, the selected example or square bucket is noted below.
 A dash means no recipe override: the available primary sources did not establish a fixed value, or the size depends on the input image.
 Ordinary DiT recipes explicitly use Euler; PiD retains its LCM sampling path.
@@ -27,7 +30,7 @@ For community edit adapters and other optional extensions, these are the base-mo
 | `flux1-canny` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Canny-dev)  |
 | `flux1-depth` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Depth-dev)  |
 | `flux1-dev` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-dev)  |
-| `flux1-fill` | 1 | 20 | 1232 × 1632 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Fill-dev)  |
+| `flux1-fill` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Fill-dev)  |
 | `flux1-kontext` | 1 | 20 | — | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/flux/pipeline_flux_kontext.py) Output size normally follows the reference image; no fixed size is imposed by this recipe. |
 | `flux1-pulid` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-dev) FLUX.1-dev defaults; identity conditioning is provided by PuLID. |
 | `flux1-schnell` | 1 | 4 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-schnell)  |
@@ -38,7 +41,9 @@ For community edit adapters and other optional extensions, these are the base-mo
 | `flux2-klein-base-9b` | 4 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B)  |
 | `flux3-klein` | 1 | 16 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/black-forest-labs/flux-3-image-klein/main/README.md)  |
 | `flux3-klein-base` | 4 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/black-forest-labs/flux-3-image-klein/main/src/flux3/pipeline.py)  |
-| `hidream-o1-dev` | 1 | 20 | 2048 × 2048 | [Source](https://huggingface.co/HiDream-ai/HiDream-O1-Image-Dev)  |
+| `hidream-o1-dev` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/HiDream-ai/HiDream-O1-Image-Dev)  |
+| `hyperflow-h3-flash-fl2va` | 1 | 6 | 1344 × 768 | [Source](https://huggingface.co/VideoRebirthAI/hyperflow-h3-flash) Two-time FL2VA checkpoint; native text and first/last-frame generation. Fixed 6-step video sigma grid with shift 12, 124 frames at 24 FPS. Reuses MiniMax-H3 components. |
+| `hyperflow-h3-flash-ref2va` | 1 | 6 | 1344 × 768 | [Source](https://huggingface.co/VideoRebirthAI/hyperflow-h3-flash) Separate two-time Ref2VA checkpoint; image references. Same schedule and components as FL2VA; video/audio references remain unverified. |
 | `hunyuan-video-15-i2v` | 6 | 20 | 1280 × 720 | [Source](https://huggingface.co/tencent/HunyuanVideo-1.5) 720p base I2V defaults; CFG/step-distilled and 480p checkpoints use different settings. |
 | `hunyuan-video-15-t2v` | 6 | 20 | 1280 × 720 | [Source](https://huggingface.co/tencent/HunyuanVideo-1.5) 720p base T2V defaults; CFG-distilled / 480p checkpoints need their published settings. |
 | `ideogram4` | 7 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/ideogram-oss/ideogram4/main/docs/inference.md) Constant-guidance pipeline defaults; this recipe does not reproduce the CLI’s per-step guidance schedule. |
@@ -46,10 +51,10 @@ For community edit adapters and other optional extensions, these are the base-mo
 | `iris-3b` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/speridlabs/iris-3b)  |
 | `krea2-raw` | 3.5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/krea/Krea-2-Raw) Base model defaults; edit requires a matching community LoRA with its own settings. |
 | `krea2-raw-edit` | 3.5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/krea/Krea-2-Raw) Base model defaults; edit requires a matching community LoRA with its own settings. |
-| `krea2-turbo` | 1 | 8 | 2048 × 2048 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. |
-| `krea2-turbo-edit` | 1 | 8 | 2048 × 2048 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. |
-| `lens` | 5 | 20 | 1440 × 1440 | [Source](https://huggingface.co/microsoft/Lens) Official model card: RL model, default base_resolution 1440. |
-| `lens-turbo` | 1 | 4 | 1440 × 1440 | [Source](https://huggingface.co/microsoft/Lens-Turbo) Official model card: 4-step distilled model, default base_resolution 1440. |
+| `krea2-turbo` | 1 | 8 | 1024 × 1024 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. |
+| `krea2-turbo-edit` | 1 | 8 | 1024 × 1024 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. |
+| `lens` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Lens) Official model card: RL model, default base_resolution 1440. |
+| `lens-turbo` | 1 | 4 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Lens-Turbo) Official model card: 4-step distilled model, default base_resolution 1440. |
 | `lingbot-video-dense` | 3 | 20 | 832 × 480 | [Source](https://huggingface.co/robbyant/lingbot-video-dense-1.3b) Official first-stage inference example; optional refiner is separate. |
 | `lingbot-video-moe` | 3 | 20 | 832 × 480 | [Source](https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b) Official inference example. |
 | `llada-image` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/inclusionAI/LLaDA-Image)  |
@@ -62,12 +67,13 @@ For community edit adapters and other optional extensions, these are the base-mo
 | `mage-flow` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow)  |
 | `mage-flow-edit` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow)  |
 | `mage-flow-turbo` | 1 | 4 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow)  |
-| `ming-image` | 1 | 12 | 2048 × 2048 | [Source](https://huggingface.co/inclusionAI/Ming-Image-0.1-Design)  |
+| `ming-image` | 1 | 12 | 1024 × 1024 | [Source](https://huggingface.co/inclusionAI/Ming-Image-0.1-Design)  |
 | `minimax-h3-fl2va` | 1 | — | 1344 × 768 | [Source](https://huggingface.co/docs/diffusers/main/en/api/pipelines/minimax_h3) Guidance-distilled; official 768p canvas. No fixed step count is prescribed here; the upstream API counts sigma grid points rather than sd.cpp denoising steps. |
 | `minimax-h3-ref2va` | 1 | — | 1344 × 768 | [Source](https://huggingface.co/docs/diffusers/main/en/api/pipelines/minimax_h3) Guidance-distilled; official 768p canvas. No fixed step count is prescribed here; the upstream API counts sigma grid points rather than sd.cpp denoising steps. |
 | `minit2i` | 2.5 | 20 | 512 × 512 | [Source](https://huggingface.co/MiniT2I/MiniT2I/raw/main/pipeline.py) B/16 example and native model size; L/16 uses CFG 6. |
 | `moss-image-base` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base)  |
 | `moss-image-sft` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT)  |
+| `p-h3-image` | 1 | 8 | 1024 × 1024 | [Runtime guide](../../../docs/p_h3_image.md) Fixed P-H3 sigma schedule; upstream 2912 × 2912 reduced to the catalog default. |
 | `nitrosd-realism` | 1 | 1 | 1024 × 1024 | [Source](https://huggingface.co/ChenDY/NitroFusion)  |
 | `ovis-image` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/ATH-MaaS/Ovis-Image-7B)  |
 | `pid` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
@@ -81,9 +87,9 @@ For community edit adapters and other optional extensions, these are the base-mo
 | `pixart-alpha-1024` | 4.5 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py)  |
 | `pixart-alpha-512` | 4.5 | 20 | 512 × 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py)  |
 | `pixart-sigma` | 4.5 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py)  |
-| `qwen-image` | 4 | 20 | 1328 × 1328 | [Source](https://huggingface.co/Qwen/Qwen-Image) Official square resolution bucket; true_cfg_scale maps to cfg_scale. |
-| `qwen-image-21` | 1 | 20 | 2048 × 2048 | [Source](https://raw.githubusercontent.com/QwenLM/Qwen-Image-2.1/main/README.md) Original checkpoint defaults; select the matching TAEQI2.1 to replace its VAE. |
-| `qwen-image-21-turbo` | 1 | 8 | 2048 × 2048 | [Model card](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo), [fixed sigmas](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo/blob/main/model_index.json). Standalone Turbo weights; the fixed 8-step schedule is applied on model load. |
+| `qwen-image` | 4 | 20 | 1024 × 1024 | [Source](https://huggingface.co/Qwen/Qwen-Image) The source uses a larger square resolution bucket; true_cfg_scale maps to cfg_scale. |
+| `qwen-image-21` | 1 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/QwenLM/Qwen-Image-2.1/main/README.md) Original checkpoint sampling defaults; select the matching TAEQI2.1 to replace its VAE. |
+| `qwen-image-21-turbo` | 1 | 8 | 1024 × 1024 | [Model card](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo), [fixed sigmas](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo/blob/main/model_index.json). Standalone Turbo weights; the fixed 8-step schedule is applied on model load. |
 | `qwen-image-edit` | 4 | 20 | — | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit) Official editing example; output dimensions follow reference-image aspect ratio. |
 | `qwen-image-edit-2509` | 4 | 20 | — | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit-2509) Official editing example; output dimensions follow reference-image aspect ratio. |
 | `qwen-image-edit-2511` | 4 | 20 | — | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit-2511) Official editing example; output dimensions follow reference-image aspect ratio. |
@@ -107,7 +113,7 @@ For community edit adapters and other optional extensions, these are the base-mo
 | `sdxl-turbo` | 1 | 1 | 512 × 512 | [Source](https://huggingface.co/stabilityai/sdxl-turbo)  |
 | `sdxs` | 1 | 1 | 512 × 512 | [Source](https://huggingface.co/IDKiro/sdxs-512-dreamshaper)  |
 | `sefi-image` | 4 | 20 | — | [Source](https://huggingface.co/SeFi-Image/SeFi-Image-5B-Base) Base/RL family defaults; Turbo checkpoints require 4 steps / CFG 1. |
-| `sensenova-u15` | 4 | 20 | 2048 × 2048 | [Source](https://raw.githubusercontent.com/OpenSenseNova/SenseNova-U1/main/examples/README.md)  |
+| `sensenova-u15` | 4 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/OpenSenseNova/SenseNova-U1/main/examples/README.md)  |
 | `wan21-flf2v` | 5 | 20 | 1280 × 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py)  |
 | `wan21-i2v` | 5 | 20 | 832 × 480 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py) 480p base defaults; also accepts compatible SkyReels V2 and single Wan 2.2 I2V experts. |
 | `wan21-t2v` | 5 | 20 | 1280 × 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py) 14B base defaults; single Wan 2.2 experts and acceleration LoRAs need matching settings. |

examples/server/recipes/ernie-image-turbo.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "ernie-image-turbo",
   "name": "ERNIE Image Turbo",
+  "size_multiple": 16,
   "model_url": "https://huggingface.co/baidu/ERNIE-Image-Turbo",
   "slots": [
     {

examples/server/recipes/ernie-image.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "ernie-image",
   "name": "ERNIE Image",
+  "size_multiple": 16,
   "model_url": "https://huggingface.co/baidu/ERNIE-Image",
   "slots": [
     {

examples/server/recipes/fibo2-dev.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "fibo2-dev",
   "name": "Fibo 2.0 Dev",
+  "size_multiple": 16,
   "description": "Text-to-image generation and native reference-image editing. Select a separate vision projector for editing when it is not bundled in the text encoder.",
   "model_url": "https://huggingface.co/briaai/Fibo-2.0-dev",
   "slots": [

examples/server/recipes/fibo2-edit-turbo.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "fibo2-edit-turbo",
   "name": "Fibo 2.0 Edit Turbo",
+  "size_multiple": 16,
   "description": "Dedicated reference-image editing checkpoint; supply reference images. Select a separate vision projector when it is not bundled in the text encoder.",
   "model_url": "https://huggingface.co/briaai/Fibo-2.0-edit-turbo",
   "slots": [

examples/server/recipes/flux1-fill.json

@@ -39,8 +39,8 @@
     "scheduler": "flux",
     "cfg_scale": 1,
     "steps": 20,
-    "width": 1232,
-    "height": 1632
+    "width": 1024,
+    "height": 1024
   },
   "features": [
     "clip_skip",

examples/server/recipes/flux2-dev.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "flux2-dev",
   "name": "FLUX.2 dev",
+  "size_multiple": 16,
   "description": "Supports text-to-image and reference-image editing.",
   "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-dev",
   "slots": [

examples/server/recipes/flux2-klein-4b.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "flux2-klein-4b",
   "name": "FLUX.2 Klein 4B",
+  "size_multiple": 16,
   "description": "Supports reference-image editing. The FLUX.2 small decoder is an alternative VAE.",
   "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-4B",
   "slots": [

examples/server/recipes/flux2-klein-9b.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "flux2-klein-9b",
   "name": "FLUX.2 Klein 9B",
+  "size_multiple": 16,
   "description": "Supports text-to-image and reference-image editing.",
   "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-9B",
   "slots": [

examples/server/recipes/flux2-klein-base-4b.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "flux2-klein-base-4b",
   "name": "FLUX.2 Klein Base 4B",
+  "size_multiple": 16,
   "description": "Supports text-to-image and reference-image editing.",
   "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4B",
   "slots": [

examples/server/recipes/flux2-klein-base-9b.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "flux2-klein-base-9b",
   "name": "FLUX.2 Klein Base 9B",
+  "size_multiple": 16,
   "description": "Use the 9B base transformer with Qwen 3 8B. Supports reference-image editing.",
   "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B",
   "slots": [

examples/server/recipes/flux3-klein-base.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "flux3-klein-base",
   "name": "FLUX.3 Image Klein Base",
+  "size_multiple": 16,
   "model_url": "https://huggingface.co/black-forest-labs/flux-3-image-klein-base",
   "slots": [
     {

examples/server/recipes/flux3-klein.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "flux3-klein",
   "name": "FLUX.3 Image Klein",
+  "size_multiple": 16,
   "description": "Supports text-to-image and reference-image editing.",
   "model_url": "https://huggingface.co/black-forest-labs/flux-3-image-klein",
   "slots": [

examples/server/recipes/hidream-o1-dev.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "hidream-o1-dev",
   "name": "HiDream O1 Image dev",
+  "size_multiple": 1,
   "description": "Bundled checkpoint; also supports reference-image editing.",
   "model_url": "https://huggingface.co/HiDream-ai/HiDream-O1-Image-Dev",
   "slots": [
@@ -13,8 +14,8 @@
   "defaults": {
     "cfg_scale": 1,
     "steps": 20,
-    "width": 2048,
-    "height": 2048,
+    "width": 1024,
+    "height": 1024,
     "sampling_method": "euler"
   },
   "features": [

examples/server/recipes/hunyuan-video-15-i2v.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "hunyuan-video-15-i2v",
   "name": "HunyuanVideo 1.5 I2V",
+  "size_multiple": 16,
   "description": "Select HunyuanVideo 1.5 I2V weights and supply a start image in Video Generation.",
   "model_url": "https://huggingface.co/tencent/HunyuanVideo-1.5",
   "slots": [
@@ -36,6 +37,12 @@
       "tae"
     ]
   ],
+  "video_constraints": {
+    "frame_step": 4,
+    "frame_offset": 1,
+    "frame_rounding": "floor",
+    "min_frames": 1
+  },
   "defaults": {
     "sampling_method": "euler",
     "video_frames": 33,

examples/server/recipes/hunyuan-video-15-t2v.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "hunyuan-video-15-t2v",
   "name": "HunyuanVideo 1.5 T2V",
+  "size_multiple": 16,
   "description": "Select HunyuanVideo 1.5 T2V weights and use Video Generation. Use the separate I2V recipe for I2V weights.",
   "model_url": "https://huggingface.co/tencent/HunyuanVideo-1.5",
   "slots": [
@@ -36,6 +37,12 @@
       "tae"
     ]
   ],
+  "video_constraints": {
+    "frame_step": 4,
+    "frame_offset": 1,
+    "frame_rounding": "floor",
+    "min_frames": 1
+  },
   "defaults": {
     "sampling_method": "euler",
     "video_frames": 33,

examples/server/recipes/hyperflow-h3-flash-fl2va.json

@@ -0,0 +1,75 @@
+{
+  "schema_version": 1,
+  "id": "hyperflow-h3-flash-fl2va",
+  "name": "HyperFlow H3 Flash FL2VA",
+  "size_multiple": 32,
+  "description": "Generate video and audio from text, optionally with start/end images. Use the HyperFlow H3 Flash FL2VA transformer. Reuses MiniMax-H3 components with the HyperFlow 6-step schedule. Audio VAE is optional for silent video.",
+  "model_url": "https://huggingface.co/VideoRebirthAI/hyperflow-h3-flash",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true,
+      "label": "HyperFlow H3 Flash FL2VA Transformer"
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "MiniMax-H3 Qwen 3 VL 32B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "MiniMax H3 Video VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    },
+    {
+      "id": "audio_vae",
+      "required": false,
+      "label": "MiniMax H3 Audio VAE"
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "video_constraints": {
+    "frame_step": 17,
+    "frame_offset": 5,
+    "frame_rounding": "ceil",
+    "min_frames": 5,
+    "fixed_fps": 24
+  },
+  "defaults": {
+    "video_frames": 124,
+    "fps": 24,
+    "cfg_scale": 1,
+    "width": 1344,
+    "height": 768,
+    "sampling_method": "euler",
+    "steps": 6,
+    "flow_shift": 12,
+    "custom_sigmas": [
+      1.0,
+      0.982655942,
+      0.914642572,
+      0.674755692,
+      0.385223746,
+      0.159055337,
+      0.0
+    ]
+  },
+  "features": [
+    "end_image",
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}

examples/server/recipes/hyperflow-h3-flash-ref2va.json

@@ -0,0 +1,74 @@
+{
+  "schema_version": 1,
+  "id": "hyperflow-h3-flash-ref2va",
+  "name": "HyperFlow H3 Flash Ref2VA",
+  "size_multiple": 32,
+  "description": "Generate video and audio with reference images. Use the HyperFlow H3 Flash Ref2VA transformer; video/audio references are not validated for these weights. Reuses MiniMax-H3 components with the HyperFlow 6-step schedule. Audio VAE is optional for silent video.",
+  "model_url": "https://huggingface.co/VideoRebirthAI/hyperflow-h3-flash",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true,
+      "label": "HyperFlow H3 Flash REF2VA Transformer"
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "MiniMax-H3 Qwen 3 VL 32B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "MiniMax H3 Video VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    },
+    {
+      "id": "audio_vae",
+      "required": false,
+      "label": "MiniMax H3 Audio VAE"
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "video_constraints": {
+    "frame_step": 17,
+    "frame_offset": 5,
+    "frame_rounding": "ceil",
+    "min_frames": 5,
+    "fixed_fps": 24
+  },
+  "defaults": {
+    "video_frames": 124,
+    "fps": 24,
+    "cfg_scale": 1,
+    "width": 1344,
+    "height": 768,
+    "sampling_method": "euler",
+    "steps": 6,
+    "flow_shift": 12,
+    "custom_sigmas": [
+      1.0,
+      0.982655942,
+      0.914642572,
+      0.674755692,
+      0.385223746,
+      0.159055337,
+      0.0
+    ]
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}

examples/server/recipes/ideogram4.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "ideogram4",
   "name": "Ideogram 4",
+  "size_multiple": 16,
   "description": "Select the matching unconditional transformer for classifier-free guidance.",
   "model_url": "https://huggingface.co/ideogram-ai/ideogram-4-fp8",
   "slots": [

examples/server/recipes/instruct-pix2pix.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "instruct-pix2pix",
   "name": "Instruct-Pix2Pix",
+  "size_multiple": 64,
   "description": "Supply the image to edit as a reference image.",
   "model_url": "https://huggingface.co/timbrooks/instruct-pix2pix",
   "slots": [

examples/server/recipes/iris-3b.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "iris-3b",
   "name": "Iris 3B",
+  "size_multiple": 16,
   "description": "Pixel-space model; no VAE is required.",
   "model_url": "https://huggingface.co/speridlabs/iris-3b",
   "slots": [

examples/server/recipes/krea2-turbo-edit.json

@@ -39,8 +39,8 @@
     "sampling_method": "euler",
     "cfg_scale": 1,
     "steps": 8,
-    "width": 2048,
-    "height": 2048
+    "width": 1024,
+    "height": 1024
   },
   "features": [
     "flow_shift",

examples/server/recipes/krea2-turbo.json

@@ -32,8 +32,8 @@
   "defaults": {
     "cfg_scale": 1,
     "steps": 8,
-    "width": 2048,
-    "height": 2048,
+    "width": 1024,
+    "height": 1024,
     "sampling_method": "euler"
   },
   "features": [

examples/server/recipes/lens-turbo.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "lens-turbo",
   "name": "LENS Turbo",
+  "size_multiple": 16,
   "model_url": "https://huggingface.co/microsoft/Lens-Turbo",
   "slots": [
     {
@@ -36,8 +37,8 @@
   "defaults": {
     "cfg_scale": 1,
     "steps": 4,
-    "width": 1440,
-    "height": 1440,
+    "width": 1024,
+    "height": 1024,
     "sampling_method": "euler"
   },
   "features": [

examples/server/recipes/lens.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "lens",
   "name": "LENS",
+  "size_multiple": 16,
   "model_url": "https://huggingface.co/microsoft/Lens",
   "slots": [
     {
@@ -36,8 +37,8 @@
   "defaults": {
     "cfg_scale": 5,
     "steps": 20,
-    "width": 1440,
-    "height": 1440,
+    "width": 1024,
+    "height": 1024,
     "sampling_method": "euler"
   },
   "features": [

examples/server/recipes/lingbot-video-dense.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "lingbot-video-dense",
   "name": "LingBot Video Dense 1.3B",
+  "size_multiple": 16,
   "description": "Use Video Generation.",
   "model_url": "https://huggingface.co/robbyant/lingbot-video-dense-1.3b",
   "slots": [
@@ -30,6 +31,12 @@
       "tae"
     ]
   ],
+  "video_constraints": {
+    "frame_step": 4,
+    "frame_offset": 1,
+    "frame_rounding": "floor",
+    "min_frames": 1
+  },
   "defaults": {
     "video_frames": 33,
     "cfg_scale": 3,

examples/server/recipes/lingbot-video-moe.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "lingbot-video-moe",
   "name": "LingBot Video MoE 30B A3B",
+  "size_multiple": 16,
   "description": "Use Video Generation.",
   "model_url": "https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b",
   "slots": [
@@ -30,6 +31,12 @@
       "tae"
     ]
   ],
+  "video_constraints": {
+    "frame_step": 4,
+    "frame_offset": 1,
+    "frame_rounding": "floor",
+    "min_frames": 1
+  },
   "defaults": {
     "video_frames": 33,
     "cfg_scale": 3,

examples/server/recipes/llada-image-turbo.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "llada-image-turbo",
   "name": "LLaDA Image Turbo",
+  "size_multiple": 16,
   "description": "Use matching turbo text encoder and connectors. Editing needs connectors that include the SigVQ encoder.",
   "model_url": "https://huggingface.co/inclusionAI/LLaDA-Image-Turbo",
   "slots": [

examples/server/recipes/llada-image.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "llada-image",
   "name": "LLaDA Image (base)",
+  "size_multiple": 16,
   "description": "Use matching base transformer, text encoder and connectors; do not mix them with Turbo components. Editing needs connectors with SigVQ and 1024x1024 output.",
   "model_url": "https://huggingface.co/inclusionAI/LLaDA-Image",
   "slots": [

examples/server/recipes/ltx23-dev.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "ltx23-dev",
   "name": "LTX 2.3 dev",
+  "size_multiple": 32,
   "description": "Supports text, start-image and start/end-image video. Audio VAE is optional for silent video. Configure latent upscaling separately in generation settings. Optional distillation LoRAs use this recipe; select the LoRA and adjust the sampling settings separately.",
   "model_url": "https://huggingface.co/Lightricks/LTX-2.3",
   "slots": [
@@ -39,6 +40,12 @@
       "tae"
     ]
   ],
+  "video_constraints": {
+    "frame_step": 8,
+    "frame_offset": 1,
+    "frame_rounding": "floor",
+    "min_frames": 1
+  },
   "defaults": {
     "sampling_method": "euler",
     "video_frames": 33,

examples/server/recipes/ltx23-distilled.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "ltx23-distilled",
   "name": "LTX 2.3 Distilled 1.1",
+  "size_multiple": 32,
   "description": "Supports text, start-image and start/end-image video. Audio VAE is optional for silent video. Configure latent upscaling separately in generation settings.",
   "model_url": "https://huggingface.co/Lightricks/LTX-2.3",
   "slots": [
@@ -39,6 +40,12 @@
       "tae"
     ]
   ],
+  "video_constraints": {
+    "frame_step": 8,
+    "frame_offset": 1,
+    "frame_rounding": "floor",
+    "min_frames": 1
+  },
   "defaults": {
     "sampling_method": "euler",
     "video_frames": 121,

examples/server/recipes/ltx25-dev.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "ltx25-dev",
   "name": "LTX 2.5 dev",
+  "size_multiple": 32,
   "description": "Use the conv video VAE and LTX-specific Gemma 4 with bundled projection. Audio VAE is optional for silent video.",
   "model_url": "https://huggingface.co/Lightricks/LTX-2.5",
   "slots": [
@@ -35,6 +36,12 @@
       "tae"
     ]
   ],
+  "video_constraints": {
+    "frame_step": 8,
+    "frame_offset": 1,
+    "frame_rounding": "floor",
+    "min_frames": 1
+  },
   "defaults": {
     "sampling_method": "euler",
     "fps": 24,

examples/server/recipes/mage-flow-edit.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "mage-flow-edit",
   "name": "Mage Flow Edit",
+  "size_multiple": 16,
   "description": "Supply reference images. Select a separate vision projector when it is not bundled in the text encoder.",
   "model_url": "https://huggingface.co/microsoft/Mage-Flow-Edit",
   "slots": [

examples/server/recipes/mage-flow-turbo.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "mage-flow-turbo",
   "name": "Mage Flow Turbo",
+  "size_multiple": 16,
   "model_url": "https://huggingface.co/microsoft/Mage-Flow-Turbo",
   "slots": [
     {

examples/server/recipes/mage-flow.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "mage-flow",
   "name": "Mage Flow",
+  "size_multiple": 16,
   "model_url": "https://huggingface.co/microsoft/Mage-Flow",
   "slots": [
     {

examples/server/recipes/ming-image.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "ming-image",
   "name": "Ming Image 0.1",
+  "size_multiple": 16,
   "model_url": "https://huggingface.co/inclusionAI/Ming-Image-0.1-Design",
   "slots": [
     {
@@ -36,8 +37,8 @@
   "defaults": {
     "cfg_scale": 1,
     "steps": 12,
-    "width": 2048,
-    "height": 2048,
+    "width": 1024,
+    "height": 1024,
     "sampling_method": "euler"
   },
   "features": [

examples/server/recipes/minimax-h3-fl2va.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "minimax-h3-fl2va",
   "name": "MiniMax H3 FL2VA",
+  "size_multiple": 32,
   "description": "Generate video and audio, optionally with start/end images. Supports full and pruned FL2VA weights. Audio VAE is optional for silent video.",
   "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-H3",
   "slots": [
@@ -35,6 +36,13 @@
       "tae"
     ]
   ],
+  "video_constraints": {
+    "frame_step": 17,
+    "frame_offset": 5,
+    "frame_rounding": "ceil",
+    "min_frames": 5,
+    "fixed_fps": 24
+  },
   "defaults": {
     "video_frames": 56,
     "fps": 24,

examples/server/recipes/minimax-h3-ref2va.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "minimax-h3-ref2va",
   "name": "MiniMax H3 Ref2VA",
+  "size_multiple": 32,
   "description": "Supply image, video or audio references with compatible Ref2VA weights. Audio VAE is optional for silent video.",
   "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-H3",
   "slots": [
@@ -35,6 +36,13 @@
       "tae"
     ]
   ],
+  "video_constraints": {
+    "frame_step": 17,
+    "frame_offset": 5,
+    "frame_rounding": "ceil",
+    "min_frames": 5,
+    "fixed_fps": 24
+  },
   "defaults": {
     "video_frames": 56,
     "fps": 24,
@@ -45,7 +53,10 @@
   },
   "features": [
     "flow_shift",
-    "ref_images"
+    "ref_images",
+    "ref_videos",
+    "ref_audios",
+    "ref_video_audios"
   ],
   "disabled_features": [
     "img2img",

examples/server/recipes/minit2i.json

@@ -2,6 +2,7 @@
   "schema_version": 1,
   "id": "minit2i",
   "name": "MiniT2I",
+  "size_multiple": 1,
   "description": "Pixel-space model; no VAE is required.",
   "model_url": "https://huggingface.co/MiniT2I/MiniT2I",
   "slots": [