Hopefully fixed CUDA 11 build

Vulkan backend

.github/workflows/backends.yml

@@ -15,7 +15,10 @@ on:
 jobs:
   windows:
     runs-on: windows-latest
-          
+    
+    env:
+      VULKAN_VERSION: 1.3.261.1
+    
     strategy:
       matrix:
         include:
@@ -27,12 +30,12 @@ jobs:
             defines: '-DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON'
           - build: 'avx512'
             defines: '-DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON'
-          - build: 'cuda11'
-            defines: '-DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler"'
           - build: 'cuda12'
             defines: '-DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON'
           - build: 'rocm5'
             defines: '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS="gfx1100;gfx1102;gfx1030" -DSD_BUILD_SHARED_LIBS=ON'
+          - build: 'vulkan'
+            defines: "-DSD_VULKAN=ON -DSD_BUILD_SHARED_LIBS=ON"
 
     steps:
       - name: Checkout
@@ -43,17 +46,6 @@ jobs:
           ref: '${{ github.event.inputs.commit }}'
           submodules: recursive
 
-      - name: Install cuda-toolkit
-        id: cuda-toolkit-11
-        if: ${{ matrix.build == 'cuda11' }}
-        uses: Jimver/cuda-toolkit@v0.2.14
-        with:
-          cuda: '11.8.0'
-          method: network
-          sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
-          use-github-cache: false
-          use-local-cache: false
-
       - name: Install cuda-toolkit
         id: cuda-toolkit-12
         if: ${{ matrix.build == 'cuda12' }}
@@ -79,6 +71,15 @@ jobs:
         with:
           version: 1.11.1
 
+      - name: Install Vulkan SDK
+        id: get_vulkan
+        if: ${{ matrix.build == 'vulkan' }}
+        run: |
+          curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/VulkanSDK-${env:VULKAN_VERSION}-Installer.exe"
+          & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
+          Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
+          Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
+
       - name: Build
         id: cmake_build
         run: |
@@ -101,6 +102,96 @@ jobs:
         uses: actions/upload-artifact@v4.3.1
         with:
           name: windows-${{ matrix.build }}
+          path: .\build\bin\stable-diffusion.dll  
+
+  windows-2019:
+    runs-on: windows-2019
+        
+    strategy:
+      matrix:
+        include:
+          - build: 'cuda11'
+            defines: '-DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler"'
+
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@v4.1.2
+        with:
+          repository: 'leejet/stable-diffusion.cpp'
+          ref: '${{ github.event.inputs.commit }}'
+          submodules: recursive
+
+      - name: Install cuda-toolkit
+        id: cuda-toolkit-11
+        if: ${{ matrix.build == 'cuda11' }}
+        uses: Jimver/cuda-toolkit@v0.2.16
+        with:
+          cuda: '11.8.0'
+          method: network
+          sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
+          use-github-cache: false
+          use-local-cache: false
+
+      - name: Build
+        id: cmake_build
+        run: |
+          mkdir build
+          cd build
+          cmake .. ${{ matrix.defines }}
+          cmake --build . --config Release
+      
+      - name: Upload artifact
+        id: upload_artifact
+        if: ${{ matrix.build != 'rocm5' }}
+        uses: actions/upload-artifact@v4.3.1
+        with:
+          name: windows-${{ matrix.build }}
+          path: .\build\bin\Release\stable-diffusion.dll
+          
+  windows-sycl:
+    runs-on: windows-latest
+    
+    defaults:
+      run:
+        shell: bash
+
+    env:
+      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/7dff44ba-e3af-4448-841c-0d616c8da6e7/w_BaseKit_p_2024.1.0.595_offline.exe
+      WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel
+
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@v4.1.2
+        with:
+          repository: 'leejet/stable-diffusion.cpp'
+          ref: '${{ github.event.inputs.commit }}'
+          submodules: recursive            
+          
+      - name: Checkout Install Script
+        id: checkoutInstallScript
+        uses: actions/checkout@v4.1.2
+        with:
+          path: scripts
+          sparse-checkout: |
+            Backends/install-oneapi.bat
+            Backends/win-build-sycl.bat
+          sparse-checkout-cone-mode: false
+        
+      - name: Install Sycl tools
+        id: installSyclCompiler
+        run:  scripts/Backends/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL
+          
+      - name: Build
+        id: cmake_build
+        run: scripts/Backends/win-build-sycl.bat
+              
+      - name: Upload artifact
+        id: upload_artifact
+        uses: actions/upload-artifact@v4.3.1
+        with:
+          name: windows-sycl
           path: .\build\bin\stable-diffusion.dll
 
   linux:
@@ -119,7 +210,9 @@ jobs:
           - build: 'cuda11'
             defines: '-DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON'
           - build: 'cuda12'
-            defines: '-DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON'
+            defines: '-DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON'            
+          - build: 'sycl'
+            defines: '-DSD_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DSD_BUILD_SHARED_LIBS=ON'
             
     steps:
       - name: Checkout
@@ -150,8 +243,32 @@ jobs:
           use-github-cache: false
           use-local-cache: false
    
+      - name: Install Sycl tools
+        id: installSyclCompiler
+        if: ${{ matrix.build == 'sycl' }}
+        shell: bash
+        run: |        
+          cd /tmp
+          wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
+          sudo apt update
+          sudo apt install intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel
+   
+      - name: Build
+        id: cmake_build_sycl
+        if: ${{ matrix.build == 'sycl' }}
+        run: |
+          source /opt/intel/oneapi/setvars.sh
+          mkdir build
+          cd build
+          cmake .. ${{ matrix.defines }}
+          cmake --build . --config Release
+   
       - name: Build
         id: cmake_build
+        if: ${{ matrix.build != 'sycl' }}
         run: |
           mkdir build
           cd build
@@ -207,13 +324,13 @@ jobs:
       matrix:
         include:
           - build: 'noavx'
-            defines: '-DCMAKE_OSX_ARCHITECTURES="x86_64" -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON'
+            defines: '-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON'
           - build: 'avx'
-            defines: '-DCMAKE_OSX_ARCHITECTURES="x86_64" -DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON'
+            defines: '-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON'
           - build: 'avx2'
-            defines: '-DCMAKE_OSX_ARCHITECTURES="x86_64" -DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON'
+            defines: '-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON'
           - build: 'avx512'
-            defines: '-DCMAKE_OSX_ARCHITECTURES="x86_64" -DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON'
+            defines: '-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON'
    
     steps:          
       - name: Checkout
@@ -244,6 +361,7 @@ jobs:
 
     needs:
       - windows
+      - windows-sycl
       - linux
       - linux-hip
       - osx
@@ -270,8 +388,14 @@ jobs:
         id: pack
         run: |
           nuget pack ./Backends/StableDiffusion.NET.Backend.Cpu.nuspec -version ${{ github.event.inputs.version }}
+          nuget pack ./Backends/StableDiffusion.NET.Backend.Cuda11.Windows.nuspec -version ${{ github.event.inputs.version }}
+          nuget pack ./Backends/StableDiffusion.NET.Backend.Cuda12.Windows.nuspec -version ${{ github.event.inputs.version }}
+          nuget pack ./Backends/StableDiffusion.NET.Backend.Cuda11.Linux.nuspec -version ${{ github.event.inputs.version }}
+          nuget pack ./Backends/StableDiffusion.NET.Backend.Cuda12.Linux.nuspec -version ${{ github.event.inputs.version }}
           nuget pack ./Backends/StableDiffusion.NET.Backend.Cuda.nuspec -version ${{ github.event.inputs.version }}
           nuget pack ./Backends/StableDiffusion.NET.Backend.Rocm.nuspec -version ${{ github.event.inputs.version }}
+          nuget pack ./Backends/StableDiffusion.NET.Backend.Sycl.nuspec -version ${{ github.event.inputs.version }}
+          nuget pack ./Backends/StableDiffusion.NET.Backend.Vulkan.nuspec -version ${{ github.event.inputs.version }}
           
       - name: Upload artifacts
         id: upload_artifacts

Backends/StableDiffusion.NET.Backend.Cuda.nuspec

@@ -13,20 +13,17 @@
     <releaseNotes></releaseNotes>
     <copyright>Copyright © Darth Affe 2024</copyright>
     <readme>readme.md</readme>
+  
+    <dependencies>
+      <dependency id="StableDiffusion.NET.Backend.Cuda11.Windows" version="$version$" />
+      <dependency id="StableDiffusion.NET.Backend.Cuda12.Windows" version="$version$" />
+      <dependency id="StableDiffusion.NET.Backend.Cuda11.Linux" version="$version$" />
+      <dependency id="StableDiffusion.NET.Backend.Cuda12.Linux" version="$version$" />
+    </dependencies>
   </metadata>
 
   <files>
-    <file src="StableDiffusion.NET.Backend.props" target="build/net8.0/StableDiffusion.NET.Backend.Cuda.props" />
-
-    <file src="windows-cuda11/stable-diffusion.dll" target="runtimes\win-x64\native\cuda11\stable-diffusion.dll" />
-    <file src="windows-cuda12/stable-diffusion.dll" target="runtimes\win-x64\native\cuda12\stable-diffusion.dll" />
-    
-    <file src="linux-cuda11/libstable-diffusion.so" target="runtimes\linux-x64\native\cuda11\libstable-diffusion.so" />
-    <file src="linux-cuda12/libstable-diffusion.so" target="runtimes\linux-x64\native\cuda12\libstable-diffusion.so" />
-    
     <file src="sd_net_cuda.png" target="sd_net_cuda.png" />
     <file src="readme.md" target="readme.md" />
-    <file src="ggml.txt" target="ggml.txt" />
-    <file src="stable-diffusion.cpp.txt" target="stable-diffusion.cpp.txt" />
   </files>
 </package>

Backends/StableDiffusion.NET.Backend.Cuda11.Linux.nuspec

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package>
+  <metadata>
+    <id>StableDiffusion.NET.Backend.Cuda11.Linux</id>
+    <version>$version$</version>
+    <title>StableDiffusion.NET.Backend.Cuda11.Linux</title>
+    <authors>Darth Affe &amp; stable-diffusion.cpp Authors</authors>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <license type="expression">MIT</license>
+    <icon>sd_net_cuda.png</icon>
+    <projectUrl>https://github.com/DarthAffe/StableDiffusion.NET</projectUrl>
+    <description>CUDA 11 Linux Backend for StableDiffusion.NET.</description>
+    <releaseNotes></releaseNotes>
+    <copyright>Copyright © Darth Affe 2024</copyright>
+    <readme>readme.md</readme>
+  </metadata>
+
+  <files>
+    <file src="StableDiffusion.NET.Backend.props" target="build/net8.0/StableDiffusion.NET.Backend.Cuda.props" />
+    
+    <file src="linux-cuda11/libstable-diffusion.so" target="runtimes\linux-x64\native\cuda11\libstable-diffusion.so" />
+    
+    <file src="sd_net_cuda.png" target="sd_net_cuda.png" />
+    <file src="readme.md" target="readme.md" />
+    <file src="ggml.txt" target="ggml.txt" />
+    <file src="stable-diffusion.cpp.txt" target="stable-diffusion.cpp.txt" />
+  </files>
+</package>

Backends/StableDiffusion.NET.Backend.Cuda11.Windows.nuspec

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package>
+  <metadata>
+    <id>StableDiffusion.NET.Backend.Cuda11.Windows</id>
+    <version>$version$</version>
+    <title>StableDiffusion.NET.Backend.Cuda11.Windows</title>
+    <authors>Darth Affe &amp; stable-diffusion.cpp Authors</authors>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <license type="expression">MIT</license>
+    <icon>sd_net_cuda.png</icon>
+    <projectUrl>https://github.com/DarthAffe/StableDiffusion.NET</projectUrl>
+    <description>CUDA 11 Windows Backend for StableDiffusion.NET.</description>
+    <releaseNotes></releaseNotes>
+    <copyright>Copyright © Darth Affe 2024</copyright>
+    <readme>readme.md</readme>
+  </metadata>
+
+  <files>
+    <file src="StableDiffusion.NET.Backend.props" target="build/net8.0/StableDiffusion.NET.Backend.Cuda.props" />
+
+    <file src="windows-cuda11/stable-diffusion.dll" target="runtimes\win-x64\native\cuda11\stable-diffusion.dll" />
+    
+    <file src="sd_net_cuda.png" target="sd_net_cuda.png" />
+    <file src="readme.md" target="readme.md" />
+    <file src="ggml.txt" target="ggml.txt" />
+    <file src="stable-diffusion.cpp.txt" target="stable-diffusion.cpp.txt" />
+  </files>
+</package>

Backends/StableDiffusion.NET.Backend.Cuda12.Linux.nuspec

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package>
+  <metadata>
+    <id>StableDiffusion.NET.Backend.Cuda12.Linux</id>
+    <version>$version$</version>
+    <title>StableDiffusion.NET.Backend.Cuda12.Linux</title>
+    <authors>Darth Affe &amp; stable-diffusion.cpp Authors</authors>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <license type="expression">MIT</license>
+    <icon>sd_net_cuda.png</icon>
+    <projectUrl>https://github.com/DarthAffe/StableDiffusion.NET</projectUrl>
+    <description>CUDA 12 Linux Backend for StableDiffusion.NET.</description>
+    <releaseNotes></releaseNotes>
+    <copyright>Copyright © Darth Affe 2024</copyright>
+    <readme>readme.md</readme>
+  </metadata>
+
+  <files>
+    <file src="StableDiffusion.NET.Backend.props" target="build/net8.0/StableDiffusion.NET.Backend.Cuda.props" />
+    
+    <file src="linux-cuda12/libstable-diffusion.so" target="runtimes\linux-x64\native\cuda12\libstable-diffusion.so" />
+    
+    <file src="sd_net_cuda.png" target="sd_net_cuda.png" />
+    <file src="readme.md" target="readme.md" />
+    <file src="ggml.txt" target="ggml.txt" />
+    <file src="stable-diffusion.cpp.txt" target="stable-diffusion.cpp.txt" />
+  </files>
+</package>

Backends/StableDiffusion.NET.Backend.Cuda12.Windows.nuspec

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package>
+  <metadata>
+    <id>StableDiffusion.NET.Backend.Cuda12.Windows</id>
+    <version>$version$</version>
+    <title>StableDiffusion.NET.Backend.Cuda12.Windows</title>
+    <authors>Darth Affe &amp; stable-diffusion.cpp Authors</authors>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <license type="expression">MIT</license>
+    <icon>sd_net_cuda.png</icon>
+    <projectUrl>https://github.com/DarthAffe/StableDiffusion.NET</projectUrl>
+    <description>CUDA 12 Windows Backend for StableDiffusion.NET.</description>
+    <releaseNotes></releaseNotes>
+    <copyright>Copyright © Darth Affe 2024</copyright>
+    <readme>readme.md</readme>
+  </metadata>
+
+  <files>
+    <file src="StableDiffusion.NET.Backend.props" target="build/net8.0/StableDiffusion.NET.Backend.Cuda.props" />
+
+    <file src="windows-cuda12/stable-diffusion.dll" target="runtimes\win-x64\native\cuda12\stable-diffusion.dll" />
+    
+    <file src="sd_net_cuda.png" target="sd_net_cuda.png" />
+    <file src="readme.md" target="readme.md" />
+    <file src="ggml.txt" target="ggml.txt" />
+    <file src="stable-diffusion.cpp.txt" target="stable-diffusion.cpp.txt" />
+  </files>
+</package>

Backends/StableDiffusion.NET.Backend.Sycl.nuspec

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package>
+  <metadata>
+    <id>StableDiffusion.NET.Backend.Sycl</id>
+    <version>$version$</version>
+    <title>StableDiffusion.NET.Backend.Sycl</title>
+    <authors>Darth Affe &amp; stable-diffusion.cpp Authors</authors>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <license type="expression">MIT</license>
+    <icon>sd_net_sycl.png</icon>
+    <projectUrl>https://github.com/DarthAffe/StableDiffusion.NET</projectUrl>
+    <description>SYCL-Backend for StableDiffusion.NET.</description>
+    <releaseNotes></releaseNotes>
+    <copyright>Copyright © Darth Affe 2024</copyright>
+    <readme>readme.md</readme>
+  </metadata>
+
+  <files>
+    <file src="StableDiffusion.NET.Backend.props" target="build/net8.0/StableDiffusion.NET.Backend.Sycl.props" />
+
+    <file src="windows-sycl/stable-diffusion.dll" target="runtimes\win-x64\native\sycl\stable-diffusion.dll" />
+    
+    <file src="linux-sycl/libstable-diffusion.so" target="runtimes\linux-x64\native\sycl\libstable-diffusion.so" />
+    
+    <file src="sd_net_sycl.png" target="sd_net_sycl.png" />
+    <file src="readme.md" target="readme.md" />
+    <file src="ggml.txt" target="ggml.txt" />
+    <file src="stable-diffusion.cpp.txt" target="stable-diffusion.cpp.txt" />
+  </files>
+</package>

Backends/StableDiffusion.NET.Backend.Vulkan.nuspec

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package>
+  <metadata>
+    <id>StableDiffusion.NET.Backend.Vulkan</id>
+    <version>$version$</version>
+    <title>StableDiffusion.NET.Backend.Vulkan</title>
+    <authors>Darth Affe &amp; stable-diffusion.cpp Authors</authors>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <license type="expression">MIT</license>
+    <icon>sd_net_vulkan.png</icon>
+    <projectUrl>https://github.com/DarthAffe/StableDiffusion.NET</projectUrl>
+    <description>Vulkan-Backend for StableDiffusion.NET.</description>
+    <releaseNotes></releaseNotes>
+    <copyright>Copyright © Darth Affe 2024</copyright>
+    <readme>readme.md</readme>
+  </metadata>
+
+  <files>
+    <file src="StableDiffusion.NET.Backend.props" target="build/net8.0/StableDiffusion.NET.Backend.Vulkan.props" />
+
+    <file src="windows-vulkan/stable-diffusion.dll" target="runtimes\win-x64\native\vulkan\stable-diffusion.dll" />
+    
+    <file src="sd_net_vulkan.png" target="sd_net_vulkan.png" />
+    <file src="readme.md" target="readme.md" />
+    <file src="ggml.txt" target="ggml.txt" />
+    <file src="stable-diffusion.cpp.txt" target="stable-diffusion.cpp.txt" />
+  </files>
+</package>

Backends/install-oneapi.bat

@@ -0,0 +1,19 @@
+::  MIT license
+::  Copyright (C) 2024 Intel Corporation
+::  SPDX-License-Identifier: MIT
+
+
+set URL=%1
+set COMPONENTS=%2
+
+curl.exe --output %TEMP%\webimage.exe --url %URL% --retry 5 --retry-delay 5
+start /b /wait %TEMP%\webimage.exe -s -x -f webimage_extracted --log extract.log
+del %TEMP%\webimage.exe
+if "%COMPONENTS%"=="" (
+  webimage_extracted\bootstrapper.exe -s --action install --eula=accept -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 -p=NEED_VS2022_INTEGRATION=0 --log-dir=.
+) else (
+  webimage_extracted\bootstrapper.exe -s --action install --components=%COMPONENTS% --eula=accept -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 -p=NEED_VS2022_INTEGRATION=0 --log-dir=.
+)
+set installer_exit_code=%ERRORLEVEL%
+rd /s/q "webimage_extracted"
+exit /b %installer_exit_code%

Backends/win-build-sycl.bat

@@ -0,0 +1,24 @@
+::  MIT license
+::  Copyright (C) 2024 Intel Corporation
+::  SPDX-License-Identifier: MIT
+
+
+IF not exist build (mkdir build)
+cd build
+if %errorlevel% neq 0 goto ERROR
+
+@call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force
+if %errorlevel% neq 0 goto ERROR
+
+cmake -G "Ninja" ..  -DGGML_SYCL=ON -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=icx -DSD_BUILD_SHARED_LIBS=ON
+if %errorlevel% neq 0 goto ERROR
+
+cmake --build . -j --config Release
+if %errorlevel% neq 0 goto ERROR
+
+cd ..
+exit /B 0
+
+:ERROR
+echo comomand error: %errorlevel%
+exit /B %errorlevel%

Examples/ImageCreationUI/ImageCreationUI.csproj

@@ -11,10 +11,13 @@
 
   <ItemGroup>
     <PackageReference Include="HPPH.System.Drawing" Version="1.0.0" />
-    <PackageReference Include="StableDiffusion.NET" Version="2.0.0" />
-    <PackageReference Include="StableDiffusion.NET.Backend.Cpu" Version="2.0.0" />
-    <PackageReference Include="StableDiffusion.NET.Backend.Cuda" Version="2.0.0" />
-    <PackageReference Include="StableDiffusion.NET.Backend.Rocm" Version="2.0.0" />
+    <PackageReference Include="StableDiffusion.NET.Backend.Cpu" Version="3.0.0" />
+    <PackageReference Include="StableDiffusion.NET.Backend.Cuda" Version="3.0.0" />
+    <PackageReference Include="StableDiffusion.NET.Backend.Rocm" Version="3.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\StableDiffusion.NET\StableDiffusion.NET.csproj" />
   </ItemGroup>
 
 </Project>

Examples/ImageCreationUI/MainWindow.xaml

@@ -8,7 +8,7 @@
         xmlns:sys="clr-namespace:System;assembly=System.Runtime"
         xmlns:sd="clr-namespace:StableDiffusion.NET;assembly=StableDiffusion.NET"
         mc:Ignorable="d"
-        Title="StableDiffusion.NET" Width="1280" Height="800">
+        Title="StableDiffusion.NET" Width="1706" Height="960">
     <Window.DataContext>
         <local:MainWindowViewModel />
     </Window.DataContext>
@@ -42,81 +42,112 @@
             <ColumnDefinition Width="400" />
         </Grid.ColumnDefinitions>
 
-        <StackPanel Margin="4,0" Orientation="Vertical">
-            <TextBlock Margin="8" Text="This is just an example - inputs are not validated!&#10;Make sure everything is correct before pressing 'Create Image'." />
+        <ScrollViewer VerticalScrollBarVisibility="Auto" HorizontalScrollBarVisibility="Disabled">
+            <StackPanel Margin="4,0" Orientation="Vertical">
+                <TextBlock Margin="8" Text="This is just an example - inputs are not validated!&#10;Make sure everything is correct before pressing 'Create Image'." />
 
-            <Separator />
+                <Separator />
 
-            <Label Content="Model-Path" />
-            <DockPanel>
-                <Button DockPanel.Dock="Right" Width="24" Margin="2,0,0,0" Content="..." Command="{Binding SelectModelCommand}" IsEnabled="{Binding IsReady}" />
-                <TextBox Text="{Binding ModelPath}" />
-            </DockPanel>
-
-            <Label Content="Vae-Path (Optional)" />
-            <DockPanel>
-                <Button DockPanel.Dock="Right" Width="24" Margin="2,0,0,0" Content="..." Command="{Binding SelectVaeCommand}" IsEnabled="{Binding IsReady}" />
-                <TextBox Text="{Binding VaePath}" />
-            </DockPanel>
-
-            <Label Content="Schedule" />
-            <ComboBox ItemsSource="{Binding Source={StaticResource ScheduleDataSource}}" SelectedItem="{Binding Schedule}" />
-
-            <Button Margin="0,8"  Content="Load Model" Command="{Binding LoadModelCommand}" IsEnabled="{Binding IsReady}" />
-
-            <Separator />
-
-            <Label Margin="0,8,0,0" Content="Prompt" />
-            <TextBox Height="80" TextWrapping="Wrap" Text="{Binding Prompt}" />
-
-            <Label Content="AntiPrompt" />
-            <TextBox Height="80" TextWrapping="Wrap" Text="{Binding AntiPrompt}" />
-
-            <DockPanel LastChildFill="True">
-                <StackPanel DockPanel.Dock="Left">
-                    <StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,8,0,0">
-                        <Label Width="50" Content="Width" />
-                        <TextBox HorizontalAlignment="Left" Width="60" Text="{Binding Width}" />
-                    </StackPanel>
-
-                    <StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,8,0,0">
-                        <Label Width="50" Content="Height" />
-                        <TextBox HorizontalAlignment="Left" Width="60" Text="{Binding Height}" />
-                    </StackPanel>
-
-                    <StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,8,0,0">
-                        <Label Width="50" Content="Cfg" />
-                        <TextBox HorizontalAlignment="Left" Width="60" Text="{Binding Cfg}" />
-                    </StackPanel>
-
-                    <StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,8,0,0">
-                        <Label Width="50" Content="Steps" />
-                        <TextBox HorizontalAlignment="Left" Width="60" Text="{Binding Steps}" />
-                    </StackPanel>
-
-                    <StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,8,0,0">
-                        <Label Width="50" Content="Seed" />
-                        <TextBox HorizontalAlignment="Left" Width="60" Text="{Binding Seed}" />
-                    </StackPanel>
+                <Label Content="Model-Type" />
+                <StackPanel Orientation="Horizontal" Margin="4,0,0,0">
+                    <RadioButton Content="Stable Diffusion" IsChecked="{Binding IsStableDiffusionSelected}" GroupName="ModelType" />
+                    <RadioButton Margin="16,0,0,0" Content="Flux" IsChecked="{Binding IsFluxSelected}" GroupName="ModelType" />
                 </StackPanel>
 
-                <DockPanel Margin="16,0,0,0" LastChildFill="True">
-                    <Label DockPanel.Dock="Top" Content="Image2Image Source" />
-                    <DockPanel DockPanel.Dock="Top" >
-                        <Button DockPanel.Dock="Right" Width="24" Margin="2,0,0,0" Content="..." Command="{Binding SelectImage2ImageSourceCommand}" IsEnabled="{Binding IsReady}" />
-                        <TextBox Text="{Binding Image2ImageSourcePath}" />
+                <StackPanel Orientation="Vertical" IsEnabled="{Binding IsStableDiffusionSelected}">
+                    <Label Content="Model-Path" />
+                    <DockPanel>
+                        <Button DockPanel.Dock="Right" Width="24" Margin="2,0,0,0" Content="..." Command="{Binding SelectModelCommand}" IsEnabled="{Binding IsReady}" />
+                        <TextBox Text="{Binding ModelPath}" />
+                    </DockPanel>
+                </StackPanel>
+
+                <StackPanel Orientation="Vertical" IsEnabled="{Binding IsFluxSelected}">
+                    <Label Content="DiffusionModel-Path" />
+                    <DockPanel>
+                        <Button DockPanel.Dock="Right" Width="24" Margin="2,0,0,0" Content="..." Command="{Binding SelectDiffusionModelCommand}" IsEnabled="{Binding IsReady}" />
+                        <TextBox Text="{Binding DiffusionModelPath}" />
                     </DockPanel>
 
-                    <Image Source="{Binding Image2ImageSource, Converter={StaticResource ImageToImageSourceConverter}}" />
+                    <Label Content="ClipL-Path" />
+                    <DockPanel>
+                        <Button DockPanel.Dock="Right" Width="24" Margin="2,0,0,0" Content="..." Command="{Binding SelectClipLCommand}" IsEnabled="{Binding IsReady}" />
+                        <TextBox Text="{Binding ClipLPath}" />
+                    </DockPanel>
+
+                    <Label Content="T5xxl-Path" />
+                    <DockPanel>
+                        <Button DockPanel.Dock="Right" Width="24" Margin="2,0,0,0" Content="..." Command="{Binding SelectT5xxlCommand}" IsEnabled="{Binding IsReady}" />
+                        <TextBox Text="{Binding T5xxlPath}" />
+                    </DockPanel>
+                </StackPanel>
+
+                <Label Content="Vae-Path (Optional)" />
+                <DockPanel>
+                    <Button DockPanel.Dock="Right" Width="24" Margin="2,0,0,0" Content="..." Command="{Binding SelectVaeCommand}" IsEnabled="{Binding IsReady}" />
+                    <TextBox Text="{Binding VaePath}" />
                 </DockPanel>
-            </DockPanel>
 
-            <Label Content="Sample-Method" />
-            <ComboBox ItemsSource="{Binding Source={StaticResource SamplerDataSource}}" SelectedItem="{Binding SampleMethod}" />
+                <Label Content="Schedule" />
+                <ComboBox ItemsSource="{Binding Source={StaticResource ScheduleDataSource}}" SelectedItem="{Binding Schedule}" />
 
-            <Button Margin="0,16,0,0" Content="Create Image" Command="{Binding CreateImageCommand}" IsEnabled="{Binding IsReady}" />
-            <Button Margin="0,16,0,0" Content="Save Image" Command="{Binding SaveImageCommand}" IsEnabled="{Binding IsReady}" />
-        </StackPanel>
+                <Button Margin="0,8"  Content="Load Model" Command="{Binding LoadModelCommand}" IsEnabled="{Binding IsReady}" />
+
+                <Separator />
+
+                <Label Margin="0,8,0,0" Content="Prompt" />
+                <TextBox Height="80" TextWrapping="Wrap" Text="{Binding Prompt}" />
+
+                <Label Content="AntiPrompt" />
+                <TextBox Height="60" TextWrapping="Wrap" Text="{Binding AntiPrompt}" />
+
+                <DockPanel LastChildFill="True">
+                    <StackPanel DockPanel.Dock="Left">
+                        <StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,8,0,0">
+                            <Label Width="50" Content="Width" />
+                            <TextBox HorizontalAlignment="Left" Width="60" Text="{Binding Width}" />
+                        </StackPanel>
+
+                        <StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,8,0,0">
+                            <Label Width="50" Content="Height" />
+                            <TextBox HorizontalAlignment="Left" Width="60" Text="{Binding Height}" />
+                        </StackPanel>
+
+                        <StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,8,0,0">
+                            <Label Width="50" Content="Cfg" />
+                            <TextBox HorizontalAlignment="Left" Width="60" Text="{Binding Cfg}" />
+                        </StackPanel>
+
+                        <StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,8,0,0">
+                            <Label Width="50" Content="Steps" />
+                            <TextBox HorizontalAlignment="Left" Width="60" Text="{Binding Steps}" />
+                        </StackPanel>
+
+                        <StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,8,0,0">
+                            <Label Width="50" Content="Seed" />
+                            <TextBox HorizontalAlignment="Left" Width="60" Text="{Binding Seed}" />
+                        </StackPanel>
+                    </StackPanel>
+
+                    <DockPanel Margin="16,0,0,0" LastChildFill="True">
+                        <Label DockPanel.Dock="Top" Content="Image2Image Source" />
+                        <DockPanel DockPanel.Dock="Top" >
+                            <Button DockPanel.Dock="Right" Width="24" Margin="2,0,0,0" Content="..." Command="{Binding SelectImage2ImageSourceCommand}" IsEnabled="{Binding IsReady}" />
+                            <TextBox Text="{Binding Image2ImageSourcePath}" />
+                        </DockPanel>
+
+                        <Image Source="{Binding Image2ImageSource, Converter={StaticResource ImageToImageSourceConverter}}" />
+                    </DockPanel>
+                </DockPanel>
+
+                <Label Content="Sample-Method" />
+                <ComboBox ItemsSource="{Binding Source={StaticResource SamplerDataSource}}" SelectedItem="{Binding SampleMethod}" />
+
+                <Button Margin="0,16,0,0" Content="Create Image" Command="{Binding CreateImageCommand}" IsEnabled="{Binding IsReady}" />
+                <Button Margin="0,16,0,0" Content="Save Image" Command="{Binding SaveImageCommand}" IsEnabled="{Binding IsReady}" />
+            </StackPanel>
+
+        </ScrollViewer>
 
         <GridSplitter Grid.Column="1" Margin="2,0" Width="2" Background="DimGray" VerticalAlignment="Stretch" HorizontalAlignment="Center" />
 

Examples/ImageCreationUI/MainWindowViewModel.cs

@@ -13,7 +13,21 @@ public class MainWindowViewModel : INotifyPropertyChanged
 {
     #region Properties & Fields
 
-    private StableDiffusionModel? _model;
+    private DiffusionModel? _model;
+
+    private bool _isStableDiffusionSelected = true;
+    public bool IsStableDiffusionSelected
+    {
+        get => _isStableDiffusionSelected;
+        set => SetProperty(ref _isStableDiffusionSelected, value);
+    }
+
+    private bool _isFluxSelected;
+    public bool IsFluxSelected
+    {
+        get => _isFluxSelected;
+        set => SetProperty(ref _isFluxSelected, value);
+    }
 
     private string _modelPath = string.Empty;
     public string ModelPath
@@ -22,6 +36,27 @@ public class MainWindowViewModel : INotifyPropertyChanged
         set => SetProperty(ref _modelPath, value);
     }
 
+    private string _clipLPath;
+    public string ClipLPath
+    {
+        get => _clipLPath;
+        set => SetProperty(ref _clipLPath, value);
+    }
+
+    private string _t5xxlPath;
+    public string T5xxlPath
+    {
+        get => _t5xxlPath;
+        set => SetProperty(ref _t5xxlPath, value);
+    }
+
+    private string _diffusionModelPath;
+    public string DiffusionModelPath
+    {
+        get => _diffusionModelPath;
+        set => SetProperty(ref _diffusionModelPath, value);
+    }
+
     private string _vaePath = string.Empty;
     public string VaePath
     {
@@ -78,8 +113,8 @@ public class MainWindowViewModel : INotifyPropertyChanged
         set => SetProperty(ref _steps, value);
     }
 
-    private int _seed = -1;
-    public int Seed
+    private long _seed = -1;
+    public long Seed
     {
         get => _seed;
         set => SetProperty(ref _seed, value);
@@ -156,6 +191,15 @@ public class MainWindowViewModel : INotifyPropertyChanged
     private ActionCommand? _selectModelCommand;
     public ActionCommand SelectModelCommand => _selectModelCommand ??= new ActionCommand(SelectModel);
 
+    private ActionCommand? _selectDiffusionModelCommand;
+    public ActionCommand SelectDiffusionModelCommand => _selectDiffusionModelCommand ??= new ActionCommand(SelectDiffusionModel);
+
+    private ActionCommand? _selectClipLCommand;
+    public ActionCommand SelectClipLCommand => _selectClipLCommand ??= new ActionCommand(SelectClipL);
+
+    private ActionCommand? _selectT5xxlCommand;
+    public ActionCommand SelectT5xxlCommand => _selectT5xxlCommand ??= new ActionCommand(SelectT5xxl);
+
     private ActionCommand? _selectVaeCommand;
     public ActionCommand SelectVaeCommand => _selectVaeCommand ??= new ActionCommand(SelectVae);
 
@@ -170,12 +214,12 @@ public class MainWindowViewModel : INotifyPropertyChanged
     {
         try
         {
-            StableDiffusionModel.Log += (_, args) => LogLine($"LOG [{args.Level}]: {args.Text}", false);
-            StableDiffusionModel.Progress += (_, args) => LogLine($"PROGRESS {args.Step} / {args.Steps} ({(args.Progress * 100):N2} %) {args.IterationsPerSecond:N2} it/s ({args.Time})");
+            StableDiffusionCpp.Log += (_, args) => LogLine($"LOG [{args.Level}]: {args.Text}", false);
+            StableDiffusionCpp.Progress += (_, args) => LogLine($"PROGRESS {args.Step} / {args.Steps} ({(args.Progress * 100):N2} %) {args.IterationsPerSecond:N2} it/s ({args.Time})");
         }
         catch (Exception ex)
         {
-            LogLine($"Failed to load stable-diffussion.cpp libraries!{Environment.NewLine}{ex.Message}");
+            LogLine($"Failed to load stable-diffusion.cpp libraries!{Environment.NewLine}{ex.Message}");
         }
     }
 
@@ -191,8 +235,38 @@ public class MainWindowViewModel : INotifyPropertyChanged
 
             _model?.Dispose();
 
-            LogLine($"Loading model '{ModelPath}'");
-            _model = await Task.Run(() => new StableDiffusionModel(ModelPath, new ModelParameter { VaePath = VaePath, Schedule = Schedule }));
+            bool restoreDefaultParameters = false;
+
+            if (IsStableDiffusionSelected)
+            {
+                restoreDefaultParameters = _model?.ModelParameter.DiffusionModelType != DiffusionModelType.StableDiffusion;
+
+                LogLine($"Loading stable diffusion-model '{ModelPath}'");
+                _model = await Task.Run(() => ModelBuilder.StableDiffusion(ModelPath).WithMultithreading().WithVae(VaePath).WithSchedule(Schedule).Build());
+            }
+            else if (IsFluxSelected)
+            {
+                restoreDefaultParameters = _model?.ModelParameter.DiffusionModelType != DiffusionModelType.Flux;
+
+                LogLine($"Loading flux-model '{DiffusionModelPath}'");
+                _model = await Task.Run(() => ModelBuilder.Flux(DiffusionModelPath, ClipLPath, T5xxlPath, VaePath).WithMultithreading().WithSchedule(Schedule).Build());
+            }
+            else
+            {
+                LogLine("No model-type selected :(");
+            }
+
+            if (restoreDefaultParameters && (_model != null))
+            {
+                DiffusionParameter parameter = _model.GetDefaultParameter();
+                AntiPrompt = parameter.NegativePrompt;
+                Width = parameter.Width;
+                Height = parameter.Height;
+                Steps = parameter.SampleSteps;
+                Seed = parameter.Seed;
+                SampleMethod = parameter.SampleMethod;
+                Cfg = _model.ModelParameter.DiffusionModelType == DiffusionModelType.Flux ? parameter.Guidance : parameter.CfgScale;
+            }
         }
         catch (Exception ex)
         {
@@ -206,37 +280,32 @@ public class MainWindowViewModel : INotifyPropertyChanged
 
     private async void CreateImage()
     {
+        if (_model == null) return;
+
         try
         {
             IsReady = false;
 
+            DiffusionParameter parameter = _model.GetDefaultParameter()
+                                                 .WithNegativePrompt(AntiPrompt)
+                                                 .WithSize(Width, Height)
+                                                 .WithSteps(Steps)
+                                                 .WithSeed(Seed)
+                                                 .WithSampler(SampleMethod);
+            if (_model.ModelParameter.DiffusionModelType == DiffusionModelType.StableDiffusion)
+                parameter = parameter.WithCfg(Cfg);
+            else if (_model.ModelParameter.DiffusionModelType == DiffusionModelType.Flux)
+                parameter = parameter.WithGuidance(Cfg);
+
             if (Image2ImageSource == null)
             {
                 LogLine("Creating image ...");
-                Image = await Task.Run(() => _model?.TextToImage(Prompt, new StableDiffusionParameter
-                {
-                    NegativePrompt = AntiPrompt,
-                    Width = Width,
-                    Height = Height,
-                    CfgScale = Cfg,
-                    SampleSteps = Steps,
-                    Seed = Seed,
-                    SampleMethod = SampleMethod
-                }));
+                Image = await Task.Run(() => _model?.TextToImage(Prompt, parameter));
             }
             else
             {
                 LogLine("Manipulating image ...");
-                Image = await Task.Run(() => _model?.ImageToImage(Prompt, Image2ImageSource, new StableDiffusionParameter
-                {
-                    NegativePrompt = AntiPrompt,
-                    Width = Width,
-                    Height = Height,
-                    CfgScale = Cfg,
-                    SampleSteps = Steps,
-                    Seed = Seed,
-                    SampleMethod = SampleMethod
-                }));
+                Image = await Task.Run(() => _model?.ImageToImage(Prompt, Image2ImageSource, parameter));
             }
 
             LogLine("done!");
@@ -277,9 +346,30 @@ public class MainWindowViewModel : INotifyPropertyChanged
             ModelPath = openFileDialog.FileName;
     }
 
+    private void SelectDiffusionModel()
+    {
+        OpenFileDialog openFileDialog = new() { Filter = "Diffusion Model|*.*" };
+        if (openFileDialog.ShowDialog() == true)
+            DiffusionModelPath = openFileDialog.FileName;
+    }
+
+    private void SelectClipL()
+    {
+        OpenFileDialog openFileDialog = new() { Filter = "ClipL|*.*" };
+        if (openFileDialog.ShowDialog() == true)
+            ClipLPath = openFileDialog.FileName;
+    }
+
+    private void SelectT5xxl()
+    {
+        OpenFileDialog openFileDialog = new() { Filter = "T5xxl|*.*" };
+        if (openFileDialog.ShowDialog() == true)
+            T5xxlPath = openFileDialog.FileName;
+    }
+
     private void SelectVae()
     {
-        OpenFileDialog openFileDialog = new() { Filter = "Stable Diffusion VAE|*.*" };
+        OpenFileDialog openFileDialog = new() { Filter = "VAE|*.*" };
         if (openFileDialog.ShowDialog() == true)
             VaePath = openFileDialog.FileName;
     }

README.md

@@ -13,12 +13,43 @@ If GPU-support is available it will prefer this over CPU.
 If you want to add your own native-libraries or need more control over which backend to load, check the static `Backends` class.   
 
 ### Example
+#### 1. Create a model
+
+stable diffusion:
 ```csharp
-using StableDiffusionModel sd = new(@"<path_to_model>", new ModelParameter());
-IImage<ColorRGB> image = sd.TextToImage("<prompt>", new StableDiffusionParameter());
+using DiffusionModel model = ModelBuilder.StableDiffusion(@"<path to model")
+                                         .WithVae(@"<optional path to vae>")
+                                         .WithMultithreading()
+                                         .Build();
 ```
 
+flux:
+```csharp
+using DiffusionModel model = ModelBuilder.Flux(@"<path to flux-model.gguf>",
+                                               @"<path to clip_l.safetensors>",
+                                               @"<path to t5xxl_fp16.safetensors>",
+                                               @"<path to ae.safetensors>")
+                                         .WithMultithreading()
+                                         .Build();
+```
+
+#### 2. create image
+
+with default parameters:
+```csharp
+IImage<ColorRGB> image = model.TextToImage("<prompt>");
+```
+
+with custom parameters:
+```csharp
+IImage<ColorRGB> image = model.TextToImage("<prompt>", model.GetDefaultParameter().WithSeed(1234).WithSize(1344, 768));
+```
+
+#### 3. (optional) save the image (requires System.Dawing or SkiaSharp extension)
+```csharp
+File.WriteAllBytes("output.png", image.ToPng());
+```
 
 To process the resulting image further you can write your own extensions or install one of the [HPPH](https://github.com/DarthAffe/HPPH)-extension sets:   
 [HPPH.System.Drawing](https://www.nuget.org/packages/HPPH.System.Drawing)   
-[HPPH.SkiaSharp](https://www.nuget.org/packages/HPPH.SkiaSharp)
+[HPPH.SkiaSharp](https://www.nuget.org/packages/HPPH.SkiaSharp)

StableDiffusion.NET/Backends/Backends.cs

@@ -14,11 +14,13 @@ public static class Backends
     public static CpuBackend CpuBackend { get; } = new();
     public static CudaBackend CudaBackend { get; } = new();
     public static RocmBackend RocmBackend { get; } = new();
+    public static SyclBackend SyclBackend { get; } = new();
+    public static VulkanBackend VulkanBackend { get; } = new();
 
     private static readonly List<IBackend> CUSTOM_BACKENDS = [];
     public static IReadOnlyList<IBackend> CustomBackends => CUSTOM_BACKENDS.AsReadOnly();
 
-    public static IEnumerable<IBackend> RegisteredBackends => [CpuBackend, CudaBackend, RocmBackend, .. CUSTOM_BACKENDS];
+    public static IEnumerable<IBackend> RegisteredBackends => [CpuBackend, CudaBackend, RocmBackend, SyclBackend, VulkanBackend, .. CUSTOM_BACKENDS];
     public static IEnumerable<IBackend> AvailableBackends => RegisteredBackends.Where(x => x.IsAvailable);
     public static IEnumerable<IBackend> ActiveBackends => AvailableBackends.Where(x => x.IsEnabled);
 
@@ -36,7 +38,7 @@ public static class Backends
 
     public static bool RegisterBackend(IBackend backend)
     {
-        if (backend is NET.CpuBackend or NET.CudaBackend or NET.RocmBackend)
+        if (backend is NET.CpuBackend or NET.CudaBackend or NET.RocmBackend or NET.SyclBackend or NET.VulkanBackend)
             throw new ArgumentException("Default backends can't be registered again.");
 
         if (CUSTOM_BACKENDS.Contains(backend))

StableDiffusion.NET/Backends/CpuBackend.cs

@@ -13,7 +13,7 @@ public class CpuBackend : IBackend
 
     public bool IsEnabled { get; set; } = true;
 
-    public int Priority => 0;
+    public int Priority { get; set; } = 0;
 
     public bool IsAvailable => (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
                              || RuntimeInformation.IsOSPlatform(OSPlatform.Linux)

StableDiffusion.NET/Backends/CudaBackend.cs

@@ -22,7 +22,7 @@ public partial class CudaBackend : IBackend
 
     public bool IsEnabled { get; set; } = true;
 
-    public int Priority => 10;
+    public int Priority { get; set; } = 10;
 
     public bool IsAvailable => (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
                              || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))

StableDiffusion.NET/Backends/IBackend.cs

@@ -6,7 +6,7 @@ namespace StableDiffusion.NET;
 public interface IBackend
 {
     bool IsEnabled { get; set; }
-    public int Priority { get; }
+    public int Priority { get; set; }
     bool IsAvailable { get; }
     string PathPart { get; }
 }

StableDiffusion.NET/Backends/RocmBackend.cs

@@ -12,7 +12,7 @@ public partial class RocmBackend : IBackend
 
     public bool IsEnabled { get; set; } = true;
 
-    public int Priority => 10;
+    public int Priority { get; set; } = 10;
 
     public bool IsAvailable => ((RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
                               && RocmVersion is 5)

StableDiffusion.NET/Backends/SyclBackend.cs

@@ -0,0 +1,30 @@
+using System.Runtime.InteropServices;
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public class SyclBackend : IBackend
+{
+    #region Properties & Fields
+
+    //TODO DarthAffe 10.08.2024: tbh I'm not really sure how to detect a sycl-compatible system so for now it's disabled by default
+    public bool IsEnabled { get; set; } = false;
+
+    public int Priority { get; set; } = 5;
+
+    public bool IsAvailable => (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+                             || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+                            && (RuntimeInformation.OSArchitecture == Architecture.X64);
+
+    public string PathPart => "sycl";
+
+    #endregion
+
+    #region Constructors
+
+    internal SyclBackend()
+    { }
+
+    #endregion
+}

StableDiffusion.NET/Backends/VulkanBackend.cs

@@ -0,0 +1,29 @@
+using System.Runtime.InteropServices;
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public class VulkanBackend : IBackend
+{
+    #region Properties & Fields
+
+    //TODO DarthAffe 28.08.2024: Find a way to detect vulkan compatibility
+    public bool IsEnabled { get; set; } = false;
+
+    public int Priority { get; set; } = 5;
+
+    public bool IsAvailable => RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+                            && (RuntimeInformation.OSArchitecture == Architecture.X64);
+
+    public string PathPart => "vulkan";
+
+    #endregion
+
+    #region Constructors
+
+    internal VulkanBackend()
+    { }
+
+    #endregion
+}

StableDiffusion.NET/Enums/DiffusionModelType.cs

@@ -0,0 +1,8 @@
+namespace StableDiffusion.NET;
+
+public enum DiffusionModelType
+{
+    None = 0,
+    StableDiffusion = 1,
+    Flux = 2
+}

StableDiffusion.NET/Enums/Quantization.cs

@@ -8,7 +8,7 @@ public enum Quantization
     Q4_0 = 2,
     Q4_1 = 3,
     // Q4_2 = 4, support has been removed
-    // Q4_3 (5) support has been removed
+    // Q4_3 = 5, support has been removed
     Q5_0 = 6,
     Q5_1 = 7,
     Q8_0 = 8,
@@ -18,7 +18,7 @@ public enum Quantization
     Q4_K = 12,
     Q5_K = 13,
     Q6_K = 14,
-    Q8_K = 15, 
+    Q8_K = 15,
     IQ2_XXS = 16,
     IQ2_XS = 17,
     IQ3_XXS = 18,
@@ -27,7 +27,16 @@ public enum Quantization
     IQ3_S = 21,
     IQ2_S = 22,
     IQ4_XS = 23,
-    I8,
-    I16,
-    I32,
+    I8 = 24,
+    I16 = 25,
+    I32 = 26,
+    I64 = 27,
+    F64 = 28,
+    IQ1_M = 29,
+    BF16 = 30,
+    Q4_0_4_4 = 31,
+    Q4_0_4_8 = 32,
+    Q4_0_8_8 = 33,
+
+    Unspecified
 }

StableDiffusion.NET/Enums/Sampler.cs

@@ -9,6 +9,7 @@ public enum Sampler
     DPMPP2SA,
     DPMPP2M,
     DPMPP2Mv2,
-    LCM,
-    N_Sample_Methods
+    IPNDM,
+    IPNDM_V,
+    LCM
 }

StableDiffusion.NET/Enums/Schedule.cs

@@ -5,6 +5,7 @@ public enum Schedule
     Default,
     Discrete,
     Karras,
+    Exponential,
     AYS,
-    N_Schedules
+    GITS
 }

StableDiffusion.NET/Extensions/ParameterExtension.cs

@@ -6,65 +6,87 @@ namespace StableDiffusion.NET;
 
 public static class ParameterExtension
 {
-    public static void Validate(this StableDiffusionParameter parameter)
+    public static void Validate(this UpscaleModelParameter parameter)
     {
         ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
-        ArgumentNullException.ThrowIfNull(parameter.ControlNet, nameof(StableDiffusionParameter.ControlNet));
-        ArgumentNullException.ThrowIfNull(parameter.PhotoMaker, nameof(StableDiffusionParameter.PhotoMaker));
-        ArgumentNullException.ThrowIfNull(parameter.NegativePrompt, nameof(StableDiffusionParameter.NegativePrompt));
+        ArgumentNullException.ThrowIfNull(parameter.ModelPath, nameof(UpscaleModelParameter.ModelPath));
 
-        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Width, nameof(StableDiffusionParameter.Width));
-        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Height, nameof(StableDiffusionParameter.Height));
-        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.SampleSteps, nameof(StableDiffusionParameter.SampleSteps));
+        if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(UpscaleModelParameter.Quantization));
+    }
 
-        ArgumentOutOfRangeException.ThrowIfNegative(parameter.CfgScale, nameof(StableDiffusionParameter.CfgScale));
-        ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(StableDiffusionParameter.Strength));
+    public static void Validate(this DiffusionModelParameter parameter)
+    {
+        ((IQuantizedModelParameter)parameter).Validate();
+        ((IDiffusionModelParameter)parameter).Validate();
+        ((IPhotomakerModelParameter)parameter).Validate();
+    }
 
-        if (!Enum.IsDefined(parameter.SampleMethod)) throw new ArgumentOutOfRangeException(nameof(StableDiffusionParameter.SampleMethod));
+    public static void Validate(this IQuantizedModelParameter parameter)
+    {
+        ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
+        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.ThreadCount, nameof(IQuantizedModelParameter.ThreadCount));
+
+        if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(IQuantizedModelParameter.Quantization));
+    }
+
+    public static void Validate(this IPhotomakerModelParameter parameter)
+    {
+        ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
+        ArgumentNullException.ThrowIfNull(parameter.StackedIdEmbeddingsDirectory, nameof(IPhotomakerModelParameter.StackedIdEmbeddingsDirectory));
+    }
+
+    public static void Validate(this IDiffusionModelParameter parameter)
+    {
+        ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
+        ArgumentNullException.ThrowIfNull(parameter.VaePath, nameof(IDiffusionModelParameter.VaePath));
+        ArgumentNullException.ThrowIfNull(parameter.TaesdPath, nameof(IDiffusionModelParameter.TaesdPath));
+        ArgumentNullException.ThrowIfNull(parameter.LoraModelDirectory, nameof(IDiffusionModelParameter.LoraModelDirectory));
+        ArgumentNullException.ThrowIfNull(parameter.ControlNetPath, nameof(IDiffusionModelParameter.ControlNetPath));
+        ArgumentNullException.ThrowIfNull(parameter.EmbeddingsDirectory, nameof(IDiffusionModelParameter.EmbeddingsDirectory));
+
+        if (!string.IsNullOrWhiteSpace(parameter.VaePath) && !string.IsNullOrWhiteSpace(parameter.TaesdPath)) throw new ArgumentException("VAE and TAESD are mutually exclusive.");
+
+        if (!Enum.IsDefined(parameter.RngType)) throw new ArgumentOutOfRangeException(nameof(IDiffusionModelParameter.RngType));
+        if (!Enum.IsDefined(parameter.Schedule)) throw new ArgumentOutOfRangeException(nameof(IDiffusionModelParameter.Schedule));
+    }
+
+    public static void Validate(this DiffusionParameter parameter)
+    {
+        ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
+        ArgumentNullException.ThrowIfNull(parameter.ControlNet, nameof(DiffusionParameter.ControlNet));
+        ArgumentNullException.ThrowIfNull(parameter.PhotoMaker, nameof(DiffusionParameter.PhotoMaker));
+        ArgumentNullException.ThrowIfNull(parameter.NegativePrompt, nameof(DiffusionParameter.NegativePrompt));
+
+        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Width, nameof(DiffusionParameter.Width));
+        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.Height, nameof(DiffusionParameter.Height));
+        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parameter.SampleSteps, nameof(DiffusionParameter.SampleSteps));
+
+        ArgumentOutOfRangeException.ThrowIfNegative(parameter.CfgScale, nameof(DiffusionParameter.CfgScale));
+        ArgumentOutOfRangeException.ThrowIfNegative(parameter.Guidance, nameof(DiffusionParameter.Guidance));
+        ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(DiffusionParameter.Strength));
+
+        if (!Enum.IsDefined(parameter.SampleMethod)) throw new ArgumentOutOfRangeException(nameof(DiffusionParameter.SampleMethod));
 
         parameter.ControlNet.Validate();
         parameter.PhotoMaker.Validate();
     }
 
-    public static void Validate(this StableDiffusionControlNetParameter parameter)
+    public static void Validate(this ControlNetParameter parameter)
     {
-        ArgumentNullException.ThrowIfNull(parameter, nameof(StableDiffusionParameter.ControlNet));
+        ArgumentNullException.ThrowIfNull(parameter, nameof(DiffusionParameter.ControlNet));
 
-        ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(StableDiffusionControlNetParameter.Strength));
-        ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyHighThreshold, nameof(StableDiffusionControlNetParameter.CannyHighThreshold));
-        ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyLowThreshold, nameof(StableDiffusionControlNetParameter.CannyLowThreshold));
-        ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyWeak, nameof(StableDiffusionControlNetParameter.CannyWeak));
-        ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyStrong, nameof(StableDiffusionControlNetParameter.CannyStrong));
+        ArgumentOutOfRangeException.ThrowIfNegative(parameter.Strength, nameof(ControlNetParameter.Strength));
+        ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyHighThreshold, nameof(ControlNetParameter.CannyHighThreshold));
+        ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyLowThreshold, nameof(ControlNetParameter.CannyLowThreshold));
+        ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyWeak, nameof(ControlNetParameter.CannyWeak));
+        ArgumentOutOfRangeException.ThrowIfNegative(parameter.CannyStrong, nameof(ControlNetParameter.CannyStrong));
     }
 
     public static void Validate(this PhotoMakerParameter parameter)
     {
-        ArgumentNullException.ThrowIfNull(parameter, nameof(StableDiffusionParameter.PhotoMaker));
+        ArgumentNullException.ThrowIfNull(parameter, nameof(DiffusionParameter.PhotoMaker));
         ArgumentNullException.ThrowIfNull(parameter.InputIdImageDirectory, nameof(PhotoMakerParameter.InputIdImageDirectory));
 
         ArgumentOutOfRangeException.ThrowIfNegative(parameter.StyleRatio, nameof(PhotoMakerParameter.StyleRatio));
     }
-
-    public static void Validate(this ModelParameter parameter)
-    {
-        ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
-        ArgumentNullException.ThrowIfNull(parameter.TaesdPath, nameof(ModelParameter.TaesdPath));
-        ArgumentNullException.ThrowIfNull(parameter.LoraModelDir, nameof(ModelParameter.LoraModelDir));
-        ArgumentNullException.ThrowIfNull(parameter.VaePath, nameof(ModelParameter.VaePath));
-        ArgumentNullException.ThrowIfNull(parameter.ControlNetPath, nameof(ModelParameter.ControlNetPath));
-        ArgumentNullException.ThrowIfNull(parameter.EmbeddingsDirectory, nameof(ModelParameter.EmbeddingsDirectory));
-        ArgumentNullException.ThrowIfNull(parameter.StackedIdEmbeddingsDirectory, nameof(ModelParameter.StackedIdEmbeddingsDirectory));
-
-        if (!Enum.IsDefined(parameter.RngType)) throw new ArgumentOutOfRangeException(nameof(ModelParameter.RngType));
-        if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(ModelParameter.Quantization));
-        if (!Enum.IsDefined(parameter.Schedule)) throw new ArgumentOutOfRangeException(nameof(ModelParameter.Schedule));
-    }
-
-    public static void Validate(this UpscalerModelParameter parameter)
-    {
-        ArgumentNullException.ThrowIfNull(parameter, nameof(parameter));
-        ArgumentNullException.ThrowIfNull(parameter.ESRGANPath, nameof(UpscalerModelParameter.ESRGANPath));
-
-        if (!Enum.IsDefined(parameter.Quantization)) throw new ArgumentOutOfRangeException(nameof(ModelParameter.Quantization));
-    }
 }

StableDiffusion.NET/Models/Builder/ESRGANModelBuilder.cs

@@ -0,0 +1,29 @@
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public sealed class ESRGANModelBuilder : IQuantizedModelBuilder
+{
+    #region Properties & Fields
+
+    public UpscaleModelParameter Parameter { get; }
+    IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter;
+
+    #endregion
+
+    #region Constructors
+
+    public ESRGANModelBuilder(string modelPath)
+    {
+        Parameter = new UpscaleModelParameter { ModelPath = modelPath };
+    }
+
+    #endregion
+
+    #region Methods
+
+    public UpscaleModel Build() => new(Parameter);
+
+    #endregion
+}

StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs

@@ -0,0 +1,122 @@
+using System;
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public static class DiffusionModelBuilderExtension
+{
+    public static T WithVae<T>(this T builder, string vaePath)
+        where T : IDiffusionModelBuilder
+    {
+        ArgumentNullException.ThrowIfNull(vaePath);
+
+        if (!string.IsNullOrWhiteSpace(builder.Parameter.TaesdPath)) throw new ArgumentException("TAESD is already enabled. VAE and TAESD are mutually exclusive.", nameof(vaePath));
+
+        builder.Parameter.VaePath = vaePath;
+
+        return builder;
+    }
+
+    public static T WithTaesd<T>(this T builder, string taesdPath)
+        where T : IDiffusionModelBuilder
+    {
+        ArgumentNullException.ThrowIfNull(taesdPath);
+
+        if (!string.IsNullOrWhiteSpace(builder.Parameter.VaePath)) throw new ArgumentException("VAE is already enabled. TAESD and VAE are mutually exclusive.", nameof(taesdPath));
+
+        builder.Parameter.TaesdPath = taesdPath;
+
+        return builder;
+    }
+
+    public static T WithLoraSupport<T>(this T builder, string loraModelDirectory)
+        where T : IDiffusionModelBuilder
+    {
+        ArgumentNullException.ThrowIfNull(loraModelDirectory);
+
+        builder.Parameter.LoraModelDirectory = loraModelDirectory;
+
+        return builder;
+    }
+
+    public static T WithEmbeddingSupport<T>(this T builder, string embeddingsDirectory)
+        where T : IDiffusionModelBuilder
+    {
+        ArgumentNullException.ThrowIfNull(embeddingsDirectory);
+
+        builder.Parameter.EmbeddingsDirectory = embeddingsDirectory;
+
+        return builder;
+    }
+
+    public static T WithControlNet<T>(this T builder, string controlNetPath)
+        where T : IDiffusionModelBuilder
+    {
+        ArgumentNullException.ThrowIfNull(controlNetPath);
+
+        builder.Parameter.ControlNetPath = controlNetPath;
+
+        return builder;
+    }
+
+    public static T WithVaeDecodeOnly<T>(this T builder, bool vaeDecodeOnly = true)
+        where T : IDiffusionModelBuilder
+    {
+        builder.Parameter.VaeDecodeOnly = vaeDecodeOnly;
+
+        return builder;
+    }
+
+    public static T WithVaeTiling<T>(this T builder, bool vaeTiling = true)
+        where T : IDiffusionModelBuilder
+    {
+        builder.Parameter.VaeTiling = vaeTiling;
+
+        return builder;
+    }
+
+    public static T KeepVaeOnCpu<T>(this T builder, bool keepVaeOnCpu = true)
+        where T : IDiffusionModelBuilder
+    {
+        builder.Parameter.KeepVaeOnCPU = keepVaeOnCpu;
+
+        return builder;
+    }
+
+    public static T KeepClipNetOnCpu<T>(this T builder, bool keepClipNetOnCpu = true)
+        where T : IDiffusionModelBuilder
+    {
+        builder.Parameter.KeepClipOnCPU = keepClipNetOnCpu;
+
+        return builder;
+    }
+
+    public static T KeepControlNetOnCpu<T>(this T builder, bool keepControlNetOnCpu = true)
+        where T : IDiffusionModelBuilder
+    {
+        builder.Parameter.KeepControlNetOnCPU = keepControlNetOnCpu;
+
+        return builder;
+    }
+
+    public static T WithRngType<T>(this T builder, RngType rngType)
+        where T : IDiffusionModelBuilder
+    {
+        if (!Enum.IsDefined(rngType)) throw new ArgumentOutOfRangeException(nameof(rngType));
+
+        builder.Parameter.RngType = rngType;
+
+        return builder;
+    }
+
+    public static T WithSchedule<T>(this T builder, Schedule schedule)
+        where T : IDiffusionModelBuilder
+    {
+        if (!Enum.IsDefined(schedule)) throw new ArgumentOutOfRangeException(nameof(schedule));
+
+        builder.Parameter.Schedule = schedule;
+
+        return builder;
+    }
+}

StableDiffusion.NET/Models/Builder/Extensions/PhotomakerModelBuilderExtension.cs

@@ -0,0 +1,18 @@
+using System;
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public static class PhotomakerModelBuilderExtension
+{
+    public static T WithPhotomaker<T>(this T builder, string stackedIdEmbeddingsDirectory)
+        where T : IPhotomakerModelBuilder
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(stackedIdEmbeddingsDirectory, nameof(stackedIdEmbeddingsDirectory));
+
+        builder.Parameter.StackedIdEmbeddingsDirectory = stackedIdEmbeddingsDirectory;
+
+        return builder;
+    }
+}

StableDiffusion.NET/Models/Builder/Extensions/QuantizedModelBuilderExtension.cs

@@ -0,0 +1,36 @@
+using System;
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public static class QuantizedModelBuilderExtension
+{
+    public static T WithoutMultithreading<T>(this T builder)
+        where T : IQuantizedModelBuilder
+    {
+        builder.Parameter.ThreadCount = 1;
+
+        return builder;
+    }
+
+    public static T WithMultithreading<T>(this T builder, int threadCount = 0)
+        where T : IQuantizedModelBuilder
+    {
+        ArgumentOutOfRangeException.ThrowIfLessThan(threadCount, 0, nameof(threadCount));
+
+        if (threadCount == 0) threadCount = Environment.ProcessorCount;
+
+        builder.Parameter.ThreadCount = threadCount;
+
+        return builder;
+    }
+
+    public static T WithQuantization<T>(this T builder, Quantization quantization)
+        where T : IQuantizedModelBuilder
+    {
+        builder.Parameter.Quantization = quantization;
+
+        return builder;
+    }
+}

StableDiffusion.NET/Models/Builder/FluxModelBuilder.cs

@@ -0,0 +1,30 @@
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public sealed class FluxModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder
+{
+    #region Properties & Fields
+
+    public DiffusionModelParameter Parameter { get; }
+    IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter;
+    IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter;
+
+    #endregion
+
+    #region Constructors
+
+    public FluxModelBuilder(string diffusionModelPath, string clipLPath, string t5xxlPath, string vaePath)
+    {
+        Parameter = new DiffusionModelParameter { DiffusionModelType = DiffusionModelType.Flux, DiffusionModelPath = diffusionModelPath, ClipLPath = clipLPath, T5xxlPath = t5xxlPath, VaePath = vaePath };
+    }
+
+    #endregion
+
+    #region Methods
+
+    public DiffusionModel Build() => new(Parameter);
+
+    #endregion
+}

StableDiffusion.NET/Models/Builder/Interfaces/IDiffusionModelBuilder.cs

@@ -0,0 +1,6 @@
+namespace StableDiffusion.NET;
+
+public interface IDiffusionModelBuilder
+{
+    IDiffusionModelParameter Parameter { get; }
+}

StableDiffusion.NET/Models/Builder/Interfaces/IPhotomakerModelBuilder.cs

@@ -0,0 +1,6 @@
+namespace StableDiffusion.NET;
+
+public interface IPhotomakerModelBuilder
+{
+    IPhotomakerModelParameter Parameter { get; }
+}

StableDiffusion.NET/Models/Builder/Interfaces/IQuantizedModelBuilder.cs

@@ -0,0 +1,6 @@
+namespace StableDiffusion.NET;
+
+public interface IQuantizedModelBuilder
+{
+    IQuantizedModelParameter Parameter { get; }
+}

StableDiffusion.NET/Models/Builder/ModelBuilder.cs

@@ -0,0 +1,11 @@
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public static class ModelBuilder
+{
+    public static StableDiffusionModelBuilder StableDiffusion(string modelPath) => new(modelPath);
+    public static FluxModelBuilder Flux(string diffusionModelPath, string clipLPath, string t5xxlPath, string vaePath) => new(diffusionModelPath, clipLPath, t5xxlPath, vaePath);
+    public static ESRGANModelBuilder ESRGAN(string modelPath) => new(modelPath);
+}

StableDiffusion.NET/Models/Builder/StableDiffusionModelBuilder.cs

@@ -0,0 +1,31 @@
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public sealed class StableDiffusionModelBuilder : IDiffusionModelBuilder, IQuantizedModelBuilder, IPhotomakerModelBuilder
+{
+    #region Properties & Fields
+
+    public DiffusionModelParameter Parameter { get; }
+    IDiffusionModelParameter IDiffusionModelBuilder.Parameter => Parameter;
+    IQuantizedModelParameter IQuantizedModelBuilder.Parameter => Parameter;
+    IPhotomakerModelParameter IPhotomakerModelBuilder.Parameter => Parameter;
+
+    #endregion
+
+    #region Constructors
+
+    public StableDiffusionModelBuilder(string modelPath)
+    {
+        Parameter = new DiffusionModelParameter { DiffusionModelType = DiffusionModelType.StableDiffusion, ModelPath = modelPath };
+    }
+
+    #endregion
+
+    #region Methods
+
+    public DiffusionModel Build() => new(Parameter);
+
+    #endregion
+}

StableDiffusion.NET/Models/DiffusionModel.cs

@@ -1,61 +1,37 @@
 using System;
-using System.Runtime.InteropServices;
 using HPPH;
+using System.Runtime.InteropServices;
 using JetBrains.Annotations;
 
 namespace StableDiffusion.NET;
 
 [PublicAPI]
-public sealed unsafe class StableDiffusionModel : IDisposable
+public sealed unsafe class DiffusionModel : IDisposable
 {
     #region Properties & Fields
 
-    // ReSharper disable NotAccessedField.Local - They are important, the delegate can be collected if it's not stored!
-    private static readonly Native.sd_log_cb_t LOG_CALLBACK;
-    private static readonly Native.sd_progress_cb_t PROGRESS_CALLBACK;
-    // ReSharper restore NotAccessedField.Local
-
     private bool _disposed;
 
-    private readonly string _modelPath;
-    private readonly ModelParameter _parameter;
-    private readonly UpscalerModelParameter? _upscalerParameter;
+    public DiffusionModelParameter ModelParameter { get; }
 
     private Native.sd_ctx_t* _ctx;
-    private Native.upscaler_ctx_t* _upscalerCtx;
-
-    #endregion
-
-    #region Events
-
-    public static event EventHandler<StableDiffusionLogEventArgs>? Log;
-    public static event EventHandler<StableDiffusionProgressEventArgs>? Progress;
 
     #endregion
 
     #region Constructors
 
-    static StableDiffusionModel()
+    public DiffusionModel(DiffusionModelParameter modelParameter)
     {
-        Native.sd_set_log_callback(LOG_CALLBACK = OnNativeLog, null);
-        Native.sd_set_progress_callback(PROGRESS_CALLBACK = OnNativeProgress, null);
-    }
+        ArgumentNullException.ThrowIfNull(modelParameter, nameof(modelParameter));
 
-    public StableDiffusionModel(string modelPath, ModelParameter parameter, UpscalerModelParameter? upscalerParameter = null)
-    {
-        ArgumentException.ThrowIfNullOrWhiteSpace(modelPath, nameof(modelPath));
+        modelParameter.Validate();
 
-        parameter.Validate();
-        upscalerParameter?.Validate();
-
-        this._modelPath = modelPath;
-        this._parameter = parameter;
-        this._upscalerParameter = upscalerParameter;
+        this.ModelParameter = modelParameter;
 
         Initialize();
     }
 
-    ~StableDiffusionModel() => Dispose();
+    ~DiffusionModel() => Dispose();
 
     #endregion
 
@@ -63,36 +39,42 @@ public sealed unsafe class StableDiffusionModel : IDisposable
 
     private void Initialize()
     {
-        _ctx = Native.new_sd_ctx(_modelPath,
-                                 _parameter.VaePath,
-                                 _parameter.TaesdPath,
-                                 _parameter.ControlNetPath,
-                                 _parameter.LoraModelDir,
-                                 _parameter.EmbeddingsDirectory,
-                                 _parameter.StackedIdEmbeddingsDirectory,
-                                 _parameter.VaeDecodeOnly,
-                                 _parameter.VaeTiling,
-                                  false,
-                                 _parameter.ThreadCount,
-                                 _parameter.Quantization,
-                                 _parameter.RngType,
-                                 _parameter.Schedule,
-                                 _parameter.KeepClipOnCPU,
-                                 _parameter.KeepControlNetOnCPU,
-                                 _parameter.KeepVaeOnCPU);
-        if (_ctx == null) throw new NullReferenceException("Failed to initialize Stable Diffusion");
+        _ctx = Native.new_sd_ctx(ModelParameter.ModelPath,
+                                 ModelParameter.ClipLPath,
+                                 ModelParameter.T5xxlPath,
+                                 ModelParameter.DiffusionModelPath,
+                                 ModelParameter.VaePath,
+                                 ModelParameter.TaesdPath,
+                                 ModelParameter.ControlNetPath,
+                                 ModelParameter.LoraModelDirectory,
+                                 ModelParameter.EmbeddingsDirectory,
+                                 ModelParameter.StackedIdEmbeddingsDirectory,
+                                 ModelParameter.VaeDecodeOnly,
+                                 ModelParameter.VaeTiling,
+                                 false,
+                                 ModelParameter.ThreadCount,
+                                 ModelParameter.Quantization,
+                                 ModelParameter.RngType,
+                                 ModelParameter.Schedule,
+                                 ModelParameter.KeepClipOnCPU,
+                                 ModelParameter.KeepControlNetOnCPU,
+                                 ModelParameter.KeepVaeOnCPU);
 
-        if (_upscalerParameter != null)
-        {
-            _upscalerCtx = Native.new_upscaler_ctx(_upscalerParameter.ESRGANPath,
-                                                   _upscalerParameter.ThreadCount,
-                                                   _upscalerParameter.Quantization);
-            if (_upscalerCtx == null) throw new NullReferenceException("Failed to initialize Stable Diffusion");
-        }
+        if (_ctx == null) throw new NullReferenceException("Failed to initialize diffusion-model.");
     }
 
-    public IImage<ColorRGB> TextToImage(string prompt, StableDiffusionParameter parameter)
+    public DiffusionParameter GetDefaultParameter() => ModelParameter.DiffusionModelType switch
     {
+        DiffusionModelType.None => new DiffusionParameter(),
+        DiffusionModelType.StableDiffusion => DiffusionParameter.SDXLDefault,
+        DiffusionModelType.Flux => DiffusionParameter.FluxDefault,
+        _ => throw new ArgumentOutOfRangeException()
+    };
+
+    public IImage<ColorRGB> TextToImage(string prompt, DiffusionParameter? parameter = null)
+    {
+        parameter ??= GetDefaultParameter();
+
         ObjectDisposedException.ThrowIf(_disposed, this);
         ArgumentNullException.ThrowIfNull(prompt);
 
@@ -101,15 +83,18 @@ public sealed unsafe class StableDiffusionModel : IDisposable
         Native.sd_image_t* result;
         if (parameter.ControlNet.IsEnabled)
         {
-            fixed (byte* imagePtr = parameter.ControlNet.Image!.ToRawArray())
+            if (parameter.ControlNet.Image is not IImage<ColorRGB> controlNetImage)
+                controlNetImage = parameter.ControlNet.Image!.ConvertTo<ColorRGB>();
+
+            fixed (byte* imagePtr = controlNetImage.ToRawArray())
             {
                 if (parameter.ControlNet.CannyPreprocess)
                 {
-                    Native.sd_image_t controlNetImage = new()
+                    Native.sd_image_t nativeControlNetImage = new()
                     {
-                        width = (uint)parameter.ControlNet.Image.Width,
-                        height = (uint)parameter.ControlNet.Image.Height,
-                        channel = (uint)parameter.ControlNet.Image.ColorFormat.BytesPerPixel,
+                        width = (uint)controlNetImage.Width,
+                        height = (uint)controlNetImage.Height,
+                        channel = (uint)controlNetImage.ColorFormat.BytesPerPixel,
                         data = Native.preprocess_canny(imagePtr,
                                                        parameter.Width,
                                                        parameter.Height,
@@ -125,27 +110,28 @@ public sealed unsafe class StableDiffusionModel : IDisposable
                                             parameter.NegativePrompt,
                                             parameter.ClipSkip,
                                             parameter.CfgScale,
+                                            parameter.Guidance,
                                             parameter.Width,
                                             parameter.Height,
                                             parameter.SampleMethod,
                                             parameter.SampleSteps,
                                             parameter.Seed,
                                             1,
-                                            &controlNetImage,
+                                            &nativeControlNetImage,
                                             parameter.ControlNet.Strength,
                                             parameter.PhotoMaker.StyleRatio,
                                             parameter.PhotoMaker.NormalizeInput,
                                             parameter.PhotoMaker.InputIdImageDirectory);
 
-                    Marshal.FreeHGlobal((nint)controlNetImage.data);
+                    Marshal.FreeHGlobal((nint)nativeControlNetImage.data);
                 }
                 else
                 {
-                    Native.sd_image_t controlNetImage = new()
+                    Native.sd_image_t nativeControlNetImage = new()
                     {
-                        width = (uint)parameter.ControlNet.Image.Width,
-                        height = (uint)parameter.ControlNet.Image.Height,
-                        channel = (uint)parameter.ControlNet.Image.ColorFormat.BytesPerPixel,
+                        width = (uint)controlNetImage.Width,
+                        height = (uint)controlNetImage.Height,
+                        channel = (uint)controlNetImage.ColorFormat.BytesPerPixel,
                         data = imagePtr
                     };
 
@@ -154,13 +140,14 @@ public sealed unsafe class StableDiffusionModel : IDisposable
                                             parameter.NegativePrompt,
                                             parameter.ClipSkip,
                                             parameter.CfgScale,
+                                            parameter.Guidance,
                                             parameter.Width,
                                             parameter.Height,
                                             parameter.SampleMethod,
                                             parameter.SampleSteps,
                                             parameter.Seed,
                                             1,
-                                            &controlNetImage,
+                                            &nativeControlNetImage,
                                             parameter.ControlNet.Strength,
                                             parameter.PhotoMaker.StyleRatio,
                                             parameter.PhotoMaker.NormalizeInput,
@@ -175,6 +162,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable
                                     parameter.NegativePrompt,
                                     parameter.ClipSkip,
                                     parameter.CfgScale,
+                                    parameter.Guidance,
                                     parameter.Width,
                                     parameter.Height,
                                     parameter.SampleMethod,
@@ -191,18 +179,23 @@ public sealed unsafe class StableDiffusionModel : IDisposable
         return ImageHelper.ToImage(result);
     }
 
-    public IImage<ColorRGB> ImageToImage(string prompt, IImage<ColorRGB> image, StableDiffusionParameter parameter)
+    public IImage<ColorRGB> ImageToImage(string prompt, IImage image, DiffusionParameter? parameter = null)
     {
+        parameter ??= GetDefaultParameter();
+
         ObjectDisposedException.ThrowIf(_disposed, this);
         ArgumentNullException.ThrowIfNull(prompt);
 
         parameter.Validate();
 
-        fixed (byte* imagePtr = image.AsRefImage())
-            return ImageToImage(prompt, image.ToSdImage(imagePtr), parameter);
+        if (image is not IImage<ColorRGB> refImage)
+            refImage = image.ConvertTo<ColorRGB>();
+
+        fixed (byte* imagePtr = refImage.AsRefImage())
+            return ImageToImage(prompt, refImage.ToSdImage(imagePtr), parameter);
     }
 
-    private IImage<ColorRGB> ImageToImage(string prompt, Native.sd_image_t image, StableDiffusionParameter parameter)
+    private IImage<ColorRGB> ImageToImage(string prompt, Native.sd_image_t image, DiffusionParameter parameter)
     {
         ObjectDisposedException.ThrowIf(_disposed, this);
         ArgumentNullException.ThrowIfNull(prompt);
@@ -212,15 +205,18 @@ public sealed unsafe class StableDiffusionModel : IDisposable
         Native.sd_image_t* result;
         if (parameter.ControlNet.IsEnabled)
         {
-            fixed (byte* imagePtr = parameter.ControlNet.Image!.ToRawArray())
+            if (parameter.ControlNet.Image is not IImage<ColorRGB> controlNetImage)
+                controlNetImage = parameter.ControlNet.Image!.ConvertTo<ColorRGB>();
+
+            fixed (byte* imagePtr = controlNetImage.ToRawArray())
             {
                 if (parameter.ControlNet.CannyPreprocess)
                 {
-                    Native.sd_image_t controlNetImage = new()
+                    Native.sd_image_t nativeControlNetImage = new()
                     {
-                        width = (uint)parameter.ControlNet.Image.Width,
-                        height = (uint)parameter.ControlNet.Image.Height,
-                        channel = (uint)parameter.ControlNet.Image.ColorFormat.BytesPerPixel,
+                        width = (uint)controlNetImage.Width,
+                        height = (uint)controlNetImage.Height,
+                        channel = (uint)controlNetImage.ColorFormat.BytesPerPixel,
                         data = Native.preprocess_canny(imagePtr,
                                                        parameter.Width,
                                                        parameter.Height,
@@ -237,6 +233,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable
                                             parameter.NegativePrompt,
                                             parameter.ClipSkip,
                                             parameter.CfgScale,
+                                            parameter.Guidance,
                                             parameter.Width,
                                             parameter.Height,
                                             parameter.SampleMethod,
@@ -244,17 +241,17 @@ public sealed unsafe class StableDiffusionModel : IDisposable
                                             parameter.Strength,
                                             parameter.Seed,
                                             1,
-                                            &controlNetImage,
+                                            &nativeControlNetImage,
                                             parameter.ControlNet.Strength,
                                             parameter.PhotoMaker.StyleRatio,
                                             parameter.PhotoMaker.NormalizeInput,
                                             parameter.PhotoMaker.InputIdImageDirectory);
 
-                    Marshal.FreeHGlobal((nint)controlNetImage.data);
+                    Marshal.FreeHGlobal((nint)nativeControlNetImage.data);
                 }
                 else
                 {
-                    Native.sd_image_t controlNetImage = new()
+                    Native.sd_image_t nativeControlNetImage = new()
                     {
                         width = (uint)parameter.ControlNet.Image.Width,
                         height = (uint)parameter.ControlNet.Image.Height,
@@ -268,6 +265,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable
                                             parameter.NegativePrompt,
                                             parameter.ClipSkip,
                                             parameter.CfgScale,
+                                            parameter.Guidance,
                                             parameter.Width,
                                             parameter.Height,
                                             parameter.SampleMethod,
@@ -275,7 +273,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable
                                             parameter.Strength,
                                             parameter.Seed,
                                             1,
-                                            &controlNetImage,
+                                            &nativeControlNetImage,
                                             parameter.ControlNet.Strength,
                                             parameter.PhotoMaker.StyleRatio,
                                             parameter.PhotoMaker.NormalizeInput,
@@ -291,6 +289,7 @@ public sealed unsafe class StableDiffusionModel : IDisposable
                                     parameter.NegativePrompt,
                                     parameter.ClipSkip,
                                     parameter.CfgScale,
+                                    parameter.Guidance,
                                     parameter.Width,
                                     parameter.Height,
                                     parameter.SampleMethod,
@@ -308,74 +307,16 @@ public sealed unsafe class StableDiffusionModel : IDisposable
         return ImageHelper.ToImage(result);
     }
 
-    public IImage<ColorRGB> Upscale(IImage<ColorRGB> image, int upscaleFactor)
-    {
-        ObjectDisposedException.ThrowIf(_disposed, this);
-        ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(upscaleFactor, 0, nameof(upscaleFactor));
-
-        if (_upscalerCtx == null) throw new NullReferenceException("The upscaler is not initialized.");
-
-        fixed (byte* imagePtr = image.ConvertTo<ColorRGB>().AsRefImage())
-        {
-            Native.sd_image_t result = Native.upscale(_upscalerCtx, image.ToSdImage(imagePtr), upscaleFactor);
-            return ImageHelper.ToImage(&result);
-        }
-    }
-
-    private IImage<ColorRGB> Upscale(Native.sd_image_t image, int upscaleFactor)
-    {
-        Native.sd_image_t result = Native.upscale(_upscalerCtx, image, upscaleFactor);
-        return ImageHelper.ToImage(&result);
-    }
-
     public void Dispose()
     {
         if (_disposed) return;
 
-        Native.free_sd_ctx(_ctx);
-
-        if (_upscalerCtx != null)
-            Native.free_upscaler_ctx(_upscalerCtx);
+        if (_ctx != null)
+            Native.free_sd_ctx(_ctx);
 
         GC.SuppressFinalize(this);
         _disposed = true;
     }
 
-    public static void Convert(string modelPath, string vaePath, Quantization quantization, string outputPath)
-    {
-        ArgumentException.ThrowIfNullOrWhiteSpace(nameof(modelPath));
-        ArgumentException.ThrowIfNullOrWhiteSpace(nameof(outputPath));
-        ArgumentNullException.ThrowIfNull(vaePath);
-        if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization));
-
-        Native.convert(modelPath, vaePath, outputPath, quantization);
-    }
-
-    public static string GetSystemInfo()
-    {
-        void* s = Native.sd_get_system_info();
-        return Marshal.PtrToStringUTF8((nint)s) ?? "";
-    }
-
-    public static int GetNumPhysicalCores() => Native.get_num_physical_cores();
-
-    private static void OnNativeLog(LogLevel level, string text, void* data)
-    {
-        try
-        {
-            Log?.Invoke(null, new StableDiffusionLogEventArgs(level, text));
-        }
-        catch { /**/ }
-    }
-
-    private static void OnNativeProgress(int step, int steps, float time, void* data)
-    {
-        try
-        {
-            Progress?.Invoke(null, new StableDiffusionProgressEventArgs(step, steps, time));
-        }
-        catch { /**/ }
-    }
-
     #endregion
-}
+}

StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs

@@ -0,0 +1,19 @@
+using HPPH;
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public sealed class ControlNetParameter
+{
+    public bool IsEnabled => Image != null;
+
+    public IImage? Image { get; set; } = null;
+    public float Strength { get; set; } = 0.9f;
+    public bool CannyPreprocess { get; set; } = false;
+    public float CannyHighThreshold { get; set; } = 0.08f;
+    public float CannyLowThreshold { get; set; } = 0.08f;
+    public float CannyWeak { get; set; } = 0.8f;
+    public float CannyStrong { get; set; } = 1.0f;
+    public bool CannyInverse { get; set; } = false;
+}

StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs

@@ -1,30 +1,35 @@
-using JetBrains.Annotations;
+namespace StableDiffusion.NET;
 
-namespace StableDiffusion.NET;
-
-[PublicAPI]
-public class ModelParameter
+public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantizedModelParameter, IPhotomakerModelParameter
 {
-    #region Properties & Fields
+    public DiffusionModelType DiffusionModelType { get; set; } = DiffusionModelType.None;
+
+    public string VaePath { get; set; } = string.Empty;
+    public string TaesdPath { get; set; } = string.Empty;
+
+    public string LoraModelDirectory { get; set; } = string.Empty;
+    public string EmbeddingsDirectory { get; set; } = string.Empty;
+    public string ControlNetPath { get; set; } = string.Empty;
+
+    public int ThreadCount { get; set; } = 1;
 
-    public int ThreadCount { get; set; } = 8;
     public bool VaeDecodeOnly { get; set; } = false;
     public bool VaeTiling { get; set; } = false;
-    public string TaesdPath { get; set; } = string.Empty;
-    public string LoraModelDir { get; set; } = string.Empty;
-    public RngType RngType { get; set; } = RngType.Standard;
-    public string VaePath { get; set; } = string.Empty;
-    public string ControlNetPath { get; set; } = string.Empty;
-    public string EmbeddingsDirectory { get; set; } = string.Empty;
-    public string StackedIdEmbeddingsDirectory { get; set; } = string.Empty;
     public bool KeepControlNetOnCPU { get; set; } = false;
     public bool KeepClipOnCPU { get; set; } = false;
     public bool KeepVaeOnCPU { get; set; } = false;
 
-    //TODO DarthAffe 01.01.2024: K-Quants doesn't seem to work so far
-    public Quantization Quantization { get; set; } = Quantization.F16;
-
+    public RngType RngType { get; set; } = RngType.Standard;
     public Schedule Schedule { get; set; } = Schedule.Default;
 
-    #endregion
-}
+    public Quantization Quantization { get; set; } = Quantization.Unspecified;
+
+    // Stable Diffusion only
+    public string ModelPath { get; set; } = string.Empty;
+    public string StackedIdEmbeddingsDirectory { get; set; } = string.Empty;
+
+    // Flux only
+    public string DiffusionModelPath { get; set; } = string.Empty;
+    public string ClipLPath { get; set; } = string.Empty;
+    public string T5xxlPath { get; set; } = string.Empty;
+}

StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs

@@ -0,0 +1,33 @@
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public sealed class DiffusionParameter
+{
+    #region Properties & Fields
+
+    public static DiffusionParameter SD1Default => new() { Width = 512, Height = 512, CfgScale = 7.5f, Guidance = 1f, SampleSteps = 25, SampleMethod = Sampler.Euler_A };
+    public static DiffusionParameter SDXLDefault => new() { Width = 1024, Height = 1024, CfgScale = 7f, Guidance = 1f, SampleSteps = 30, SampleMethod = Sampler.Euler_A };
+    public static DiffusionParameter FluxDefault => new() { Width = 1024, Height = 1024, CfgScale = 1, Guidance = 3.5f, SampleSteps = 20, SampleMethod = Sampler.Euler };
+
+    public string NegativePrompt { get; set; } = string.Empty;
+    public int Width { get; set; } = 512;
+    public int Height { get; set; } = 512;
+    public Sampler SampleMethod { get; set; } = Sampler.Euler_A;
+    public int SampleSteps { get; set; } = 25;
+    public long Seed { get; set; } = -1;
+    public float Strength { get; set; } = 0.7f;
+    public int ClipSkip { get; set; } = -1;
+
+    public ControlNetParameter ControlNet { get; } = new();
+
+    // Stable Diffusion only
+    public float CfgScale { get; set; } = 7.5f;
+    public PhotoMakerParameter PhotoMaker { get; } = new();
+
+    // Flux only
+    public float Guidance { get; set; } = 3.5f;
+
+    #endregion
+}

StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs

@@ -0,0 +1,114 @@
+using HPPH;
+
+namespace StableDiffusion.NET;
+
+public static class DiffusionParameterExtension
+{
+    public static DiffusionParameter WithSize(this DiffusionParameter parameter, int? width = null, int? height = null)
+    {
+        if (width != null)
+            parameter.Width = width.Value;
+
+        if (height != null)
+            parameter.Height = height.Value;
+
+        return parameter;
+    }
+
+    public static DiffusionParameter WithSampler(this DiffusionParameter parameter, Sampler sampler)
+    {
+        parameter.SampleMethod = sampler;
+
+        return parameter;
+    }
+
+    public static DiffusionParameter WithSteps(this DiffusionParameter parameter, int steps)
+    {
+        parameter.SampleSteps = steps;
+
+        return parameter;
+    }
+
+    public static DiffusionParameter WithSeed(this DiffusionParameter parameter, long seed)
+    {
+        parameter.Seed = seed;
+
+        return parameter;
+    }
+
+    public static DiffusionParameter WithClipSkip(this DiffusionParameter parameter, int clipSkip)
+    {
+        parameter.ClipSkip = clipSkip;
+
+        return parameter;
+    }
+
+    public static DiffusionParameter WithCfg(this DiffusionParameter parameter, float cfg)
+    {
+        parameter.CfgScale = cfg;
+
+        return parameter;
+    }
+
+    public static DiffusionParameter WithGuidance(this DiffusionParameter parameter, float guidance)
+    {
+        parameter.Guidance = guidance;
+
+        return parameter;
+    }
+
+    public static DiffusionParameter WithNegativePrompt(this DiffusionParameter parameter, string negativePrompt)
+    {
+        parameter.NegativePrompt = negativePrompt;
+
+        return parameter;
+    }
+
+    public static DiffusionParameter WithControlNet(this DiffusionParameter parameter, IImage image, float? strength = null)
+    {
+        parameter.ControlNet.Image = image;
+
+        if (strength != null)
+            parameter.ControlNet.Strength = strength.Value;
+
+        return parameter;
+    }
+
+    public static DiffusionParameter WithCannyPreprocessing(this DiffusionParameter parameter,
+                                                            float? cannyHighThreshold = null, float? cannyLowThreshold = null,
+                                                            float? cannyWeak = null, float? cannyStrong = null,
+                                                            bool? cannyInverse = null)
+    {
+        parameter.ControlNet.CannyPreprocess = true;
+
+        if (cannyHighThreshold != null)
+            parameter.ControlNet.CannyHighThreshold = cannyHighThreshold.Value;
+
+        if (cannyLowThreshold != null)
+            parameter.ControlNet.CannyLowThreshold = cannyLowThreshold.Value;
+
+        if (cannyWeak != null)
+            parameter.ControlNet.CannyWeak = cannyWeak.Value;
+
+        if (cannyStrong != null)
+            parameter.ControlNet.CannyStrong = cannyStrong.Value;
+
+        if (cannyInverse != null)
+            parameter.ControlNet.CannyInverse = cannyInverse.Value;
+
+        return parameter;
+    }
+
+    public static DiffusionParameter WithPhotomaker(this DiffusionParameter parameter, string inputIdImageDirectory, float? styleRatio = null, bool? normalizeInput = null)
+    {
+        parameter.PhotoMaker.InputIdImageDirectory = inputIdImageDirectory;
+
+        if (styleRatio != null)
+            parameter.PhotoMaker.StyleRatio = styleRatio.Value;
+
+        if (normalizeInput != null)
+            parameter.PhotoMaker.NormalizeInput = normalizeInput.Value;
+
+        return parameter;
+    }
+}

StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs

@@ -0,0 +1,22 @@
+namespace StableDiffusion.NET;
+
+public interface IDiffusionModelParameter
+{
+    DiffusionModelType DiffusionModelType { get; set; }
+
+    string VaePath { get; set; }
+    string TaesdPath { get; set; }
+
+    string LoraModelDirectory { get; set; }
+    string EmbeddingsDirectory { get; set; }
+    string ControlNetPath { get; set; }
+
+    bool VaeDecodeOnly { get; set; }
+    bool VaeTiling { get; set; }
+    bool KeepControlNetOnCPU { get; set; }
+    bool KeepClipOnCPU { get; set; }
+    bool KeepVaeOnCPU { get; set; }
+
+    RngType RngType { get; set; }
+    Schedule Schedule { get; set; }
+}

StableDiffusion.NET/Models/Parameter/Interfaces/IPhotomakerModelParameter.cs

@@ -0,0 +1,6 @@
+namespace StableDiffusion.NET;
+
+public interface IPhotomakerModelParameter
+{
+    string StackedIdEmbeddingsDirectory { get; set; }
+}

StableDiffusion.NET/Models/Parameter/Interfaces/IQuantizedModelParameter.cs

@@ -0,0 +1,8 @@
+namespace StableDiffusion.NET;
+
+public interface IQuantizedModelParameter
+{
+    int ThreadCount { get; set; }
+
+    Quantization Quantization { get; set; }
+}

StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs

@@ -0,0 +1,11 @@
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public sealed class PhotoMakerParameter
+{
+    public string InputIdImageDirectory { get; set; } = string.Empty;
+    public float StyleRatio { get; set; } = 20f;
+    public bool NormalizeInput { get; set; } = false;
+}

StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs

@@ -0,0 +1,12 @@
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public sealed class UpscaleModelParameter : IQuantizedModelParameter
+{
+    public string ModelPath { get; set; } = string.Empty;
+    public int ThreadCount { get; set; } = 1;
+
+    public Quantization Quantization { get; set; } = Quantization.F16;
+}

StableDiffusion.NET/Models/UpscaleModel.cs

@@ -0,0 +1,83 @@
+using HPPH;
+using JetBrains.Annotations;
+using System;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public sealed unsafe class UpscaleModel : IDisposable
+{
+    #region Properties & Fields
+
+    private bool _disposed;
+
+    public UpscaleModelParameter ModelParameter { get; }
+
+    private Native.upscaler_ctx_t* _ctx;
+
+    #endregion
+
+    #region Constructors
+
+    public UpscaleModel(UpscaleModelParameter modelParameter)
+    {
+        ArgumentNullException.ThrowIfNull(modelParameter, nameof(modelParameter));
+
+        modelParameter.Validate();
+
+        this.ModelParameter = modelParameter;
+
+        Initialize();
+    }
+
+    ~UpscaleModel() => Dispose();
+
+    #endregion
+
+    #region Methods
+
+    private void Initialize()
+    {
+        _ctx = Native.new_upscaler_ctx(ModelParameter.ModelPath,
+                                       ModelParameter.ThreadCount,
+                                       ModelParameter.Quantization);
+
+        if (_ctx == null) throw new NullReferenceException("Failed to initialize upscale-model.");
+    }
+
+    public IImage<ColorRGB> Upscale(IImage image, int upscaleFactor)
+    {
+        ObjectDisposedException.ThrowIf(_disposed, this);
+        ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(upscaleFactor, 0, nameof(upscaleFactor));
+
+        if (_ctx == null) throw new NullReferenceException("The model is not initialized.");
+
+        if (image is not IImage<ColorRGB> sourceImage)
+            sourceImage = image.ConvertTo<ColorRGB>();
+
+        fixed (byte* imagePtr = sourceImage.AsRefImage())
+        {
+            Native.sd_image_t result = Native.upscale(_ctx, sourceImage.ToSdImage(imagePtr), upscaleFactor);
+            return ImageHelper.ToImage(&result);
+        }
+    }
+
+    private IImage<ColorRGB> Upscale(Native.sd_image_t image, int upscaleFactor)
+    {
+        Native.sd_image_t result = Native.upscale(_ctx, image, upscaleFactor);
+        return ImageHelper.ToImage(&result);
+    }
+
+    public void Dispose()
+    {
+        if (_disposed) return;
+
+        if (_ctx != null)
+            Native.free_upscaler_ctx(_ctx);
+
+        GC.SuppressFinalize(this);
+        _disposed = true;
+    }
+
+    #endregion
+}

StableDiffusion.NET/Native/Native.Load.cs

@@ -26,6 +26,18 @@ internal static partial class Native
 
     #region Methods
 
+    internal static bool LoadNativeLibrary(string libraryPath)
+    {
+        if (_loadedLibraryHandle != nint.Zero) return true;
+        if (NativeLibrary.TryLoad(libraryPath, out nint handle))
+        {
+            _loadedLibraryHandle = handle;
+            return true;
+        }
+
+        return false;
+    }
+
     private static nint ResolveDllImport(string libraryname, Assembly assembly, DllImportSearchPath? searchpath)
     {
         if (libraryname != LIB_NAME) return nint.Zero;

StableDiffusion.NET/Native/Native.cs

@@ -50,6 +50,9 @@ internal unsafe partial class Native
 
     [LibraryImport(LIB_NAME, EntryPoint = "new_sd_ctx")]
     internal static partial sd_ctx_t* new_sd_ctx([MarshalAs(UnmanagedType.LPStr)] string model_path,
+                                                 [MarshalAs(UnmanagedType.LPStr)] string clip_l_path,
+                                                 [MarshalAs(UnmanagedType.LPStr)] string t5xxl_path,
+                                                 [MarshalAs(UnmanagedType.LPStr)] string diffusion_model_path,
                                                  [MarshalAs(UnmanagedType.LPStr)] string vae_path,
                                                  [MarshalAs(UnmanagedType.LPStr)] string taesd_path,
                                                  [MarshalAs(UnmanagedType.LPStr)] string control_net_path_c_str,
@@ -76,6 +79,7 @@ internal unsafe partial class Native
                                                 [MarshalAs(UnmanagedType.LPStr)] string negative_prompt,
                                                 int clip_skip,
                                                 float cfg_scale,
+                                                float guidance,
                                                 int width,
                                                 int height,
                                                 sample_method_t sample_method,
@@ -95,6 +99,7 @@ internal unsafe partial class Native
                                                 [MarshalAs(UnmanagedType.LPStr)] string negative_prompt,
                                                 int clip_skip,
                                                 float cfg_scale,
+                                                float guidance,
                                                 int width,
                                                 int height,
                                                 sample_method_t sample_method,

StableDiffusion.NET/StableDiffusion.NET.csproj.DotSettings

@@ -1,8 +1,16 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
 	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=attributes/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=backends/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=builder/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=enums/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=eventargs/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=extensions/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=helper/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cbuilder/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cbuilder_005Cextensions/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cbuilder_005Cinterfaces/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cparameter/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cparameter_005Cextensions/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cparameter_005Cinterfaces/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=native/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

StableDiffusion.NET/StableDiffusionCpp.cs

@@ -0,0 +1,77 @@
+using System;
+using System.Runtime.InteropServices;
+using JetBrains.Annotations;
+
+namespace StableDiffusion.NET;
+
+[PublicAPI]
+public static unsafe class StableDiffusionCpp
+{
+    #region Properties & Fields
+
+    // ReSharper disable NotAccessedField.Local - They are important, the delegate can be collected if it's not stored!
+    private static readonly Native.sd_log_cb_t LOG_CALLBACK;
+    private static readonly Native.sd_progress_cb_t PROGRESS_CALLBACK;
+    // ReSharper restore NotAccessedField.Local
+
+    #endregion
+
+    #region Events
+
+    public static event EventHandler<StableDiffusionLogEventArgs>? Log;
+    public static event EventHandler<StableDiffusionProgressEventArgs>? Progress;
+
+    #endregion
+
+    #region Constructors
+
+    static StableDiffusionCpp()
+    {
+        Native.sd_set_log_callback(LOG_CALLBACK = OnNativeLog, null);
+        Native.sd_set_progress_callback(PROGRESS_CALLBACK = OnNativeProgress, null);
+    }
+
+    #endregion
+
+    #region Methods
+
+    public static bool LoadNativeLibrary(string libraryPath) => Native.LoadNativeLibrary(libraryPath);
+
+    public static void Convert(string modelPath, string vaePath, Quantization quantization, string outputPath)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(nameof(modelPath));
+        ArgumentException.ThrowIfNullOrWhiteSpace(nameof(outputPath));
+        ArgumentNullException.ThrowIfNull(vaePath);
+        if (!Enum.IsDefined(quantization)) throw new ArgumentOutOfRangeException(nameof(quantization));
+
+        Native.convert(modelPath, vaePath, outputPath, quantization);
+    }
+
+    public static string GetSystemInfo()
+    {
+        void* s = Native.sd_get_system_info();
+        return Marshal.PtrToStringUTF8((nint)s) ?? "";
+    }
+
+    public static int GetNumPhysicalCores() => Native.get_num_physical_cores();
+
+    private static void OnNativeLog(LogLevel level, string text, void* data)
+    {
+        try
+        {
+            Log?.Invoke(null, new StableDiffusionLogEventArgs(level, text));
+        }
+        catch { /**/ }
+    }
+
+    private static void OnNativeProgress(int step, int steps, float time, void* data)
+    {
+        try
+        {
+            Progress?.Invoke(null, new StableDiffusionProgressEventArgs(step, steps, time));
+        }
+        catch { /**/ }
+    }
+
+    #endregion
+}

StableDiffusion.NET/StableDiffusionParameter.cs

@@ -1,48 +0,0 @@
-using HPPH;
-using JetBrains.Annotations;
-
-namespace StableDiffusion.NET;
-
-[PublicAPI]
-public sealed class StableDiffusionParameter
-{
-    #region Properties & Fields
-
-    public string NegativePrompt { get; set; } = string.Empty;
-    public float CfgScale { get; set; } = 7.5f;
-    public int Width { get; set; } = 512;
-    public int Height { get; set; } = 512;
-    public Sampler SampleMethod { get; set; } = Sampler.Euler_A;
-    public int SampleSteps { get; set; } = 25;
-    public long Seed { get; set; } = -1;
-    public float Strength { get; set; } = 0.7f;
-    public int ClipSkip { get; set; } = -1;
-
-    public StableDiffusionControlNetParameter ControlNet { get; } = new();
-    public PhotoMakerParameter PhotoMaker { get; } = new();
-
-    #endregion
-}
-
-[PublicAPI]
-public sealed class StableDiffusionControlNetParameter
-{
-    public bool IsEnabled => Image != null;
-
-    public IImage<ColorRGB>? Image { get; set; } = null;
-    public float Strength { get; set; } = 0.9f;
-    public bool CannyPreprocess { get; set; } = false;
-    public float CannyHighThreshold { get; set; } = 0.08f;
-    public float CannyLowThreshold { get; set; } = 0.08f;
-    public float CannyWeak { get; set; } = 0.8f;
-    public float CannyStrong { get; set; } = 1.0f;
-    public bool CannyInverse { get; set; } = false;
-}
-
-[PublicAPI]
-public sealed class PhotoMakerParameter
-{
-    public string InputIdImageDirectory { get; set; } = string.Empty;
-    public float StyleRatio { get; set; } = 20f;
-    public bool NormalizeInput { get; set; } = false;
-}

StableDiffusion.NET/UpscalerModelParameter.cs

@@ -1,17 +0,0 @@
-using JetBrains.Annotations;
-
-namespace StableDiffusion.NET;
-
-[PublicAPI]
-public class UpscalerModelParameter
-{
-    #region Properties & Fields
-
-    public int ThreadCount { get; set; } = 8;
-    public string ESRGANPath { get; set; } = string.Empty;
-
-    //TODO DarthAffe 01.01.2024: K-Quants doesn't seem to work so far
-    public Quantization Quantization { get; set; } = Quantization.F16;
-
-    #endregion
-}