DarthAffe/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-08-11 06:36:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into hidream-o1

Browse Source
This commit is contained in:
leejet 2026-05-15 00:37:13 +08:00
commit 237b0258f5
15 changed files with 550 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
CMakeLists.txt
View File

@ -106,19 +106,28 @@ if(SD_WEBP)
"Or link against system library:\n cmake (...) -DSD_USE_SYSTEM_WEBP=ON") "Or link against system library:\n cmake (...) -DSD_USE_SYSTEM_WEBP=ON")
endif() endif()
if(SD_USE_SYSTEM_WEBP) if(SD_USE_SYSTEM_WEBP)
find_package(WebP REQUIRED) find_package(WebP)
add_library(webp ALIAS WebP::webp) if(WebP_FOUND)
# libwebp CMake target naming is not consistent across versions/distros. add_library(webp ALIAS WebP::webp)
# Some export WebP::libwebpmux, others export WebP::webpmux. # libwebp CMake target naming is not consistent across versions/distros.
if(TARGET WebP::libwebpmux) # Some export WebP::libwebpmux, others export WebP::webpmux.
add_library(libwebpmux ALIAS WebP::libwebpmux) if(TARGET WebP::libwebpmux)
elseif(TARGET WebP::webpmux) add_library(libwebpmux ALIAS WebP::libwebpmux)
add_library(libwebpmux ALIAS WebP::webpmux) elseif(TARGET WebP::webpmux)
add_library(libwebpmux ALIAS WebP::webpmux)
else()
message(FATAL_ERROR
"Could not find a compatible webpmux target in system WebP package. "
"Expected WebP::libwebpmux or WebP::webpmux."
)
endif()
else() else()
message(FATAL_ERROR find_package(PkgConfig REQUIRED)
"Could not find a compatible webpmux target in system WebP package. " pkg_check_modules(WebP REQUIRED IMPORTED_TARGET GLOBAL libwebp)
"Expected WebP::libwebpmux or WebP::webpmux." pkg_check_modules(WebPMux REQUIRED IMPORTED_TARGET GLOBAL libwebpmux)
) link_libraries(PkgConfig::WebP)
link_libraries(PkgConfig::WebPMux)
add_library(libwebpmux ALIAS PkgConfig::WebPMux)
endif() endif()
endif() endif()
endif() endif()
@ -133,18 +142,26 @@ if(SD_WEBM)
"Or link against system library:\n cmake (...) -DSD_USE_SYSTEM_WEBM=ON") "Or link against system library:\n cmake (...) -DSD_USE_SYSTEM_WEBM=ON")
endif() endif()
if(SD_USE_SYSTEM_WEBM) if(SD_USE_SYSTEM_WEBM)
find_path(WEBM_INCLUDE_DIR find_package(PkgConfig)
NAMES mkvmuxer/mkvmuxer.h mkvparser/mkvparser.h common/webmids.h if(PkgConfig_FOUND)
PATH_SUFFIXES webm pkg_check_modules(WebM REQUIRED IMPORTED_TARGET GLOBAL libwebm)
REQUIRED) endif()
find_library(WEBM_LIBRARY if(PkgConfig_FOUND AND WebM_FOUND)
NAMES webm libwebm link_libraries(PkgConfig::WebM)
REQUIRED) else()
find_path(WEBM_INCLUDE_DIR
NAMES mkvmuxer/mkvmuxer.h mkvparser/mkvparser.h common/webmids.h
PATH_SUFFIXES webm
REQUIRED)
find_library(WEBM_LIBRARY
NAMES webm libwebm
REQUIRED)
add_library(webm UNKNOWN IMPORTED) add_library(webm UNKNOWN IMPORTED)
set_target_properties(webm PROPERTIES set_target_properties(webm PROPERTIES
IMPORTED_LOCATION "${WEBM_LIBRARY}" IMPORTED_LOCATION "${WEBM_LIBRARY}"
INTERFACE_INCLUDE_DIRECTORIES "${WEBM_INCLUDE_DIR}") INTERFACE_INCLUDE_DIRECTORIES "${WEBM_INCLUDE_DIR}")
endif()
endif() endif()
endif() endif()

1
README.md
View File

@ -165,6 +165,7 @@ These projects wrap `stable-diffusion.cpp` for easier use in other languages/fra
* Golang (non-cgo): [seasonjs/stable-diffusion](https://github.com/seasonjs/stable-diffusion) * Golang (non-cgo): [seasonjs/stable-diffusion](https://github.com/seasonjs/stable-diffusion)
* Golang (cgo): [Binozo/GoStableDiffusion](https://github.com/Binozo/GoStableDiffusion) * Golang (cgo): [Binozo/GoStableDiffusion](https://github.com/Binozo/GoStableDiffusion)
* Golang (non-cgo): [l8bloom/gosd](https://github.com/l8bloom/gosd)
* C#: [DarthAffe/StableDiffusion.NET](https://github.com/DarthAffe/StableDiffusion.NET) * C#: [DarthAffe/StableDiffusion.NET](https://github.com/DarthAffe/StableDiffusion.NET)
* Python: [william-murray1204/stable-diffusion-cpp-python](https://github.com/william-murray1204/stable-diffusion-cpp-python) * Python: [william-murray1204/stable-diffusion-cpp-python](https://github.com/william-murray1204/stable-diffusion-cpp-python)
* Rust: [newfla/diffusion-rs](https://github.com/newfla/diffusion-rs) * Rust: [newfla/diffusion-rs](https://github.com/newfla/diffusion-rs)

4
examples/cli/README.md
View File

@ -164,10 +164,10 @@ Generation Options:
-s, --seed RNG seed (default: 42, use random seed for < 0) -s, --seed RNG seed (default: 42, use random seed for < 0)
--sampling-method sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, --sampling-method sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m,
dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s,
er_sde] (default: euler for Flux/SD3/Wan, euler_a otherwise) er_sde, euler_cfg_pp, euler_a_cfg_pp] (default: euler for Flux/SD3/Wan, euler_a otherwise)
--high-noise-sampling-method (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, --high-noise-sampling-method (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a,
dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep,
res_2s, er_sde] default: euler for Flux/SD3/Wan, euler_a otherwise res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for Flux/SD3/Wan, euler_a otherwise
--scheduler denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, --scheduler denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits,
smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent], default: smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent], default:
discrete discrete

4
examples/common/common.cpp
View File

@ -1248,12 +1248,12 @@ ArgOptions SDGenerationParams::get_options() {
on_seed_arg}, on_seed_arg},
{"", {"",
"--sampling-method", "--sampling-method",
"sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde] " "sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]"
"(default: euler for Flux/SD3/Wan, euler_a otherwise)", "(default: euler for Flux/SD3/Wan, euler_a otherwise)",
on_sample_method_arg}, on_sample_method_arg},
{"", {"",
"--high-noise-sampling-method", "--high-noise-sampling-method",
"(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde]" "(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]"
" default: euler for Flux/SD3/Wan, euler_a otherwise", " default: euler for Flux/SD3/Wan, euler_a otherwise",
on_high_noise_sample_method_arg}, on_high_noise_sample_method_arg},
{"", {"",

4
examples/server/README.md
View File

@ -266,10 +266,10 @@ Default Generation Options:
-s, --seed RNG seed (default: 42, use random seed for < 0) -s, --seed RNG seed (default: 42, use random seed for < 0)
--sampling-method sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, --sampling-method sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m,
dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s,
er_sde] (default: euler for Flux/SD3/Wan, euler_a otherwise) er_sde, euler_cfg_pp, euler_a_cfg_pp] (default: euler for Flux/SD3/Wan, euler_a otherwise)
--high-noise-sampling-method (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, --high-noise-sampling-method (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a,
dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep,
res_2s, er_sde] default: euler for Flux/SD3/Wan, euler_a otherwise res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for Flux/SD3/Wan, euler_a otherwise
--scheduler denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, --scheduler denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits,
smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent], default: smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent], default:
discrete discrete

2
examples/server/main.cpp
View File

@ -145,7 +145,7 @@ int main(int argc, const char** argv) {
register_sdapi_endpoints(svr, runtime); register_sdapi_endpoints(svr, runtime);
register_sdcpp_api_endpoints(svr, runtime); register_sdcpp_api_endpoints(svr, runtime);
LOG_INFO("listening on: %s:%d\n", svr_params.listen_ip.c_str(), svr_params.listen_port); LOG_INFO("listening on: http://%s:%d\n", svr_params.listen_ip.c_str(), svr_params.listen_port);
svr.listen(svr_params.listen_ip, svr_params.listen_port); svr.listen(svr_params.listen_ip, svr_params.listen_port);
{ {

4
examples/server/routes_sdapi.cpp
View File

@ -67,6 +67,10 @@ static enum sample_method_t get_sdapi_sample_method(std::string name) {
{"k_res_multistep", RES_MULTISTEP_SAMPLE_METHOD}, {"k_res_multistep", RES_MULTISTEP_SAMPLE_METHOD},
{"res 2s", RES_2S_SAMPLE_METHOD}, {"res 2s", RES_2S_SAMPLE_METHOD},
{"k_res_2s", RES_2S_SAMPLE_METHOD}, {"k_res_2s", RES_2S_SAMPLE_METHOD},
{"euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
{"k_euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
{"euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
{"k_euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
}; };
auto it = hardcoded.find(name); auto it = hardcoded.find(name);
return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT; return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT;

2
include/stable-diffusion.h
View File

@ -51,6 +51,8 @@ enum sample_method_t {
RES_MULTISTEP_SAMPLE_METHOD, RES_MULTISTEP_SAMPLE_METHOD,
RES_2S_SAMPLE_METHOD, RES_2S_SAMPLE_METHOD,
ER_SDE_SAMPLE_METHOD, ER_SDE_SAMPLE_METHOD,
EULER_CFG_PP_SAMPLE_METHOD,
EULER_A_CFG_PP_SAMPLE_METHOD,
SAMPLE_METHOD_COUNT SAMPLE_METHOD_COUNT
}; };

104
src/denoiser.hpp
View File

@ -753,7 +753,7 @@ struct Flux2FlowDenoiser : public FluxFlowDenoiser {
} }
}; };
typedef std::function<sd::Tensor<float>(const sd::Tensor<float>&, float, int)> denoise_cb_t; typedef std::function<sd::Tensor<float>(const sd::Tensor<float>&, float, int, sd::Tensor<float>*)> denoise_cb_t;
static std::pair<float, float> get_ancestral_step(float sigma_from, static std::pair<float, float> get_ancestral_step(float sigma_from,
float sigma_to, float sigma_to,
@ -829,7 +829,7 @@ static sd::Tensor<float> sample_euler_ancestral(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1; int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) { for (int i = 0; i < steps; i++) {
float sigma = sigmas[i]; float sigma = sigmas[i];
auto denoised_opt = model(x, sigma, i + 1); auto denoised_opt = model(x, sigma, i + 1, nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -852,7 +852,7 @@ static sd::Tensor<float> sample_euler_flow(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1; int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) { for (int i = 0; i < steps; i++) {
float sigma = sigmas[i]; float sigma = sigmas[i];
auto denoised_opt = model(x, sigma, i + 1); auto denoised_opt = model(x, sigma, i + 1, nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -874,7 +874,7 @@ static sd::Tensor<float> sample_euler(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1; int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) { for (int i = 0; i < steps; i++) {
float sigma = sigmas[i]; float sigma = sigmas[i];
auto denoised_opt = model(x, sigma, i + 1); auto denoised_opt = model(x, sigma, i + 1, nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -890,7 +890,7 @@ static sd::Tensor<float> sample_heun(denoise_cb_t model,
const std::vector<float>& sigmas) { const std::vector<float>& sigmas) {
int steps = static_cast<int>(sigmas.size()) - 1; int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) { for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], -(i + 1)); auto denoised_opt = model(x, sigmas[i], -(i + 1), nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -901,7 +901,7 @@ static sd::Tensor<float> sample_heun(denoise_cb_t model,
x += d * dt; x += d * dt;
} else { } else {
sd::Tensor<float> x2 = x + d * dt; sd::Tensor<float> x2 = x + d * dt;
auto denoised2_opt = model(x2, sigmas[i + 1], i + 1); auto denoised2_opt = model(x2, sigmas[i + 1], i + 1, nullptr);
if (denoised2_opt.empty()) { if (denoised2_opt.empty()) {
return {}; return {};
} }
@ -918,7 +918,7 @@ static sd::Tensor<float> sample_dpm2(denoise_cb_t model,
const std::vector<float>& sigmas) { const std::vector<float>& sigmas) {
int steps = static_cast<int>(sigmas.size()) - 1; int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) { for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], -(i + 1)); auto denoised_opt = model(x, sigmas[i], -(i + 1), nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -931,7 +931,7 @@ static sd::Tensor<float> sample_dpm2(denoise_cb_t model,
float dt_1 = sigma_mid - sigmas[i]; float dt_1 = sigma_mid - sigmas[i];
float dt_2 = sigmas[i + 1] - sigmas[i]; float dt_2 = sigmas[i + 1] - sigmas[i];
sd::Tensor<float> x2 = x + d * dt_1; sd::Tensor<float> x2 = x + d * dt_1;
auto denoised2_opt = model(x2, sigma_mid, i + 1); auto denoised2_opt = model(x2, sigma_mid, i + 1, nullptr);
if (denoised2_opt.empty()) { if (denoised2_opt.empty()) {
return {}; return {};
} }
@ -952,7 +952,7 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1; int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) { for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], -(i + 1)); auto denoised_opt = model(x, sigmas[i], -(i + 1), nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -968,7 +968,7 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral(denoise_cb_t model,
float s = t + 0.5f * h; float s = t + 0.5f * h;
float sigma_s = sigma_fn(s); float sigma_s = sigma_fn(s);
sd::Tensor<float> x2 = (sigma_s / sigma_fn(t)) * x - (exp(-h * 0.5f) - 1) * denoised; sd::Tensor<float> x2 = (sigma_s / sigma_fn(t)) * x - (exp(-h * 0.5f) - 1) * denoised;
auto denoised2_opt = model(x2, sigma_s, i + 1); auto denoised2_opt = model(x2, sigma_s, i + 1, nullptr);
if (denoised2_opt.empty()) { if (denoised2_opt.empty()) {
return {}; return {};
} }
@ -995,7 +995,7 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral_flow(denoise_cb_t model,
bool opt_first_step = (1.0 - sigma < 1e-6); bool opt_first_step = (1.0 - sigma < 1e-6);
auto denoised_opt = model(x, sigma, (opt_first_step ? 1 : -1) * (i + 1)); auto denoised_opt = model(x, sigma, (opt_first_step ? 1 : -1) * (i + 1), nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -1024,8 +1024,8 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral_flow(denoise_cb_t model,
// so sigma_s = 1 = sigma, and sigma_s_i_ratio = sigma_s / sigma = 1 // so sigma_s = 1 = sigma, and sigma_s_i_ratio = sigma_s / sigma = 1
// u = (x*sigma_s_i_ratio)+(denoised*(1.0f-sigma_s_i_ratio)) // u = (x*sigma_s_i_ratio)+(denoised*(1.0f-sigma_s_i_ratio))
// = (x*1)+(denoised*0) = x // = (x*1)+(denoised*0) = x
// so D_i = model(u, sigma_s, i + 1) // so D_i = model(u, sigma_s, i + 1, nullptr)
// = model(x, sigma, i + 1) // = model(x, sigma, i + 1, nullptr)
// = denoised // = denoised
D_i = denoised; D_i = denoised;
@ -1058,7 +1058,7 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral_flow(denoise_cb_t model,
float sigma_s_i_ratio = sigma_s / sigma; float sigma_s_i_ratio = sigma_s / sigma;
sd::Tensor<float> u = (x * sigma_s_i_ratio) + (denoised * (1.0f - sigma_s_i_ratio)); sd::Tensor<float> u = (x * sigma_s_i_ratio) + (denoised * (1.0f - sigma_s_i_ratio));
auto denoised2_opt = model(u, sigma_s, i + 1); auto denoised2_opt = model(u, sigma_s, i + 1, nullptr);
if (denoised2_opt.empty()) { if (denoised2_opt.empty()) {
return {}; return {};
} }
@ -1085,7 +1085,7 @@ static sd::Tensor<float> sample_dpmpp_2m(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1; int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) { for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], i + 1); auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -1117,7 +1117,7 @@ static sd::Tensor<float> sample_dpmpp_2m_v2(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1; int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) { for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], i + 1); auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -1225,7 +1225,7 @@ static sd::Tensor<float> sample_lcm(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1; int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) { for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], i + 1); auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -1271,7 +1271,7 @@ static sd::Tensor<float> sample_ipndm(denoise_cb_t model,
float sigma = sigmas[i]; float sigma = sigmas[i];
float sigma_next = sigmas[i + 1]; float sigma_next = sigmas[i + 1];
auto denoised_opt = model(x, sigma, i + 1); auto denoised_opt = model(x, sigma, i + 1, nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -1315,7 +1315,7 @@ static sd::Tensor<float> sample_ipndm_v(denoise_cb_t model,
float sigma = sigmas[i]; float sigma = sigmas[i];
float t_next = sigmas[i + 1]; float t_next = sigmas[i + 1];
auto denoised_opt = model(x, sigma, i + 1); auto denoised_opt = model(x, sigma, i + 1, nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -1377,7 +1377,7 @@ static sd::Tensor<float> sample_res_multistep(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1; int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) { for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], i + 1); auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -1454,7 +1454,7 @@ static sd::Tensor<float> sample_res_2s(denoise_cb_t model,
float sigma_from = sigmas[i]; float sigma_from = sigmas[i];
float sigma_to = sigmas[i + 1]; float sigma_to = sigmas[i + 1];
auto denoised_opt = model(x, sigma_from, -(i + 1)); auto denoised_opt = model(x, sigma_from, -(i + 1), nullptr);
if (denoised_opt.empty()) { if (denoised_opt.empty()) {
return {}; return {};
} }
@ -1480,7 +1480,7 @@ static sd::Tensor<float> sample_res_2s(denoise_cb_t model,
sd::Tensor<float> eps1 = denoised - x0; sd::Tensor<float> eps1 = denoised - x0;
sd::Tensor<float> x2 = x0 + eps1 * (h * a21); sd::Tensor<float> x2 = x0 + eps1 * (h * a21);
auto denoised2_opt = model(x2, sigma_c2, i + 1); auto denoised2_opt = model(x2, sigma_c2, i + 1, nullptr);
if (denoised2_opt.empty()) { if (denoised2_opt.empty()) {
return {}; return {};
} }
@ -1557,7 +1557,7 @@ static sd::Tensor<float> sample_er_sde(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1; int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) { for (int i = 0; i < steps; i++) {
sd::Tensor<float> denoised = model(x, sigmas[i], i + 1); sd::Tensor<float> denoised = model(x, sigmas[i], i + 1, nullptr);
if (denoised.empty()) { if (denoised.empty()) {
return {}; return {};
} }
@ -1643,7 +1643,7 @@ static sd::Tensor<float> sample_ddim_trailing(denoise_cb_t model,
float sigma = sigmas[i]; float sigma = sigmas[i];
float sigma_to = sigmas[i + 1]; float sigma_to = sigmas[i + 1];
auto model_output_opt = model(x, sigma, i + 1); auto model_output_opt = model(x, sigma, i + 1, nullptr);
if (model_output_opt.empty()) { if (model_output_opt.empty()) {
return {}; return {};
} }
@ -1715,7 +1715,7 @@ static sd::Tensor<float> sample_tcd(denoise_cb_t model,
int timestep_s = (int)floor((1 - eta) * prev_timestep); int timestep_s = (int)floor((1 - eta) * prev_timestep);
float sigma = sigmas[i]; float sigma = sigmas[i];
auto model_output_opt = model(x, sigma, i + 1); auto model_output_opt = model(x, sigma, i + 1, nullptr);
if (model_output_opt.empty()) { if (model_output_opt.empty()) {
return {}; return {};
} }
@ -1743,6 +1743,56 @@ static sd::Tensor<float> sample_tcd(denoise_cb_t model,
return x; return x;
} }
static sd::Tensor<float> sample_euler_cfg_pp(denoise_cb_t model,
sd::Tensor<float> x,
const std::vector<float>& sigmas) {
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
float sigma = sigmas[i];
sd::Tensor<float> uncond_denoised;
auto denoised_opt = model(x, sigma, i + 1, &uncond_denoised);
if (denoised_opt.empty() || uncond_denoised.empty()) {
return {};
}
sd::Tensor<float> denoised = std::move(denoised_opt);
sd::Tensor<float> d = (x - uncond_denoised) / sigma;
x = denoised + d * sigmas[i + 1];
}
return x;
}
static sd::Tensor<float> sample_euler_ancestral_cfg_pp(denoise_cb_t model,
sd::Tensor<float> x,
const std::vector<float>& sigmas,
std::shared_ptr<RNG> rng,
float eta) {
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
float sigma = sigmas[i];
sd::Tensor<float> uncond_denoised;
auto denoised_opt = model(x, sigma, i + 1, &uncond_denoised);
if (denoised_opt.empty() || uncond_denoised.empty()) {
return {};
}
sd::Tensor<float> denoised = std::move(denoised_opt);
sd::Tensor<float> d = (x - uncond_denoised) / sigma;
auto [sigma_down, sigma_up] = get_ancestral_step(sigmas[i], sigmas[i + 1], eta);
x = denoised + d * sigma_down;
if (sigmas[i + 1] > 0) {
x += sd::Tensor<float>::randn_like(x, rng) * sigma_up;
}
}
return x;
}
// k diffusion reverse ODE: dx = (x - D(x;\sigma)) / \sigma dt; \sigma(t) = t // k diffusion reverse ODE: dx = (x - D(x;\sigma)) / \sigma dt; \sigma(t) = t
static sd::Tensor<float> sample_k_diffusion(sample_method_t method, static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
denoise_cb_t model, denoise_cb_t model,
@ -1789,6 +1839,10 @@ static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
return sample_ddim_trailing(model, std::move(x), sigmas, rng, eta); return sample_ddim_trailing(model, std::move(x), sigmas, rng, eta);
case TCD_SAMPLE_METHOD: case TCD_SAMPLE_METHOD:
return sample_tcd(model, std::move(x), sigmas, rng, eta); return sample_tcd(model, std::move(x), sigmas, rng, eta);
case EULER_CFG_PP_SAMPLE_METHOD:
return sample_euler_cfg_pp(model, std::move(x), sigmas);
case EULER_A_CFG_PP_SAMPLE_METHOD:
return sample_euler_ancestral_cfg_pp(model, std::move(x), sigmas, rng, eta);
default: default:
return {}; return {};
} }

19
src/ggml_extend.hpp
View File

@ -2716,7 +2716,24 @@ public:
bool alloc_params_buffer() { bool alloc_params_buffer() {
size_t num_tensors = ggml_tensor_num(params_ctx); size_t num_tensors = ggml_tensor_num(params_ctx);
params_buffer = ggml_backend_alloc_ctx_tensors(params_ctx, params_backend); if (num_tensors > 0) {
// ggml_backend_alloc_ctx_tensors fails when all tensors are already allocated
// (typical for memory-mapped weights). See ggml-alloc.c n_buffers==0 branch.
bool all_have_data = true;
for (ggml_tensor* t = ggml_get_first_tensor(params_ctx); t != nullptr; t = ggml_get_next_tensor(params_ctx, t)) {
if (t->data == nullptr) {
all_have_data = false;
break;
}
}
if (all_have_data) {
LOG_DEBUG("%s all params already mmap-allocated (no separate buffer needed)", get_desc().c_str());
params_buffer = nullptr;
rebuild_params_tensor_set();
return true;
}
}
params_buffer = ggml_backend_alloc_ctx_tensors(params_ctx, params_backend);
if (params_buffer == nullptr) { if (params_buffer == nullptr) {
LOG_ERROR("%s alloc params backend buffer failed, num_tensors = %i", LOG_ERROR("%s alloc params backend buffer failed, num_tensors = %i",
get_desc().c_str(), get_desc().c_str(),

195
src/model.cpp
View File

@ -734,16 +734,10 @@ void ModelLoader::set_wtype_override(ggml_type wtype, std::string tensor_type_ru
} }
} }
bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads_p, bool enable_mmap) { void ModelLoader::process_model_files(bool enable_mmap, bool writable_mmap) {
int64_t process_time_ms = 0; if (model_files_processed) {
std::atomic<int64_t> read_time_ms(0); return;
std::atomic<int64_t> memcpy_time_ms(0); }
std::atomic<int64_t> copy_to_backend_time_ms(0);
std::atomic<int64_t> convert_time_ms(0);
std::atomic<uint64_t> bytes_processed(0);
int num_threads_to_use = n_threads_p > 0 ? n_threads_p : sd_get_num_physical_cores();
LOG_DEBUG("using %d threads for model loading", num_threads_to_use);
int64_t start_time = ggml_time_ms(); int64_t start_time = ggml_time_ms();
@ -755,22 +749,13 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
processed_tensor_storages.push_back(tensor_storage); processed_tensor_storages.push_back(tensor_storage);
} }
process_time_ms = ggml_time_ms() - start_time;
bool success = true;
size_t total_tensors_processed = 0;
const size_t total_tensors_to_process = processed_tensor_storages.size();
const int64_t t_start = ggml_time_ms();
int last_n_threads = 1;
for (size_t file_index = 0; file_index < file_paths_.size(); file_index++) { for (size_t file_index = 0; file_index < file_paths_.size(); file_index++) {
std::string file_path = file_paths_[file_index]; std::string file_path = file_paths_[file_index];
LOG_DEBUG("loading tensors from %s", file_path.c_str());
std::vector<const TensorStorage*> file_tensors; std::vector<TensorStorage> file_tensors;
for (const auto& ts : processed_tensor_storages) { for (const auto& ts : processed_tensor_storages) {
if (ts.file_index == file_index) { if (ts.file_index == file_index) {
file_tensors.push_back(&ts); file_tensors.push_back(ts);
} }
} }
if (file_tensors.empty()) { if (file_tensors.empty()) {
@ -779,21 +764,169 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
bool is_zip = false; bool is_zip = false;
for (auto const& ts : file_tensors) { for (auto const& ts : file_tensors) {
if (ts->index_in_zip >= 0) { if (ts.index_in_zip >= 0) {
is_zip = true; is_zip = true;
break; break;
} }
} }
std::unique_ptr<MmapWrapper> mmapped; ModelFileData fdata = {};
fdata.path = file_path;
fdata.is_zip = is_zip;
fdata.tensors = std::move(file_tensors);
if (enable_mmap && !is_zip) { if (enable_mmap && !is_zip) {
LOG_DEBUG("using mmap for I/O"); LOG_DEBUG("using mmap for I/O");
mmapped = MmapWrapper::create(file_path); std::unique_ptr<MmapWrapper> mmapped = MmapWrapper::create(file_path, writable_mmap);
if (!mmapped) { if (mmapped) {
LOG_WARN("failed to memory-map '%s'", file_path.c_str()); uint8_t* mmap_data = static_cast<uint8_t*>(mmapped->writable_data());
ggml_backend_buffer_t buf_mmap = ggml_backend_cpu_buffer_from_ptr(mmap_data, mmapped->size());
if (buf_mmap) {
LOG_INFO("using mmap for '%s'", file_path.c_str());
fdata.mmbuffer = std::shared_ptr<struct ggml_backend_buffer>(buf_mmap, ggml_backend_buffer_free);
} else {
LOG_WARN("mmap: failed to create backend buffer for file %s", fdata.path.c_str());
}
fdata.mmapped = std::shared_ptr<MmapWrapper>(std::move(mmapped));
} else {
LOG_WARN("failed to memory-map '%s' (falling back to read())", file_path.c_str());
} }
} else if (!is_zip) {
LOG_INFO("NOT using mmap for '%s' (mmap disabled by caller)",
file_path.c_str());
} }
file_data.push_back(std::move(fdata));
}
model_files_processed = true;
int64_t end_time = ggml_time_ms();
int64_t process_time_ms = end_time - start_time;
LOG_INFO("model files processing completed in %.2fs", process_time_ms / 1000.f);
}
std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggml_tensor*>& tensors,
std::set<std::string> ignore_tensors,
bool writable_mmap) {
process_model_files(true, writable_mmap);
std::vector<MmapTensorStore> result;
uint64_t mapped_bytes = 0;
size_t mapped_tensors = 0;
LOG_DEBUG("memory-mapping tensors...");
int64_t t_start = ggml_time_ms();
for (auto& fdata : file_data) {
if (!fdata.mmbuffer)
continue;
const std::vector<TensorStorage>& file_tensors = fdata.tensors;
size_t file_mapped_bytes = 0;
size_t file_mapped_tensors = 0;
for (const auto& tensor_storage : file_tensors) {
const std::string& name = tensor_storage.name;
bool is_ignored = false;
for (const auto& ignore_prefix : ignore_tensors) {
if (starts_with(name, ignore_prefix)) {
is_ignored = true;
break;
}
}
if (is_ignored)
continue;
auto it = tensors.find(name);
if (it == tensors.end())
continue;
ggml_tensor* dst_tensor = it->second;
if (dst_tensor == nullptr)
continue;
if (tensor_storage.type != dst_tensor->type)
continue;
size_t tensor_size = tensor_storage.nbytes();
size_t tensor_offset = tensor_storage.offset;
if (tensor_storage.ne[0] != dst_tensor->ne[0] ||
tensor_storage.ne[1] != dst_tensor->ne[1] ||
tensor_storage.ne[2] != dst_tensor->ne[2] ||
tensor_storage.ne[3] != dst_tensor->ne[3] ||
tensor_size != ggml_nbytes(dst_tensor)) {
// let load_tensors worry about this
continue;
}
ggml_backend_buffer_t buf_mmap = fdata.mmbuffer.get();
uint8_t* mmap_data = static_cast<uint8_t*>(ggml_backend_buffer_get_base(buf_mmap));
dst_tensor->buffer = buf_mmap;
dst_tensor->data = mmap_data + tensor_offset;
file_mapped_bytes += tensor_size;
file_mapped_tensors++;
}
if (file_mapped_bytes > 0) {
mapped_tensors += file_mapped_tensors;
mapped_bytes += file_mapped_bytes;
result.push_back({fdata.mmapped, fdata.mmbuffer});
}
}
int64_t t_end = ggml_time_ms();
int64_t duration_ms = t_end - t_start;
LOG_INFO("memory-mapped %zu tensors in %zu files (%.2f MB), taking %.2fs",
mapped_tensors,
result.size(),
mapped_bytes / (1024.0 * 1024.0),
duration_ms / 1000.0);
return result;
}
bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads_p, bool enable_mmap) {
process_model_files(enable_mmap, false);
std::atomic<int64_t> read_time_ms(0);
std::atomic<int64_t> memcpy_time_ms(0);
std::atomic<int64_t> copy_to_backend_time_ms(0);
std::atomic<int64_t> convert_time_ms(0);
std::atomic<uint64_t> bytes_processed(0);
int num_threads_to_use = n_threads_p > 0 ? n_threads_p : sd_get_num_physical_cores();
LOG_DEBUG("using %d threads for model loading", num_threads_to_use);
int64_t start_time = ggml_time_ms();
size_t total_tensors_to_process = 0;
for (const auto& fdata : file_data) {
total_tensors_to_process += fdata.tensors.size();
}
bool success = true;
size_t total_tensors_processed = 0;
const int64_t t_start = start_time;
int last_n_threads = 1;
for (auto& fdata : file_data) {
const std::string& file_path = fdata.path;
LOG_DEBUG("loading tensors from %s", file_path.c_str());
const std::vector<TensorStorage>& file_tensors = fdata.tensors;
bool is_zip = fdata.is_zip;
std::shared_ptr<MmapWrapper> mmapped = fdata.mmapped;
int n_threads = is_zip ? 1 : std::min(num_threads_to_use, (int)file_tensors.size()); int n_threads = is_zip ? 1 : std::min(num_threads_to_use, (int)file_tensors.size());
if (n_threads < 1) { if (n_threads < 1) {
n_threads = 1; n_threads = 1;
@ -834,7 +967,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
break; break;
} }
const TensorStorage& tensor_storage = *file_tensors[idx]; const TensorStorage& tensor_storage = file_tensors[idx];
ggml_tensor* dst_tensor = nullptr; ggml_tensor* dst_tensor = nullptr;
t0 = ggml_time_ms(); t0 = ggml_time_ms();
@ -851,6 +984,11 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
continue; continue;
} }
// skip mmapped tensors
if (dst_tensor->buffer != nullptr && dst_tensor->buffer == fdata.mmbuffer.get()) {
continue;
}
size_t nbytes_to_read = tensor_storage.nbytes_to_read(); size_t nbytes_to_read = tensor_storage.nbytes_to_read();
auto read_data = [&](char* buf, size_t n) { auto read_data = [&](char* buf, size_t n) {
@ -994,9 +1132,8 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
} }
int64_t end_time = ggml_time_ms(); int64_t end_time = ggml_time_ms();
LOG_INFO("loading tensors completed, taking %.2fs (process: %.2fs, read: %.2fs, memcpy: %.2fs, convert: %.2fs, copy_to_backend: %.2fs)", LOG_INFO("loading tensors completed, taking %.2fs (read: %.2fs, memcpy: %.2fs, convert: %.2fs, copy_to_backend: %.2fs)",
(end_time - start_time) / 1000.f, (end_time - start_time) / 1000.f,
process_time_ms / 1000.f,
(read_time_ms.load() / (float)last_n_threads) / 1000.f, (read_time_ms.load() / (float)last_n_threads) / 1000.f,
(memcpy_time_ms.load() / (float)last_n_threads) / 1000.f, (memcpy_time_ms.load() / (float)last_n_threads) / 1000.f,
(convert_time_ms.load() / (float)last_n_threads) / 1000.f, (convert_time_ms.load() / (float)last_n_threads) / 1000.f,

21
src/model.h
View File

@ -195,10 +195,27 @@ using TensorTypeRules = std::vector<std::pair<std::string, ggml_type>>;
TensorTypeRules parse_tensor_type_rules(const std::string& tensor_type_rules); TensorTypeRules parse_tensor_type_rules(const std::string& tensor_type_rules);
class MmapWrapper;
struct ModelFileData {
std::string path;
std::vector<TensorStorage> tensors;
std::shared_ptr<MmapWrapper> mmapped;
std::shared_ptr<struct ggml_backend_buffer> mmbuffer;
bool is_zip;
};
struct MmapTensorStore {
std::shared_ptr<MmapWrapper> mmapped;
std::shared_ptr<struct ggml_backend_buffer> mmbuffer;
};
class ModelLoader { class ModelLoader {
protected: protected:
SDVersion version_ = VERSION_COUNT; SDVersion version_ = VERSION_COUNT;
std::vector<std::string> file_paths_; std::vector<std::string> file_paths_;
std::vector<ModelFileData> file_data;
bool model_files_processed = false;
String2TensorStorage tensor_storage_map; String2TensorStorage tensor_storage_map;
void add_tensor_storage(const TensorStorage& tensor_storage); void add_tensor_storage(const TensorStorage& tensor_storage);
@ -222,6 +239,10 @@ public:
std::map<ggml_type, uint32_t> get_vae_wtype_stat(); std::map<ggml_type, uint32_t> get_vae_wtype_stat();
String2TensorStorage& get_tensor_storage_map() { return tensor_storage_map; } String2TensorStorage& get_tensor_storage_map() { return tensor_storage_map; }
void set_wtype_override(ggml_type wtype, std::string tensor_type_rules = ""); void set_wtype_override(ggml_type wtype, std::string tensor_type_rules = "");
void process_model_files(bool enable_mmap = false, bool writable_mmap = true);
std::vector<MmapTensorStore> mmap_tensors(std::map<std::string, ggml_tensor*>& tensors,
std::set<std::string> ignore_tensors = {},
bool writable = true);
bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads = 0, bool use_mmap = false); bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads = 0, bool use_mmap = false);
bool load_tensors(std::map<std::string, ggml_tensor*>& tensors, bool load_tensors(std::map<std::string, ggml_tensor*>& tensors,
std::set<std::string> ignore_tensors = {}, std::set<std::string> ignore_tensors = {},

152
src/stable-diffusion.cpp
View File

@ -74,6 +74,8 @@ const char* sampling_methods_str[] = {
"Res Multistep", "Res Multistep",
"Res 2s", "Res 2s",
"ER-SDE", "ER-SDE",
"Euler CFG++",
"Euler A CFG++",
}; };
/*================================================== Helper Functions ================================================*/ /*================================================== Helper Functions ================================================*/
@ -109,6 +111,7 @@ static float get_cache_reuse_threshold(const sd_cache_params_t& params) {
class StableDiffusionGGML { class StableDiffusionGGML {
public: public:
std::vector<MmapTensorStore> mmap_tensor_store;
ggml_backend_t backend = nullptr; // general backend ggml_backend_t backend = nullptr; // general backend
ggml_backend_t clip_backend = nullptr; ggml_backend_t clip_backend = nullptr;
ggml_backend_t control_net_backend = nullptr; ggml_backend_t control_net_backend = nullptr;
@ -361,6 +364,51 @@ public:
apply_lora_immediately = false; apply_lora_immediately = false;
} }
std::map<std::string, ggml_tensor*> mmap_able_tensors;
bool enable_mmap_tensors = false;
bool main_backend_mmap = false;
bool needs_writable_mmap = false;
if (sd_ctx_params->enable_mmap) {
if (apply_lora_immediately) {
needs_writable_mmap = true;
LOG_WARN("in mode 'immediately', LoRAs will cause extra memory usage with mmap");
}
enable_mmap_tensors = true;
if (offload_params_to_cpu) {
main_backend_mmap = true;
} else {
ggml_backend_dev_t dev = ggml_backend_get_device(backend);
struct ggml_backend_dev_props props;
ggml_backend_dev_get_props(dev, &props);
main_backend_mmap = props.caps.buffer_from_host_ptr;
}
}
// split definition to avoid msvc choking on the extra parameter handling
auto get_param_tensors_p = [&](auto&& model, bool force_cpu, const char* prefix) {
std::map<std::string, ggml_tensor*> temp;
model->get_param_tensors(temp, prefix);
bool do_mmap = enable_mmap_tensors && (main_backend_mmap || force_cpu);
for (const auto& [key, tensor] : temp) {
tensors[key] = tensor;
if (do_mmap) {
mmap_able_tensors[key] = tensor;
}
}
};
auto get_param_tensors = [&](auto&& model, bool force_cpu = false) {
std::map<std::string, ggml_tensor*> temp;
model->get_param_tensors(temp);
bool do_mmap = enable_mmap_tensors && (main_backend_mmap || force_cpu);
for (const auto& [key, tensor] : temp) {
tensors[key] = tensor;
if (do_mmap) {
mmap_able_tensors[key] = tensor;
}
}
};
if (sd_version_is_control(version)) { if (sd_version_is_control(version)) {
// Might need vae encode for control cond // Might need vae encode for control cond
vae_decode_only = false; vae_decode_only = false;
@ -472,8 +520,7 @@ public:
offload_params_to_cpu, offload_params_to_cpu,
tensor_storage_map); tensor_storage_map);
clip_vision->set_max_graph_vram_bytes(max_graph_vram_bytes); clip_vision->set_max_graph_vram_bytes(max_graph_vram_bytes);
clip_vision->alloc_params_buffer(); get_param_tensors(clip_vision);
clip_vision->get_param_tensors(tensors);
} }
} else if (sd_version_is_qwen_image(version)) { } else if (sd_version_is_qwen_image(version)) {
bool enable_vision = false; bool enable_vision = false;
@ -557,12 +604,10 @@ public:
} }
cond_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes); cond_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes);
cond_stage_model->alloc_params_buffer(); get_param_tensors(cond_stage_model, clip_on_cpu);
cond_stage_model->get_param_tensors(tensors);
diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes); diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes);
diffusion_model->alloc_params_buffer(); get_param_tensors(diffusion_model);
diffusion_model->get_param_tensors(tensors);
if (sd_version_is_unet_edit(version)) { if (sd_version_is_unet_edit(version)) {
vae_decode_only = false; vae_decode_only = false;
@ -570,8 +615,7 @@ public:
if (high_noise_diffusion_model) { if (high_noise_diffusion_model) {
high_noise_diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes); high_noise_diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes);
high_noise_diffusion_model->alloc_params_buffer(); get_param_tensors(high_noise_diffusion_model);
high_noise_diffusion_model->get_param_tensors(tensors);
} }
if (sd_ctx_params->keep_vae_on_cpu && !ggml_backend_is_cpu(backend)) { if (sd_ctx_params->keep_vae_on_cpu && !ggml_backend_is_cpu(backend)) {
@ -634,6 +678,8 @@ public:
} }
}; };
bool force_vae_cpu = sd_ctx_params->keep_vae_on_cpu;
if (version == VERSION_CHROMA_RADIANCE || version == VERSION_HIDREAM_O1) { if (version == VERSION_CHROMA_RADIANCE || version == VERSION_HIDREAM_O1) {
LOG_INFO("using FakeVAE"); LOG_INFO("using FakeVAE");
first_stage_model = std::make_shared<FakeVAE>(version, first_stage_model = std::make_shared<FakeVAE>(version,
@ -643,20 +689,17 @@ public:
LOG_INFO("using TAE for encoding / decoding"); LOG_INFO("using TAE for encoding / decoding");
first_stage_model = create_tae(); first_stage_model = create_tae();
first_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes); first_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes);
first_stage_model->alloc_params_buffer(); get_param_tensors_p(first_stage_model, force_vae_cpu, "tae");
first_stage_model->get_param_tensors(tensors, "tae");
} else { } else {
LOG_INFO("using VAE for encoding / decoding"); LOG_INFO("using VAE for encoding / decoding");
first_stage_model = create_vae(); first_stage_model = create_vae();
first_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes); first_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes);
first_stage_model->alloc_params_buffer(); get_param_tensors_p(first_stage_model, force_vae_cpu, "first_stage_model");
first_stage_model->get_param_tensors(tensors, "first_stage_model");
if (use_tae && tae_preview_only) { if (use_tae && tae_preview_only) {
LOG_INFO("using TAE for preview"); LOG_INFO("using TAE for preview");
preview_vae = create_tae(); preview_vae = create_tae();
preview_vae->set_max_graph_vram_bytes(max_graph_vram_bytes); preview_vae->set_max_graph_vram_bytes(max_graph_vram_bytes);
preview_vae->alloc_params_buffer(); get_param_tensors_p(first_stage_model, force_vae_cpu, "vae");
preview_vae->get_param_tensors(tensors, "tae");
} }
} }
@ -721,11 +764,7 @@ public:
} }
} }
if (use_pmid) { if (use_pmid) {
if (!pmid_model->alloc_params_buffer()) { get_param_tensors_p(pmid_model, false, "pmid");
LOG_ERROR(" pmid model params buffer allocation failed");
return false;
}
pmid_model->get_param_tensors(tensors, "pmid");
} }
if (sd_ctx_params->flash_attn) { if (sd_ctx_params->flash_attn) {
@ -809,6 +848,41 @@ public:
ignore_tensors.insert("lm_head."); ignore_tensors.insert("lm_head.");
ignore_tensors.insert("model.visual.deepstack_merger_list."); ignore_tensors.insert("model.visual.deepstack_merger_list.");
} }
if (enable_mmap_tensors) {
if (mmap_able_tensors.empty()) {
LOG_DEBUG("no tensors could be memory-mapped");
} else {
mmap_tensor_store = model_loader.mmap_tensors(mmap_able_tensors, ignore_tensors, needs_writable_mmap);
}
}
if (clip_vision) {
clip_vision->alloc_params_buffer();
}
if (cond_stage_model) {
cond_stage_model->alloc_params_buffer();
}
if (diffusion_model) {
diffusion_model->alloc_params_buffer();
}
if (high_noise_diffusion_model) {
high_noise_diffusion_model->alloc_params_buffer();
}
if (first_stage_model) {
first_stage_model->alloc_params_buffer();
}
if (preview_vae) {
preview_vae->alloc_params_buffer();
}
if (use_pmid && pmid_model) {
if (!pmid_model->alloc_params_buffer()) {
LOG_ERROR(" pmid model params buffer allocation failed");
ggml_free(ctx);
return false;
}
}
bool success = model_loader.load_tensors(tensors, ignore_tensors, n_threads, sd_ctx_params->enable_mmap); bool success = model_loader.load_tensors(tensors, ignore_tensors, n_threads, sd_ctx_params->enable_mmap);
if (!success) { if (!success) {
LOG_ERROR("load tensors from model loader failed"); LOG_ERROR("load tensors from model loader failed");
@ -1618,8 +1692,17 @@ public:
cache_params, cache_params,
denoiser.get(), denoiser.get(),
sigmas); sigmas);
size_t steps = sigmas.size() - 1;
bool has_skiplayer = slg_scale != 0.0f && !skip_layers.empty(); // Spectrum cache is not supported for CFG++ samplers
if (method == EULER_CFG_PP_SAMPLE_METHOD || method == EULER_A_CFG_PP_SAMPLE_METHOD) {
if (cache_runtime.spectrum_enabled) {
LOG_WARN("Spectrum cache requested but not supported for CFG++ samplers");
cache_runtime.spectrum_enabled = false;
}
}
size_t steps = sigmas.size() - 1;
bool has_skiplayer = slg_scale != 0.0f && !skip_layers.empty();
if (has_skiplayer && !sd_version_is_dit(version)) { if (has_skiplayer && !sd_version_is_dit(version)) {
has_skiplayer = false; has_skiplayer = false;
LOG_WARN("SLG is incompatible with this model type"); LOG_WARN("SLG is incompatible with this model type");
@ -1636,7 +1719,7 @@ public:
sd::Tensor<float> denoised = x_t; sd::Tensor<float> denoised = x_t;
SamplePreviewContext preview = prepare_sample_preview_context(); SamplePreviewContext preview = prepare_sample_preview_context();
auto denoise = [&](const sd::Tensor<float>& x, float sigma, int step) -> sd::Tensor<float> { auto denoise = [&](const sd::Tensor<float>& x, float sigma, int step, sd::Tensor<float>* out_uncond_denoised = nullptr) -> sd::Tensor<float> {
if (step == 1 || step == -1) { if (step == 1 || step == -1) {
pretty_progress(0, (int)steps, 0); pretty_progress(0, (int)steps, 0);
} }
@ -1659,15 +1742,17 @@ public:
} }
if (cache_runtime.spectrum_enabled && cache_runtime.spectrum.should_predict()) { if (cache_runtime.spectrum_enabled && cache_runtime.spectrum.should_predict()) {
cache_runtime.spectrum.predict(&denoised); if (out_uncond_denoised == nullptr) {
if (!denoise_mask.empty()) { cache_runtime.spectrum.predict(&denoised);
denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask); if (!denoise_mask.empty()) {
denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask);
}
if (sd_should_preview_denoised() && preview.callback != nullptr) {
preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
}
report_sample_progress(step, steps, t0);
return denoised;
} }
if (sd_should_preview_denoised() && preview.callback != nullptr) {
preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
}
report_sample_progress(step, steps, t0);
return denoised;
} }
if (sd_should_preview_noisy() && preview.callback != nullptr) { if (sd_should_preview_noisy() && preview.callback != nullptr) {
@ -1796,6 +1881,10 @@ public:
latent_result += (cond_out - skip_cond_out) * slg_scale; latent_result += (cond_out - skip_cond_out) * slg_scale;
} }
denoised = latent_result * c_out + x * c_skip; denoised = latent_result * c_out + x * c_skip;
if (out_uncond_denoised != nullptr) {
sd::Tensor<float> base_uncond = !uncond_out.empty() ? uncond_out : cond_out;
*out_uncond_denoised = base_uncond * c_out + x * c_skip;
}
if (cache_runtime.spectrum_enabled) { if (cache_runtime.spectrum_enabled) {
cache_runtime.spectrum.update(denoised); cache_runtime.spectrum.update(denoised);
} }
@ -1995,6 +2084,8 @@ const char* sample_method_to_str[] = {
"res_multistep", "res_multistep",
"res_2s", "res_2s",
"er_sde", "er_sde",
"euler_cfg_pp",
"euler_a_cfg_pp",
}; };
const char* sd_sample_method_name(enum sample_method_t sample_method) { const char* sd_sample_method_name(enum sample_method_t sample_method) {
@ -2563,6 +2654,7 @@ static float resolve_eta(sd_ctx_t* sd_ctx,
case EULER_A_SAMPLE_METHOD: case EULER_A_SAMPLE_METHOD:
case DPMPP2S_A_SAMPLE_METHOD: case DPMPP2S_A_SAMPLE_METHOD:
case ER_SDE_SAMPLE_METHOD: case ER_SDE_SAMPLE_METHOD:
case EULER_A_CFG_PP_SAMPLE_METHOD:
return 1.0f; return 1.0f;
default:; default:;
} }

108
src/util.cpp
View File

@ -112,7 +112,7 @@ private:
HANDLE hmapping_; HANDLE hmapping_;
}; };
std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename) { std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename, bool writable) {
void* mapped_data = nullptr; void* mapped_data = nullptr;
size_t file_size = 0; size_t file_size = 0;
@ -137,14 +137,18 @@ std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename) {
file_size = static_cast<size_t>(size.QuadPart); file_size = static_cast<size_t>(size.QuadPart);
HANDLE mapping_handle = CreateFileMapping(file_handle, nullptr, PAGE_READONLY, 0, 0, nullptr); DWORD page_prot = writable ? PAGE_WRITECOPY : PAGE_READONLY;
HANDLE mapping_handle = CreateFileMapping(file_handle, nullptr, page_prot, 0, 0, nullptr);
if (mapping_handle == nullptr) { if (mapping_handle == nullptr) {
CloseHandle(file_handle); CloseHandle(file_handle);
return nullptr; return nullptr;
} }
mapped_data = MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_size); DWORD view_access = writable ? FILE_MAP_COPY : FILE_MAP_READ;
mapped_data = MapViewOfFile(mapping_handle, view_access, 0, 0, file_size);
if (mapped_data == nullptr) { if (mapped_data == nullptr) {
CloseHandle(mapping_handle); CloseHandle(mapping_handle);
@ -172,28 +176,85 @@ bool is_directory(const std::string& path) {
return (stat(path.c_str(), &buffer) == 0 && S_ISDIR(buffer.st_mode)); return (stat(path.c_str(), &buffer) == 0 && S_ISDIR(buffer.st_mode));
} }
class MmapWrapperImpl : public MmapWrapper { struct MmapFlags {
public: bool sequential;
MmapWrapperImpl(void* data, size_t size) bool populate;
: MmapWrapper(data, size) {} bool willneed;
bool dontneed;
~MmapWrapperImpl() override {
munmap(data_, size_);
}
}; };
std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename) { static MmapFlags get_mmap_flags() {
MmapFlags result = {};
const char* SD_MMAP_FLAGS = std::getenv("SD_MMAP_FLAGS");
if (SD_MMAP_FLAGS && *SD_MMAP_FLAGS) {
std::stringstream ss(SD_MMAP_FLAGS);
std::string token;
while (std::getline(ss, token, ',')) {
std::string ntoken = trim(token);
std::transform(ntoken.begin(), ntoken.end(), ntoken.begin(), ::tolower);
if (ntoken == "sequential") {
result.sequential = true;
} else if (ntoken == "populate") {
result.populate = true;
} else if (ntoken == "willneed") {
result.willneed = true;
} else if (ntoken == "dontneed") {
result.dontneed = true;
}
}
}
return result;
}
class MmapWrapperImpl : public MmapWrapper {
public:
MmapWrapperImpl(void* data, size_t size, int fd)
: MmapWrapper(data, size), fd_(fd) {}
~MmapWrapperImpl() override {
#ifdef __linux__
auto cfg_flags = get_mmap_flags();
// Drop the kernel pagecache pages for this file. madvise(DONTNEED)
// alone only unmaps from the process address space; pagecache
// entries persist (`free` reports them as buff/cache and the OOM
// killer doesn't touch them, but they ARE counted against
// overcommit and can starve other allocations on tight-RAM
// systems). posix_fadvise(POSIX_FADV_DONTNEED) is the documented
// way to evict pagecache for a specific fd's pages.
if (cfg_flags.dontneed) {
madvise(data_, size_, MADV_DONTNEED);
posix_fadvise(fd_, 0, 0, POSIX_FADV_DONTNEED);
}
#endif
munmap(data_, size_);
close(fd_);
}
private:
int fd_;
};
std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename, bool writable) {
int file_descriptor = open(filename.c_str(), O_RDONLY); int file_descriptor = open(filename.c_str(), O_RDONLY);
if (file_descriptor == -1) { if (file_descriptor == -1) {
return nullptr; return nullptr;
} }
auto cfg_flags = get_mmap_flags();
int mmap_flags = MAP_PRIVATE; int mmap_flags = MAP_PRIVATE;
#ifdef __linux__ #ifdef __linux__
// performance flags used by llama.cpp // Sequential access hint helps the kernel read-ahead efficiently and
// posix_fadvise(file_descriptor, 0, 0, POSIX_FADV_SEQUENTIAL); // also encourages eviction of already-read pages (the kernel keeps
// mmap_flags |= MAP_POPULATE; // a smaller working set when this is set).
if (cfg_flags.sequential) {
posix_fadvise(file_descriptor, 0, 0, POSIX_FADV_SEQUENTIAL);
}
if (cfg_flags.populate) {
mmap_flags |= MAP_POPULATE;
}
#endif #endif
struct stat sb; struct stat sb;
@ -204,20 +265,27 @@ std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename) {
size_t file_size = sb.st_size; size_t file_size = sb.st_size;
void* mapped_data = mmap(nullptr, file_size, PROT_READ, mmap_flags, file_descriptor, 0); if (file_size == 0) {
close(file_descriptor);
return nullptr;
}
close(file_descriptor); int mmap_prot = PROT_READ | (writable ? PROT_WRITE : 0);
void* mapped_data = mmap(nullptr, file_size, mmap_prot, mmap_flags, file_descriptor, 0);
if (mapped_data == MAP_FAILED) { if (mapped_data == MAP_FAILED) {
close(file_descriptor);
return nullptr; return nullptr;
} }
#ifdef __linux__ #ifdef __linux__
// performance flags used by llama.cpp if (cfg_flags.willneed) {
// posix_madvise(mapped_data, file_size, POSIX_MADV_WILLNEED); posix_madvise(mapped_data, file_size, POSIX_MADV_WILLNEED);
}
#endif #endif
return std::make_unique<MmapWrapperImpl>(mapped_data, file_size); return std::make_unique<MmapWrapperImpl>(mapped_data, file_size, file_descriptor);
} }
#endif #endif

3
src/util.h
View File

@ -42,7 +42,7 @@ sd::Tensor<float> clip_preprocess(const sd::Tensor<float>& image, int target_wid
class MmapWrapper { class MmapWrapper {
public: public:
static std::unique_ptr<MmapWrapper> create(const std::string& filename); static std::unique_ptr<MmapWrapper> create(const std::string& filename, bool writable = false);
virtual ~MmapWrapper() = default; virtual ~MmapWrapper() = default;
@ -52,6 +52,7 @@ public:
MmapWrapper& operator=(MmapWrapper&&) = delete; MmapWrapper& operator=(MmapWrapper&&) = delete;
const uint8_t* data() const { return static_cast<uint8_t*>(data_); } const uint8_t* data() const { return static_cast<uint8_t*>(data_); }
uint8_t* writable_data() { return static_cast<uint8_t*>(data_); }
size_t size() const { return size_; } size_t size() const { return size_; }
bool copy_data(void* buf, size_t n, size_t offset) const; bool copy_data(void* buf, size_t n, size_t offset) const;