DarthAffe/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-08-11 06:36:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into hidream-o1

Browse Source
This commit is contained in:
leejet 2026-05-15 00:37:13 +08:00
commit 237b0258f5
15 changed files with 550 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
CMakeLists.txt
View File

@ -106,19 +106,28 @@ if(SD_WEBP)
"Or link against system library:\n cmake (...) -DSD_USE_SYSTEM_WEBP=ON")
endif()
if(SD_USE_SYSTEM_WEBP)
find_package(WebP REQUIRED)
add_library(webp ALIAS WebP::webp)
# libwebp CMake target naming is not consistent across versions/distros.
# Some export WebP::libwebpmux, others export WebP::webpmux.
if(TARGET WebP::libwebpmux)
add_library(libwebpmux ALIAS WebP::libwebpmux)
elseif(TARGET WebP::webpmux)
add_library(libwebpmux ALIAS WebP::webpmux)
find_package(WebP)
if(WebP_FOUND)
add_library(webp ALIAS WebP::webp)
# libwebp CMake target naming is not consistent across versions/distros.
# Some export WebP::libwebpmux, others export WebP::webpmux.
if(TARGET WebP::libwebpmux)
add_library(libwebpmux ALIAS WebP::libwebpmux)
elseif(TARGET WebP::webpmux)
add_library(libwebpmux ALIAS WebP::webpmux)
else()
message(FATAL_ERROR
"Could not find a compatible webpmux target in system WebP package. "
"Expected WebP::libwebpmux or WebP::webpmux."
)
endif()
else()
message(FATAL_ERROR
"Could not find a compatible webpmux target in system WebP package. "
"Expected WebP::libwebpmux or WebP::webpmux."
)
find_package(PkgConfig REQUIRED)
pkg_check_modules(WebP REQUIRED IMPORTED_TARGET GLOBAL libwebp)
pkg_check_modules(WebPMux REQUIRED IMPORTED_TARGET GLOBAL libwebpmux)
link_libraries(PkgConfig::WebP)
link_libraries(PkgConfig::WebPMux)
add_library(libwebpmux ALIAS PkgConfig::WebPMux)
endif()
endif()
endif()
@ -133,18 +142,26 @@ if(SD_WEBM)
"Or link against system library:\n cmake (...) -DSD_USE_SYSTEM_WEBM=ON")
endif()
if(SD_USE_SYSTEM_WEBM)
find_path(WEBM_INCLUDE_DIR
NAMES mkvmuxer/mkvmuxer.h mkvparser/mkvparser.h common/webmids.h
PATH_SUFFIXES webm
REQUIRED)
find_library(WEBM_LIBRARY
NAMES webm libwebm
REQUIRED)
find_package(PkgConfig)
if(PkgConfig_FOUND)
pkg_check_modules(WebM REQUIRED IMPORTED_TARGET GLOBAL libwebm)
endif()
if(PkgConfig_FOUND AND WebM_FOUND)
link_libraries(PkgConfig::WebM)
else()
find_path(WEBM_INCLUDE_DIR
NAMES mkvmuxer/mkvmuxer.h mkvparser/mkvparser.h common/webmids.h
PATH_SUFFIXES webm
REQUIRED)
find_library(WEBM_LIBRARY
NAMES webm libwebm
REQUIRED)
add_library(webm UNKNOWN IMPORTED)
set_target_properties(webm PROPERTIES
IMPORTED_LOCATION "${WEBM_LIBRARY}"
INTERFACE_INCLUDE_DIRECTORIES "${WEBM_INCLUDE_DIR}")
add_library(webm UNKNOWN IMPORTED)
set_target_properties(webm PROPERTIES
IMPORTED_LOCATION "${WEBM_LIBRARY}"
INTERFACE_INCLUDE_DIRECTORIES "${WEBM_INCLUDE_DIR}")
endif()
endif()
endif()

1
README.md
View File

@ -165,6 +165,7 @@ These projects wrap `stable-diffusion.cpp` for easier use in other languages/fra
* Golang (non-cgo): [seasonjs/stable-diffusion](https://github.com/seasonjs/stable-diffusion)
* Golang (cgo): [Binozo/GoStableDiffusion](https://github.com/Binozo/GoStableDiffusion)
* Golang (non-cgo): [l8bloom/gosd](https://github.com/l8bloom/gosd)
* C#: [DarthAffe/StableDiffusion.NET](https://github.com/DarthAffe/StableDiffusion.NET)
* Python: [william-murray1204/stable-diffusion-cpp-python](https://github.com/william-murray1204/stable-diffusion-cpp-python)
* Rust: [newfla/diffusion-rs](https://github.com/newfla/diffusion-rs)

4
examples/cli/README.md
View File

@ -164,10 +164,10 @@ Generation Options:
-s, --seed RNG seed (default: 42, use random seed for < 0)
--sampling-method sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m,
dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s,
er_sde] (default: euler for Flux/SD3/Wan, euler_a otherwise)
er_sde, euler_cfg_pp, euler_a_cfg_pp] (default: euler for Flux/SD3/Wan, euler_a otherwise)
--high-noise-sampling-method (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a,
dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep,
res_2s, er_sde] default: euler for Flux/SD3/Wan, euler_a otherwise
res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for Flux/SD3/Wan, euler_a otherwise
--scheduler denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits,
smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent], default:
discrete

4
examples/common/common.cpp
View File

@ -1248,12 +1248,12 @@ ArgOptions SDGenerationParams::get_options() {
on_seed_arg},
{"",
"--sampling-method",
"sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde] "
"sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]"
"(default: euler for Flux/SD3/Wan, euler_a otherwise)",
on_sample_method_arg},
{"",
"--high-noise-sampling-method",
"(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde]"
"(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]"
" default: euler for Flux/SD3/Wan, euler_a otherwise",
on_high_noise_sample_method_arg},
{"",

4
examples/server/README.md
View File

@ -266,10 +266,10 @@ Default Generation Options:
-s, --seed RNG seed (default: 42, use random seed for < 0)
--sampling-method sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m,
dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s,
er_sde] (default: euler for Flux/SD3/Wan, euler_a otherwise)
er_sde, euler_cfg_pp, euler_a_cfg_pp] (default: euler for Flux/SD3/Wan, euler_a otherwise)
--high-noise-sampling-method (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a,
dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep,
res_2s, er_sde] default: euler for Flux/SD3/Wan, euler_a otherwise
res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for Flux/SD3/Wan, euler_a otherwise
--scheduler denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits,
smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent], default:
discrete

2
examples/server/main.cpp
View File

@ -145,7 +145,7 @@ int main(int argc, const char** argv) {
register_sdapi_endpoints(svr, runtime);
register_sdcpp_api_endpoints(svr, runtime);
LOG_INFO("listening on: %s:%d\n", svr_params.listen_ip.c_str(), svr_params.listen_port);
LOG_INFO("listening on: http://%s:%d\n", svr_params.listen_ip.c_str(), svr_params.listen_port);
svr.listen(svr_params.listen_ip, svr_params.listen_port);
{

4
examples/server/routes_sdapi.cpp
View File

@ -67,6 +67,10 @@ static enum sample_method_t get_sdapi_sample_method(std::string name) {
{"k_res_multistep", RES_MULTISTEP_SAMPLE_METHOD},
{"res 2s", RES_2S_SAMPLE_METHOD},
{"k_res_2s", RES_2S_SAMPLE_METHOD},
{"euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
{"k_euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
{"euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
{"k_euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
};
auto it = hardcoded.find(name);
return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT;

2
include/stable-diffusion.h
View File

@ -51,6 +51,8 @@ enum sample_method_t {
RES_MULTISTEP_SAMPLE_METHOD,
RES_2S_SAMPLE_METHOD,
ER_SDE_SAMPLE_METHOD,
EULER_CFG_PP_SAMPLE_METHOD,
EULER_A_CFG_PP_SAMPLE_METHOD,
SAMPLE_METHOD_COUNT
};

104
src/denoiser.hpp
View File

@ -753,7 +753,7 @@ struct Flux2FlowDenoiser : public FluxFlowDenoiser {
}
};
typedef std::function<sd::Tensor<float>(const sd::Tensor<float>&, float, int)> denoise_cb_t;
typedef std::function<sd::Tensor<float>(const sd::Tensor<float>&, float, int, sd::Tensor<float>*)> denoise_cb_t;
static std::pair<float, float> get_ancestral_step(float sigma_from,
float sigma_to,
@ -829,7 +829,7 @@ static sd::Tensor<float> sample_euler_ancestral(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
float sigma = sigmas[i];
auto denoised_opt = model(x, sigma, i + 1);
auto denoised_opt = model(x, sigma, i + 1, nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -852,7 +852,7 @@ static sd::Tensor<float> sample_euler_flow(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
float sigma = sigmas[i];
auto denoised_opt = model(x, sigma, i + 1);
auto denoised_opt = model(x, sigma, i + 1, nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -874,7 +874,7 @@ static sd::Tensor<float> sample_euler(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
float sigma = sigmas[i];
auto denoised_opt = model(x, sigma, i + 1);
auto denoised_opt = model(x, sigma, i + 1, nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -890,7 +890,7 @@ static sd::Tensor<float> sample_heun(denoise_cb_t model,
const std::vector<float>& sigmas) {
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], -(i + 1));
auto denoised_opt = model(x, sigmas[i], -(i + 1), nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -901,7 +901,7 @@ static sd::Tensor<float> sample_heun(denoise_cb_t model,
x += d * dt;
} else {
sd::Tensor<float> x2 = x + d * dt;
auto denoised2_opt = model(x2, sigmas[i + 1], i + 1);
auto denoised2_opt = model(x2, sigmas[i + 1], i + 1, nullptr);
if (denoised2_opt.empty()) {
return {};
}
@ -918,7 +918,7 @@ static sd::Tensor<float> sample_dpm2(denoise_cb_t model,
const std::vector<float>& sigmas) {
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], -(i + 1));
auto denoised_opt = model(x, sigmas[i], -(i + 1), nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -931,7 +931,7 @@ static sd::Tensor<float> sample_dpm2(denoise_cb_t model,
float dt_1 = sigma_mid - sigmas[i];
float dt_2 = sigmas[i + 1] - sigmas[i];
sd::Tensor<float> x2 = x + d * dt_1;
auto denoised2_opt = model(x2, sigma_mid, i + 1);
auto denoised2_opt = model(x2, sigma_mid, i + 1, nullptr);
if (denoised2_opt.empty()) {
return {};
}
@ -952,7 +952,7 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], -(i + 1));
auto denoised_opt = model(x, sigmas[i], -(i + 1), nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -968,7 +968,7 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral(denoise_cb_t model,
float s = t + 0.5f * h;
float sigma_s = sigma_fn(s);
sd::Tensor<float> x2 = (sigma_s / sigma_fn(t)) * x - (exp(-h * 0.5f) - 1) * denoised;
auto denoised2_opt = model(x2, sigma_s, i + 1);
auto denoised2_opt = model(x2, sigma_s, i + 1, nullptr);
if (denoised2_opt.empty()) {
return {};
}
@ -995,7 +995,7 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral_flow(denoise_cb_t model,
bool opt_first_step = (1.0 - sigma < 1e-6);
auto denoised_opt = model(x, sigma, (opt_first_step ? 1 : -1) * (i + 1));
auto denoised_opt = model(x, sigma, (opt_first_step ? 1 : -1) * (i + 1), nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -1024,8 +1024,8 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral_flow(denoise_cb_t model,
// so sigma_s = 1 = sigma, and sigma_s_i_ratio = sigma_s / sigma = 1
// u = (x*sigma_s_i_ratio)+(denoised*(1.0f-sigma_s_i_ratio))
// = (x*1)+(denoised*0) = x
// so D_i = model(u, sigma_s, i + 1)
// = model(x, sigma, i + 1)
// so D_i = model(u, sigma_s, i + 1, nullptr)
// = model(x, sigma, i + 1, nullptr)
// = denoised
D_i = denoised;
@ -1058,7 +1058,7 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral_flow(denoise_cb_t model,
float sigma_s_i_ratio = sigma_s / sigma;
sd::Tensor<float> u = (x * sigma_s_i_ratio) + (denoised * (1.0f - sigma_s_i_ratio));
auto denoised2_opt = model(u, sigma_s, i + 1);
auto denoised2_opt = model(u, sigma_s, i + 1, nullptr);
if (denoised2_opt.empty()) {
return {};
}
@ -1085,7 +1085,7 @@ static sd::Tensor<float> sample_dpmpp_2m(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], i + 1);
auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -1117,7 +1117,7 @@ static sd::Tensor<float> sample_dpmpp_2m_v2(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], i + 1);
auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -1225,7 +1225,7 @@ static sd::Tensor<float> sample_lcm(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], i + 1);
auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -1271,7 +1271,7 @@ static sd::Tensor<float> sample_ipndm(denoise_cb_t model,
float sigma = sigmas[i];
float sigma_next = sigmas[i + 1];
auto denoised_opt = model(x, sigma, i + 1);
auto denoised_opt = model(x, sigma, i + 1, nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -1315,7 +1315,7 @@ static sd::Tensor<float> sample_ipndm_v(denoise_cb_t model,
float sigma = sigmas[i];
float t_next = sigmas[i + 1];
auto denoised_opt = model(x, sigma, i + 1);
auto denoised_opt = model(x, sigma, i + 1, nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -1377,7 +1377,7 @@ static sd::Tensor<float> sample_res_multistep(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
auto denoised_opt = model(x, sigmas[i], i + 1);
auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -1454,7 +1454,7 @@ static sd::Tensor<float> sample_res_2s(denoise_cb_t model,
float sigma_from = sigmas[i];
float sigma_to = sigmas[i + 1];
auto denoised_opt = model(x, sigma_from, -(i + 1));
auto denoised_opt = model(x, sigma_from, -(i + 1), nullptr);
if (denoised_opt.empty()) {
return {};
}
@ -1480,7 +1480,7 @@ static sd::Tensor<float> sample_res_2s(denoise_cb_t model,
sd::Tensor<float> eps1 = denoised - x0;
sd::Tensor<float> x2 = x0 + eps1 * (h * a21);
auto denoised2_opt = model(x2, sigma_c2, i + 1);
auto denoised2_opt = model(x2, sigma_c2, i + 1, nullptr);
if (denoised2_opt.empty()) {
return {};
}
@ -1557,7 +1557,7 @@ static sd::Tensor<float> sample_er_sde(denoise_cb_t model,
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
sd::Tensor<float> denoised = model(x, sigmas[i], i + 1);
sd::Tensor<float> denoised = model(x, sigmas[i], i + 1, nullptr);
if (denoised.empty()) {
return {};
}
@ -1643,7 +1643,7 @@ static sd::Tensor<float> sample_ddim_trailing(denoise_cb_t model,
float sigma = sigmas[i];
float sigma_to = sigmas[i + 1];
auto model_output_opt = model(x, sigma, i + 1);
auto model_output_opt = model(x, sigma, i + 1, nullptr);
if (model_output_opt.empty()) {
return {};
}
@ -1715,7 +1715,7 @@ static sd::Tensor<float> sample_tcd(denoise_cb_t model,
int timestep_s = (int)floor((1 - eta) * prev_timestep);
float sigma = sigmas[i];
auto model_output_opt = model(x, sigma, i + 1);
auto model_output_opt = model(x, sigma, i + 1, nullptr);
if (model_output_opt.empty()) {
return {};
}
@ -1743,6 +1743,56 @@ static sd::Tensor<float> sample_tcd(denoise_cb_t model,
return x;
}
static sd::Tensor<float> sample_euler_cfg_pp(denoise_cb_t model,
sd::Tensor<float> x,
const std::vector<float>& sigmas) {
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
float sigma = sigmas[i];
sd::Tensor<float> uncond_denoised;
auto denoised_opt = model(x, sigma, i + 1, &uncond_denoised);
if (denoised_opt.empty() || uncond_denoised.empty()) {
return {};
}
sd::Tensor<float> denoised = std::move(denoised_opt);
sd::Tensor<float> d = (x - uncond_denoised) / sigma;
x = denoised + d * sigmas[i + 1];
}
return x;
}
static sd::Tensor<float> sample_euler_ancestral_cfg_pp(denoise_cb_t model,
sd::Tensor<float> x,
const std::vector<float>& sigmas,
std::shared_ptr<RNG> rng,
float eta) {
int steps = static_cast<int>(sigmas.size()) - 1;
for (int i = 0; i < steps; i++) {
float sigma = sigmas[i];
sd::Tensor<float> uncond_denoised;
auto denoised_opt = model(x, sigma, i + 1, &uncond_denoised);
if (denoised_opt.empty() || uncond_denoised.empty()) {
return {};
}
sd::Tensor<float> denoised = std::move(denoised_opt);
sd::Tensor<float> d = (x - uncond_denoised) / sigma;
auto [sigma_down, sigma_up] = get_ancestral_step(sigmas[i], sigmas[i + 1], eta);
x = denoised + d * sigma_down;
if (sigmas[i + 1] > 0) {
x += sd::Tensor<float>::randn_like(x, rng) * sigma_up;
}
}
return x;
}
// k diffusion reverse ODE: dx = (x - D(x;\sigma)) / \sigma dt; \sigma(t) = t
static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
denoise_cb_t model,
@ -1789,6 +1839,10 @@ static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
return sample_ddim_trailing(model, std::move(x), sigmas, rng, eta);
case TCD_SAMPLE_METHOD:
return sample_tcd(model, std::move(x), sigmas, rng, eta);
case EULER_CFG_PP_SAMPLE_METHOD:
return sample_euler_cfg_pp(model, std::move(x), sigmas);
case EULER_A_CFG_PP_SAMPLE_METHOD:
return sample_euler_ancestral_cfg_pp(model, std::move(x), sigmas, rng, eta);
default:
return {};
}

19
src/ggml_extend.hpp
View File

@ -2716,7 +2716,24 @@ public:
bool alloc_params_buffer() {
size_t num_tensors = ggml_tensor_num(params_ctx);
params_buffer = ggml_backend_alloc_ctx_tensors(params_ctx, params_backend);
if (num_tensors > 0) {
// ggml_backend_alloc_ctx_tensors fails when all tensors are already allocated
// (typical for memory-mapped weights). See ggml-alloc.c n_buffers==0 branch.
bool all_have_data = true;
for (ggml_tensor* t = ggml_get_first_tensor(params_ctx); t != nullptr; t = ggml_get_next_tensor(params_ctx, t)) {
if (t->data == nullptr) {
all_have_data = false;
break;
}
}
if (all_have_data) {
LOG_DEBUG("%s all params already mmap-allocated (no separate buffer needed)", get_desc().c_str());
params_buffer = nullptr;
rebuild_params_tensor_set();
return true;
}
}
params_buffer = ggml_backend_alloc_ctx_tensors(params_ctx, params_backend);
if (params_buffer == nullptr) {
LOG_ERROR("%s alloc params backend buffer failed, num_tensors = %i",
get_desc().c_str(),

195
src/model.cpp
View File

@ -734,16 +734,10 @@ void ModelLoader::set_wtype_override(ggml_type wtype, std::string tensor_type_ru
}
}
bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads_p, bool enable_mmap) {
int64_t process_time_ms = 0;
std::atomic<int64_t> read_time_ms(0);
std::atomic<int64_t> memcpy_time_ms(0);
std::atomic<int64_t> copy_to_backend_time_ms(0);
std::atomic<int64_t> convert_time_ms(0);
std::atomic<uint64_t> bytes_processed(0);
int num_threads_to_use = n_threads_p > 0 ? n_threads_p : sd_get_num_physical_cores();
LOG_DEBUG("using %d threads for model loading", num_threads_to_use);
void ModelLoader::process_model_files(bool enable_mmap, bool writable_mmap) {
if (model_files_processed) {
return;
}
int64_t start_time = ggml_time_ms();
@ -755,22 +749,13 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
processed_tensor_storages.push_back(tensor_storage);
}
process_time_ms = ggml_time_ms() - start_time;
bool success = true;
size_t total_tensors_processed = 0;
const size_t total_tensors_to_process = processed_tensor_storages.size();
const int64_t t_start = ggml_time_ms();
int last_n_threads = 1;
for (size_t file_index = 0; file_index < file_paths_.size(); file_index++) {
std::string file_path = file_paths_[file_index];
LOG_DEBUG("loading tensors from %s", file_path.c_str());
std::vector<const TensorStorage*> file_tensors;
std::vector<TensorStorage> file_tensors;
for (const auto& ts : processed_tensor_storages) {
if (ts.file_index == file_index) {
file_tensors.push_back(&ts);
file_tensors.push_back(ts);
}
}
if (file_tensors.empty()) {
@ -779,21 +764,169 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
bool is_zip = false;
for (auto const& ts : file_tensors) {
if (ts->index_in_zip >= 0) {
if (ts.index_in_zip >= 0) {
is_zip = true;
break;
}
}
std::unique_ptr<MmapWrapper> mmapped;
ModelFileData fdata = {};
fdata.path = file_path;
fdata.is_zip = is_zip;
fdata.tensors = std::move(file_tensors);
if (enable_mmap && !is_zip) {
LOG_DEBUG("using mmap for I/O");
mmapped = MmapWrapper::create(file_path);
if (!mmapped) {
LOG_WARN("failed to memory-map '%s'", file_path.c_str());
std::unique_ptr<MmapWrapper> mmapped = MmapWrapper::create(file_path, writable_mmap);
if (mmapped) {
uint8_t* mmap_data = static_cast<uint8_t*>(mmapped->writable_data());
ggml_backend_buffer_t buf_mmap = ggml_backend_cpu_buffer_from_ptr(mmap_data, mmapped->size());
if (buf_mmap) {
LOG_INFO("using mmap for '%s'", file_path.c_str());
fdata.mmbuffer = std::shared_ptr<struct ggml_backend_buffer>(buf_mmap, ggml_backend_buffer_free);
} else {
LOG_WARN("mmap: failed to create backend buffer for file %s", fdata.path.c_str());
}
fdata.mmapped = std::shared_ptr<MmapWrapper>(std::move(mmapped));
} else {
LOG_WARN("failed to memory-map '%s' (falling back to read())", file_path.c_str());
}
} else if (!is_zip) {
LOG_INFO("NOT using mmap for '%s' (mmap disabled by caller)",
file_path.c_str());
}
file_data.push_back(std::move(fdata));
}
model_files_processed = true;
int64_t end_time = ggml_time_ms();
int64_t process_time_ms = end_time - start_time;
LOG_INFO("model files processing completed in %.2fs", process_time_ms / 1000.f);
}
std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggml_tensor*>& tensors,
std::set<std::string> ignore_tensors,
bool writable_mmap) {
process_model_files(true, writable_mmap);
std::vector<MmapTensorStore> result;
uint64_t mapped_bytes = 0;
size_t mapped_tensors = 0;
LOG_DEBUG("memory-mapping tensors...");
int64_t t_start = ggml_time_ms();
for (auto& fdata : file_data) {
if (!fdata.mmbuffer)
continue;
const std::vector<TensorStorage>& file_tensors = fdata.tensors;
size_t file_mapped_bytes = 0;
size_t file_mapped_tensors = 0;
for (const auto& tensor_storage : file_tensors) {
const std::string& name = tensor_storage.name;
bool is_ignored = false;
for (const auto& ignore_prefix : ignore_tensors) {
if (starts_with(name, ignore_prefix)) {
is_ignored = true;
break;
}
}
if (is_ignored)
continue;
auto it = tensors.find(name);
if (it == tensors.end())
continue;
ggml_tensor* dst_tensor = it->second;
if (dst_tensor == nullptr)
continue;
if (tensor_storage.type != dst_tensor->type)
continue;
size_t tensor_size = tensor_storage.nbytes();
size_t tensor_offset = tensor_storage.offset;
if (tensor_storage.ne[0] != dst_tensor->ne[0] ||
tensor_storage.ne[1] != dst_tensor->ne[1] ||
tensor_storage.ne[2] != dst_tensor->ne[2] ||
tensor_storage.ne[3] != dst_tensor->ne[3] ||
tensor_size != ggml_nbytes(dst_tensor)) {
// let load_tensors worry about this
continue;
}
ggml_backend_buffer_t buf_mmap = fdata.mmbuffer.get();
uint8_t* mmap_data = static_cast<uint8_t*>(ggml_backend_buffer_get_base(buf_mmap));
dst_tensor->buffer = buf_mmap;
dst_tensor->data = mmap_data + tensor_offset;
file_mapped_bytes += tensor_size;
file_mapped_tensors++;
}
if (file_mapped_bytes > 0) {
mapped_tensors += file_mapped_tensors;
mapped_bytes += file_mapped_bytes;
result.push_back({fdata.mmapped, fdata.mmbuffer});
}
}
int64_t t_end = ggml_time_ms();
int64_t duration_ms = t_end - t_start;
LOG_INFO("memory-mapped %zu tensors in %zu files (%.2f MB), taking %.2fs",
mapped_tensors,
result.size(),
mapped_bytes / (1024.0 * 1024.0),
duration_ms / 1000.0);
return result;
}
bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads_p, bool enable_mmap) {
process_model_files(enable_mmap, false);
std::atomic<int64_t> read_time_ms(0);
std::atomic<int64_t> memcpy_time_ms(0);
std::atomic<int64_t> copy_to_backend_time_ms(0);
std::atomic<int64_t> convert_time_ms(0);
std::atomic<uint64_t> bytes_processed(0);
int num_threads_to_use = n_threads_p > 0 ? n_threads_p : sd_get_num_physical_cores();
LOG_DEBUG("using %d threads for model loading", num_threads_to_use);
int64_t start_time = ggml_time_ms();
size_t total_tensors_to_process = 0;
for (const auto& fdata : file_data) {
total_tensors_to_process += fdata.tensors.size();
}
bool success = true;
size_t total_tensors_processed = 0;
const int64_t t_start = start_time;
int last_n_threads = 1;
for (auto& fdata : file_data) {
const std::string& file_path = fdata.path;
LOG_DEBUG("loading tensors from %s", file_path.c_str());
const std::vector<TensorStorage>& file_tensors = fdata.tensors;
bool is_zip = fdata.is_zip;
std::shared_ptr<MmapWrapper> mmapped = fdata.mmapped;
int n_threads = is_zip ? 1 : std::min(num_threads_to_use, (int)file_tensors.size());
if (n_threads < 1) {
n_threads = 1;
@ -834,7 +967,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
break;
}
const TensorStorage& tensor_storage = *file_tensors[idx];
const TensorStorage& tensor_storage = file_tensors[idx];
ggml_tensor* dst_tensor = nullptr;
t0 = ggml_time_ms();
@ -851,6 +984,11 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
continue;
}
// skip mmapped tensors
if (dst_tensor->buffer != nullptr && dst_tensor->buffer == fdata.mmbuffer.get()) {
continue;
}
size_t nbytes_to_read = tensor_storage.nbytes_to_read();
auto read_data = [&](char* buf, size_t n) {
@ -994,9 +1132,8 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
}
int64_t end_time = ggml_time_ms();
LOG_INFO("loading tensors completed, taking %.2fs (process: %.2fs, read: %.2fs, memcpy: %.2fs, convert: %.2fs, copy_to_backend: %.2fs)",
LOG_INFO("loading tensors completed, taking %.2fs (read: %.2fs, memcpy: %.2fs, convert: %.2fs, copy_to_backend: %.2fs)",
(end_time - start_time) / 1000.f,
process_time_ms / 1000.f,
(read_time_ms.load() / (float)last_n_threads) / 1000.f,
(memcpy_time_ms.load() / (float)last_n_threads) / 1000.f,
(convert_time_ms.load() / (float)last_n_threads) / 1000.f,

21
src/model.h
View File

@ -195,10 +195,27 @@ using TensorTypeRules = std::vector<std::pair<std::string, ggml_type>>;
TensorTypeRules parse_tensor_type_rules(const std::string& tensor_type_rules);
class MmapWrapper;
struct ModelFileData {
std::string path;
std::vector<TensorStorage> tensors;
std::shared_ptr<MmapWrapper> mmapped;
std::shared_ptr<struct ggml_backend_buffer> mmbuffer;
bool is_zip;
};
struct MmapTensorStore {
std::shared_ptr<MmapWrapper> mmapped;
std::shared_ptr<struct ggml_backend_buffer> mmbuffer;
};
class ModelLoader {
protected:
SDVersion version_ = VERSION_COUNT;
std::vector<std::string> file_paths_;
std::vector<ModelFileData> file_data;
bool model_files_processed = false;
String2TensorStorage tensor_storage_map;
void add_tensor_storage(const TensorStorage& tensor_storage);
@ -222,6 +239,10 @@ public:
std::map<ggml_type, uint32_t> get_vae_wtype_stat();
String2TensorStorage& get_tensor_storage_map() { return tensor_storage_map; }
void set_wtype_override(ggml_type wtype, std::string tensor_type_rules = "");
void process_model_files(bool enable_mmap = false, bool writable_mmap = true);
std::vector<MmapTensorStore> mmap_tensors(std::map<std::string, ggml_tensor*>& tensors,
std::set<std::string> ignore_tensors = {},
bool writable = true);
bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads = 0, bool use_mmap = false);
bool load_tensors(std::map<std::string, ggml_tensor*>& tensors,
std::set<std::string> ignore_tensors = {},

152
src/stable-diffusion.cpp
View File

@ -74,6 +74,8 @@ const char* sampling_methods_str[] = {
"Res Multistep",
"Res 2s",
"ER-SDE",
"Euler CFG++",
"Euler A CFG++",
};
/*================================================== Helper Functions ================================================*/
@ -109,6 +111,7 @@ static float get_cache_reuse_threshold(const sd_cache_params_t& params) {
class StableDiffusionGGML {
public:
std::vector<MmapTensorStore> mmap_tensor_store;
ggml_backend_t backend = nullptr; // general backend
ggml_backend_t clip_backend = nullptr;
ggml_backend_t control_net_backend = nullptr;
@ -361,6 +364,51 @@ public:
apply_lora_immediately = false;
}
std::map<std::string, ggml_tensor*> mmap_able_tensors;
bool enable_mmap_tensors = false;
bool main_backend_mmap = false;
bool needs_writable_mmap = false;
if (sd_ctx_params->enable_mmap) {
if (apply_lora_immediately) {
needs_writable_mmap = true;
LOG_WARN("in mode 'immediately', LoRAs will cause extra memory usage with mmap");
}
enable_mmap_tensors = true;
if (offload_params_to_cpu) {
main_backend_mmap = true;
} else {
ggml_backend_dev_t dev = ggml_backend_get_device(backend);
struct ggml_backend_dev_props props;
ggml_backend_dev_get_props(dev, &props);
main_backend_mmap = props.caps.buffer_from_host_ptr;
}
}
// split definition to avoid msvc choking on the extra parameter handling
auto get_param_tensors_p = [&](auto&& model, bool force_cpu, const char* prefix) {
std::map<std::string, ggml_tensor*> temp;
model->get_param_tensors(temp, prefix);
bool do_mmap = enable_mmap_tensors && (main_backend_mmap || force_cpu);
for (const auto& [key, tensor] : temp) {
tensors[key] = tensor;
if (do_mmap) {
mmap_able_tensors[key] = tensor;
}
}
};
auto get_param_tensors = [&](auto&& model, bool force_cpu = false) {
std::map<std::string, ggml_tensor*> temp;
model->get_param_tensors(temp);
bool do_mmap = enable_mmap_tensors && (main_backend_mmap || force_cpu);
for (const auto& [key, tensor] : temp) {
tensors[key] = tensor;
if (do_mmap) {
mmap_able_tensors[key] = tensor;
}
}
};
if (sd_version_is_control(version)) {
// Might need vae encode for control cond
vae_decode_only = false;
@ -472,8 +520,7 @@ public:
offload_params_to_cpu,
tensor_storage_map);
clip_vision->set_max_graph_vram_bytes(max_graph_vram_bytes);
clip_vision->alloc_params_buffer();
clip_vision->get_param_tensors(tensors);
get_param_tensors(clip_vision);
}
} else if (sd_version_is_qwen_image(version)) {
bool enable_vision = false;
@ -557,12 +604,10 @@ public:
}
cond_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes);
cond_stage_model->alloc_params_buffer();
cond_stage_model->get_param_tensors(tensors);
get_param_tensors(cond_stage_model, clip_on_cpu);
diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes);
diffusion_model->alloc_params_buffer();
diffusion_model->get_param_tensors(tensors);
get_param_tensors(diffusion_model);
if (sd_version_is_unet_edit(version)) {
vae_decode_only = false;
@ -570,8 +615,7 @@ public:
if (high_noise_diffusion_model) {
high_noise_diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes);
high_noise_diffusion_model->alloc_params_buffer();
high_noise_diffusion_model->get_param_tensors(tensors);
get_param_tensors(high_noise_diffusion_model);
}
if (sd_ctx_params->keep_vae_on_cpu && !ggml_backend_is_cpu(backend)) {
@ -634,6 +678,8 @@ public:
}
};
bool force_vae_cpu = sd_ctx_params->keep_vae_on_cpu;
if (version == VERSION_CHROMA_RADIANCE || version == VERSION_HIDREAM_O1) {
LOG_INFO("using FakeVAE");
first_stage_model = std::make_shared<FakeVAE>(version,
@ -643,20 +689,17 @@ public:
LOG_INFO("using TAE for encoding / decoding");
first_stage_model = create_tae();
first_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes);
first_stage_model->alloc_params_buffer();
first_stage_model->get_param_tensors(tensors, "tae");
get_param_tensors_p(first_stage_model, force_vae_cpu, "tae");
} else {
LOG_INFO("using VAE for encoding / decoding");
first_stage_model = create_vae();
first_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes);
first_stage_model->alloc_params_buffer();
first_stage_model->get_param_tensors(tensors, "first_stage_model");
get_param_tensors_p(first_stage_model, force_vae_cpu, "first_stage_model");
if (use_tae && tae_preview_only) {
LOG_INFO("using TAE for preview");
preview_vae = create_tae();
preview_vae->set_max_graph_vram_bytes(max_graph_vram_bytes);
preview_vae->alloc_params_buffer();
preview_vae->get_param_tensors(tensors, "tae");
get_param_tensors_p(first_stage_model, force_vae_cpu, "vae");
}
}
@ -721,11 +764,7 @@ public:
}
}
if (use_pmid) {
if (!pmid_model->alloc_params_buffer()) {
LOG_ERROR(" pmid model params buffer allocation failed");
return false;
}
pmid_model->get_param_tensors(tensors, "pmid");
get_param_tensors_p(pmid_model, false, "pmid");
}
if (sd_ctx_params->flash_attn) {
@ -809,6 +848,41 @@ public:
ignore_tensors.insert("lm_head.");
ignore_tensors.insert("model.visual.deepstack_merger_list.");
}
if (enable_mmap_tensors) {
if (mmap_able_tensors.empty()) {
LOG_DEBUG("no tensors could be memory-mapped");
} else {
mmap_tensor_store = model_loader.mmap_tensors(mmap_able_tensors, ignore_tensors, needs_writable_mmap);
}
}
if (clip_vision) {
clip_vision->alloc_params_buffer();
}
if (cond_stage_model) {
cond_stage_model->alloc_params_buffer();
}
if (diffusion_model) {
diffusion_model->alloc_params_buffer();
}
if (high_noise_diffusion_model) {
high_noise_diffusion_model->alloc_params_buffer();
}
if (first_stage_model) {
first_stage_model->alloc_params_buffer();
}
if (preview_vae) {
preview_vae->alloc_params_buffer();
}
if (use_pmid && pmid_model) {
if (!pmid_model->alloc_params_buffer()) {
LOG_ERROR(" pmid model params buffer allocation failed");
ggml_free(ctx);
return false;
}
}
bool success = model_loader.load_tensors(tensors, ignore_tensors, n_threads, sd_ctx_params->enable_mmap);
if (!success) {
LOG_ERROR("load tensors from model loader failed");
@ -1618,8 +1692,17 @@ public:
cache_params,
denoiser.get(),
sigmas);
size_t steps = sigmas.size() - 1;
bool has_skiplayer = slg_scale != 0.0f && !skip_layers.empty();
// Spectrum cache is not supported for CFG++ samplers
if (method == EULER_CFG_PP_SAMPLE_METHOD || method == EULER_A_CFG_PP_SAMPLE_METHOD) {
if (cache_runtime.spectrum_enabled) {
LOG_WARN("Spectrum cache requested but not supported for CFG++ samplers");
cache_runtime.spectrum_enabled = false;
}
}
size_t steps = sigmas.size() - 1;
bool has_skiplayer = slg_scale != 0.0f && !skip_layers.empty();
if (has_skiplayer && !sd_version_is_dit(version)) {
has_skiplayer = false;
LOG_WARN("SLG is incompatible with this model type");
@ -1636,7 +1719,7 @@ public:
sd::Tensor<float> denoised = x_t;
SamplePreviewContext preview = prepare_sample_preview_context();
auto denoise = [&](const sd::Tensor<float>& x, float sigma, int step) -> sd::Tensor<float> {
auto denoise = [&](const sd::Tensor<float>& x, float sigma, int step, sd::Tensor<float>* out_uncond_denoised = nullptr) -> sd::Tensor<float> {
if (step == 1 || step == -1) {
pretty_progress(0, (int)steps, 0);
}
@ -1659,15 +1742,17 @@ public:
}
if (cache_runtime.spectrum_enabled && cache_runtime.spectrum.should_predict()) {
cache_runtime.spectrum.predict(&denoised);
if (!denoise_mask.empty()) {
denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask);
if (out_uncond_denoised == nullptr) {
cache_runtime.spectrum.predict(&denoised);
if (!denoise_mask.empty()) {
denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask);
}
if (sd_should_preview_denoised() && preview.callback != nullptr) {
preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
}
report_sample_progress(step, steps, t0);
return denoised;
}
if (sd_should_preview_denoised() && preview.callback != nullptr) {
preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
}
report_sample_progress(step, steps, t0);
return denoised;
}
if (sd_should_preview_noisy() && preview.callback != nullptr) {
@ -1796,6 +1881,10 @@ public:
latent_result += (cond_out - skip_cond_out) * slg_scale;
}
denoised = latent_result * c_out + x * c_skip;
if (out_uncond_denoised != nullptr) {
sd::Tensor<float> base_uncond = !uncond_out.empty() ? uncond_out : cond_out;
*out_uncond_denoised = base_uncond * c_out + x * c_skip;
}
if (cache_runtime.spectrum_enabled) {
cache_runtime.spectrum.update(denoised);
}
@ -1995,6 +2084,8 @@ const char* sample_method_to_str[] = {
"res_multistep",
"res_2s",
"er_sde",
"euler_cfg_pp",
"euler_a_cfg_pp",
};
const char* sd_sample_method_name(enum sample_method_t sample_method) {
@ -2563,6 +2654,7 @@ static float resolve_eta(sd_ctx_t* sd_ctx,
case EULER_A_SAMPLE_METHOD:
case DPMPP2S_A_SAMPLE_METHOD:
case ER_SDE_SAMPLE_METHOD:
case EULER_A_CFG_PP_SAMPLE_METHOD:
return 1.0f;
default:;
}

108
src/util.cpp
View File

@ -112,7 +112,7 @@ private:
HANDLE hmapping_;
};
std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename) {
std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename, bool writable) {
void* mapped_data = nullptr;
size_t file_size = 0;
@ -137,14 +137,18 @@ std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename) {
file_size = static_cast<size_t>(size.QuadPart);
HANDLE mapping_handle = CreateFileMapping(file_handle, nullptr, PAGE_READONLY, 0, 0, nullptr);
DWORD page_prot = writable ? PAGE_WRITECOPY : PAGE_READONLY;
HANDLE mapping_handle = CreateFileMapping(file_handle, nullptr, page_prot, 0, 0, nullptr);
if (mapping_handle == nullptr) {
CloseHandle(file_handle);
return nullptr;
}
mapped_data = MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_size);
DWORD view_access = writable ? FILE_MAP_COPY : FILE_MAP_READ;
mapped_data = MapViewOfFile(mapping_handle, view_access, 0, 0, file_size);
if (mapped_data == nullptr) {
CloseHandle(mapping_handle);
@ -172,28 +176,85 @@ bool is_directory(const std::string& path) {
return (stat(path.c_str(), &buffer) == 0 && S_ISDIR(buffer.st_mode));
}
class MmapWrapperImpl : public MmapWrapper {
public:
MmapWrapperImpl(void* data, size_t size)
: MmapWrapper(data, size) {}
~MmapWrapperImpl() override {
munmap(data_, size_);
}
struct MmapFlags {
bool sequential;
bool populate;
bool willneed;
bool dontneed;
};
std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename) {
static MmapFlags get_mmap_flags() {
MmapFlags result = {};
const char* SD_MMAP_FLAGS = std::getenv("SD_MMAP_FLAGS");
if (SD_MMAP_FLAGS && *SD_MMAP_FLAGS) {
std::stringstream ss(SD_MMAP_FLAGS);
std::string token;
while (std::getline(ss, token, ',')) {
std::string ntoken = trim(token);
std::transform(ntoken.begin(), ntoken.end(), ntoken.begin(), ::tolower);
if (ntoken == "sequential") {
result.sequential = true;
} else if (ntoken == "populate") {
result.populate = true;
} else if (ntoken == "willneed") {
result.willneed = true;
} else if (ntoken == "dontneed") {
result.dontneed = true;
}
}
}
return result;
}
class MmapWrapperImpl : public MmapWrapper {
public:
MmapWrapperImpl(void* data, size_t size, int fd)
: MmapWrapper(data, size), fd_(fd) {}
~MmapWrapperImpl() override {
#ifdef __linux__
auto cfg_flags = get_mmap_flags();
// Drop the kernel pagecache pages for this file. madvise(DONTNEED)
// alone only unmaps from the process address space; pagecache
// entries persist (`free` reports them as buff/cache and the OOM
// killer doesn't touch them, but they ARE counted against
// overcommit and can starve other allocations on tight-RAM
// systems). posix_fadvise(POSIX_FADV_DONTNEED) is the documented
// way to evict pagecache for a specific fd's pages.
if (cfg_flags.dontneed) {
madvise(data_, size_, MADV_DONTNEED);
posix_fadvise(fd_, 0, 0, POSIX_FADV_DONTNEED);
}
#endif
munmap(data_, size_);
close(fd_);
}
private:
int fd_;
};
std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename, bool writable) {
int file_descriptor = open(filename.c_str(), O_RDONLY);
if (file_descriptor == -1) {
return nullptr;
}
auto cfg_flags = get_mmap_flags();
int mmap_flags = MAP_PRIVATE;
#ifdef __linux__
// performance flags used by llama.cpp
// posix_fadvise(file_descriptor, 0, 0, POSIX_FADV_SEQUENTIAL);
// mmap_flags |= MAP_POPULATE;
// Sequential access hint helps the kernel read-ahead efficiently and
// also encourages eviction of already-read pages (the kernel keeps
// a smaller working set when this is set).
if (cfg_flags.sequential) {
posix_fadvise(file_descriptor, 0, 0, POSIX_FADV_SEQUENTIAL);
}
if (cfg_flags.populate) {
mmap_flags |= MAP_POPULATE;
}
#endif
struct stat sb;
@ -204,20 +265,27 @@ std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename) {
size_t file_size = sb.st_size;
void* mapped_data = mmap(nullptr, file_size, PROT_READ, mmap_flags, file_descriptor, 0);
if (file_size == 0) {
close(file_descriptor);
return nullptr;
}
close(file_descriptor);
int mmap_prot = PROT_READ | (writable ? PROT_WRITE : 0);
void* mapped_data = mmap(nullptr, file_size, mmap_prot, mmap_flags, file_descriptor, 0);
if (mapped_data == MAP_FAILED) {
close(file_descriptor);
return nullptr;
}
#ifdef __linux__
// performance flags used by llama.cpp
// posix_madvise(mapped_data, file_size, POSIX_MADV_WILLNEED);
if (cfg_flags.willneed) {
posix_madvise(mapped_data, file_size, POSIX_MADV_WILLNEED);
}
#endif
return std::make_unique<MmapWrapperImpl>(mapped_data, file_size);
return std::make_unique<MmapWrapperImpl>(mapped_data, file_size, file_descriptor);
}
#endif

3
src/util.h
View File

@ -42,7 +42,7 @@ sd::Tensor<float> clip_preprocess(const sd::Tensor<float>& image, int target_wid
class MmapWrapper {
public:
static std::unique_ptr<MmapWrapper> create(const std::string& filename);
static std::unique_ptr<MmapWrapper> create(const std::string& filename, bool writable = false);
virtual ~MmapWrapper() = default;
@ -52,6 +52,7 @@ public:
MmapWrapper& operator=(MmapWrapper&&) = delete;
const uint8_t* data() const { return static_cast<uint8_t*>(data_); }
uint8_t* writable_data() { return static_cast<uint8_t*>(data_); }
size_t size() const { return size_; }
bool copy_data(void* buf, size_t n, size_t offset) const;