From e6281b631876baf46ddb184630e91681abf91a98 Mon Sep 17 00:00:00 2001 From: leejet Date: Wed, 23 Sep 2026 22:25:42 +0800 Subject: [PATCH] feat: add configurable conditioning cache for all models (#2034) --- docs/caching.md | 10 +++ examples/cli/main.cpp | 1 + examples/common/common.cpp | 10 +++ examples/common/common.h | 3 +- include/stable-diffusion.h | 1 + src/conditioning/conditioner.hpp | 31 -------- src/conditioning/conditioning_cache.h | 107 ++++++++++++++++++++++++++ src/pipeline/diffusion_engine.cpp | 30 +++++++- src/pipeline/diffusion_engine.h | 8 +- src/pipeline/image.cpp | 9 +-- src/pipeline/video.cpp | 10 +-- src/stable-diffusion.cpp | 3 + 12 files changed, 172 insertions(+), 51 deletions(-) create mode 100644 src/conditioning/conditioning_cache.h diff --git a/docs/caching.md b/docs/caching.md index 01f01974..f69cfbec 100644 --- a/docs/caching.md +++ b/docs/caching.md @@ -2,6 +2,16 @@ Caching methods accelerate diffusion inference by reusing intermediate computations when changes between steps are small. +### Conditioning Cache + +Conditioning results are cached per model context using an LRU cache. The default +capacity is **0 (disabled) for `sd-cli`** and **4 entries for `sd-server` and the C +API**. Set `--conditioning-cache-size N` to change the limit; `0` disables caching. +For example, `sd-cli -m model.safetensors -p "a cat" --conditioning-cache-size 4` +enables the cache in the CLI. The C API option is +`sd_ctx_params_t::conditioning_cache_size`, initialized by `sd_ctx_params_init()`. +This cache is independent of the diffusion-step `--cache-mode` options below. + ### Cache Modes | Mode | Target | Description | diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index a2791dda..dc79b8ce 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -644,6 +644,7 @@ int main(int argc, const char* argv[]) { SDCliParams cli_params; SDContextParams ctx_params; + ctx_params.conditioning_cache_size = 0; SDGenerationParams gen_params; sd_set_log_callback(sd_log_cb, (void*)&cli_params); diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 1eadc17c..81345199 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -571,6 +571,10 @@ ArgOptions SDContextParams::get_options() { "number of threads to use during computation (default: -1). " "If threads <= 0, then threads will be set to the number of CPU physical cores", &n_threads}, + {"", + "--conditioning-cache-size", + "maximum number of conditioning results cached per model context (default: " + std::to_string(conditioning_cache_size) + ", 0 disables caching)", + &conditioning_cache_size}, }; options.bool_options = { @@ -822,6 +826,10 @@ bool SDContextParams::resolve(SDMode mode) { } bool SDContextParams::validate(SDMode mode) { + if (conditioning_cache_size < 0) { + LOG_ERROR("error: conditioning-cache-size must be non-negative"); + return false; + } if (mode == CONVERT) { const bool has_convert_input = model_path.length() != 0 || clip_l_path.length() != 0 || @@ -898,6 +906,7 @@ std::string SDContextParams::to_string() const { std::ostringstream oss; oss << "SDContextParams {\n" << " n_threads: " << n_threads << ",\n" + << " conditioning_cache_size: " << conditioning_cache_size << ",\n" << " model_path: \"" << model_path << "\",\n" << " clip_l_path: \"" << clip_l_path << "\",\n" << " clip_g_path: \"" << clip_g_path << "\",\n" @@ -992,6 +1001,7 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) { sd_ctx_params.pulid_weights_path = pulid_weights_path.c_str(); sd_ctx_params.tensor_type_rules = tensor_type_rules.c_str(); sd_ctx_params.n_threads = n_threads; + sd_ctx_params.conditioning_cache_size = conditioning_cache_size; sd_ctx_params.wtype = wtype; sd_ctx_params.rng_type = rng_type; sd_ctx_params.sampler_rng_type = sampler_rng_type; diff --git a/examples/common/common.h b/examples/common/common.h index b46308ac..d0d39b73 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -116,7 +116,8 @@ bool decode_base64_image(const std::string& encoded_input, SDImageOwner& out_image); struct SDContextParams { - int n_threads = -1; + int n_threads = -1; + int conditioning_cache_size = 4; std::string model_path; std::string clip_l_path; std::string clip_g_path; diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 64a91fcb..a2bfdb68 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -247,6 +247,7 @@ typedef struct { float attn_scale; // Override flash-attention K/V scaling; 0 keeps the model default const char* tokenizer; // tokenizer.json path or main=FILE,clip-l=FILE,clip-g=FILE assignments; required for PiD and Lens bool sage_attn; + int conditioning_cache_size; // Maximum cached conditioning entries per context; 0 disables caching (default: 4) } sd_ctx_params_t; typedef struct { diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index 53ea9df8..6b750f93 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -137,7 +137,6 @@ struct ConditionerParams { const std::vector>* ref_images = nullptr; // for qwen image edit const std::vector* minimax_h3_references = nullptr; RefImageParams ref_image_params; - bool allow_cache = false; }; struct Conditioner { @@ -1955,10 +1954,6 @@ struct LLMEmbedder : public Conditioner { std::shared_ptr llm; std::shared_ptr byt5; - bool h3_text_cache_valid = false; - std::string h3_text_cache_text; - SDCondition h3_text_cache; - LLMEmbedder(ggml_backend_t backend, const String2TensorStorage& tensor_storage_map = {}, SDVersion version = VERSION_QWEN_IMAGE, @@ -2303,25 +2298,6 @@ struct LLMEmbedder : public Conditioner { SDCondition get_learned_condition(int n_threads, const ConditionerParams& conditioner_params) override { - const bool h3_text_cacheable = - sd_version_is_minimax_h3(version) && - conditioner_params.allow_cache && - (conditioner_params.minimax_h3_references == nullptr || - conditioner_params.minimax_h3_references->empty()) && - (conditioner_params.ref_images == nullptr || - conditioner_params.ref_images->empty()); - - if (sd_version_is_minimax_h3(version) && !h3_text_cacheable) { - h3_text_cache_valid = false; - } - - if (h3_text_cacheable && - h3_text_cache_valid && - h3_text_cache_text == conditioner_params.text) { - LOG_INFO("H3 conditioning cache hit"); - return h3_text_cache; - } - std::string prompt; std::pair prompt_attn_range; std::vector extra_prompts; @@ -3191,13 +3167,6 @@ struct LLMEmbedder : public Conditioner { result.c_token_types = sd::Tensor({tag_count}, std::move(tags)); } - if (h3_text_cacheable) { - h3_text_cache_text = conditioner_params.text; - h3_text_cache = result; - h3_text_cache_valid = true; - LOG_INFO("H3 conditioning cache stored"); - } - return result; } }; diff --git a/src/conditioning/conditioning_cache.h b/src/conditioning/conditioning_cache.h new file mode 100644 index 00000000..6e6cbcaf --- /dev/null +++ b/src/conditioning/conditioning_cache.h @@ -0,0 +1,107 @@ +#ifndef __SD_CONDITIONING_CONDITIONING_CACHE_H__ +#define __SD_CONDITIONING_CONDITIONING_CACHE_H__ + +#include +#include +#include + +#include "conditioning/conditioner.hpp" + +class ConditioningCache { + struct Entry { + ConditionerParams params; + std::vector> ref_images; + std::vector references; + SDCondition condition; + + Entry(const ConditionerParams& input, const SDCondition& output) + : params(input), condition(output) { + // Request-owned reference pointers must not outlive the request. + if (input.ref_images != nullptr) { + ref_images = *input.ref_images; + params.ref_images = &ref_images; + } + if (input.minimax_h3_references != nullptr) { + references = *input.minimax_h3_references; + params.minimax_h3_references = &references; + } + } + + Entry(const Entry&) = delete; + Entry& operator=(const Entry&) = delete; + }; + + size_t capacity_ = 4; + std::list entries_; + + static bool same_images(const std::vector>& a, + const std::vector>& b) { + return std::equal(a.begin(), a.end(), b.begin(), b.end(), + [](const sd::Tensor& x, const sd::Tensor& y) { + return x.shape() == y.shape() && x.values() == y.values(); + }); + } + + static bool same_params(const ConditionerParams& a, const ConditionerParams& b) { + const auto fields = [](const ConditionerParams& p) { + const auto& r = p.ref_image_params; + return std::tie(p.text, p.clip_skip, p.width, p.height, p.zero_out_masked, + r.pass_to_vlm, r.pass_to_dit, r.ref_index_mode, + r.force_ref_timestep_zero, r.resize_before_vae, r.vae_input_max_pixels, + r.vlm_resize_mode, r.vlm_min_size, r.vlm_max_size, r.resize_vae_to_target); + }; + if (fields(a) != fields(b) || + (a.ref_images == nullptr) != (b.ref_images == nullptr) || + (a.minimax_h3_references == nullptr) != (b.minimax_h3_references == nullptr)) { + return false; + } + if (a.ref_images != nullptr && !same_images(*a.ref_images, *b.ref_images)) { + return false; + } + if (a.minimax_h3_references != nullptr && + !std::equal(a.minimax_h3_references->begin(), a.minimax_h3_references->end(), + b.minimax_h3_references->begin(), b.minimax_h3_references->end(), + [](const MiniMaxH3PresentationItem& x, const MiniMaxH3PresentationItem& y) { + return x.kind == y.kind && x.timestamps == y.timestamps && same_images(x.frames, y.frames); + })) { + return false; + } + return true; + } + +public: + void set_capacity(size_t capacity) { + capacity_ = capacity; + while (entries_.size() > capacity_) { + entries_.pop_back(); + } + } + + void clear() { + entries_.clear(); + } + + SDCondition get(Conditioner& conditioner, int n_threads, const ConditionerParams& params) { + if (capacity_ == 0) { + return conditioner.get_learned_condition(n_threads, params); + } + for (auto it = entries_.begin(); it != entries_.end(); ++it) { + if (same_params(it->params, params)) { + entries_.splice(entries_.begin(), entries_, it); + LOG_INFO("conditioning cache hit"); + return entries_.front().condition; + } + } + auto condition = conditioner.get_learned_condition(n_threads, params); + if (!condition.empty()) { + if (entries_.size() == capacity_) { + entries_.pop_back(); + } + entries_.emplace_front(params, condition); + LOG_VERBOSE("conditioning cache stored (%zu/%zu)", entries_.size(), capacity_); + } + return condition; + } +}; + +#endif // __SD_CONDITIONING_CONDITIONING_CACHE_H__ diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 2b49ede9..78426f08 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -29,6 +29,7 @@ #include "stable-diffusion.h" #include "conditioning/conditioner.hpp" +#include "conditioning/conditioning_cache.h" #include "core/backend_fit.h" #include "extensions/generation_extension.h" #include "model/adapter/ip_adapter.hpp" @@ -135,6 +136,7 @@ static_assert(std::atomic::is_always_lock_free, StableDiffusionGGML::StableDiffusionGGML() : rng(std::make_shared()), + conditioning_cache_(std::make_unique()), denoiser(std::make_shared()) {} StableDiffusionGGML::~StableDiffusionGGML() = default; @@ -204,6 +206,8 @@ void StableDiffusionGGML::end_runners() { } bool StableDiffusionGGML::reset_runners(const RunnerGroups& groups) { + conditioning_cache_->clear(); + conditioning_loras_.clear(); end_runners(); clear_lora_adapters(); runtime_lora_models.clear(); @@ -915,6 +919,11 @@ bool StableDiffusionGGML::init(const sd_ctx_params_t* sd_ctx_params) { return false; } } + if (sd_ctx_params->conditioning_cache_size < 0) { + LOG_ERROR("conditioning_cache_size must be non-negative"); + return false; + } + conditioning_cache_->set_capacity(static_cast(sd_ctx_params->conditioning_cache_size)); auto configuration = std::make_unique(*sd_ctx_params); n_threads = sd_ctx_params->n_threads; tensor_executor = std::make_unique(n_threads > 0 ? n_threads : sd_get_num_physical_cores()); @@ -1762,13 +1771,22 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun extension->collect_loras(all_loras); } - conditioning_cache_allowed_ = all_loras.empty(); - int64_t t0 = ggml_time_ms(); end_runners(); clear_lora_adapters(); - if (!model_manager->prepare_lora_sources(all_loras)) + if (!model_manager->prepare_lora_sources(all_loras)) { + conditioning_cache_->clear(); return false; + } + if (!std::equal(all_loras.begin(), all_loras.end(), + conditioning_loras_.begin(), conditioning_loras_.end(), + [](const ModelManager::LoraSpec& a, const ModelManager::LoraSpec& b) { + return a.file_id == b.file_id && a.file_revision == b.file_revision && + a.multiplier == b.multiplier && a.is_high_noise == b.is_high_noise && + a.tensor_name_prefix_filter == b.tensor_name_prefix_filter; + })) { + conditioning_cache_->clear(); + } runtime_lora_models.erase(std::remove_if(runtime_lora_models.begin(), runtime_lora_models.end(), [&](const RuntimeLora& entry) { return std::none_of(all_loras.begin(), all_loras.end(), [&](const ModelManager::LoraSpec& spec) { return entry.matches(spec); @@ -1778,6 +1796,7 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun const bool success = apply_lora_immediately ? apply_loras_immediately(all_loras) : apply_loras_at_runtime(all_loras); if (!success) { + conditioning_cache_->clear(); clear_lora_adapters(); runtime_lora_models.clear(); return false; @@ -1787,9 +1806,14 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun if (!all_loras.empty()) { LOG_INFO("apply_loras completed, taking %.2fs", (t1 - t0) * 1.0f / 1000); } + conditioning_loras_ = std::move(all_loras); return true; } +SDCondition StableDiffusionGGML::get_learned_condition(const ConditionerParams& params) { + return conditioning_cache_->get(*cond_stage_model, n_threads, params); +} + void StableDiffusionGGML::reset_generation_extensions() { for (auto& extension : generation_extensions) { extension->reset_runtime_condition(); diff --git a/src/pipeline/diffusion_engine.h b/src/pipeline/diffusion_engine.h index 2e38a9e2..d1a8866b 100644 --- a/src/pipeline/diffusion_engine.h +++ b/src/pipeline/diffusion_engine.h @@ -26,6 +26,7 @@ class RNG; struct Denoiser; struct LoraModel; struct ConditionerParams; +class ConditioningCache; struct SDCondition; struct RefImageParams; namespace Wav2Vec2 { @@ -178,8 +179,9 @@ public: std::recursive_mutex execution_mutex; std::unique_ptr config_; RunnerState runner_state_; - bool conditioning_cache_allowed_ = false; - bool executing_ = false; + std::unique_ptr conditioning_cache_; + std::vector conditioning_loras_; + bool executing_ = false; std::shared_ptr denoiser; std::vector file_alphas_cumprod; @@ -362,6 +364,8 @@ public: bool apply_loras(const sd_lora_t* loras, uint32_t lora_count); + SDCondition get_learned_condition(const ConditionerParams& params); + void reset_generation_extensions(); void prepare_generation_extensions(const sd_pm_params_t& pm_params, diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp index 3ef5318b..38b8f086 100644 --- a/src/pipeline/image.cpp +++ b/src/pipeline/image.cpp @@ -441,8 +441,7 @@ namespace sd::pipeline { sd->compute_ip_adapter_tokens(sd_img_gen_params->ip_adapter_image, sd_img_gen_params->ip_adapter_strength); int64_t prepare_start_ms = ggml_time_ms(); condition_params.zero_out_masked = false; - auto cond = sd->cond_stage_model->get_learned_condition(sd->n_threads, - condition_params); + auto cond = sd->get_learned_condition(condition_params); if (cond.empty()) { LOG_ERROR("failed to encode prompt"); return std::nullopt; @@ -480,8 +479,7 @@ namespace sd::pipeline { // LLaDA-Image CFG keeps the source latent but drops its SigVQ features. condition_params.ref_images = nullptr; } - uncond = sd->cond_stage_model->get_learned_condition(sd->n_threads, - condition_params); + uncond = sd->get_learned_condition(condition_params); if (uncond.empty()) { LOG_ERROR("failed to encode negative prompt"); return std::nullopt; @@ -509,8 +507,7 @@ namespace sd::pipeline { if (use_ref_latent_img_cfg) { condition_params.ref_images = &empty_ref_images; } - img_uncond = sd->cond_stage_model->get_learned_condition(sd->n_threads, - condition_params); + img_uncond = sd->get_learned_condition(condition_params); if (img_uncond.empty()) { LOG_ERROR("failed to encode image guidance prompt"); return std::nullopt; diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index 5f227cee..2fcff4c9 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -1157,17 +1157,12 @@ namespace sd::pipeline { condition_params.zero_out_masked = true; condition_params.ref_images = &latents.ref_images; condition_params.minimax_h3_references = &latents.minimax_presentation_refs; - condition_params.allow_cache = - sd_version_is_minimax_h3(sd->version) && - sd->conditioning_cache_allowed_ && - !request.use_uncond; if (sd_version_is_lingbot_video(sd->version) || sd_version_is_minimax_h3(sd->version)) { condition_params.ref_image_params.vlm_resize_mode = RefImageResizeMode::AREA; } int64_t prepare_start_ms = ggml_time_ms(); - embeds.cond = sd->cond_stage_model->get_learned_condition(sd->n_threads, - condition_params); + embeds.cond = sd->get_learned_condition(condition_params); if (embeds.cond.empty()) { LOG_ERROR("failed to encode video prompt"); return std::nullopt; @@ -1192,8 +1187,7 @@ namespace sd::pipeline { } if (request.use_uncond) { condition_params.text = request.negative_prompt; - embeds.uncond = sd->cond_stage_model->get_learned_condition(sd->n_threads, - condition_params); + embeds.uncond = sd->get_learned_condition(condition_params); if (embeds.uncond.empty()) { LOG_ERROR("failed to encode negative video prompt"); return std::nullopt; diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index f2f0a01d..b13c055f 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -326,6 +326,7 @@ void sd_hires_params_init(sd_hires_params_t* hires_params) { void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params) { *sd_ctx_params = {}; sd_ctx_params->n_threads = sd_get_num_physical_cores(); + sd_ctx_params->conditioning_cache_size = 4; sd_ctx_params->wtype = SD_TYPE_COUNT; sd_ctx_params->rng_type = CUDA_RNG; sd_ctx_params->sampler_rng_type = RNG_TYPE_COUNT; @@ -378,6 +379,7 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { "pulid_weights_path: %s\n" "tensor_type_rules: %s\n" "n_threads: %d\n" + "conditioning_cache_size: %d\n" "wtype: %s\n" "rng_type: %s\n" "sampler_rng_type: %s\n" @@ -418,6 +420,7 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { SAFE_STR(sd_ctx_params->pulid_weights_path), SAFE_STR(sd_ctx_params->tensor_type_rules), sd_ctx_params->n_threads, + sd_ctx_params->conditioning_cache_size, sd_type_name(sd_ctx_params->wtype), sd_rng_type_name(sd_ctx_params->rng_type), sd_rng_type_name(sd_ctx_params->sampler_rng_type),