perf: cache MiniMax H3 text conditioning (#1966)

src/conditioning/conditioner.hpp

@@ -137,6 +137,7 @@ struct ConditionerParams {
     const std::vector<sd::Tensor<float>>* ref_images                    = nullptr;  // for qwen image edit
     const std::vector<MiniMaxH3PresentationItem>* minimax_h3_references = nullptr;
     RefImageParams ref_image_params;
+    bool allow_cache = false;
 };
 
 struct Conditioner {
@@ -1954,6 +1955,10 @@ struct LLMEmbedder : public Conditioner {
     std::shared_ptr<LLM::LLMRunner> llm;
     std::shared_ptr<T5Runner> byt5;
 
+    bool h3_text_cache_valid = false;
+    std::string h3_text_cache_text;
+    SDCondition h3_text_cache;
+
     LLMEmbedder(ggml_backend_t backend,
                 const String2TensorStorage& tensor_storage_map      = {},
                 SDVersion version                                   = VERSION_QWEN_IMAGE,
@@ -2298,6 +2303,25 @@ struct LLMEmbedder : public Conditioner {
 
     SDCondition get_learned_condition(int n_threads,
                                       const ConditionerParams& conditioner_params) override {
+        const bool h3_text_cacheable =
+            sd_version_is_minimax_h3(version) &&
+            conditioner_params.allow_cache &&
+            (conditioner_params.minimax_h3_references == nullptr ||
+             conditioner_params.minimax_h3_references->empty()) &&
+            (conditioner_params.ref_images == nullptr ||
+             conditioner_params.ref_images->empty());
+
+        if (sd_version_is_minimax_h3(version) && !h3_text_cacheable) {
+            h3_text_cache_valid = false;
+        }
+
+        if (h3_text_cacheable &&
+            h3_text_cache_valid &&
+            h3_text_cache_text == conditioner_params.text) {
+            LOG_INFO("H3 conditioning cache hit");
+            return h3_text_cache;
+        }
+
         std::string prompt;
         std::pair<int, int> prompt_attn_range;
         std::vector<std::string> extra_prompts;
@@ -3166,6 +3190,14 @@ struct LLMEmbedder : public Conditioner {
             int64_t tag_count    = static_cast<int64_t>(tags.size());
             result.c_token_types = sd::Tensor<int32_t>({tag_count}, std::move(tags));
         }
+
+        if (h3_text_cacheable) {
+            h3_text_cache_text  = conditioner_params.text;
+            h3_text_cache       = result;
+            h3_text_cache_valid = true;
+            LOG_INFO("H3 conditioning cache stored");
+        }
+
         return result;
     }
 };

src/pipeline/diffusion_engine.cpp

@@ -1762,6 +1762,8 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun
         extension->collect_loras(all_loras);
     }
 
+    conditioning_cache_allowed_ = all_loras.empty();
+
     int64_t t0 = ggml_time_ms();
     end_runners();
     clear_lora_adapters();

src/pipeline/diffusion_engine.h

@@ -178,6 +178,7 @@ public:
     std::recursive_mutex execution_mutex;
     std::unique_ptr<ModelConfig> config_;
     RunnerState runner_state_;
+    bool conditioning_cache_allowed_ = false;
     bool executing_ = false;
 
     std::shared_ptr<Denoiser> denoiser;

src/pipeline/video.cpp

@@ -1156,6 +1156,10 @@ namespace sd::pipeline {
         condition_params.zero_out_masked       = true;
         condition_params.ref_images            = &latents.ref_images;
         condition_params.minimax_h3_references = &latents.minimax_presentation_refs;
+        condition_params.allow_cache =
+            sd_version_is_minimax_h3(sd->version) &&
+            sd->conditioning_cache_allowed_ &&
+            !request.use_uncond;
         if (sd_version_is_lingbot_video(sd->version) || sd_version_is_minimax_h3(sd->version)) {
             condition_params.ref_image_params.vlm_resize_mode = RefImageResizeMode::AREA;
         }