refactor: split generation pipeline out of stable-diffusion.cpp (#1957)

CMakeLists.txt

@@ -229,6 +229,8 @@ file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
     "src/model/*/*.h"
     "src/model/*/*.cpp"
     "src/model/*/*.hpp"
+    "src/pipeline/*.h"
+    "src/pipeline/*.cpp"
     "src/runtime/*.h"
     "src/runtime/*.cpp"
     "src/runtime/*.hpp"

scripts/format-code.ps1

@@ -11,6 +11,8 @@ $patterns = @(
     "src/extensions/*.cpp"
     "src/extensions/*.h"
     "src/extensions/*.hpp"
+    "src/pipeline/*.cpp"
+    "src/pipeline/*.h"
     "src/runtime/*.cpp"
     "src/runtime/*.h"
     "src/runtime/*.hpp"

scripts/format-code.sh

@@ -9,6 +9,7 @@ for f in src/*.cpp src/*.h src/*.hpp \
          src/conditioning/*.cpp src/conditioning/*.h src/conditioning/*.hpp \
          src/core/*.cpp src/core/*.h src/core/*.hpp \
          src/extensions/*.cpp src/extensions/*.h src/extensions/*.hpp \
+         src/pipeline/*.cpp src/pipeline/*.h \
          src/runtime/*.cpp src/runtime/*.h src/runtime/*.hpp \
          src/model/*/*.cpp src/model/*/*.h src/model/*/*.hpp \
          src/tokenizers/*.h src/tokenizers/*.cpp src/tokenizers/vocab/*.h src/tokenizers/vocab/*.cpp \

src/pipeline/diffusion_engine.h

@@ -0,0 +1,482 @@
+#ifndef __SD_PIPELINE_DIFFUSION_ENGINE_H__
+#define __SD_PIPELINE_DIFFUSION_ENGINE_H__
+
+#include <atomic>
+#include <cmath>
+#include <functional>
+#include <list>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "core/ggml_extend_backend.h"
+#include "core/ggml_graph_cut.h"
+#include "core/tensor.hpp"
+#include "core/util.h"
+#include "model/adapter/lora.hpp"
+#include "model_builders.h"
+#include "model_manager.h"
+#include "stable-diffusion.h"
+
+class RNG;
+struct Denoiser;
+struct LoraModel;
+struct ConditionerParams;
+struct SDCondition;
+struct RefImageParams;
+
+extern const char* model_version_to_str[];
+
+static inline bool sd_version_supports_ref_latent_img_cfg(SDVersion version) {
+    return version == VERSION_FLUX ||
+           sd_version_is_flux2(version) ||
+           sd_version_is_qwen_image(version) ||
+           sd_version_is_mage_flow(version) ||
+           sd_version_is_longcat(version) ||
+           sd_version_is_z_image(version) ||
+           sd_version_is_boogu_image(version);
+}
+
+class StableDiffusionGGML {
+public:
+    SDBackendManager backend_manager;
+
+    SDVersion version;
+    bool external_vae_is_invalid = false;
+
+    bool circular_x = false;
+    bool circular_y = false;
+
+    std::shared_ptr<RNG> rng;
+    std::shared_ptr<RNG> sampler_rng = nullptr;
+    int n_threads                    = -1;
+    float default_flow_shift         = INFINITY;
+    float active_flow_shift          = INFINITY;
+
+    std::shared_ptr<Conditioner> cond_stage_model;
+    std::shared_ptr<FrozenCLIPVisionEmbedder> clip_vision;  // for svd or wan2.1 i2v
+    std::shared_ptr<DiffusionModelRunner> diffusion_model;
+    std::shared_ptr<DiffusionModelRunner> high_noise_diffusion_model;
+    std::shared_ptr<VAE> first_stage_model;
+    std::shared_ptr<VAE> preview_vae;
+    std::shared_ptr<AudioVAERunner> audio_vae_model;
+    std::shared_ptr<ControlNet> control_net;
+    std::shared_ptr<IPAdapter::IPAdapterRunner> ip_adapter;
+    sd::Tensor<float> ip_adapter_tokens;
+    sd::Tensor<float> ip_adapter_uncond_tokens;
+    float ip_adapter_strength = 1.0f;
+    std::vector<std::shared_ptr<GenerationExtension>> generation_extensions;
+    struct RuntimeLora {
+        ModelManager::LoraSpec spec;
+        SDBackendModule module;
+        std::shared_ptr<LoraModel> model;
+
+        bool matches(const ModelManager::LoraSpec& other) const {
+            return spec.file_id == other.file_id && spec.file_revision == other.file_revision &&
+                   spec.tensor_name_prefix_filter == other.tensor_name_prefix_filter;
+        }
+    };
+    std::vector<RuntimeLora> runtime_lora_models;
+    bool apply_lora_immediately = false;
+    int animatediff_num_frames  = 0;
+
+    std::string taesd_path;
+    sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr};
+    bool enable_mmap                     = false;
+    sd::ggml_graph_cut::MaxVramAssignment max_vram_assignment;
+    bool disable_prefetch          = false;
+    bool disable_segmented_compute = false;
+    bool eager_load                = false;
+    std::string backend_spec;
+    std::string params_backend_spec;
+    std::string split_mode_spec;
+    bool auto_fit_enabled = false;
+
+    bool diffusion_conv_direct = false;
+
+    bool is_using_v_parameterization     = false;
+    bool is_using_edm_v_parameterization = false;
+
+    std::shared_ptr<ModelManager> model_manager;
+
+    enum class RunnerGroup { Core,
+                             VAE,
+                             ControlNet,
+                             Extensions };
+    using RunnerGroups = std::set<RunnerGroup>;
+
+    struct ModelConfig {
+        sd_ctx_params_t params{};
+        std::list<std::string> strings;
+        std::vector<sd_embedding_t> embeddings;
+        ModelLoader::FileId control_net_file = 0;
+        bool use_tae                         = false;
+        bool use_audio_vae                   = false;
+        bool photomaker_source_available     = false;
+        bool animatediff_loaded              = false;
+
+        explicit ModelConfig(const sd_ctx_params_t& initial)
+            : params(initial) {
+            for (auto member : {&sd_ctx_params_t::model_path, &sd_ctx_params_t::clip_l_path,
+                                &sd_ctx_params_t::clip_g_path, &sd_ctx_params_t::clip_vision_path,
+                                &sd_ctx_params_t::t5xxl_path, &sd_ctx_params_t::llm_path,
+                                &sd_ctx_params_t::llm_vision_path, &sd_ctx_params_t::diffusion_model_path,
+                                &sd_ctx_params_t::high_noise_diffusion_model_path, &sd_ctx_params_t::uncond_diffusion_model_path,
+                                &sd_ctx_params_t::embeddings_connectors_path, &sd_ctx_params_t::vae_path,
+                                &sd_ctx_params_t::audio_vae_path, &sd_ctx_params_t::taesd_path,
+                                &sd_ctx_params_t::control_net_path, &sd_ctx_params_t::ip_adapter_path,
+                                &sd_ctx_params_t::motion_module_path, &sd_ctx_params_t::photo_maker_path,
+                                &sd_ctx_params_t::pulid_weights_path, &sd_ctx_params_t::tensor_type_rules,
+                                &sd_ctx_params_t::max_vram, &sd_ctx_params_t::backend,
+                                &sd_ctx_params_t::params_backend, &sd_ctx_params_t::split_mode,
+                                &sd_ctx_params_t::rpc_servers, &sd_ctx_params_t::model_args}) {
+                strings.emplace_back(SAFE_STR(initial.*member));
+                params.*member = strings.back().c_str();
+            }
+            for (uint32_t i = 0; i < initial.embedding_count; ++i) {
+                strings.emplace_back(SAFE_STR(initial.embeddings[i].name));
+                const char* name = strings.back().c_str();
+                strings.emplace_back(SAFE_STR(initial.embeddings[i].path));
+                embeddings.push_back({name, strings.back().c_str()});
+            }
+            params.embeddings = embeddings.data();
+        }
+
+        ModelConfig(const ModelConfig& other)
+            : ModelConfig(other.params) {
+            control_net_file            = other.control_net_file;
+            use_tae                     = other.use_tae;
+            use_audio_vae               = other.use_audio_vae;
+            photomaker_source_available = other.photomaker_source_available;
+            animatediff_loaded          = other.animatediff_loaded;
+        }
+        ModelConfig& operator=(const ModelConfig&) = delete;
+
+        void set_control_net(ModelLoader::FileId id, const std::string& path) {
+            control_net_file = id;
+            strings.push_back(path);
+            params.control_net_path = strings.back().c_str();
+        }
+    };
+
+    struct RunnerState {
+        bool ready                = false;
+        uint64_t catalog_revision = 0;
+        std::map<RunnerGroup, ModelLoader::FileVersions> sources;
+    };
+
+    std::recursive_mutex execution_mutex;
+    std::unique_ptr<ModelConfig> config_;
+    RunnerState runner_state_;
+    bool executing_ = false;
+
+    std::shared_ptr<Denoiser> denoiser;
+    std::vector<float> file_alphas_cumprod;
+
+    StableDiffusionGGML();
+    ~StableDiffusionGGML();
+
+    static const std::map<RunnerGroup, std::set<ModelComponent>>& runner_components();
+
+    static RunnerGroups all_runner_groups();
+
+    ModelLoader::FileVersions runner_source_versions(RunnerGroup group, const ModelLoader& loader) const;
+
+    void capture_runner_sources();
+
+    void end_runners();
+
+    bool reset_runners(const RunnerGroups& groups);
+
+    bool refresh_model_sources();
+
+    bool apply_model_update(ModelLoader candidate,
+                            std::unique_ptr<ModelConfig> next_config = nullptr,
+                            RunnerGroups groups                      = {});
+
+    struct ContextOperation {
+        StableDiffusionGGML& sd;
+        std::unique_lock<std::recursive_mutex> lock;
+        bool acquired = false;
+
+        explicit ContextOperation(StableDiffusionGGML& sd)
+            : sd(sd), lock(sd.execution_mutex, std::try_to_lock) {
+            if (!lock.owns_lock() || sd.executing_) {
+                // The caller may be a log callback, so rejecting it must not log.
+                return;
+            }
+            sd.executing_ = true;
+            acquired      = true;
+        }
+
+        ~ContextOperation() {
+            if (acquired) {
+                sd.executing_ = false;
+            }
+        }
+    };
+
+    struct ExecutionScope {
+        ContextOperation operation;
+        bool ready = false;
+
+        explicit ExecutionScope(StableDiffusionGGML& sd)
+            : operation(sd) {
+            ready = operation.acquired && sd.refresh_model_sources();
+        }
+
+        ~ExecutionScope() {
+            if (ready) {
+                operation.sd.end_runners();
+            }
+        }
+    };
+
+    ggml_backend_t backend_for(SDBackendModule module);
+
+    ggml_backend_t params_backend_for(SDBackendModule module);
+
+    std::atomic<sd_cancel_mode_t> cancellation_flag = SD_CANCEL_RESET;
+
+    void set_cancel_flag(enum sd_cancel_mode_t flag);
+
+    void reset_cancel_flag();
+
+    enum sd_cancel_mode_t get_cancel_flag();
+
+    size_t max_graph_vram_bytes_for_module(SDBackendModule module);
+
+    std::vector<size_t> layer_split_vram_limits_for_backends(const std::vector<ggml_backend_t>& backends);
+
+    bool ensure_backend_pair(SDBackendModule module);
+
+    template <typename T>
+    bool register_runner_params(ModelComponent component,
+                                const std::shared_ptr<T>& model,
+                                SDBackendModule module,
+                                size_t* params_mem_size = nullptr);
+
+    template <typename T>
+    bool register_row_split_runner_params(ModelComponent component,
+                                          const std::shared_ptr<T>& model,
+                                          SDBackendModule module,
+                                          const std::vector<ggml_backend_t>& module_backends,
+                                          std::map<std::string, ggml_tensor*> group_tensors,
+                                          const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
+                                          ModelManager::ResidencyMode residency_mode,
+                                          size_t* params_mem_size);
+
+    // Register graph-cut layer-split tensors on the primary backend first.
+    // The first real graph assigns each param tensor to a runtime backend
+    // before weights are loaded or staged.
+    template <typename T>
+    bool register_layer_split_runner_params(ModelComponent component,
+                                            const std::shared_ptr<T>& model,
+                                            SDBackendModule module,
+                                            const std::vector<ggml_backend_t>& module_backends,
+                                            std::map<std::string, ggml_tensor*> group_tensors,
+                                            const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
+                                            ModelManager::ResidencyMode residency_mode,
+                                            size_t* params_mem_size);
+
+    bool unload_control_net();
+
+    bool load_control_net_from_file(const std::string& path);
+
+    void apply_circular_axes(bool circular_x, bool circular_y);
+
+    bool init_backend();
+
+    bool row_split_active();
+
+    bool graph_cut_layer_split_active();
+
+    std::shared_ptr<RNG> get_rng(rng_type_t rng_type);
+
+    void refresh_compvis_denoiser_sigmas();
+
+    void load_alphas_cumprod();
+
+    bool init_model_loader(ModelLoader& model_loader, ModelConfig& configuration);
+
+    bool init(const sd_ctx_params_t* sd_ctx_params);
+
+    bool uses_tae() const;
+
+    bool tae_preview_only() const;
+
+    void configure_weight_loading();
+
+    sd::model_builders::Context model_build_context();
+
+    bool build_core_runners();
+
+    bool build_vae_runners();
+
+    bool build_control_net_runner();
+
+    bool build_extension_runners();
+
+    bool validate_and_load_runners();
+
+    bool build_denoiser();
+
+    bool build_runners(const RunnerGroups& groups);
+
+    bool is_using_v_parameterization_for_sd2(bool is_inpaint = false);
+
+    static std::string lora_log_id(const ModelManager::LoraSpec& lora);
+
+    std::shared_ptr<LoraModel> load_lora_model(const ModelManager::LoraSpec& lora_spec,
+                                               SDBackendModule module,
+                                               LoraModel::filter_t module_filter = nullptr);
+
+    void clear_lora_adapters();
+
+    std::vector<std::shared_ptr<LoraModel>> load_runtime_loras_for_module(const std::vector<ModelManager::LoraSpec>& loras,
+                                                                          const std::set<std::string>& model_tensor_names,
+                                                                          SDBackendModule module,
+                                                                          LoraModel::filter_t module_filter,
+                                                                          bool& success,
+                                                                          std::vector<RuntimeLora>& next_models);
+
+    bool apply_loras_immediately(const std::vector<ModelManager::LoraSpec>& loras);
+
+    bool apply_loras_at_runtime(const std::vector<ModelManager::LoraSpec>& loras);
+
+    void lora_stat();
+
+    bool apply_loras(const sd_lora_t* loras, uint32_t lora_count);
+
+    void reset_generation_extensions();
+
+    void prepare_generation_extensions(const sd_pm_params_t& pm_params,
+                                       const sd_pulid_params_t& pulid_params,
+                                       ConditionerParams& condition_params,
+                                       int total_steps);
+
+    sd::Tensor<float> get_clip_vision_output(const sd::Tensor<float>& image,
+                                             bool return_pooled   = true,
+                                             int clip_skip        = -1,
+                                             bool zero_out_masked = false);
+
+    void compute_ip_adapter_tokens(const sd_image_t& image, float strength);
+
+    std::vector<float> process_timesteps(const std::vector<float>& timesteps,
+                                         const sd::Tensor<float>& init_latent,
+                                         const sd::Tensor<float>& denoise_mask,
+                                         int step);
+
+    std::vector<float> process_ltxav_video_timesteps(const std::vector<float>& timesteps,
+                                                     const sd::Tensor<float>& init_latent,
+                                                     const sd::Tensor<float>& denoise_mask);
+
+    void preview_image(int step,
+                       const sd::Tensor<float>& latents,
+                       enum SDVersion version,
+                       preview_t preview_mode,
+                       std::function<void(int, int, sd_image_t*, bool, void*)> step_callback,
+                       void* step_callback_data,
+                       bool is_noisy);
+
+    std::vector<float> prepare_sample_timesteps(float sigma,
+                                                int shifted_timestep);
+
+    void adjust_sample_step_scalings(int shifted_timestep,
+                                     const std::vector<float>& timesteps_vec,
+                                     float c_in,
+                                     float* c_skip,
+                                     float* c_out);
+
+    struct SamplePreviewContext {
+        sd_preview_cb_t callback = nullptr;
+        void* data               = nullptr;
+        preview_t mode           = PREVIEW_NONE;
+    };
+
+    SamplePreviewContext prepare_sample_preview_context();
+
+    void report_sample_progress(int step,
+                                size_t total_steps,
+                                bool terminal_sigma_is_zero,
+                                int64_t* last_progress_us);
+
+    void compute_sample_controls(const sd::Tensor<float>& control_image,
+                                 const sd::Tensor<float>& noised_input,
+                                 const sd::Tensor<float>& timesteps_tensor,
+                                 const SDCondition& condition,
+                                 std::vector<sd::Tensor<float>>* controls);
+
+    sd::Tensor<float> sample(const std::shared_ptr<DiffusionModelRunner>& work_diffusion_model,
+                             bool inverse_noise_scaling,
+                             const sd::Tensor<float>& init_latent,
+                             sd::Tensor<float> noise,
+                             const SDCondition& cond,
+                             const SDCondition& uncond,
+                             const SDCondition& img_uncond,
+                             const sd::Tensor<float>& control_image,
+                             float control_strength,
+                             const sd_guidance_params_t& guidance,
+                             float eta,
+                             int shifted_timestep,
+                             sample_method_t method,
+                             bool is_flow_denoiser,
+                             const char* extra_sample_args,
+                             const std::vector<float>& sigmas,
+                             const std::vector<sd::Tensor<float>>& ref_latents,
+                             const RefImageParams& ref_image_params,
+                             const sd::Tensor<float>& denoise_mask,
+                             const sd::Tensor<float>& vace_context,
+                             float vace_strength,
+                             int audio_length,
+                             float frame_rate,
+                             const sd_cache_params_t* cache_params,
+                             bool preview_final_step,
+                             const sd::Tensor<float>& video_positions = {});
+
+    int get_vae_scale_factor();
+
+    int get_diffusion_model_down_factor();
+
+    int get_latent_channel();
+
+    int get_image_channels() const;
+
+    int get_image_seq_len(int h, int w);
+
+    sd::Tensor<float> generate_init_latent(int width,
+                                           int height,
+                                           int frames = 1,
+                                           bool video = false);
+
+    int video_frames_to_latent_frames(int frames);
+
+    int latent_frames_to_video_frames(int latent_frames);
+
+    int align_video_frames(int frames);
+
+    sd::Tensor<float> encode_to_vae_latents(const sd::Tensor<float>& x);
+
+    sd::Tensor<float> encode_first_stage(const sd::Tensor<float>& x);
+
+    sd::Tensor<float> decode_first_stage(const sd::Tensor<float>& x, bool decode_video = false);
+
+    sd::Tensor<float> normalize_ltx_video_latents(const sd::Tensor<float>& x);
+
+    sd::Tensor<float> un_normalize_ltx_video_latents(const sd::Tensor<float>& x);
+
+    sd::Tensor<float> decode_ltx_audio_latent(const sd::Tensor<float>& audio_latent);
+
+    void set_flow_shift(float flow_shift = INFINITY);
+
+    bool is_flow_denoiser();
+
+    std::string get_default_ref_image_preset(SDVersion version) const;
+
+    RefImageParams resolve_ref_image_params(const char* ref_image_args) const;
+};
+
+#endif  // __SD_PIPELINE_DIFFUSION_ENGINE_H__

src/pipeline/generation.h

@@ -0,0 +1,73 @@
+#ifndef __SD_PIPELINE_GENERATION_H__
+#define __SD_PIPELINE_GENERATION_H__
+
+#include "conditioning/conditioner.hpp"
+#include "stable-diffusion.h"
+
+class StableDiffusionGGML;
+
+static inline bool sd_version_supports_animatediff(SDVersion version) {
+    return version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX;
+}
+
+namespace sd::pipeline {
+
+    struct ImageGenerationLatents {
+        sd::Tensor<float> init_latent;
+        sd::Tensor<float> concat_latent;
+        sd::Tensor<float> img_uncond_concat_latent;
+        sd::Tensor<float> audio_latent;
+        sd::Tensor<float> video_positions;
+        sd::Tensor<float> control_image;
+        std::vector<sd::Tensor<float>> ref_images;
+        std::vector<sd::Tensor<float>> ref_latents;
+        std::vector<sd::Tensor<float>> reference_audio_latents;
+        std::vector<MiniMaxH3ReferenceBlock> minimax_reference_blocks;
+        std::vector<MiniMaxH3PresentationItem> minimax_presentation_refs;
+        std::vector<int32_t> keyframe_indices;
+        sd::Tensor<float> denoise_mask;
+        sd::Tensor<float> clip_vision_output;
+        sd::Tensor<float> vace_context;
+        int64_t ref_image_num                  = 0;
+        int64_t video_conditioning_frame_count = 0;
+        int64_t video_target_frame_count       = 0;
+        int audio_length                       = 0;
+    };
+
+    struct ImageGenerationEmbeds {
+        SDCondition cond;
+        SDCondition uncond;
+        SDCondition img_uncond;
+    };
+
+    struct ConditionerRunnerEndOnExit {
+        Conditioner* conditioner = nullptr;
+        ~ConditionerRunnerEndOnExit() {
+            if (conditioner != nullptr) {
+                conditioner->runner_end();
+            }
+        }
+    };
+
+    // Callers hold ExecutionScope; AnimateDiff reuses the image path within the same scope.
+    bool generate_image(StableDiffusionGGML* sd,
+                        const sd_img_gen_params_t* sd_img_gen_params,
+                        sd_image_t** images_out,
+                        int* num_images_out);
+
+    bool generate_video(StableDiffusionGGML* sd,
+                        const sd_vid_gen_params_t* sd_vid_gen_params,
+                        sd_image_t** frames_out,
+                        int* num_frames_out,
+                        sd_audio_t** audio_out);
+
+    sd::Tensor<float> upscale_ltx_spatial_video_latent(StableDiffusionGGML* sd,
+                                                       const char* model_path,
+                                                       const sd::Tensor<float>& packed_latent,
+                                                       int audio_length);
+
+    sd::Tensor<float> ensure_image_tensor_channels(sd::Tensor<float> image, int channels);
+
+}  // namespace sd::pipeline
+
+#endif  // __SD_PIPELINE_GENERATION_H__

src/pipeline/model_builders.h

@@ -1,5 +1,5 @@
-#ifndef __SD_MODEL_BUILDERS_H__
+#ifndef __SD_PIPELINE_MODEL_BUILDERS_H__
-#define __SD_MODEL_BUILDERS_H__
+#define __SD_PIPELINE_MODEL_BUILDERS_H__
 
 #include <memory>
 #include <vector>
@@ -60,4 +60,4 @@ namespace sd::model_builders {
 
 }  // namespace sd::model_builders
 
-#endif  // __SD_MODEL_BUILDERS_H__
+#endif  // __SD_PIPELINE_MODEL_BUILDERS_H__

src/pipeline/request.cpp

@@ -0,0 +1,471 @@
+#include "request.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdlib>
+#include <ctime>
+
+#include "diffusion_engine.h"
+#include "runtime/denoiser.hpp"
+
+namespace sd::pipeline {
+
+    const char* sampling_methods_str[] = {
+        "Euler",
+        "Euler A",
+        "Heun",
+        "DPM2",
+        "DPM++ (2s)",
+        "DPM++ (2M)",
+        "modified DPM++ (2M)",
+        "iPNDM",
+        "iPNDM_v",
+        "LCM",
+        "DDIM \"trailing\"",
+        "TCD",
+        "Res Multistep",
+        "Res 2s",
+        "ER-SDE",
+        "Euler CFG++",
+        "Euler A CFG++",
+        "Euler GE",
+        "DPM++ (2M) SDE",
+        "DPM++ (2M) SDE BT",
+        "LMS",
+    };
+
+    static_assert(SAMPLE_METHOD_COUNT == sizeof(sampling_methods_str) / sizeof(sampling_methods_str[0]),
+                  "\nnumber of elements in sampling_methods_str[] != SAMPLE_METHOD_COUNT");
+
+    static bool sd_version_supports_img_cfg(SDVersion version, bool has_ref_images) {
+        return sd_version_is_inpaint_or_unet_edit(version) ||
+               (has_ref_images && sd_version_supports_ref_latent_img_cfg(version));
+    }
+
+    enum sample_method_t default_sample_method(const StableDiffusionGGML* sd) {
+        if (sd != nullptr) {
+            if (sd_version_is_pid(sd->version)) {
+                return LCM_SAMPLE_METHOD;
+            }
+            if (sd_version_is_dit(sd->version)) {
+                return EULER_SAMPLE_METHOD;
+            }
+        }
+        return EULER_A_SAMPLE_METHOD;
+    }
+
+    enum scheduler_t default_scheduler(const StableDiffusionGGML* sd, enum sample_method_t sample_method) {
+        if (sd != nullptr) {
+            auto edm_v_denoiser = std::dynamic_pointer_cast<EDMVDenoiser>(sd->denoiser);
+            if (edm_v_denoiser) {
+                return EXPONENTIAL_SCHEDULER;
+            }
+        }
+        if (sample_method == LCM_SAMPLE_METHOD || sample_method == TCD_SAMPLE_METHOD) {
+            return LCM_SCHEDULER;
+        } else if (sample_method == DDIM_TRAILING_SAMPLE_METHOD) {
+            return SIMPLE_SCHEDULER;
+        } else if (sd != nullptr && sd_version_is_flux(sd->version)) {
+            return FLUX_SCHEDULER;
+        } else if (sd != nullptr && sd_version_is_flux2(sd->version)) {
+            return FLUX2_SCHEDULER;
+        } else if (sd != nullptr && sd_version_is_ltxav(sd->version)) {
+            return LTX2_SCHEDULER;
+        } else if (sd != nullptr && sd_version_is_ideogram4(sd->version)) {
+            return LOGIT_NORMAL_SCHEDULER;
+        }
+        return DISCRETE_SCHEDULER;
+    }
+
+    static int64_t resolve_seed(int64_t seed) {
+        if (seed >= 0) {
+            return seed;
+        }
+        srand((int)time(nullptr));
+        return rand();
+    }
+
+    static enum sample_method_t resolve_sample_method(StableDiffusionGGML* sd, enum sample_method_t sample_method) {
+        if (sample_method == SAMPLE_METHOD_COUNT) {
+            return default_sample_method(sd);
+        }
+        return sample_method;
+    }
+
+    static scheduler_t resolve_scheduler(StableDiffusionGGML* sd,
+                                         scheduler_t scheduler,
+                                         enum sample_method_t sample_method) {
+        if (scheduler == SCHEDULER_COUNT) {
+            return default_scheduler(sd, sample_method);
+        }
+        return scheduler;
+    }
+
+    float resolve_eta(StableDiffusionGGML* sd,
+                      float eta,
+                      enum sample_method_t sample_method) {
+        if (eta == INFINITY) {
+            if (sd->version == VERSION_HIDREAM_O1) {
+                return 8.f;
+            }
+            switch (sample_method) {
+                case DDIM_TRAILING_SAMPLE_METHOD:
+                case TCD_SAMPLE_METHOD:
+                case RES_MULTISTEP_SAMPLE_METHOD:
+                case RES_2S_SAMPLE_METHOD:
+                    return 0.0f;
+                case EULER_A_SAMPLE_METHOD:
+                case DPMPP2S_A_SAMPLE_METHOD:
+                case ER_SDE_SAMPLE_METHOD:
+                case EULER_A_CFG_PP_SAMPLE_METHOD:
+                case DPMPP2M_SDE_SAMPLE_METHOD:
+                case DPMPP2M_SDE_BT_SAMPLE_METHOD:
+                    return 1.0f;
+                default:;
+            }
+            return 0.0f;
+        }
+        return eta;
+    }
+
+    GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params) {
+        prompt                      = SAFE_STR(sd_img_gen_params->prompt);
+        negative_prompt             = SAFE_STR(sd_img_gen_params->negative_prompt);
+        width                       = sd_img_gen_params->width;
+        height                      = sd_img_gen_params->height;
+        vae_scale_factor            = sd->get_vae_scale_factor();
+        diffusion_model_down_factor = sd->get_diffusion_model_down_factor();
+        seed                        = sd_img_gen_params->seed;
+        batch_count                 = sd_img_gen_params->batch_count;
+        qwen_image_layers           = std::max(0, sd_img_gen_params->qwen_image_layers);
+        clip_skip                   = sd_img_gen_params->clip_skip;
+        shifted_timestep            = sd_img_gen_params->sample_params.shifted_timestep;
+        strength                    = sd_img_gen_params->strength;
+        control_strength            = sd_img_gen_params->control_strength;
+        eta                         = sd_img_gen_params->sample_params.eta;
+        has_ref_images              = sd_img_gen_params->ref_images_count > 0;
+        guidance                    = sd_img_gen_params->sample_params.guidance;
+        pm_params                   = sd_img_gen_params->pm_params;
+        pulid_params                = sd_img_gen_params->pulid_params;
+        hires                       = sd_img_gen_params->hires;
+        cache_params                = &sd_img_gen_params->cache;
+        resolve(sd);
+    }
+
+    GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params) {
+        prompt           = SAFE_STR(sd_vid_gen_params->prompt);
+        negative_prompt  = SAFE_STR(sd_vid_gen_params->negative_prompt);
+        width            = sd_vid_gen_params->width;
+        height           = sd_vid_gen_params->height;
+        requested_frames = std::max(1, sd_vid_gen_params->video_frames);
+        frames           = sd->align_video_frames(requested_frames);
+        clip_skip        = sd_vid_gen_params->clip_skip;
+        fps              = std::max(1, sd_vid_gen_params->fps);
+        if (sd_version_is_minimax_h3(sd->version) && fps != 24) {
+            LOG_WARN("MiniMax-H3 uses 24 fps; overriding requested fps %d", fps);
+            fps = 24;
+        }
+        vae_scale_factor            = sd->get_vae_scale_factor();
+        diffusion_model_down_factor = sd->get_diffusion_model_down_factor();
+        seed                        = sd_vid_gen_params->seed;
+        strength                    = sd_vid_gen_params->strength;
+        cache_params                = &sd_vid_gen_params->cache;
+        vace_strength               = sd_vid_gen_params->vace_strength;
+        guidance                    = sd_vid_gen_params->sample_params.guidance;
+        high_noise_guidance         = sd_vid_gen_params->high_noise_sample_params.guidance;
+        hires                       = sd_vid_gen_params->hires;
+        resolve(sd);
+        if (frames != requested_frames) {
+            LOG_WARN("align video frames from %d to %d for %s",
+                     requested_frames,
+                     frames,
+                     model_version_to_str[sd->version]);
+        }
+    }
+
+    void GenerationRequest::align_generation_request_size() {
+        align_image_size(&width, &height, "generation request");
+    }
+
+    void GenerationRequest::align_image_size(int* target_width, int* target_height, const char* label) {
+        int spatial_multiple = vae_scale_factor * diffusion_model_down_factor;
+        int width_offset     = align_up_offset(*target_width, spatial_multiple);
+        int height_offset    = align_up_offset(*target_height, spatial_multiple);
+        if (width_offset <= 0 && height_offset <= 0) {
+            return;
+        }
+
+        int original_width  = *target_width;
+        int original_height = *target_height;
+
+        *target_width += width_offset;
+        *target_height += height_offset;
+        LOG_WARN("align %s up %dx%d to %dx%d (multiple=%d)",
+                 label,
+                 original_width,
+                 original_height,
+                 *target_width,
+                 *target_height,
+                 spatial_multiple);
+    }
+
+    void GenerationRequest::resolve_hires() {
+        if (!hires.enabled) {
+            return;
+        }
+        if (hires.upscaler == SD_HIRES_UPSCALER_NONE) {
+            hires.enabled = false;
+            return;
+        }
+        if (hires.upscaler < SD_HIRES_UPSCALER_NONE || hires.upscaler >= SD_HIRES_UPSCALER_COUNT) {
+            LOG_WARN("hires upscaler '%d' is invalid, disabling hires", hires.upscaler);
+            hires.enabled = false;
+            return;
+        }
+        if (hires.upscaler == SD_HIRES_UPSCALER_MODEL && strlen(SAFE_STR(hires.model_path)) == 0) {
+            LOG_WARN("hires model upscaler requires a model path, disabling hires");
+            hires.enabled = false;
+            return;
+        }
+        if (hires.scale <= 0.f && hires.target_width <= 0 && hires.target_height <= 0) {
+            LOG_WARN("hires scale must be positive when no target size is set, disabling hires");
+            hires.enabled = false;
+            return;
+        }
+        if (hires.custom_sigmas_count < 0) {
+            LOG_WARN("hires custom sigmas count is negative, ignoring custom sigmas");
+            hires.custom_sigmas       = nullptr;
+            hires.custom_sigmas_count = 0;
+        }
+        if (hires.custom_sigmas_count > 0 && hires.custom_sigmas == nullptr) {
+            LOG_WARN("hires custom sigmas count is positive but custom sigmas are null, ignoring custom sigmas");
+            hires.custom_sigmas_count = 0;
+        }
+        if (hires.custom_sigmas_count == 1) {
+            LOG_WARN("hires custom sigmas requires at least two values, ignoring custom sigmas");
+            hires.custom_sigmas       = nullptr;
+            hires.custom_sigmas_count = 0;
+        }
+        hires.denoising_strength = std::clamp(hires.denoising_strength, 0.0001f, 1.f);
+        hires.steps              = std::max(0, hires.steps);
+
+        if (hires.target_width > 0 && hires.target_height > 0) {
+            // pass
+        } else if (hires.target_width > 0) {
+            hires.target_height = hires.target_width;
+        } else if (hires.target_height > 0) {
+            hires.target_width = hires.target_height;
+        } else {
+            hires.target_width  = static_cast<int>(std::round(width * hires.scale));
+            hires.target_height = static_cast<int>(std::round(height * hires.scale));
+        }
+
+        if (hires.target_width <= 0 || hires.target_height <= 0) {
+            LOG_WARN("hires target size is not positive, disabling hires");
+            hires.enabled = false;
+            return;
+        }
+        align_image_size(&hires.target_width, &hires.target_height, "hires target");
+    }
+
+    void GenerationRequest::resolve_guidance(StableDiffusionGGML* sd,
+                                             sd_guidance_params_t* guidance,
+                                             bool* use_uncond,
+                                             bool* use_img_uncond,
+                                             bool has_ref_images,
+                                             const char* stage_name) {
+        GGML_ASSERT(guidance != nullptr);
+        GGML_ASSERT(use_uncond != nullptr);
+        GGML_ASSERT(use_img_uncond != nullptr);
+        // out_img_uncond + text_cfg_scale * (out_cond - out_uncond) + image_cfg_scale * (out_uncond - out_img_uncond)
+        // -> text_cfg_scale * out_cond + (image_cfg_scale - text_cfg_scale) * out_uncond + (1 - image_cfg_scale) * out_img_uncond
+        // out_cond       : prompt, image latent
+        // out_uncond     : negative prompt, image latent
+        // out_img_uncond : negative prompt, zero image latent
+        // image_cfg_scale == 1 reduces 3-cond CFG to 2-cond CFG.
+        bool img_cfg_was_set = std::isfinite(guidance->img_cfg);
+        if (!img_cfg_was_set) {
+            guidance->img_cfg = 1.f;
+        }
+
+        if (!sd_version_supports_img_cfg(sd->version, has_ref_images)) {
+            if (img_cfg_was_set && guidance->img_cfg != 1.f) {
+                LOG_WARN("3-conditioning CFG is not supported with this model, disabling it for better performance");
+            }
+            guidance->img_cfg = 1.f;
+        }
+
+        if (guidance->img_cfg != guidance->txt_cfg) {
+            *use_uncond = true;
+        }
+
+        if (guidance->img_cfg != 1.f) {
+            *use_img_uncond = true;
+        }
+
+        if (guidance->txt_cfg < 1.f) {
+            const char* prefix = stage_name == nullptr ? "" : stage_name;
+            if (guidance->txt_cfg == 0.f) {
+                LOG_WARN("%sunconditioned mode, images won't follow the prompt (use cfg-scale=1 for distilled models)",
+                         prefix);
+            } else {
+                LOG_WARN("%scfg value out of expected range may produce unexpected results", prefix);
+            }
+        }
+    }
+
+    void GenerationRequest::resolve(StableDiffusionGGML* sd) {
+        align_generation_request_size();
+        resolve_hires();
+        seed = resolve_seed(seed);
+
+        resolve_guidance(sd, &guidance, &use_uncond, &use_img_uncond, has_ref_images);
+        if (sd->high_noise_diffusion_model) {
+            resolve_guidance(sd,
+                             &high_noise_guidance,
+                             &use_high_noise_uncond,
+                             &use_high_noise_img_uncond,
+                             has_ref_images,
+                             "high noise: ");
+        }
+
+        if (shifted_timestep > 0 && !sd_version_is_sdxl(sd->version)) {
+            LOG_WARN("timestep shifting is only supported for SDXL models!");
+            shifted_timestep = 0;
+        }
+    }
+
+    SamplePlan::SamplePlan(StableDiffusionGGML* sd,
+                           const sd_img_gen_params_t* sd_img_gen_params,
+                           const GenerationRequest& request) {
+        sample_method     = sd_img_gen_params->sample_params.sample_method;
+        extra_sample_args = sd_img_gen_params->sample_params.extra_sample_args;
+        eta               = sd_img_gen_params->sample_params.eta;
+        sample_steps      = sd_img_gen_params->sample_params.sample_steps;
+        resolve(sd, &request, &sd_img_gen_params->sample_params);
+    }
+
+    SamplePlan::SamplePlan(StableDiffusionGGML* sd,
+                           const sd_vid_gen_params_t* sd_vid_gen_params,
+                           const GenerationRequest& request) {
+        sample_method     = sd_vid_gen_params->sample_params.sample_method;
+        extra_sample_args = sd_vid_gen_params->sample_params.extra_sample_args;
+        eta               = sd_vid_gen_params->sample_params.eta;
+        sample_steps      = sd_vid_gen_params->sample_params.sample_steps;
+        if (sd->high_noise_diffusion_model) {
+            high_noise_sample_steps      = sd_vid_gen_params->high_noise_sample_params.sample_steps;
+            high_noise_sample_method     = sd_vid_gen_params->high_noise_sample_params.sample_method;
+            high_noise_extra_sample_args = sd_vid_gen_params->high_noise_sample_params.extra_sample_args;
+            high_noise_eta               = sd_vid_gen_params->high_noise_sample_params.eta;
+        }
+        moe_boundary = sd_vid_gen_params->moe_boundary;
+        resolve(sd, &request, &sd_vid_gen_params->sample_params);
+    }
+
+    void SamplePlan::resolve(StableDiffusionGGML* sd,
+                             const GenerationRequest* request,
+                             const sd_sample_params_t* sample_params) {
+        sample_method = resolve_sample_method(sd, sample_method);
+
+        total_steps = sample_steps + std::max(0, high_noise_sample_steps);
+
+        if (sample_params->custom_sigmas_count > 0) {
+            sigmas      = std::vector<float>(sample_params->custom_sigmas,
+                                        sample_params->custom_sigmas + sample_params->custom_sigmas_count);
+            total_steps = static_cast<int>(sigmas.size()) - 1;
+            LOG_WARN("total_steps != custom_sigmas_count - 1, set total_steps to %d", total_steps);
+            if (sample_steps >= total_steps) {
+                sample_steps = total_steps;
+                LOG_WARN("total_steps != custom_sigmas_count - 1, set sample_steps to %d", sample_steps);
+            }
+            if (high_noise_sample_steps > 0) {
+                high_noise_sample_steps = total_steps - sample_steps;
+                LOG_WARN("total_steps != custom_sigmas_count - 1, set high_noise_sample_steps to %d", high_noise_sample_steps);
+            }
+        } else {
+            scheduler_t scheduler = resolve_scheduler(sd,
+                                                      sample_params->scheduler,
+                                                      sample_method);
+            int sample_seq_len    = sd->get_image_seq_len(request->height, request->width);
+            if (sd_version_is_ltxav(sd->version) && request->frames > 0) {
+                int latent_frames = ((request->frames - 1) / 8) + 1;
+                sample_seq_len *= latent_frames;
+            } else if (sd_version_is_minimax_h3(sd->version) && request->frames > 0) {
+                sample_seq_len *= sd->video_frames_to_latent_frames(request->frames);
+            }
+            sigmas = sd->denoiser->get_sigmas(total_steps,
+                                              sample_seq_len,
+                                              scheduler,
+                                              sd->version,
+                                              sample_params->extra_sample_args);
+        }
+
+        eta = resolve_eta(sd, eta, sample_method);
+
+        if (high_noise_sample_steps < 0) {
+            for (size_t i = 0; i < sigmas.size(); ++i) {
+                if (sigmas[i] < moe_boundary) {
+                    high_noise_sample_steps = static_cast<int>(i);
+                    break;
+                }
+            }
+            LOG_VERBOSE("switching from high noise model at step %d", high_noise_sample_steps);
+        }
+
+        LOG_INFO("sampling using %s method", sampling_methods_str[sample_method]);
+        if (high_noise_sample_steps > 0) {
+            high_noise_sample_method = resolve_sample_method(sd,
+                                                             high_noise_sample_method);
+            high_noise_eta           = resolve_eta(sd, high_noise_eta, high_noise_sample_method);
+            LOG_INFO("sampling(high noise) using %s method", sampling_methods_str[high_noise_sample_method]);
+        }
+    }
+
+    std::vector<float> make_hires_sigma_schedule(StableDiffusionGGML* sd,
+                                                 const sd_hires_params_t& hires,
+                                                 const sd_sample_params_t& sample_params,
+                                                 sample_method_t sample_method,
+                                                 int default_steps,
+                                                 int sample_seq_len,
+                                                 int* scheduler_steps_out) {
+        if (scheduler_steps_out != nullptr) {
+            *scheduler_steps_out = 0;
+        }
+
+        if (hires.custom_sigmas_count > 0 && hires.custom_sigmas != nullptr) {
+            std::vector<float> custom_sigmas(hires.custom_sigmas,
+                                             hires.custom_sigmas + hires.custom_sigmas_count);
+            if (scheduler_steps_out != nullptr) {
+                *scheduler_steps_out = static_cast<int>(custom_sigmas.size()) - 1;
+            }
+            return custom_sigmas;
+        }
+
+        int effective_steps = hires.steps > 0 ? hires.steps : default_steps;
+        effective_steps     = std::max(1, effective_steps);
+
+        // sd-webui behavior: scale up total steps so trimming by denoising_strength yields exactly hires_steps effective steps,
+        // unlike img2img which trims from a fixed step count.
+        int scheduler_steps = static_cast<int>(effective_steps / hires.denoising_strength);
+        scheduler_steps     = std::max(1, scheduler_steps);
+
+        scheduler_t scheduler     = resolve_scheduler(sd,
+                                                      sample_params.scheduler,
+                                                      sample_method);
+        std::vector<float> sigmas = sd->denoiser->get_sigmas(scheduler_steps,
+                                                             sample_seq_len,
+                                                             scheduler,
+                                                             sd->version,
+                                                             sample_params.extra_sample_args);
+        size_t t_enc              = static_cast<size_t>(scheduler_steps * hires.denoising_strength);
+        if (t_enc >= static_cast<size_t>(scheduler_steps)) {
+            t_enc = static_cast<size_t>(scheduler_steps) - 1;
+        }
+        if (scheduler_steps_out != nullptr) {
+            *scheduler_steps_out = scheduler_steps;
+        }
+        return std::vector<float>(sigmas.begin() + scheduler_steps - static_cast<int>(t_enc) - 1,
+                                  sigmas.end());
+    }
+
+}  // namespace sd::pipeline

src/pipeline/request.h

@@ -0,0 +1,110 @@
+#ifndef __SD_PIPELINE_REQUEST_H__
+#define __SD_PIPELINE_REQUEST_H__
+
+#include <string>
+#include <vector>
+
+#include "stable-diffusion.h"
+
+class StableDiffusionGGML;
+
+namespace sd::pipeline {
+
+    extern const char* sampling_methods_str[];
+
+    enum sample_method_t default_sample_method(const StableDiffusionGGML* sd);
+
+    enum scheduler_t default_scheduler(const StableDiffusionGGML* sd, enum sample_method_t sample_method);
+
+    float resolve_eta(StableDiffusionGGML* sd,
+                      float eta,
+                      enum sample_method_t sample_method);
+
+    struct GenerationRequest {
+        std::string prompt;
+        std::string negative_prompt;
+        int width                                = -1;
+        int height                               = -1;
+        int clip_skip                            = -1;
+        int vae_scale_factor                     = -1;
+        int diffusion_model_down_factor          = -1;
+        int64_t seed                             = -1;
+        bool use_uncond                          = false;
+        bool use_img_uncond                      = false;
+        bool use_high_noise_uncond               = false;
+        bool use_high_noise_img_uncond           = false;
+        bool has_ref_images                      = false;
+        const sd_cache_params_t* cache_params    = nullptr;
+        int batch_count                          = 1;
+        int qwen_image_layers                    = 3;
+        int shifted_timestep                     = 0;
+        float strength                           = 1.f;
+        float control_strength                   = 0.f;
+        float eta                                = 0.f;
+        sd_guidance_params_t guidance            = {};
+        sd_guidance_params_t high_noise_guidance = {};
+        sd_pm_params_t pm_params                 = {};
+        sd_pulid_params_t pulid_params           = {};
+        sd_hires_params_t hires                  = {};
+        int frames                               = -1;
+        int requested_frames                     = -1;
+        int fps                                  = 16;
+        float vace_strength                      = 1.f;
+
+        GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params);
+
+        GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params);
+
+        void align_generation_request_size();
+
+        void align_image_size(int* target_width, int* target_height, const char* label);
+
+        void resolve_hires();
+
+        static void resolve_guidance(StableDiffusionGGML* sd,
+                                     sd_guidance_params_t* guidance,
+                                     bool* use_uncond,
+                                     bool* use_img_uncond,
+                                     bool has_ref_images,
+                                     const char* stage_name = nullptr);
+
+        void resolve(StableDiffusionGGML* sd);
+    };
+
+    struct SamplePlan {
+        enum sample_method_t sample_method            = SAMPLE_METHOD_COUNT;
+        enum sample_method_t high_noise_sample_method = SAMPLE_METHOD_COUNT;
+        const char* extra_sample_args                 = nullptr;
+        const char* high_noise_extra_sample_args      = nullptr;
+        float eta                                     = 0.f;
+        float high_noise_eta                          = 0.f;
+        int sample_steps                              = 0;
+        int high_noise_sample_steps                   = 0;
+        int total_steps                               = 0;
+        float moe_boundary                            = 0.f;
+        std::vector<float> sigmas;
+
+        SamplePlan(StableDiffusionGGML* sd,
+                   const sd_img_gen_params_t* sd_img_gen_params,
+                   const GenerationRequest& request);
+
+        SamplePlan(StableDiffusionGGML* sd,
+                   const sd_vid_gen_params_t* sd_vid_gen_params,
+                   const GenerationRequest& request);
+
+        void resolve(StableDiffusionGGML* sd,
+                     const GenerationRequest* request,
+                     const sd_sample_params_t* sample_params);
+    };
+
+    std::vector<float> make_hires_sigma_schedule(StableDiffusionGGML* sd,
+                                                 const sd_hires_params_t& hires,
+                                                 const sd_sample_params_t& sample_params,
+                                                 sample_method_t sample_method,
+                                                 int default_steps,
+                                                 int sample_seq_len,
+                                                 int* scheduler_steps_out);
+
+}  // namespace sd::pipeline
+
+#endif  // __SD_PIPELINE_REQUEST_H__