DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 11:26:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: return generation metadata with core results (#2206)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-10-25 03:18:03 +08:00
1 parent b29cad540a
commit a85db96aa5
37 files changed
+1528 -758
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/conditioning/conditioner.hpp
+9 -12
View File
@@ -141,6 +141,7 @@ struct ConditionerParams {
};
struct Conditioner {
virtual int resolve_clip_skip(int) const { return 0; }
virtual ~Conditioner() = default;
public:
@@ -173,6 +174,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
std::vector<uint8_t> token_embed_custom2;
std::map<std::string, std::pair<int, int>> embedding_pos_map;
int resolve_clip_skip(int value) const override {
return value > 0 ? value : ((sd_version_is_sd2(version) || sd_version_is_sdxl(version)) ? 2 : 1);
}
FrozenCLIPEmbedderWithCustomWords(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map,
const std::map<std::string, std::string>& orig_embedding_map,
@@ -456,10 +461,6 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
sd::Tensor<float> hidden_states; // [n_token, hidden_size] or [n_token, hidden_size + hidden_size2]
sd::Tensor<float> pooled;
if (clip_skip <= 0) {
clip_skip = (sd_version_is_sd2(version) || sd_version_is_sdxl(version)) ? 2 : 1;
}
size_t chunk_len = 77;
size_t chunk_count = tokens.size() / chunk_len;
for (int chunk_idx = 0; chunk_idx < chunk_count; chunk_idx++) {
@@ -651,6 +652,8 @@ struct SD3CLIPEmbedder : public Conditioner {
std::shared_ptr<CLIPTextModelRunner> clip_g;
std::shared_ptr<T5Runner> t5;
int resolve_clip_skip(int value) const override { return value > 0 ? value : 2; }
SD3CLIPEmbedder(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
@@ -898,10 +901,6 @@ struct SD3CLIPEmbedder : public Conditioner {
auto& t5_tokens = token_and_weights[2].first;
auto& t5_weights = token_and_weights[2].second;
if (clip_skip <= 0) {
clip_skip = 2;
}
size_t chunk_len = 77;
int64_t t0 = ggml_time_ms();
sd::Tensor<float> hidden_states;
@@ -1097,6 +1096,8 @@ struct FluxCLIPEmbedder : public Conditioner {
std::shared_ptr<T5Runner> t5;
size_t chunk_len = 256;
int resolve_clip_skip(int value) const override { return value > 0 ? value : 2; }
FluxCLIPEmbedder(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
@@ -1293,10 +1294,6 @@ struct FluxCLIPEmbedder : public Conditioner {
auto& t5_tokens = token_and_weights[1].first;
auto& t5_weights = token_and_weights[1].second;
if (clip_skip <= 0) {
clip_skip = 2;
}
int64_t t0 = ggml_time_ms();
sd::Tensor<float> hidden_states; // [N, n_token, 4096]
sd::Tensor<float> pooled; // [768,]
src/detailer.cpp
+28 -31
View File
@@ -1,4 +1,6 @@
#include "detailer.h"
#include "generation_info.h"
#include "pipeline/request.h"
#include <algorithm>
#include <cctype>
@@ -915,14 +917,11 @@ bool adetail_image(adetailer_ctx_t* context,
sd_image_t input_image,
const sd_adetailer_params_t* public_params,
const sd_img_gen_params_t* inpaint_params,
sd_image_t** images_out,
int* num_images_out) {
if (images_out != nullptr) {
*images_out = nullptr;
}
if (num_images_out != nullptr) {
*num_images_out = 0;
}
sd_image_generation_result_t** result_out) try {
if (result_out)
*result_out = nullptr;
if (!result_out)
return false;
if (context == nullptr || context->detailer == nullptr || sd_ctx == nullptr || public_params == nullptr ||
inpaint_params == nullptr || input_image.data == nullptr) {
return false;
@@ -933,7 +932,10 @@ bool adetail_image(adetailer_ctx_t* context,
return false;
}
auto detections = context->detailer->predict(input_image, params);
auto resolved_inpaint = *inpaint_params;
resolved_inpaint.seed = sd::pipeline::resolve_seed(resolved_inpaint.seed);
const auto info = sd::adetailer_generation_info(*public_params, resolved_inpaint);
auto detections = context->detailer->predict(input_image, params);
filter_and_sort_detections(&detections, input_image.width, input_image.height, params);
auto masks = make_masks(detections, input_image.width, input_image.height, params);
@@ -942,6 +944,7 @@ bool adetail_image(adetailer_ctx_t* context,
return false;
}
std::unique_ptr<uint8_t, decltype(&free)> current_owner(current.data, &free);
int applied = 0;
for (size_t i = 0; i < masks.size(); ++i) {
std::string prompt = resolve_prompt(params.prompt, inpaint_params->prompt, static_cast<int>(i));
@@ -968,11 +971,10 @@ bool adetail_image(adetailer_ctx_t* context,
if (local_image.data == nullptr || local_mask.data == nullptr) {
free(local_image.data);
free(local_mask.data);
free(current.data);
return false;
}
sd_img_gen_params_t generation = *inpaint_params;
sd_img_gen_params_t generation = resolved_inpaint;
generation.prompt = prompt.c_str();
generation.negative_prompt = negative_prompt.c_str();
generation.init_image = local_image;
@@ -980,7 +982,7 @@ bool adetail_image(adetailer_ctx_t* context,
generation.width = params.inpaint_width;
generation.height = params.inpaint_height;
generation.strength = params.denoising_strength;
generation.seed = inpaint_params->seed + static_cast<int64_t>(i);
generation.seed = static_cast<int64_t>((static_cast<uint64_t>(resolved_inpaint.seed) + i) & INT64_MAX);
generation.batch_count = 1;
generation.ref_images = nullptr;
generation.ref_images_count = 0;
@@ -1004,37 +1006,32 @@ bool adetail_image(adetailer_ctx_t* context,
generation.sample_params.scheduler = params.scheduler;
}
sd_image_t* generated = nullptr;
int generated_count = 0;
bool success = generate_image(sd_ctx, &generation, &generated, &generated_count);
sd_image_generation_result_t* generated = nullptr;
bool success = generate_image(sd_ctx, &generation, &generated);
free(local_image.data);
free(local_mask.data);
if (!success || generated_count <= 0 || generated == nullptr || generated[0].data == nullptr) {
free_sd_images(generated, generated_count);
free(current.data);
if (!success || generated == nullptr || generated->count <= 0 || generated->items[0].image.data == nullptr) {
free_sd_image_generation_result(generated);
return false;
}
std::unique_ptr<sd_image_generation_result_t, decltype(&free_sd_image_generation_result)> generated_owner(generated, &free_sd_image_generation_result);
Mask feather = gaussian_blur_mask(masks[i], params.mask_blur);
composite_crop(&current, generated[0], feather, crop);
free_sd_images(generated, generated_count);
composite_crop(&current, generated->items[0].image, feather, crop);
++applied;
}
const std::vector<nlohmann::json> infos{info};
auto* outputs = static_cast<sd_image_t*>(calloc(1, sizeof(sd_image_t)));
if (outputs == nullptr) {
free(current.data);
if (!outputs)
return false;
}
outputs[0] = current;
if (images_out != nullptr) {
*images_out = outputs;
} else {
free_sd_images(outputs, 1);
}
if (num_images_out != nullptr) {
*num_images_out = 1;
}
current_owner.release();
if (!sd::make_image_generation_result(outputs, 1, infos, result_out))
return false;
LOG_INFO("ADetailer applied %d mask(s)", applied);
return true;
} catch (const std::exception& e) {
LOG_ERROR("ADetailer failed: %s", e.what());
return false;
}
src/generation_info.cpp
+276
View File
@@ -0,0 +1,276 @@
#include "generation_info.h"
#include <cmath>
#include <cstdlib>
#include <cstring>
#include "pipeline/request.h"
#include "runtime/guidance.h"
namespace sd {
using nlohmann::json;
static const char* text(const char* value) {
return value ? value : "";
}
static json sampling_info(const sd_sample_params_t& p, bool high_noise = false) {
const auto& g = p.guidance;
json info = {{"sample_steps", p.sample_steps}, {"guidance", {{"txt_cfg", g.txt_cfg}, {"distilled_guidance", g.distilled_guidance}}}};
if (!high_noise && p.shifted_timestep > 0)
info["shifted_timestep"] = p.shifted_timestep;
if (std::isfinite(g.img_cfg) && g.img_cfg != 1.f)
info["guidance"]["img_cfg"] = g.img_cfg;
if (p.sample_method != SAMPLE_METHOD_COUNT)
info["sample_method"] = sd_sample_method_name(p.sample_method);
if (!high_noise && p.scheduler != SCHEDULER_COUNT)
info["scheduler"] = sd_scheduler_name(p.scheduler);
if (std::isfinite(p.eta))
info["eta"] = p.eta;
if (!high_noise && std::isfinite(p.flow_shift))
info["flow_shift"] = p.flow_shift;
if (!high_noise && p.custom_sigmas && p.custom_sigmas_count > 0)
info["custom_sigmas"] = std::vector<float>(p.custom_sigmas, p.custom_sigmas + p.custom_sigmas_count);
if (p.extra_sample_args && *p.extra_sample_args)
info["extra_sample_args"] = p.extra_sample_args;
if (g.slg.layers && g.slg.layer_count > 0 &&
(g.slg.scale != 0.f || guidance::parse_skip_layer_guidance_uncond_arg(p.extra_sample_args)))
info["guidance"]["slg"] = {{"scale", g.slg.scale}, {"layer_start", g.slg.layer_start}, {"layer_end", g.slg.layer_end}, {"layers", std::vector<int>(g.slg.layers, g.slg.layers + g.slg.layer_count)}};
return info;
}
static json hires_info(const sd_hires_params_t& p) {
json info = {{"enabled", p.enabled}};
if (!p.enabled)
return info;
info.update({{"upscaler", sd_hires_upscaler_name(p.upscaler)}, {"scale", p.scale}, {"target_width", p.target_width}, {"target_height", p.target_height}, {"steps", p.steps}, {"denoising_strength", p.denoising_strength}});
if (p.upscaler == SD_HIRES_UPSCALER_MODEL) {
info["model_path"] = text(p.model_path);
info["upscale_tile_size"] = p.upscale_tile_size;
}
if (p.custom_sigmas && p.custom_sigmas_count > 0)
info["custom_sigmas"] = std::vector<float>(p.custom_sigmas, p.custom_sigmas + p.custom_sigmas_count);
return info;
}
static json cache_info(const sd_cache_params_t& p) {
static const char* modes[] = {"disabled", "easycache", "ucache", "dbcache", "taylorseer", "cache_dit", "spectrum"};
if (p.mode < SD_CACHE_DISABLED || p.mode > SD_CACHE_SPECTRUM)
return {{"mode", static_cast<int>(p.mode)}};
json info = {{"mode", modes[p.mode]}};
if (p.mode == SD_CACHE_DISABLED)
return info;
if (p.mode == SD_CACHE_EASYCACHE || p.mode == SD_CACHE_UCACHE) {
if (std::isfinite(p.reuse_threshold))
info["reuse_threshold"] = p.reuse_threshold;
info.update({{"start_percent", p.start_percent}, {"end_percent", p.end_percent}});
if (p.mode == SD_CACHE_UCACHE)
info.update({{"error_decay_rate", p.error_decay_rate}, {"use_relative_threshold", p.use_relative_threshold}, {"reset_error_on_compute", p.reset_error_on_compute}});
} else if (p.mode == SD_CACHE_DBCACHE || p.mode == SD_CACHE_TAYLORSEER || p.mode == SD_CACHE_CACHE_DIT) {
// TaylorSeer also uses the shared DBCache thresholds and limits.
info.update({{"Fn_compute_blocks", p.Fn_compute_blocks},
{"Bn_compute_blocks", p.Bn_compute_blocks},
{"residual_diff_threshold", p.residual_diff_threshold},
{"max_warmup_steps", p.max_warmup_steps},
{"max_cached_steps", p.max_cached_steps},
{"max_continuous_cached_steps", p.max_continuous_cached_steps}});
if (p.mode == SD_CACHE_TAYLORSEER || p.mode == SD_CACHE_CACHE_DIT)
info.update({{"taylorseer_n_derivatives", p.taylorseer_n_derivatives}, {"taylorseer_skip_interval", p.taylorseer_skip_interval}});
if (p.scm_mask && *p.scm_mask)
info.update({{"scm_mask", p.scm_mask}, {"scm_policy_dynamic", p.scm_policy_dynamic}});
} else if (p.mode == SD_CACHE_SPECTRUM) {
info.update({{"spectrum_w", p.spectrum_w},
{"spectrum_m", p.spectrum_m},
{"spectrum_lam", p.spectrum_lam},
{"spectrum_window_size", p.spectrum_window_size},
{"spectrum_flex_window", p.spectrum_flex_window},
{"spectrum_warmup_steps", p.spectrum_warmup_steps},
{"spectrum_stop_percent", p.spectrum_stop_percent}});
}
return info;
}
template <typename Params>
static json common_info(const Params& p, const char* kind) {
json info = {{"schema", "sdcpp.generation/v1"}, {"kind", kind}, {"prompt", text(p.prompt)}, {"negative_prompt", text(p.negative_prompt)}, {"width", p.width}, {"height", p.height}, {"clip_skip", p.clip_skip}, {"seed", std::to_string(p.seed)}, {"sample_params", sampling_info(p.sample_params)}};
if (p.hires.enabled)
info["hires"] = hires_info(p.hires);
if (p.cache.mode != SD_CACHE_DISABLED)
info["cache"] = cache_info(p.cache);
if (p.circular_x)
info["circular_x"] = true;
if (p.circular_y)
info["circular_y"] = true;
if (p.ref_image_args && *p.ref_image_args)
info["ref_image_args"] = p.ref_image_args;
if (p.image_preprocess.rules && *p.image_preprocess.rules)
info["image_preprocess"] = {{"rules", p.image_preprocess.rules}};
const auto& t = p.vae_tiling_params;
if (t.enabled || t.temporal_tiling) {
auto& tiling = info["vae_tiling_params"];
if (t.enabled)
tiling.update({{"enabled", true}, {"tile_size_w", t.tile_size_w}, {"tile_size_h", t.tile_size_h}, {"target_overlap", t.target_overlap}, {"rel_size_w", t.rel_size_w}, {"rel_size_h", t.rel_size_h}});
if (t.temporal_tiling)
tiling["temporal_tiling"] = true;
if (t.extra_tiling_args && *t.extra_tiling_args)
tiling["extra_tiling_args"] = t.extra_tiling_args;
}
if (p.lora_count > 0) {
info["loras"] = json::array();
for (uint32_t i = 0; i < p.lora_count; ++i)
info["loras"].push_back({{"path", text(p.loras[i].path)}, {"multiplier", p.loras[i].multiplier}, {"is_high_noise", p.loras[i].is_high_noise}});
}
return info;
}
static json image_info(const sd_img_gen_params_t& p) {
auto info = common_info(p, "image");
if (p.init_image.data)
info["strength"] = p.strength;
if (p.control_image.data)
info["control_strength"] = p.control_strength;
if (p.ip_adapter_image.data)
info["ip_adapter_strength"] = p.ip_adapter_strength;
if (p.pm_params.id_images && p.pm_params.id_images_count > 0) {
info["pm_params"] = {{"style_strength", p.pm_params.style_strength}};
if (p.pm_params.id_embed_path && *p.pm_params.id_embed_path)
info["pm_params"]["id_embed_path"] = p.pm_params.id_embed_path;
}
if (p.pulid_params.id_embedding_path && *p.pulid_params.id_embedding_path)
info["pulid_params"] = {{"id_embedding_path", p.pulid_params.id_embedding_path}, {"id_weight", p.pulid_params.id_weight}};
return info;
}
template <typename Params>
static Params resolved_params(const pipeline::GenerationRequest& request, Params p) {
p.width = request.width;
p.height = request.height;
p.seed = request.seed;
p.clip_skip = request.clip_skip;
p.sample_params = request.sample_params;
p.hires = request.hires;
p.vae_tiling_params = request.vae_tiling_params;
return p;
}
json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params) {
auto p = resolved_params(request, params);
p.qwen_image_layers = request.qwen_image_layers;
auto info = image_info(p);
if (!request.models.empty())
info["models"] = request.models;
if (request.is_layered_image)
info["qwen_image_layers"] = p.qwen_image_layers;
return info;
}
json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params) {
const auto p = resolved_params(request, params);
auto info = common_info(p, "video");
if (!request.models.empty())
info["models"] = request.models;
info.update({{"video_frames", request.frames}, {"fps", request.fps}});
if (p.init_image.data || p.end_image.data)
info["strength"] = p.strength;
if (request.use_vace)
info["vace_strength"] = p.vace_strength;
if (request.use_high_noise)
info.update({{"moe_boundary", p.moe_boundary}, {"high_noise_sample_params", sampling_info(request.high_noise_sample_params, true)}});
if (p.ref_videos_count > 0) {
info["ref_videos"] = json::array();
for (int i = 0; i < p.ref_videos_count; ++i)
info["ref_videos"].push_back({{"fps", p.ref_videos[i].fps}});
}
return info;
}
json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params) {
auto inpaint = image_info(inpaint_params);
inpaint["strength"] = inpaint_params.strength;
// ADetailer clears these inputs before generating each crop.
for (const auto* key : {"control_strength", "pm_params", "pulid_params", "hires", "image_preprocess"})
inpaint.erase(key);
sd_img_gen_params_t defaults;
sd_img_gen_params_init(&defaults);
if (inpaint_params.qwen_image_layers != defaults.qwen_image_layers)
inpaint["qwen_image_layers"] = inpaint_params.qwen_image_layers;
json adetailer = json::object();
if (params.prompt && *params.prompt)
adetailer["prompt"] = params.prompt;
if (params.negative_prompt && *params.negative_prompt)
adetailer["negative_prompt"] = params.negative_prompt;
if (params.extra_ad_args && *params.extra_ad_args)
adetailer["extra_ad_args"] = params.extra_ad_args;
return {{"schema", "sdcpp.generation/v1"}, {"kind", "image"}, {"adetailer", adetailer}, {"inpaint_params", inpaint}};
}
json upscale_generation_info(uint32_t upscale_factor) {
return {{"schema", "sdcpp.generation/v1"}, {"kind", "image"}, {"upscale", {{"upscale_factor", upscale_factor}}}};
}
char* copy_generation_info(const json& info) {
const std::string text = info.dump(-1, ' ', false, json::error_handler_t::replace);
char* result = static_cast<char*>(malloc(text.size() + 1));
if (result)
memcpy(result, text.c_str(), text.size() + 1);
return result;
}
bool make_image_generation_result(sd_image_t* images, int count, const std::vector<json>& infos, sd_image_generation_result_t** result_out) {
if (!result_out || !images || count <= 0 || infos.size() != static_cast<size_t>(count)) {
free_sd_images(images, count);
return false;
}
auto* result = static_cast<sd_image_generation_result_t*>(calloc(1, sizeof(sd_image_generation_result_t)));
if (!result) {
free_sd_images(images, count);
return false;
}
result->items = static_cast<sd_image_result_t*>(calloc(count, sizeof(sd_image_result_t)));
if (!result->items) {
free(result);
free_sd_images(images, count);
return false;
}
result->count = count;
try {
for (int i = 0; i < count; ++i) {
result->items[i].image = images[i];
images[i] = {};
result->items[i].generation_info = copy_generation_info(infos[i]);
if (!result->items[i].generation_info) {
free_sd_images(images, count);
free_sd_image_generation_result(result);
return false;
}
}
} catch (...) {
free_sd_images(images, count);
free_sd_image_generation_result(result);
return false;
}
free(images);
*result_out = result;
return true;
}
}
void free_sd_image_generation_result(sd_image_generation_result_t* result) {
if (!result)
return;
for (int i = 0; i < result->count; ++i) {
free(result->items[i].image.data);
free(result->items[i].generation_info);
}
free(result->items);
free(result);
}
void free_sd_video_generation_result(sd_video_generation_result_t* result) {
if (!result)
return;
free_sd_images(result->frames, result->frame_count);
free_sd_audio(result->audio);
free(result->generation_info);
free(result);
}
src/generation_info.h
+23
View File
@@ -0,0 +1,23 @@
#ifndef __SD_GENERATION_INFO_H__
#define __SD_GENERATION_INFO_H__
#include <vector>
#include "json.hpp"
#include "stable-diffusion.h"
namespace sd::pipeline {
struct GenerationRequest;
}
namespace sd {
nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params);
nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params);
nlohmann::json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params);
nlohmann::json upscale_generation_info(uint32_t upscale_factor);
char* copy_generation_info(const nlohmann::json& info);
// Consumes images even on failure.
bool make_image_generation_result(sd_image_t* images, int count, const std::vector<nlohmann::json>& infos, sd_image_generation_result_t** result_out);
}
#endif // __SD_GENERATION_INFO_H__
src/pipeline/diffusion_engine.cpp
+31 -20
View File
@@ -675,87 +675,93 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
const auto* sd_ctx_params = &configuration.params;
auto& use_tae = configuration.use_tae;
auto& use_audio_vae = configuration.use_audio_vae;
const auto load_model = [&](const char* name, const char* path, const std::string& prefix = "") {
if (!model_loader.init_from_file(path, prefix))
return false;
configuration.record_model(name, path);
return true;
};
if (strlen(SAFE_STR(sd_ctx_params->model_path)) > 0) {
LOG_INFO("loading model from '%s'", sd_ctx_params->model_path);
if (!model_loader.init_from_file(sd_ctx_params->model_path)) {
if (!load_model("model", sd_ctx_params->model_path)) {
LOG_ERROR("init model loader from file failed: '%s'", sd_ctx_params->model_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->diffusion_model_path)) > 0) {
LOG_INFO("loading diffusion model from '%s'", sd_ctx_params->diffusion_model_path);
if (!model_loader.init_from_file(sd_ctx_params->diffusion_model_path, "model.diffusion_model.")) {
if (!load_model("diffusion_model", sd_ctx_params->diffusion_model_path, "model.diffusion_model.")) {
LOG_WARN("loading diffusion model from '%s' failed", sd_ctx_params->diffusion_model_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path)) > 0) {
LOG_INFO("loading high noise diffusion model from '%s'", sd_ctx_params->high_noise_diffusion_model_path);
if (!model_loader.init_from_file(sd_ctx_params->high_noise_diffusion_model_path, "model.high_noise_diffusion_model.")) {
if (!load_model("high_noise_diffusion_model", sd_ctx_params->high_noise_diffusion_model_path, "model.high_noise_diffusion_model.")) {
LOG_WARN("loading diffusion model from '%s' failed", sd_ctx_params->high_noise_diffusion_model_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->uncond_diffusion_model_path)) > 0) {
LOG_INFO("loading unconditional diffusion model from '%s'", sd_ctx_params->uncond_diffusion_model_path);
if (!model_loader.init_from_file(sd_ctx_params->uncond_diffusion_model_path, "model.diffusion_model.uncond.")) {
if (!load_model("uncond_diffusion_model", sd_ctx_params->uncond_diffusion_model_path, "model.diffusion_model.uncond.")) {
LOG_WARN("loading unconditional diffusion model from '%s' failed", sd_ctx_params->uncond_diffusion_model_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->clip_l_path)) > 0) {
LOG_INFO("loading clip_l from '%s'", sd_ctx_params->clip_l_path);
if (!model_loader.init_from_file(sd_ctx_params->clip_l_path, "clip_l.")) {
if (!load_model("clip_l", sd_ctx_params->clip_l_path, "clip_l.")) {
LOG_WARN("loading clip_l from '%s' failed", sd_ctx_params->clip_l_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->clip_g_path)) > 0) {
LOG_INFO("loading clip_g from '%s'", sd_ctx_params->clip_g_path);
if (!model_loader.init_from_file(sd_ctx_params->clip_g_path, "clip_g.")) {
if (!load_model("clip_g", sd_ctx_params->clip_g_path, "clip_g.")) {
LOG_WARN("loading clip_g from '%s' failed", sd_ctx_params->clip_g_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->clip_vision_path)) > 0) {
LOG_INFO("loading clip_vision from '%s'", sd_ctx_params->clip_vision_path);
if (!model_loader.init_from_file(sd_ctx_params->clip_vision_path, "clip_vision.")) {
if (!load_model("clip_vision", sd_ctx_params->clip_vision_path, "clip_vision.")) {
LOG_WARN("loading clip_vision from '%s' failed", sd_ctx_params->clip_vision_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->t5xxl_path)) > 0) {
LOG_INFO("loading t5xxl from '%s'", sd_ctx_params->t5xxl_path);
if (!model_loader.init_from_file(sd_ctx_params->t5xxl_path, "text_encoders.t5xxl.transformer.")) {
if (!load_model("t5xxl", sd_ctx_params->t5xxl_path, "text_encoders.t5xxl.transformer.")) {
LOG_WARN("loading t5xxl from '%s' failed", sd_ctx_params->t5xxl_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->pulid_weights_path)) > 0) {
LOG_INFO("loading PuLID weights from '%s'", sd_ctx_params->pulid_weights_path);
if (!model_loader.init_from_file(sd_ctx_params->pulid_weights_path,
"model.diffusion_model.")) {
if (!load_model("pulid_weights", sd_ctx_params->pulid_weights_path,
"model.diffusion_model.")) {
LOG_WARN("loading PuLID weights from '%s' failed", sd_ctx_params->pulid_weights_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->llm_path)) > 0) {
LOG_INFO("loading llm from '%s'", sd_ctx_params->llm_path);
if (!model_loader.init_from_file(sd_ctx_params->llm_path, "text_encoders.llm.")) {
if (!load_model("llm", sd_ctx_params->llm_path, "text_encoders.llm.")) {
LOG_WARN("loading llm from '%s' failed", sd_ctx_params->llm_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->llm_vision_path)) > 0) {
LOG_INFO("loading llm vision from '%s'", sd_ctx_params->llm_vision_path);
if (!model_loader.init_from_file(sd_ctx_params->llm_vision_path, "text_encoders.llm.visual.")) {
if (!load_model("llm_vision", sd_ctx_params->llm_vision_path, "text_encoders.llm.visual.")) {
LOG_WARN("loading llm vision from '%s' failed", sd_ctx_params->llm_vision_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->vae_path)) > 0) {
LOG_INFO("loading vae from '%s'", sd_ctx_params->vae_path);
if (!model_loader.init_from_file(sd_ctx_params->vae_path, "vae.")) {
if (!load_model("vae", sd_ctx_params->vae_path, "vae.")) {
LOG_WARN("loading vae from '%s' failed", sd_ctx_params->vae_path);
external_vae_is_invalid = true;
}
@@ -763,7 +769,7 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
if (strlen(SAFE_STR(sd_ctx_params->taesd_path)) > 0) {
LOG_INFO("loading tae from '%s'", sd_ctx_params->taesd_path);
if (!model_loader.init_from_file(sd_ctx_params->taesd_path, "tae.")) {
if (!load_model("taesd", sd_ctx_params->taesd_path, "tae.")) {
LOG_WARN("loading tae from '%s' failed", sd_ctx_params->taesd_path);
} else {
use_tae = true;
@@ -772,14 +778,14 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
if (strlen(SAFE_STR(sd_ctx_params->embeddings_connectors_path)) > 0) {
LOG_INFO("loading embeddings connectors from '%s'", sd_ctx_params->embeddings_connectors_path);
if (!model_loader.init_from_file(sd_ctx_params->embeddings_connectors_path)) {
if (!load_model("embeddings_connectors", sd_ctx_params->embeddings_connectors_path)) {
LOG_WARN("loading embeddings connectors from '%s' failed", sd_ctx_params->embeddings_connectors_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->audio_vae_path)) > 0) {
LOG_INFO("loading audio VAE from '%s'", sd_ctx_params->audio_vae_path);
if (!model_loader.init_from_file(sd_ctx_params->audio_vae_path)) {
if (!load_model("audio_vae", sd_ctx_params->audio_vae_path)) {
LOG_WARN("loading audio VAE weights from '%s' failed", sd_ctx_params->audio_vae_path);
} else {
use_audio_vae = true;
@@ -788,15 +794,15 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
if (strlen(SAFE_STR(sd_ctx_params->audio_encoder_path)) > 0) {
LOG_INFO("loading audio encoder (wav2vec2) from '%s'", sd_ctx_params->audio_encoder_path);
if (!model_loader.init_from_file(sd_ctx_params->audio_encoder_path, "wav2vec2.")) {
if (!load_model("audio_encoder", sd_ctx_params->audio_encoder_path, "wav2vec2.")) {
LOG_WARN("loading audio encoder weights from '%s' failed", sd_ctx_params->audio_encoder_path);
}
}
if (strlen(SAFE_STR(sd_ctx_params->motion_module_path)) > 0) {
LOG_INFO("loading motion module (AnimateDiff) from '%s'", sd_ctx_params->motion_module_path);
if (!model_loader.init_from_file(sd_ctx_params->motion_module_path,
"model.diffusion_model.motion_module.")) {
if (!load_model("motion_module", sd_ctx_params->motion_module_path,
"model.diffusion_model.motion_module.")) {
LOG_WARN("loading motion module from '%s' failed", sd_ctx_params->motion_module_path);
} else {
configuration.animatediff_loaded = true;
@@ -810,8 +816,11 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
}
}
if (configuration.control_net_file)
configuration.record_model("control_net", sd_ctx_params->control_net_path);
if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0) {
if (!model_loader.init_from_file(sd_ctx_params->ip_adapter_path)) {
if (!load_model("ip_adapter", sd_ctx_params->ip_adapter_path)) {
LOG_ERROR("init ip-adapter model loader from file failed: '%s'", sd_ctx_params->ip_adapter_path);
return false;
}
@@ -819,6 +828,8 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
if (strlen(SAFE_STR(sd_ctx_params->photo_maker_path)) > 0) {
configuration.photomaker_source_available = model_loader.add_file(sd_ctx_params->photo_maker_path, "pmid.");
if (configuration.photomaker_source_available)
configuration.record_model("photo_maker", sd_ctx_params->photo_maker_path);
if (!configuration.photomaker_source_available) {
LOG_WARN("loading stacked ID embedding from '%s' failed", sd_ctx_params->photo_maker_path);
}
src/pipeline/diffusion_engine.h
+12
View File
@@ -123,6 +123,7 @@ public:
sd_ctx_params_t params{};
std::list<std::string> strings;
std::vector<sd_embedding_t> embeddings;
std::map<std::string, std::string> models;
ModelLoader::FileId control_net_file = 0;
bool use_tae = false;
bool use_audio_vae = false;
@@ -164,13 +165,24 @@ public:
use_audio_vae = other.use_audio_vae;
photomaker_source_available = other.photomaker_source_available;
animatediff_loaded = other.animatediff_loaded;
models = other.models;
}
ModelConfig& operator=(const ModelConfig&) = delete;
void record_model(const std::string& name, const std::string& path) {
if (path.empty()) {
models.erase(name);
return;
}
const auto separator = path.find_last_of("/\\");
models[name] = separator == std::string::npos ? path : path.substr(separator + 1);
}
void set_control_net(ModelLoader::FileId id, const std::string& path) {
control_net_file = id;
strings.push_back(path);
params.control_net_path = strings.back().c_str();
record_model("control_net", path);
}
};
src/pipeline/generation.h
+5 -2
View File
@@ -2,6 +2,7 @@
#define __SD_PIPELINE_GENERATION_H__
#include "conditioning/conditioner.hpp"
#include "json.hpp"
#include "stable-diffusion.h"
class StableDiffusionGGML;
@@ -54,14 +55,16 @@ namespace sd::pipeline {
bool generate_image(StableDiffusionGGML* sd,
const sd_img_gen_params_t* sd_img_gen_params,
sd_image_t** images_out,
int* num_images_out);
int* num_images_out,
std::vector<nlohmann::json>* infos_out = nullptr);
bool generate_video(StableDiffusionGGML* sd,
const sd_vid_gen_params_t* sd_vid_gen_params,
sd_image_t** frames_out,
int* num_frames_out,
sd_audio_t** audio_out,
int* fps_out);
int* fps_out,
nlohmann::json* info_out = nullptr);
sd::Tensor<float> upscale_ltx_spatial_video_latent(StableDiffusionGGML* sd,
const char* model_path,
src/pipeline/image.cpp
+23 -6
View File
@@ -1,4 +1,5 @@
#include "generation.h"
#include "generation_info.h"
#include <algorithm>
#include <cmath>
@@ -668,7 +669,7 @@ namespace sd::pipeline {
int64_t t4 = ggml_time_ms();
LOG_INFO("decode_first_stage completed, taking %.2fs", (t4 - t0) * 1.0f / 1000);
if (decoded_images.empty()) {
if (decoded_images.empty() || cancelled) {
LOG_ERROR(cancelled ? "cancelled before any latent images were decoded" : "no decoded images");
return nullptr;
}
@@ -829,7 +830,10 @@ namespace sd::pipeline {
bool generate_image(StableDiffusionGGML* sd,
const sd_img_gen_params_t* sd_img_gen_params,
sd_image_t** images_out,
int* num_images_out) {
int* num_images_out,
std::vector<nlohmann::json>* infos_out) {
if (infos_out)
infos_out->clear();
if (images_out != nullptr) {
*images_out = nullptr;
}
@@ -851,6 +855,7 @@ namespace sd::pipeline {
int64_t t0 = ggml_time_ms();
sd->vae_tiling_params = sd_img_gen_params->vae_tiling_params;
GenerationRequest request(sd, sd_img_gen_params);
const auto info = sd::generation_info(request, *sd_img_gen_params);
sd::ImagePreprocessor preprocessing(sd_img_gen_params->image_preprocess.rules);
sd_img_gen_params_t processed_params = *sd_img_gen_params;
if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
@@ -891,7 +896,6 @@ namespace sd::pipeline {
return false;
}
ImageGenerationEmbeds embeds = std::move(*embeds_opt);
std::vector<sd::Tensor<float>> final_latents;
int64_t denoise_start = ggml_time_ms();
for (int b = 0; b < request.batch_count; b++) {
@@ -908,7 +912,7 @@ namespace sd::pipeline {
}
int64_t sampling_start = ggml_time_ms();
int64_t cur_seed = request.seed + b;
int64_t cur_seed = request.seed_for_batch(b);
LOG_INFO("generating image: %i/%i - seed %" PRId64, b + 1, request.batch_count, cur_seed);
sd->rng->manual_seed(cur_seed);
@@ -994,7 +998,7 @@ namespace sd::pipeline {
std::vector<float> hires_sigma_sched =
make_hires_sigma_schedule(sd,
request.hires,
sd_img_gen_params->sample_params,
request.sample_params,
plan.sample_method,
plan.sample_steps,
sd->get_image_seq_len(request.hires.target_height, request.hires.target_width),
@@ -1035,7 +1039,7 @@ namespace sd::pipeline {
LOG_ERROR("cancelling generation during hires fix");
return false;
}
int64_t cur_seed = request.seed + b;
int64_t cur_seed = request.seed_for_batch(b);
sd->rng->manual_seed(cur_seed);
sd->sampler_rng->manual_seed(cur_seed);
@@ -1113,6 +1117,19 @@ namespace sd::pipeline {
return false;
}
if (infos_out) {
try {
infos_out->assign(num_images, info);
// Layers and frames decoded from the same latent share its seed.
const int images_per_latent = num_images / static_cast<int>(final_latents.size());
for (int i = 0; i < num_images; ++i) {
(*infos_out)[i]["seed"] = std::to_string(request.seed_for_batch(i / images_per_latent));
}
} catch (...) {
free_sd_images(result, num_images);
throw;
}
}
sd->lora_stat();
int64_t t1 = ggml_time_ms();
src/pipeline/request.cpp
+82 -19
View File
@@ -1,4 +1,5 @@
#include "request.h"
#include "conditioning/conditioner.hpp"
#include <algorithm>
#include <cmath>
@@ -6,6 +7,8 @@
#include <ctime>
#include "diffusion_engine.h"
#include "generation.h"
#include "model/vae/vae.hpp"
#include "model_definition.h"
#include "runtime/denoiser.hpp"
@@ -66,7 +69,7 @@ namespace sd::pipeline {
return DISCRETE_SCHEDULER;
}
static int64_t resolve_seed(int64_t seed) {
int64_t resolve_seed(int64_t seed) {
if (seed >= 0) {
return seed;
}
@@ -127,6 +130,7 @@ namespace sd::pipeline {
seed = sd_img_gen_params->seed;
batch_count = sd_img_gen_params->batch_count;
qwen_image_layers = std::max(0, sd_img_gen_params->qwen_image_layers);
is_layered_image = sd->version == VERSION_QWEN_IMAGE_LAYERED;
clip_skip = sd_img_gen_params->clip_skip;
shifted_timestep = sd_img_gen_params->sample_params.shifted_timestep;
strength = sd_img_gen_params->strength;
@@ -137,8 +141,18 @@ namespace sd::pipeline {
pm_params = sd_img_gen_params->pm_params;
pulid_params = sd_img_gen_params->pulid_params;
hires = sd_img_gen_params->hires;
vae_tiling_params = sd_img_gen_params->vae_tiling_params;
cache_params = &sd_img_gen_params->cache;
sample_params = sd_img_gen_params->sample_params;
resolve(sd);
if (!sd_img_gen_params->control_image.data)
models.erase("control_net");
if (!sd_img_gen_params->ip_adapter_image.data)
models.erase("ip_adapter");
if (!pm_params.id_images || pm_params.id_images_count <= 0)
models.erase("photo_maker");
if (!pulid_params.id_embedding_path || !*pulid_params.id_embedding_path)
models.erase("pulid_weights");
}
GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params) {
@@ -166,7 +180,20 @@ namespace sd::pipeline {
guidance = sd_vid_gen_params->sample_params.guidance;
high_noise_guidance = sd_vid_gen_params->high_noise_sample_params.guidance;
hires = sd_vid_gen_params->hires;
vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
sample_params = sd_vid_gen_params->sample_params;
high_noise_sample_params = sd_vid_gen_params->high_noise_sample_params;
use_high_noise = sd->high_noise_diffusion_model != nullptr;
use_vace = sd->diffusion_model && (sd->diffusion_model->get_desc() == "Wan2.1-VACE-1.3B" ||
sd->diffusion_model->get_desc() == "Wan2.x-VACE-14B");
shifted_timestep = sample_params.shifted_timestep;
if (sd->config_ && sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd->version))
frames = std::min(requested_frames, 32);
resolve(sd);
models.erase("control_net");
models.erase("ip_adapter");
models.erase("photo_maker");
models.erase("pulid_weights");
if (frames != requested_frames) {
LOG_WARN("align video frames from %d to %d for %s",
requested_frames,
@@ -201,6 +228,10 @@ namespace sd::pipeline {
spatial_multiple);
}
int64_t GenerationRequest::seed_for_batch(int index) const {
return static_cast<int64_t>((static_cast<uint64_t>(seed) + index) & INT64_MAX);
}
void GenerationRequest::resolve_hires() {
if (!hires.enabled) {
return;
@@ -307,6 +338,23 @@ namespace sd::pipeline {
}
void GenerationRequest::resolve(StableDiffusionGGML* sd) {
if (sd->config_) {
models = sd->config_->models;
if (!use_high_noise)
models.erase("high_noise_diffusion_model");
if (!sd->clip_vision)
models.erase("clip_vision");
if (!sd->audio_vae_model || frames <= 0)
models.erase("audio_vae");
if (!sd->audio_encoder || frames <= 0)
models.erase("audio_encoder");
if (!sd->config_->animatediff_loaded || !sd_version_supports_animatediff(sd->version) || frames <= 0)
models.erase("motion_module");
if (sd->uses_tae() && !sd->tae_preview_only())
models.erase("vae");
else
models.erase("taesd");
}
if (sd_version_is_minit2i(sd->version) && (width != 512 || height != 512)) {
width = 512;
height = 512;
@@ -314,7 +362,11 @@ namespace sd::pipeline {
}
align_generation_request_size();
resolve_hires();
if (sd->first_stage_model)
vae_tiling_params = sd->first_stage_model->resolve_tiling_params(vae_tiling_params);
seed = resolve_seed(seed);
if (sd->cond_stage_model)
clip_skip = sd->cond_stage_model->resolve_clip_skip(clip_skip);
resolve_guidance(sd, &guidance, &use_uncond, &use_img_uncond, has_ref_images);
if (sd->high_noise_diffusion_model) {
@@ -330,33 +382,44 @@ namespace sd::pipeline {
LOG_WARN("timestep shifting is only supported for SDXL models!");
shifted_timestep = 0;
}
sample_params.guidance = guidance;
sample_params.shifted_timestep = shifted_timestep;
sample_params.sample_method = resolve_sample_method(sd, sample_params.sample_method);
sample_params.scheduler = resolve_scheduler(sd, sample_params.scheduler, sample_params.sample_method);
sample_params.eta = resolve_eta(sd, sample_params.eta, sample_params.sample_method);
if (frames > 0) {
high_noise_sample_params.guidance = high_noise_guidance;
high_noise_sample_params.sample_method = resolve_sample_method(sd, high_noise_sample_params.sample_method);
high_noise_sample_params.scheduler = resolve_scheduler(sd, high_noise_sample_params.scheduler, high_noise_sample_params.sample_method);
high_noise_sample_params.eta = resolve_eta(sd, high_noise_sample_params.eta, high_noise_sample_params.sample_method);
}
}
SamplePlan::SamplePlan(StableDiffusionGGML* sd,
const sd_img_gen_params_t* sd_img_gen_params,
const GenerationRequest& request) {
sample_method = sd_img_gen_params->sample_params.sample_method;
extra_sample_args = sd_img_gen_params->sample_params.extra_sample_args;
eta = sd_img_gen_params->sample_params.eta;
sample_steps = sd_img_gen_params->sample_params.sample_steps;
resolve(sd, &request, &sd_img_gen_params->sample_params);
sample_method = request.sample_params.sample_method;
extra_sample_args = request.sample_params.extra_sample_args;
eta = request.sample_params.eta;
sample_steps = request.sample_params.sample_steps;
resolve(sd, &request, &request.sample_params);
}
SamplePlan::SamplePlan(StableDiffusionGGML* sd,
const sd_vid_gen_params_t* sd_vid_gen_params,
const GenerationRequest& request) {
sample_method = sd_vid_gen_params->sample_params.sample_method;
extra_sample_args = sd_vid_gen_params->sample_params.extra_sample_args;
eta = sd_vid_gen_params->sample_params.eta;
sample_steps = sd_vid_gen_params->sample_params.sample_steps;
sample_method = request.sample_params.sample_method;
extra_sample_args = request.sample_params.extra_sample_args;
eta = request.sample_params.eta;
sample_steps = request.sample_params.sample_steps;
if (sd->high_noise_diffusion_model) {
high_noise_sample_steps = sd_vid_gen_params->high_noise_sample_params.sample_steps;
high_noise_sample_method = sd_vid_gen_params->high_noise_sample_params.sample_method;
high_noise_extra_sample_args = sd_vid_gen_params->high_noise_sample_params.extra_sample_args;
high_noise_eta = sd_vid_gen_params->high_noise_sample_params.eta;
high_noise_sample_steps = request.high_noise_sample_params.sample_steps;
high_noise_sample_method = request.high_noise_sample_params.sample_method;
high_noise_extra_sample_args = request.high_noise_sample_params.extra_sample_args;
high_noise_eta = request.high_noise_sample_params.eta;
}
moe_boundary = sd_vid_gen_params->moe_boundary;
resolve(sd, &request, &sd_vid_gen_params->sample_params);
resolve(sd, &request, &request.sample_params);
}
void SamplePlan::resolve(StableDiffusionGGML* sd,
@@ -382,10 +445,10 @@ namespace sd::pipeline {
}
}
} else {
scheduler_t scheduler = resolve_scheduler(sd,
sample_params->scheduler,
sample_method);
int sample_seq_len = sd->get_image_seq_len(request->height, request->width);
scheduler = resolve_scheduler(sd,
sample_params->scheduler,
sample_method);
int sample_seq_len = sd->get_image_seq_len(request->height, request->width);
if (sd_version_is_ltxav(sd->version) && request->frames > 0) {
int latent_frames = ((request->frames - 1) / 8) + 1;
sample_seq_len *= latent_frames;
src/pipeline/request.h
+40 -27
View File
@@ -1,6 +1,7 @@
#ifndef __SD_PIPELINE_REQUEST_H__
#define __SD_PIPELINE_REQUEST_H__
#include <map>
#include <string>
#include <vector>
@@ -16,6 +17,8 @@ namespace sd::pipeline {
enum scheduler_t default_scheduler(const StableDiffusionGGML* sd, enum sample_method_t sample_method);
int64_t resolve_seed(int64_t seed);
float resolve_eta(StableDiffusionGGML* sd,
float eta,
enum sample_method_t sample_method);
@@ -23,38 +26,47 @@ namespace sd::pipeline {
struct GenerationRequest {
std::string prompt;
std::string negative_prompt;
int width = -1;
int height = -1;
int clip_skip = -1;
int vae_scale_factor = -1;
int diffusion_model_down_factor = -1;
int64_t seed = -1;
bool use_uncond = false;
bool use_img_uncond = false;
bool use_high_noise_uncond = false;
bool use_high_noise_img_uncond = false;
bool has_ref_images = false;
const sd_cache_params_t* cache_params = nullptr;
int batch_count = 1;
int qwen_image_layers = 3;
int shifted_timestep = 0;
float strength = 1.f;
float control_strength = 0.f;
float eta = 0.f;
sd_guidance_params_t guidance = {};
sd_guidance_params_t high_noise_guidance = {};
sd_pm_params_t pm_params = {};
sd_pulid_params_t pulid_params = {};
sd_hires_params_t hires = {};
int frames = -1;
int requested_frames = -1;
int fps = 16;
float vace_strength = 1.f;
std::map<std::string, std::string> models;
int width = -1;
int height = -1;
int clip_skip = -1;
int vae_scale_factor = -1;
int diffusion_model_down_factor = -1;
int64_t seed = -1;
bool use_uncond = false;
bool use_img_uncond = false;
bool use_high_noise_uncond = false;
bool use_high_noise_img_uncond = false;
bool has_ref_images = false;
bool is_layered_image = false;
bool use_high_noise = false;
bool use_vace = false;
const sd_cache_params_t* cache_params = nullptr;
int batch_count = 1;
int qwen_image_layers = 3;
int shifted_timestep = 0;
float strength = 1.f;
float control_strength = 0.f;
float eta = 0.f;
sd_sample_params_t sample_params = {};
sd_sample_params_t high_noise_sample_params = {};
sd_guidance_params_t guidance = {};
sd_guidance_params_t high_noise_guidance = {};
sd_pm_params_t pm_params = {};
sd_pulid_params_t pulid_params = {};
sd_hires_params_t hires = {};
sd_tiling_params_t vae_tiling_params = {};
int frames = -1;
int requested_frames = -1;
int fps = 16;
float vace_strength = 1.f;
GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params);
GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params);
int64_t seed_for_batch(int index) const;
void align_generation_request_size();
void align_image_size(int* target_width, int* target_height, const char* label);
@@ -81,6 +93,7 @@ namespace sd::pipeline {
int sample_steps = 0;
int high_noise_sample_steps = 0;
int total_steps = 0;
scheduler_t scheduler = SCHEDULER_COUNT;
float moe_boundary = 0.f;
std::vector<float> sigmas;
src/pipeline/video.cpp
+33 -9
View File
@@ -1,4 +1,5 @@
#include "generation.h"
#include "generation_info.h"
#include <algorithm>
#include <cmath>
@@ -1575,7 +1576,13 @@ namespace sd::pipeline {
img_gen_params.ref_image_args = sd_vid_gen_params->ref_image_args;
sd->animatediff_num_frames = n_frames;
bool ok = generate_image(sd, &img_gen_params, frames_out, num_frames_out);
bool ok = false;
try {
ok = generate_image(sd, &img_gen_params, frames_out, num_frames_out);
} catch (...) {
sd->animatediff_num_frames = 0;
throw;
}
sd->animatediff_num_frames = 0;
return ok;
}
@@ -1585,14 +1592,24 @@ namespace sd::pipeline {
sd_image_t** frames_out,
int* num_frames_out,
sd_audio_t** audio_out,
int* fps_out) {
int* fps_out,
nlohmann::json* info_out) {
if (info_out)
*info_out = nlohmann::json();
GenerationRequest request(sd, sd_vid_gen_params);
const auto info = sd::generation_info(request, *sd_vid_gen_params);
if (sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd->version)) {
LOG_INFO("AnimateDiff dispatch: %d frames, %dx%d",
sd_vid_gen_params->video_frames, sd_vid_gen_params->width, sd_vid_gen_params->height);
if (fps_out != nullptr) {
*fps_out = std::max(1, sd_vid_gen_params->fps);
}
return generate_animatediff_video(sd, sd_vid_gen_params, frames_out, num_frames_out);
auto params = *sd_vid_gen_params;
params.seed = request.seed;
const bool ok = generate_animatediff_video(sd, &params, frames_out, num_frames_out);
if (ok && info_out)
*info_out = info;
return ok;
}
sd->reset_cancel_flag();
@@ -1602,7 +1619,6 @@ namespace sd::pipeline {
int64_t t0 = ggml_time_ms();
sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
sd->apply_circular_axes(sd_vid_gen_params->circular_x, sd_vid_gen_params->circular_y);
GenerationRequest request(sd, sd_vid_gen_params);
sd::ImagePreprocessor preprocessing(sd_vid_gen_params->image_preprocess.rules);
sd_vid_gen_params_t processed_params = *sd_vid_gen_params;
if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
@@ -1734,9 +1750,9 @@ namespace sd::pipeline {
embeds.img_uncond,
sd::Tensor<float>(),
0.f,
sd_vid_gen_params->sample_params.guidance,
request.guidance,
plan.eta,
sd_vid_gen_params->sample_params.shifted_timestep,
request.shifted_timestep,
plan.sample_method,
sd->is_flow_denoiser(),
plan.extra_sample_args,
@@ -1846,7 +1862,7 @@ namespace sd::pipeline {
std::vector<float> hires_sigma_sched =
make_hires_sigma_schedule(sd,
request.hires,
sd_vid_gen_params->sample_params,
request.sample_params,
hires_sample_method,
plan.sample_steps,
sd->get_image_seq_len(hires_request.height, hires_request.width) * T,
@@ -1873,9 +1889,9 @@ namespace sd::pipeline {
embeds.img_uncond,
sd::Tensor<float>(),
0.f,
sd_vid_gen_params->sample_params.guidance,
request.guidance,
hires_eta,
sd_vid_gen_params->sample_params.shifted_timestep,
request.shifted_timestep,
hires_sample_method,
sd->is_flow_denoiser(),
plan.extra_sample_args,
@@ -2007,6 +2023,14 @@ namespace sd::pipeline {
generated_audio->sample_count = want_samples;
}
}
if (info_out) {
try {
*info_out = info;
} catch (...) {
free_sd_audio(generated_audio);
throw;
}
}
if (audio_out != nullptr) {
*audio_out = generated_audio;
} else {
src/stable-diffusion.cpp
+48 -28
View File
@@ -1,4 +1,5 @@
#include "stable-diffusion.h"
#include "generation_info.h"
#include <algorithm>
#include <cinttypes>
@@ -752,43 +753,62 @@ enum scheduler_t sd_get_default_scheduler(const sd_ctx_t* sd_ctx, enum sample_me
SD_API bool generate_image(sd_ctx_t* sd_ctx,
const sd_img_gen_params_t* params,
sd_image_t** images_out,
int* num_images_out) {
if (images_out != nullptr)
*images_out = nullptr;
if (num_images_out != nullptr)
*num_images_out = 0;
if (sd_ctx == nullptr || sd_ctx->sd == nullptr || params == nullptr) {
sd_image_generation_result_t** result_out) {
if (result_out)
*result_out = nullptr;
if (!result_out || !sd_ctx || !sd_ctx->sd || !params)
return false;
StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd);
if (!execution.ready)
return false;
sd_image_t* images = nullptr;
int count = 0;
std::vector<nlohmann::json> infos;
try {
if (!sd::pipeline::generate_image(sd_ctx->sd, params, &images, &count, &infos)) {
free_sd_images(images, count);
return false;
}
} catch (const std::exception& e) {
LOG_ERROR("image generation failed: %s", e.what());
free_sd_images(images, count);
return false;
}
StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd);
return execution.ready && sd::pipeline::generate_image(sd_ctx->sd, params, images_out, num_images_out);
return sd::make_image_generation_result(images, count, infos, result_out);
}
SD_API bool generate_video(sd_ctx_t* sd_ctx,
const sd_vid_gen_params_t* sd_vid_gen_params,
sd_image_t** frames_out,
int* num_frames_out,
sd_audio_t** audio_out,
int* fps_out) {
if (frames_out != nullptr)
*frames_out = nullptr;
if (audio_out != nullptr)
*audio_out = nullptr;
if (num_frames_out != nullptr)
*num_frames_out = 0;
if (fps_out != nullptr)
*fps_out = 0;
if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_vid_gen_params == nullptr) {
const sd_vid_gen_params_t* params,
sd_video_generation_result_t** result_out) {
if (result_out)
*result_out = nullptr;
if (!result_out || !sd_ctx || !sd_ctx->sd || !params)
return false;
}
StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd);
if (!execution.ready) {
if (!execution.ready)
return false;
auto* result = static_cast<sd_video_generation_result_t*>(calloc(1, sizeof(sd_video_generation_result_t)));
if (!result)
return false;
try {
nlohmann::json info;
if (!sd::pipeline::generate_video(sd_ctx->sd, params, &result->frames, &result->frame_count,
&result->audio, &result->fps, &info)) {
free_sd_video_generation_result(result);
return false;
}
result->generation_info = sd::copy_generation_info(info);
if (!result->generation_info) {
free_sd_video_generation_result(result);
return false;
}
} catch (const std::exception& e) {
LOG_ERROR("video generation failed: %s", e.what());
free_sd_video_generation_result(result);
return false;
}
return sd::pipeline::generate_video(sd_ctx->sd, sd_vid_gen_params, frames_out, num_frames_out, audio_out, fps_out);
*result_out = result;
return true;
}
SD_API void free_sd_images(sd_image_t* result_images, int num_images) {
src/upscaler.cpp
+16 -20
View File
@@ -1,6 +1,7 @@
#include "upscaler.h"
#include "core/ggml_extend_backend.h"
#include "core/util.h"
#include "generation_info.h"
#include "model_loader.h"
#include "runtime/tiling.h"
#include "stable-diffusion.h"
@@ -259,14 +260,11 @@ upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path_c_str,
bool upscale(upscaler_ctx_t* upscaler_ctx,
sd_image_t input_image,
uint32_t upscale_factor,
sd_image_t** images_out,
int* num_images_out) {
if (images_out != nullptr) {
*images_out = nullptr;
}
if (num_images_out != nullptr) {
*num_images_out = 0;
}
sd_image_generation_result_t** result_out) try {
if (result_out)
*result_out = nullptr;
if (!result_out)
return false;
if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr) {
return false;
}
@@ -276,21 +274,17 @@ bool upscale(upscaler_ctx_t* upscaler_ctx,
return false;
}
auto release_images = [](sd_image_t* images) { free_sd_images(images, 1); };
std::unique_ptr<sd_image_t, decltype(release_images)> image_owner(result_images, release_images);
const std::vector<nlohmann::json> infos{sd::upscale_generation_info(upscale_factor)};
result_images[0] = upscaler_ctx->upscaler->upscale(input_image, upscale_factor);
if (result_images[0].data == nullptr) {
free(result_images);
if (!result_images[0].data)
return false;
}
if (num_images_out != nullptr) {
*num_images_out = 1;
}
if (images_out != nullptr) {
*images_out = result_images;
} else {
free_sd_images(result_images, 1);
}
return true;
return sd::make_image_generation_result(image_owner.release(), 1, infos, result_out);
} catch (const std::exception& e) {
LOG_ERROR("upscale failed: %s", e.what());
return false;
}
int get_upscale_factor(upscaler_ctx_t* upscaler_ctx) {
@@ -329,6 +323,8 @@ int get_upscaler_model_scale(const char* model_path) {
}
void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx) {
if (!upscaler_ctx)
return;
if (upscaler_ctx->upscaler != nullptr) {
delete upscaler_ctx->upscaler;
upscaler_ctx->upscaler = nullptr;