DarthAffe/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-09-24 20:20:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: use carrier sampling for MiniMax H3 audio (#1924)

Browse Source
This commit is contained in:
leejet 2026-08-30 21:57:02 +08:00 committed by GitHub
parent 9029655a54
commit 40e605f3f1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 98 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
src/model/diffusion/minimax_h3.hpp
View File

@ -123,25 +123,6 @@ namespace MiniMaxH3 {
return to_shift * base / (1.f + (to_shift - 1.f) * base); return to_shift * base / (1.f + (to_shift - 1.f) * base);
} }
static float time_shift_slope(float sigma, float from_shift, float to_shift) {
float base = sigma / (from_shift + sigma * (1.f - from_shift));
float a = 1.f + (from_shift - 1.f) * base;
float b = 1.f + (to_shift - 1.f) * base;
return to_shift * a * a / (from_shift * b * b);
}
static float time_shift_step_scale(float sigma,
float next_sigma,
float from_shift,
float to_shift) {
if (!std::isfinite(next_sigma) || next_sigma < 0.f || next_sigma == sigma) {
return time_shift_slope(sigma, from_shift, to_shift);
}
float shifted_sigma = time_shift_sigma(sigma, from_shift, to_shift);
float shifted_next_sigma = time_shift_sigma(next_sigma, from_shift, to_shift);
return (shifted_sigma - shifted_next_sigma) / (sigma - next_sigma);
}
struct TimeEmbedder : public GGMLBlock { struct TimeEmbedder : public GGMLBlock {
TimeEmbedder(int64_t input_dim, int64_t hidden_dim, int64_t output_dim) { TimeEmbedder(int64_t input_dim, int64_t hidden_dim, int64_t output_dim) {
blocks["proj_in"] = std::make_shared<Linear>(input_dim, hidden_dim, true, true); blocks["proj_in"] = std::make_shared<Linear>(input_dim, hidden_dim, true, true);
@ -606,8 +587,7 @@ namespace MiniMaxH3 {
const std::vector<TokenModulationSpan>& segments, const std::vector<TokenModulationSpan>& segments,
const std::vector<SequenceSegment>& sequence_segments, const std::vector<SequenceSegment>& sequence_segments,
const TokenModulationSpan& video_segment, const TokenModulationSpan& video_segment,
const TokenModulationSpan& audio_segment, const TokenModulationSpan& audio_segment) {
float audio_slope) {
auto video_proj = std::dynamic_pointer_cast<Linear>(blocks["video_patch_proj"]); auto video_proj = std::dynamic_pointer_cast<Linear>(blocks["video_patch_proj"]);
auto audio_proj = std::dynamic_pointer_cast<Linear>(blocks["audio_patch_proj"]); auto audio_proj = std::dynamic_pointer_cast<Linear>(blocks["audio_patch_proj"]);
@ -727,7 +707,7 @@ namespace MiniMaxH3 {
audio->ne[2]); audio->ne[2]);
audio_out = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, audio_out, 1, 2, 0, 3)); audio_out = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, audio_out, 1, 2, 0, 3));
video_out = ggml_ext_scale(ctx->ggml_ctx, video_out, -1.f); video_out = ggml_ext_scale(ctx->ggml_ctx, video_out, -1.f);
audio_out = ggml_ext_scale(ctx->ggml_ctx, audio_out, -audio_slope); audio_out = ggml_ext_scale(ctx->ggml_ctx, audio_out, -1.f);
return {video_out, audio_out}; return {video_out, audio_out};
} }
}; };
@ -1045,17 +1025,16 @@ namespace MiniMaxH3 {
const std::vector<MiniMaxH3ReferenceBlock>& reference_blocks, const std::vector<MiniMaxH3ReferenceBlock>& reference_blocks,
int audio_length, int audio_length,
float video_shift, float video_shift,
float audio_shift, float audio_shift) {
float next_video_sigma) {
auto split = split_av_latents(packed, audio_length); auto split = split_av_latents(packed, audio_length);
video_input_cache = std::move(split.first); video_input_cache = std::move(split.first);
audio_input_cache = std::move(split.second); audio_input_cache = std::move(split.second);
GGML_ASSERT(!audio_input_cache.empty()); GGML_ASSERT(!audio_input_cache.empty());
GGML_ASSERT(!context_tensor.empty()); GGML_ASSERT(!context_tensor.empty());
auto video = make_input(video_input_cache); auto video = make_input(video_input_cache);
auto audio = make_input(audio_input_cache); auto audio_carrier = make_input(audio_input_cache);
auto context = make_input(context_tensor); auto context = make_input(context_tensor);
std::vector<ggml_tensor*> condition_inputs; std::vector<ggml_tensor*> condition_inputs;
condition_inputs.reserve(condition_videos.size()); condition_inputs.reserve(condition_videos.size());
for (const auto& condition : condition_videos) { for (const auto& condition : condition_videos) {
@ -1067,21 +1046,26 @@ namespace MiniMaxH3 {
audio_condition_inputs.push_back(make_input(condition)); audio_condition_inputs.push_back(make_input(condition));
} }
float sigma_v = std::clamp(timestep[0] / 1000.f, 1e-6f, 1.f); float sigma_v = std::clamp(timestep[0] / 1000.f, 1e-6f, 1.f);
float t_v = 1.f - sigma_v; float sigma_a = time_shift_sigma(sigma_v, video_shift, audio_shift);
float t_a = 1.f - time_shift_sigma(sigma_v, video_shift, audio_shift); float audio_scale = video_shift / audio_shift;
auto layout = build_layout(context_tensor.shape()[1], float t_v = 1.f - sigma_v;
video_input_cache.shape()[2], float t_a = 1.f - sigma_a;
video_input_cache.shape()[1], // The sampler carries c_a = (sigma_v / sigma_a) * x_a so the packed
video_input_cache.shape()[0], // latent follows one sigma schedule. Restore x_a for the H3 network.
audio_length, auto audio = ggml_ext_scale(compute_ctx, audio_carrier, sigma_a / sigma_v);
condition_videos, auto layout = build_layout(context_tensor.shape()[1],
condition_audios, video_input_cache.shape()[2],
keyframe_indices, video_input_cache.shape()[1],
reference_blocks, video_input_cache.shape()[0],
text_tags, audio_length,
t_v, condition_videos,
t_a); condition_audios,
keyframe_indices,
reference_blocks,
text_tags,
t_v,
t_a);
position_input_cache = sd::Tensor<float>( position_input_cache = sd::Tensor<float>(
{3, static_cast<int64_t>(layout.positions.size() / 3)}, {3, static_cast<int64_t>(layout.positions.size() / 3)},
@ -1142,19 +1126,15 @@ namespace MiniMaxH3 {
layout.segments, layout.segments,
layout.sequence_segments, layout.sequence_segments,
layout.video_segment, layout.video_segment,
layout.audio_segment, layout.audio_segment);
// The generic Euler sampler advances the packed tensor by // Convert the model's audio velocity to d(c_a) / d(sigma_v).
// `next_video_sigma - sigma_v`. For that sampler, scale H3's output.second = ggml_add(compute_ctx,
// audio velocity by the exact ratio of the independent audio ggml_ext_scale(compute_ctx, audio, 1.f - audio_scale),
// step. The derivative approximation substantially oversteps ggml_ext_scale(compute_ctx,
// at low step counts (the Turbo use case). Retain the local output.second,
// slope for samplers that make extra/intermediate evaluations. 1.f + (audio_scale - 1.f) * sigma_a));
time_shift_step_scale(sigma_v, auto merged = merge_av_latents(compute_ctx, output.first, output.second);
next_video_sigma, auto graph = new_graph_custom(H3_GRAPH_SIZE);
video_shift,
audio_shift));
auto merged = merge_av_latents(compute_ctx, output.first, output.second);
auto graph = new_graph_custom(H3_GRAPH_SIZE);
ggml_build_forward_expand(graph, merged); ggml_build_forward_expand(graph, merged);
return graph; return graph;
} }
@ -1184,8 +1164,7 @@ namespace MiniMaxH3 {
reference_blocks, reference_blocks,
extra->audio_length, extra->audio_length,
extra->video_sigma_shift, extra->video_sigma_shift,
extra->audio_sigma_shift, extra->audio_sigma_shift);
extra->next_video_sigma);
}; };
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph,
n_threads, n_threads,

2
src/model/diffusion/model.hpp
View File

@ -108,8 +108,6 @@ struct MiniMaxH3DiffusionExtra {
int audio_length = 0; int audio_length = 0;
float video_sigma_shift = 12.f; float video_sigma_shift = 12.f;
float audio_sigma_shift = 3.f; float audio_sigma_shift = 3.f;
// Negative when the outer sampler is not a single-evaluation Euler step.
float next_video_sigma = -1.f;
}; };
struct MiniT2IDiffusionExtra { struct MiniT2IDiffusionExtra {

44
src/runtime/denoiser.hpp
View File

@ -1043,6 +1043,16 @@ struct Denoiser {
const sd::Tensor<float>& latent) = 0; const sd::Tensor<float>& latent) = 0;
virtual float noise_level_to_sigma(float noise_level) = 0; virtual float noise_level_to_sigma(float noise_level) = 0;
virtual sd::Tensor<float> process_latent_in(const sd::Tensor<float>& latent) {
// An empty result means the original latent can be used unchanged.
SD_UNUSED(latent);
return {};
}
virtual sd::Tensor<float> process_latent_out(sd::Tensor<float> latent) {
return latent;
}
virtual std::vector<float> get_sigmas(uint32_t n, int image_seq_len, scheduler_t scheduler_type, SDVersion version, const char* extra_sample_args = nullptr) { virtual std::vector<float> get_sigmas(uint32_t n, int image_seq_len, scheduler_t scheduler_type, SDVersion version, const char* extra_sample_args = nullptr) {
auto bound_t_to_sigma = std::bind(&Denoiser::t_to_sigma, this, std::placeholders::_1); auto bound_t_to_sigma = std::bind(&Denoiser::t_to_sigma, this, std::placeholders::_1);
std::shared_ptr<SigmaScheduler> scheduler; std::shared_ptr<SigmaScheduler> scheduler;
@ -1286,6 +1296,40 @@ struct DiscreteFlowDenoiser : public Denoiser {
} }
}; };
struct H3AVFlowDenoiser : public DiscreteFlowDenoiser {
int64_t video_channels;
float audio_shift;
H3AVFlowDenoiser(float shift, float audio_shift, int64_t video_channels)
: DiscreteFlowDenoiser(shift),
video_channels(video_channels),
audio_shift(audio_shift) {
GGML_ASSERT(shift > 0.f && audio_shift > 0.f && video_channels > 0);
}
sd::Tensor<float> process_latent_in(const sd::Tensor<float>& latent) override {
return scale_audio(latent, shift / audio_shift);
}
sd::Tensor<float> process_latent_out(sd::Tensor<float> latent) override {
auto transformed = scale_audio(latent, audio_shift / shift);
if (transformed.empty()) {
return latent;
}
return transformed;
}
private:
sd::Tensor<float> scale_audio(const sd::Tensor<float>& latent, float scale) const {
if (scale == 1.f || latent.dim() < 4 || latent.shape()[3] <= video_channels) {
return {};
}
auto video = sd::ops::slice(latent, 3, 0, video_channels);
auto audio = sd::ops::slice(latent, 3, video_channels, latent.shape()[3]) * scale;
return sd::ops::concat(video, audio, 3);
}
};
struct FluxFlowDenoiser : public DiscreteFlowDenoiser { struct FluxFlowDenoiser : public DiscreteFlowDenoiser {
FluxFlowDenoiser() = default; FluxFlowDenoiser() = default;

32
src/stable-diffusion.cpp
View File

@ -1861,6 +1861,9 @@ public:
if (sd_version_is_ltxav(version)) { if (sd_version_is_ltxav(version)) {
LOG_INFO("running in LTXAV FLOW mode"); LOG_INFO("running in LTXAV FLOW mode");
denoiser = std::make_shared<FluxFlowDenoiser>(); denoiser = std::make_shared<FluxFlowDenoiser>();
} else if (sd_version_is_minimax_h3(version)) {
LOG_INFO("running in MiniMax H3 AV FLOW mode");
denoiser = std::make_shared<H3AVFlowDenoiser>(default_flow_shift, 3.f, get_latent_channel());
} else { } else {
LOG_INFO("running in FLOW mode"); LOG_INFO("running in FLOW mode");
denoiser = std::make_shared<DiscreteFlowDenoiser>(); denoiser = std::make_shared<DiscreteFlowDenoiser>();
@ -2623,10 +2626,14 @@ public:
int64_t last_progress_us = ggml_time_us(); int64_t last_progress_us = ggml_time_us();
SamplePreviewContext preview = prepare_sample_preview_context(); SamplePreviewContext preview = prepare_sample_preview_context();
sd::Tensor<float> x_t = !noise.empty() sd::Tensor<float> processed_init_latent = denoiser->process_latent_in(init_latent);
? denoiser->noise_scaling(sigmas[0], noise, init_latent) const sd::Tensor<float>& sampling_init_latent = processed_init_latent.empty()
: init_latent; ? init_latent
sd::Tensor<float> denoised = x_t; : processed_init_latent;
sd::Tensor<float> x_t = !noise.empty()
? denoiser->noise_scaling(sigmas[0], noise, sampling_init_latent)
: sampling_init_latent;
sd::Tensor<float> denoised = x_t;
auto denoise = [&](const sd::Tensor<float>& x, float sigma, int step) -> sd::guidance::GuiderOutput { auto denoise = [&](const sd::Tensor<float>& x, float sigma, int step) -> sd::guidance::GuiderOutput {
if (get_cancel_flag() == SD_CANCEL_ALL) { if (get_cancel_flag() == SD_CANCEL_ALL) {
@ -2656,10 +2663,10 @@ public:
std::vector<float> timesteps_vec = base_timesteps_vec; std::vector<float> timesteps_vec = base_timesteps_vec;
sd::Tensor<float> audio_timesteps_tensor; sd::Tensor<float> audio_timesteps_tensor;
if (sd_version_is_ltxav(version) && !denoise_mask.empty()) { if (sd_version_is_ltxav(version) && !denoise_mask.empty()) {
timesteps_vec = process_ltxav_video_timesteps(base_timesteps_vec, init_latent, denoise_mask); timesteps_vec = process_ltxav_video_timesteps(base_timesteps_vec, sampling_init_latent, denoise_mask);
audio_timesteps_tensor = sd::Tensor<float>({static_cast<int64_t>(base_timesteps_vec.size())}, base_timesteps_vec); audio_timesteps_tensor = sd::Tensor<float>({static_cast<int64_t>(base_timesteps_vec.size())}, base_timesteps_vec);
} else { } else {
timesteps_vec = process_timesteps(timesteps_vec, init_latent, denoise_mask, step); timesteps_vec = process_timesteps(timesteps_vec, sampling_init_latent, denoise_mask, step);
} }
const std::vector<float>& scaling_timesteps_vec = (sd_version_is_ltxav(version) && !denoise_mask.empty()) const std::vector<float>& scaling_timesteps_vec = (sd_version_is_ltxav(version) && !denoise_mask.empty())
? base_timesteps_vec ? base_timesteps_vec
@ -2674,13 +2681,13 @@ public:
} }
sd::Tensor<float> noised_input = x * c_in; sd::Tensor<float> noised_input = x * c_in;
if (!denoise_mask.empty() && (version == VERSION_WAN2_2_TI2V || sd_version_is_ltxav(version) || sd_version_is_lingbot_video(version))) { if (!denoise_mask.empty() && (version == VERSION_WAN2_2_TI2V || sd_version_is_ltxav(version) || sd_version_is_lingbot_video(version))) {
noised_input = noised_input * denoise_mask + init_latent * (1.0f - denoise_mask); noised_input = noised_input * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
} }
if (cache_runtime.spectrum_enabled && cache_runtime.spectrum.should_predict()) { if (cache_runtime.spectrum_enabled && cache_runtime.spectrum.should_predict()) {
cache_runtime.spectrum.predict(&denoised); cache_runtime.spectrum.predict(&denoised);
if (!denoise_mask.empty()) { if (!denoise_mask.empty()) {
denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask); denoised = denoised * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
} }
if (preview_needed && sd_should_preview_denoised()) { if (preview_needed && sd_should_preview_denoised()) {
preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false); preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
@ -2774,11 +2781,7 @@ public:
condition.c_reference_blocks.empty() ? nullptr : &condition.c_reference_blocks, condition.c_reference_blocks.empty() ? nullptr : &condition.c_reference_blocks,
audio_length, audio_length,
std::isfinite(active_flow_shift) ? active_flow_shift : 12.f, std::isfinite(active_flow_shift) ? active_flow_shift : 12.f,
3.f, 3.f};
method == EULER_SAMPLE_METHOD && step > 0 &&
static_cast<size_t>(step) < sigmas.size()
? sigmas[step]
: -1.f};
} else if (sd_version_is_ltxav(version)) { } else if (sd_version_is_ltxav(version)) {
diffusion_params.extra = LTXAVDiffusionExtra{ diffusion_params.extra = LTXAVDiffusionExtra{
nullptr, nullptr,
@ -2903,7 +2906,7 @@ public:
cache_runtime.spectrum.update(denoised); cache_runtime.spectrum.update(denoised);
} }
if (!denoise_mask.empty()) { if (!denoise_mask.empty()) {
denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask); denoised = denoised * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
} }
if (preview_needed && sd_should_preview_denoised()) { if (preview_needed && sd_should_preview_denoised()) {
preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false); preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
@ -2931,6 +2934,7 @@ public:
if (inverse_noise_scaling) { if (inverse_noise_scaling) {
x0 = denoiser->inverse_noise_scaling(sigmas[sigmas.size() - 1], x0); x0 = denoiser->inverse_noise_scaling(sigmas[sigmas.size() - 1], x0);
} }
x0 = denoiser->process_latent_out(std::move(x0));
if (control_net) { if (control_net) {
control_net->free_control_ctx(); control_net->free_control_ctx();