mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-09-25 04:32:29 +00:00
fix: correct MiniMax H3 audio Euler steps (#1908)
This commit is contained in:
parent
c797899732
commit
dc4000d9f8
@ -130,6 +130,18 @@ namespace MiniMaxH3 {
|
|||||||
return to_shift * a * a / (from_shift * b * b);
|
return to_shift * a * a / (from_shift * b * b);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static float time_shift_step_scale(float sigma,
|
||||||
|
float next_sigma,
|
||||||
|
float from_shift,
|
||||||
|
float to_shift) {
|
||||||
|
if (!std::isfinite(next_sigma) || next_sigma < 0.f || next_sigma == sigma) {
|
||||||
|
return time_shift_slope(sigma, from_shift, to_shift);
|
||||||
|
}
|
||||||
|
float shifted_sigma = time_shift_sigma(sigma, from_shift, to_shift);
|
||||||
|
float shifted_next_sigma = time_shift_sigma(next_sigma, from_shift, to_shift);
|
||||||
|
return (shifted_sigma - shifted_next_sigma) / (sigma - next_sigma);
|
||||||
|
}
|
||||||
|
|
||||||
struct TimeEmbedder : public GGMLBlock {
|
struct TimeEmbedder : public GGMLBlock {
|
||||||
TimeEmbedder(int64_t input_dim, int64_t hidden_dim, int64_t output_dim) {
|
TimeEmbedder(int64_t input_dim, int64_t hidden_dim, int64_t output_dim) {
|
||||||
blocks["proj_in"] = std::make_shared<Linear>(input_dim, hidden_dim, true, true);
|
blocks["proj_in"] = std::make_shared<Linear>(input_dim, hidden_dim, true, true);
|
||||||
@ -1033,7 +1045,8 @@ namespace MiniMaxH3 {
|
|||||||
const std::vector<MiniMaxH3ReferenceBlock>& reference_blocks,
|
const std::vector<MiniMaxH3ReferenceBlock>& reference_blocks,
|
||||||
int audio_length,
|
int audio_length,
|
||||||
float video_shift,
|
float video_shift,
|
||||||
float audio_shift) {
|
float audio_shift,
|
||||||
|
float next_video_sigma) {
|
||||||
auto split = split_av_latents(packed, audio_length);
|
auto split = split_av_latents(packed, audio_length);
|
||||||
video_input_cache = std::move(split.first);
|
video_input_cache = std::move(split.first);
|
||||||
audio_input_cache = std::move(split.second);
|
audio_input_cache = std::move(split.second);
|
||||||
@ -1130,7 +1143,16 @@ namespace MiniMaxH3 {
|
|||||||
layout.sequence_segments,
|
layout.sequence_segments,
|
||||||
layout.video_segment,
|
layout.video_segment,
|
||||||
layout.audio_segment,
|
layout.audio_segment,
|
||||||
time_shift_slope(sigma_v, video_shift, audio_shift));
|
// The generic Euler sampler advances the packed tensor by
|
||||||
|
// `next_video_sigma - sigma_v`. For that sampler, scale H3's
|
||||||
|
// audio velocity by the exact ratio of the independent audio
|
||||||
|
// step. The derivative approximation substantially oversteps
|
||||||
|
// at low step counts (the Turbo use case). Retain the local
|
||||||
|
// slope for samplers that make extra/intermediate evaluations.
|
||||||
|
time_shift_step_scale(sigma_v,
|
||||||
|
next_video_sigma,
|
||||||
|
video_shift,
|
||||||
|
audio_shift));
|
||||||
auto merged = merge_av_latents(compute_ctx, output.first, output.second);
|
auto merged = merge_av_latents(compute_ctx, output.first, output.second);
|
||||||
auto graph = new_graph_custom(H3_GRAPH_SIZE);
|
auto graph = new_graph_custom(H3_GRAPH_SIZE);
|
||||||
ggml_build_forward_expand(graph, merged);
|
ggml_build_forward_expand(graph, merged);
|
||||||
@ -1162,7 +1184,8 @@ namespace MiniMaxH3 {
|
|||||||
reference_blocks,
|
reference_blocks,
|
||||||
extra->audio_length,
|
extra->audio_length,
|
||||||
extra->video_sigma_shift,
|
extra->video_sigma_shift,
|
||||||
extra->audio_sigma_shift);
|
extra->audio_sigma_shift,
|
||||||
|
extra->next_video_sigma);
|
||||||
};
|
};
|
||||||
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph,
|
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph,
|
||||||
n_threads,
|
n_threads,
|
||||||
|
|||||||
@ -108,6 +108,8 @@ struct MiniMaxH3DiffusionExtra {
|
|||||||
int audio_length = 0;
|
int audio_length = 0;
|
||||||
float video_sigma_shift = 12.f;
|
float video_sigma_shift = 12.f;
|
||||||
float audio_sigma_shift = 3.f;
|
float audio_sigma_shift = 3.f;
|
||||||
|
// Negative when the outer sampler is not a single-evaluation Euler step.
|
||||||
|
float next_video_sigma = -1.f;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct MiniT2IDiffusionExtra {
|
struct MiniT2IDiffusionExtra {
|
||||||
|
|||||||
@ -2765,7 +2765,11 @@ public:
|
|||||||
condition.c_reference_blocks.empty() ? nullptr : &condition.c_reference_blocks,
|
condition.c_reference_blocks.empty() ? nullptr : &condition.c_reference_blocks,
|
||||||
audio_length,
|
audio_length,
|
||||||
std::isfinite(active_flow_shift) ? active_flow_shift : 12.f,
|
std::isfinite(active_flow_shift) ? active_flow_shift : 12.f,
|
||||||
3.f};
|
3.f,
|
||||||
|
method == EULER_SAMPLE_METHOD && step > 0 &&
|
||||||
|
static_cast<size_t>(step) < sigmas.size()
|
||||||
|
? sigmas[step]
|
||||||
|
: -1.f};
|
||||||
} else if (sd_version_is_ltxav(version)) {
|
} else if (sd_version_is_ltxav(version)) {
|
||||||
diffusion_params.extra = LTXAVDiffusionExtra{
|
diffusion_params.extra = LTXAVDiffusionExtra{
|
||||||
nullptr,
|
nullptr,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user