DarthAffe/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-09-25 04:32:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: correct MiniMax H3 reference audio encoding (#1886)

Browse Source
This commit is contained in:
jk212h20 2026-08-30 07:33:59 -04:00 committed by GitHub
parent afd5306d88
commit c797899732
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 37 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/model/vae/minimax_h3_audio_vae.hpp
View File

@ -154,8 +154,9 @@ namespace MiniMaxH3 {
const String2TensorStorage& tensor_storage_map = {}, const String2TensorStorage& tensor_storage_map = {},
const std::string prefix = "") override { const std::string prefix = "") override {
GGMLBlock::init_params(ctx, tensor_storage_map, prefix); GGMLBlock::init_params(ctx, tensor_storage_map, prefix);
params["q_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); params["q_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
params["v_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels); params["zero_k_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
params["v_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
} }
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
@ -166,7 +167,7 @@ namespace MiniMaxH3 {
return ggml_reshape_4d(ctx->ggml_ctx, bias, bias->ne[0], 1, 1, 1); return ggml_reshape_4d(ctx->ggml_ctx, bias, bias->ne[0], 1, 1, 1);
}; };
auto q = ggml_add(ctx->ggml_ctx, qkv[0], bias_shape(params["q_bias"])); auto q = ggml_add(ctx->ggml_ctx, qkv[0], bias_shape(params["q_bias"]));
auto k = qkv[1]; auto k = ggml_add(ctx->ggml_ctx, qkv[1], bias_shape(params["zero_k_bias"]));
auto v = ggml_add(ctx->ggml_ctx, qkv[2], bias_shape(params["v_bias"])); auto v = ggml_add(ctx->ggml_ctx, qkv[2], bias_shape(params["v_bias"]));
int64_t sequence = x->ne[1]; int64_t sequence = x->ne[1];
@ -358,22 +359,38 @@ namespace MiniMaxH3 {
} }
ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* waveform) { ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* waveform) {
GGML_ASSERT(waveform->ne[1] == 2); GGML_ASSERT(waveform->ne[1] * waveform->ne[2] * waveform->ne[3] == 2);
auto encoder = std::dynamic_pointer_cast<AudioEncoder>(blocks["encoder"]); auto encoder = std::dynamic_pointer_cast<AudioEncoder>(blocks["encoder"]);
auto pre = std::dynamic_pointer_cast<AudioAttentionProjection>(blocks["pre_block"]); auto pre = std::dynamic_pointer_cast<AudioAttentionProjection>(blocks["pre_block"]);
auto mean_proj = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["mean_proj"]); auto mean_proj = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["mean_proj"]);
waveform = ggml_reshape_3d(ctx->ggml_ctx, waveform, waveform->ne[0], 1, waveform->ne[1]); // GGML's batched conv1d storage interleaves the stream dimension
auto x = encoder->forward(ctx, waveform); // [B*S, 2048, T] // with output channels. Subsequent layers then read stereo samples
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); // as adjacent feature channels. Run each mono stream independently,
x = pre->forward(ctx, x); // matching PyTorch's reshape(B*S, 1, samples), and concatenate only
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); // the completed normalized latents.
auto z = mean_proj->forward(ctx, x); const int64_t streams = waveform->ne[2] * waveform->ne[3];
waveform = ggml_reshape_3d(ctx->ggml_ctx,
waveform,
waveform->ne[0],
1,
streams);
ggml_tensor* stereo_z = nullptr;
for (int64_t stream = 0; stream < streams; ++stream) {
auto mono = ggml_ext_slice(ctx->ggml_ctx, waveform, 2, stream, stream + 1);
auto x = encoder->forward(ctx, mono);
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
x = pre->forward(ctx, x);
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
auto z = mean_proj->forward(ctx, x);
auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1); auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
auto std = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1); auto std = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
z = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std); z = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
return ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3)); z = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
stereo_z = stereo_z == nullptr ? z : ggml_concat(ctx->ggml_ctx, stereo_z, z, 1);
}
return stereo_z;
} }
ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* latent) { ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* latent) {

8
src/stable-diffusion.cpp
View File

@ -4743,7 +4743,11 @@ static sd::Tensor<float> prepare_minimax_h3_reference_waveform(const sd_audio_t&
static_cast<long double>(audio.sample_count) * target_sample_rate / audio.sample_rate)); static_cast<long double>(audio.sample_count) * target_sample_rate / audio.sample_rate));
output_samples = std::max<uint64_t>(1, output_samples); output_samples = std::max<uint64_t>(1, output_samples);
uint64_t padded_samples = (output_samples + 799) / 800 * 800; uint64_t padded_samples = (output_samples + 799) / 800 * 800;
sd::Tensor<float> waveform({static_cast<int64_t>(padded_samples), 2, 1, 1}); // Keep stereo streams planar for the mono-per-stream audio encoder:
// [samples, 1, stereo, batch]. This avoids flattening interleaved L/R
// storage into alternating samples when the encoder folds streams into
// its batch dimension.
sd::Tensor<float> waveform({static_cast<int64_t>(padded_samples), 1, 2, 1});
for (uint64_t i = 0; i < output_samples; ++i) { for (uint64_t i = 0; i < output_samples; ++i) {
long double source_pos = static_cast<long double>(i) * audio.sample_rate / target_sample_rate; long double source_pos = static_cast<long double>(i) * audio.sample_rate / target_sample_rate;
@ -4754,7 +4758,7 @@ static sd::Tensor<float> prepare_minimax_h3_reference_waveform(const sd_audio_t&
uint32_t source_channel = audio.channels == 1 ? 0 : std::min<uint32_t>(channel, audio.channels - 1); uint32_t source_channel = audio.channels == 1 ? 0 : std::min<uint32_t>(channel, audio.channels - 1);
float a = audio.data[source0 * audio.channels + source_channel]; float a = audio.data[source0 * audio.channels + source_channel];
float b = audio.data[source1 * audio.channels + source_channel]; float b = audio.data[source1 * audio.channels + source_channel];
waveform.index(static_cast<int64_t>(i), channel, 0, 0) = waveform.index(static_cast<int64_t>(i), 0, channel, 0) =
std::clamp(a + (b - a) * fraction, -1.f, 1.f); std::clamp(a + (b - a) * fraction, -1.f, 1.f);
} }
} }