DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 03:24:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: correct MiniMax H3 reference audio encoding (#1886)

Browse Source
This commit is contained in:
jk212h20 authored and GitHub committed 2026-08-30 19:33:59 +08:00
1 parent afd5306d88
commit c797899732
2 files changed
+37 -16
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/model/vae/minimax_h3_audio_vae.hpp
+31 -14
View File
@@ -154,8 +154,9 @@ namespace MiniMaxH3 {
const String2TensorStorage& tensor_storage_map = {},
const std::string prefix = "") override {
GGMLBlock::init_params(ctx, tensor_storage_map, prefix);
params["q_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
params["v_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
params["q_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
params["zero_k_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
params["v_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
@@ -166,7 +167,7 @@ namespace MiniMaxH3 {
return ggml_reshape_4d(ctx->ggml_ctx, bias, bias->ne[0], 1, 1, 1);
};
auto q = ggml_add(ctx->ggml_ctx, qkv[0], bias_shape(params["q_bias"]));
auto k = qkv[1];
auto k = ggml_add(ctx->ggml_ctx, qkv[1], bias_shape(params["zero_k_bias"]));
auto v = ggml_add(ctx->ggml_ctx, qkv[2], bias_shape(params["v_bias"]));
int64_t sequence = x->ne[1];
@@ -358,22 +359,38 @@ namespace MiniMaxH3 {
}
ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* waveform) {
GGML_ASSERT(waveform->ne[1] == 2);
GGML_ASSERT(waveform->ne[1] * waveform->ne[2] * waveform->ne[3] == 2);
auto encoder = std::dynamic_pointer_cast<AudioEncoder>(blocks["encoder"]);
auto pre = std::dynamic_pointer_cast<AudioAttentionProjection>(blocks["pre_block"]);
auto mean_proj = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["mean_proj"]);
waveform = ggml_reshape_3d(ctx->ggml_ctx, waveform, waveform->ne[0], 1, waveform->ne[1]);
auto x = encoder->forward(ctx, waveform); // [B*S, 2048, T]
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
x = pre->forward(ctx, x);
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
auto z = mean_proj->forward(ctx, x);
// GGML's batched conv1d storage interleaves the stream dimension
// with output channels. Subsequent layers then read stereo samples
// as adjacent feature channels. Run each mono stream independently,
// matching PyTorch's reshape(B*S, 1, samples), and concatenate only
// the completed normalized latents.
const int64_t streams = waveform->ne[2] * waveform->ne[3];
waveform = ggml_reshape_3d(ctx->ggml_ctx,
waveform,
waveform->ne[0],
1,
streams);
ggml_tensor* stereo_z = nullptr;
for (int64_t stream = 0; stream < streams; ++stream) {
auto mono = ggml_ext_slice(ctx->ggml_ctx, waveform, 2, stream, stream + 1);
auto x = encoder->forward(ctx, mono);
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
x = pre->forward(ctx, x);
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
auto z = mean_proj->forward(ctx, x);
auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
auto std = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
z = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
return ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
auto std = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
z = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
z = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
stereo_z = stereo_z == nullptr ? z : ggml_concat(ctx->ggml_ctx, stereo_z, z, 1);
}
return stereo_z;
}
ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* latent) {