fix: use carrier sampling for MiniMax H3 audio (#1924)

Co-authored-by: leejet <leejet714@gmail.com>

README.md

@@ -15,6 +15,7 @@ API and command-line option may change frequently.***
 
 ## 🔥Important News
 
+* **2026/08/20** 🚀 stable-diffusion.cpp now supports **LTX-2.5**
 * **2026/08/04** 🚀 stable-diffusion.cpp adds **Day-1 support for MiniMax-H3**
 * **2026/06/25** 🚀 stable-diffusion.cpp now supports **Krea2**
 * **2026/06/04** 🚀 stable-diffusion.cpp now supports **Ideogram4**
@@ -68,7 +69,7 @@ API and command-line option may change frequently.***
   - Video Models
     - [Wan2.1/Wan2.2](./docs/wan.md)
     - [MiniMax-H3](./docs/minimax_h3.md)
-    - [LTX-2.3](./docs/ltx2.md)
+    - [LTX-2.3/LTX-2.5](./docs/ltx2.md)
     - [HunyuanVideo 1.5](./docs/hunyuan_video.md)
     - [LingBot-Video](./docs/lingbot_video.md)
   - [PhotoMaker](./docs/photo_maker.md) support.

docs/ltx2.md

@@ -1,7 +1,17 @@
 # How to Use
 
+Both LTX-2.3 and LTX-2.5 are supported. The two share a transformer, video VAE and audio
+VAE architecture; LTX-2.5 drops the video FFN biases, adds a learned keyframe
+absolute-position embedding, and pairs with a Gemma 4 text encoder instead of Gemma 3.
+Everything is detected from the weights, so the command lines differ only in which files
+you pass.
+
+# LTX-2.3
+
 ## Download weights
 
+### LTX-2.3
+
 - Download LTX-2.3
     - safetensors: https://huggingface.co/Kijai/LTX2.3_comfy/tree/main/diffusion_models
     - gguf: https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main
@@ -16,6 +26,30 @@
 - Download LTX spatial latent upscaler
     - safetensors: https://huggingface.co/Lightricks/LTX-2.3/resolve/main/ltx-2.3-spatial-upscaler-x2-1.1.safetensors
 
+### LTX-2.5
+
+- Download LTX-2.5
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/tree/main/diffusion_models
+    - gguf: https://huggingface.co/vantagewithai/LTX-2.5-GGUF/tree/main
+- Download the text encoder. This is a Gemma 4 12B fine-tuned for LTX with the text
+  projection bundled in, so no separate `--embeddings-connectors` file is needed. Google's
+  stock Gemma 4 is not a substitute.
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/text_encoders/gemma4-12b-with-proj-ltx-2.5-bf16.safetensors
+- Download the video vae. Use the **conv** variant: `ltx-2.5-video-vae-conv-bf16.safetensors`.
+  The default `ltx-2.5-video-vae-bf16.safetensors` is a diffusion decoder, which is not
+  implemented here.
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/vae/ltx-2.5-video-vae-conv-bf16.safetensors
+- Download the audio vae
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/vae/ltx-2.5-audio-vae-bf16.safetensors
+- Download the LTX spatial latent upscaler
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors
+
+To run the text encoder quantized, convert it once with sd-cli:
+
+```
+.\bin\Release\sd-cli.exe -M convert -m ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-bf16.safetensors --type q8_0 -o ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf
+```
+
 ## Examples
 
 ### LTX-2.3 dev T2V
@@ -75,3 +109,31 @@ By default, the hires refine pass uses the main sampler and scheduler, then trim
   controls
   muted
   style="max-width: 100%; height: auto;"></video>
+
+### LTX-2.5 dev T2V
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "A wide aerial shot of a red vintage convertible driving along a coastal cliff road at sunset, waves crashing below" --cfg-scale 3.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 121 --fps 24 -o t2v.webm
+```
+
+### LTX-2.5 dev I2V
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "a lovely cat blinking slowly, gentle camera push in" --cfg-scale 3.0 --sampling-method euler -v -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 121 -i ..\assets\ernie_image\turbo_example.png -o i2v.webm
+```
+
+### LTX-2.5 spatial latent upscale
+
+Works exactly like the LTX-2.3 upscaler described below; put
+`ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors` under `--hires-upscalers-dir` and
+pass its name without path or extension to `--hires-upscaler`.
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0 --hires --hires-steps 6 -p "a lovely cat" --cfg-scale 3.0 --sampling-method euler -v -W 640 -H 360 --diffusion-fa --offload-to-cpu --video-frames 121 -o hires_t2v.webm
+```
+
+## Not implemented
+
+- The diffusion video decoder (`ltx-2.5-video-vae-bf16.safetensors`). Use the conv VAE.
+- The temporal latent upscaler and the duration head (`--auto-duration`); pass
+  `--video-frames` explicitly.

examples/cli/main.cpp

@@ -81,7 +81,7 @@ struct SDCliParams {
              &metadata_format},
             {"",
              "--preview-path",
-             "path to write preview image to (default: ./preview.png). Multi-frame previews support .avi, .webm, and animated .webp",
+             "path to write preview image to (default: ./preview.png). For image generation, the filename can have %03d placeholder for sequential numbering. Multi-frame previews support .avi, .webm, and animated .webp",
              0,
              &preview_path},
             {"",
@@ -94,7 +94,7 @@ struct SDCliParams {
         options.int_options = {
             {"",
              "--preview-interval",
-             "interval in denoising steps between consecutive updates of the image preview file (default is 1, meaning updating at every step)",
+             "preview interval: in each sampling pass, positive N updates every Nth denoiser step and -N previews only completed logical step N; 0 previews the final completed step of the first pass (base-resolution or high-noise). Default: 1",
              &preview_interval},
             {"",
              "--output-begin-idx",
@@ -377,29 +377,6 @@ bool load_images_from_dir(const std::string dir,
     return true;
 }
 
-void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy, void* data) {
-    (void)step;
-    (void)is_noisy;
-    SDCliParams* cli_params = (SDCliParams*)data;
-    // is_noisy is set to true if the preview corresponds to noisy latents, false if it's denoised latents
-    // unused in this app, it will either be always noisy or always denoised here
-    if (frame_count == 1) {
-        if (!write_image_to_file(cli_params->preview_path,
-                                 image->data,
-                                 image->width,
-                                 image->height,
-                                 image->channel,
-                                 "",
-                                 cli_params->compression_quality)) {
-            LOG_ERROR("save preview image to '%s' failed", cli_params->preview_path.c_str());
-        }
-    } else {
-        if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps, cli_params->compression_quality) != 0) {
-            LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str());
-        }
-    }
-}
-
 std::string format_frame_idx(std::string pattern, int frame_idx) {
     std::smatch match;
     std::string result = pattern;
@@ -419,6 +396,36 @@ std::string format_frame_idx(std::string pattern, int frame_idx) {
     return result;
 }
 
+int continuous_preview_counter = 0;
+
+void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy, void* data) {
+    (void)step;
+    (void)is_noisy;
+    SDCliParams* cli_params = (SDCliParams*)data;
+    // is_noisy is set to true if the preview corresponds to noisy latents, false if it's denoised latents
+    // unused in this app, it will either be always noisy or always denoised here
+    if (frame_count == 1) {
+        fs::path path = cli_params->preview_path;
+        if (encoded_image_format_from_path(path.string()) == EncodedImageFormat::UNKNOWN)
+            path += ".png";
+        if (std::regex_search(path.string(), format_specifier_regex))
+            path = fs::path(format_frame_idx(path.string(), continuous_preview_counter++));
+        if (!write_image_to_file(path.string(),
+                                 image->data,
+                                 image->width,
+                                 image->height,
+                                 image->channel,
+                                 "",
+                                 cli_params->compression_quality)) {
+            LOG_ERROR("save preview image to '%s' failed", path.string().c_str());
+        }
+    } else {
+        if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps, cli_params->compression_quality) != 0) {
+            LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str());
+        }
+    }
+}
+
 static fs::path get_video_audio_sidecar_path(const SDCliParams& cli_params) {
     fs::path out_path     = cli_params.output_path;
     fs::path base_path    = out_path;

ggml

@@ -1 +1 @@
-Subproject commit 032b6997db4c9c75dc85d8d2bb2beec77b1231b0
+Subproject commit e20c3a14aa70ee84ca58499814206dd08d8026bc

include/stable-diffusion.h

@@ -446,6 +446,9 @@ typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void*
 
 SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data);
 SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
+// In each sampling pass, a positive interval previews every Nth denoiser step, while a
+// negative interval previews only completed logical step -interval. Zero previews the final
+// completed step of the first sampling pass (base-resolution or high-noise).
 SD_API void sd_set_preview_callback(sd_preview_cb_t cb, enum preview_t mode, int interval, bool denoised, bool noisy, void* data);
 SD_API void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data);
 SD_API int32_t sd_get_num_physical_cores();

src/conditioning/conditioner.hpp

@@ -2978,15 +2978,36 @@ struct LTXAVEmbedder : public Conditioner {
     std::shared_ptr<GemmaTokenizer> tokenizer;
     std::shared_ptr<LLM::LLMRunner> llm;
     std::shared_ptr<LTXAVTextProjectionRunner> projector;
+    std::string projector_prefix;
     bool dual_projection = false;
 
+    // Gemma 4 keeps a per-layer output scalar that no Gemma 3 checkpoint has, and widens its
+    // full-attention heads to 512 so their q_proj is twice a sliding layer's.
+    static LLM::LLMArch detect_gemma_arch(const String2TensorStorage& tensor_storage_map,
+                                          const std::string& llm_prefix) {
+        if (tensor_storage_map.find(llm_prefix + ".model.layers.0.layer_scalar") != tensor_storage_map.end()) {
+            return LLM::LLMArch::GEMMA4_12B;
+        }
+        auto global_q  = tensor_storage_map.find(llm_prefix + ".model.layers.5.self_attn.q_proj.weight");
+        auto sliding_q = tensor_storage_map.find(llm_prefix + ".model.layers.0.self_attn.q_proj.weight");
+        if (global_q != tensor_storage_map.end() &&
+            sliding_q != tensor_storage_map.end() &&
+            global_q->second.ne[1] == sliding_q->second.ne[1] * 2) {
+            return LLM::LLMArch::GEMMA4_12B;
+        }
+        return LLM::LLMArch::GEMMA3_12B;
+    }
+
     LTXAVEmbedder(ggml_backend_t backend,
                   const String2TensorStorage& tensor_storage_map      = {},
                   const std::string& llm_prefix                       = "text_encoders.llm",
                   const std::string& projector_prefix                 = "text_embedding_projection",
-                  std::shared_ptr<RunnerWeightManager> weight_manager = nullptr) {
+                  std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+        : projector_prefix(projector_prefix) {
+        LLM::LLMArch arch = detect_gemma_arch(tensor_storage_map, llm_prefix);
+        LOG_INFO("ltxav text encoder: %s", arch == LLM::LLMArch::GEMMA4_12B ? "gemma 4" : "gemma 3");
         tokenizer       = std::make_shared<GemmaTokenizer>();
-        llm             = std::make_shared<LLM::LLMRunner>(LLM::LLMArch::GEMMA3_12B,
+        llm             = std::make_shared<LLM::LLMRunner>(arch,
                                                backend,
                                                tensor_storage_map,
                                                llm_prefix,
@@ -3001,7 +3022,7 @@ struct LTXAVEmbedder : public Conditioner {
 
     void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
         llm->get_param_tensors(tensors, "text_encoders.llm");
-        projector->get_param_tensors(tensors, "text_embedding_projection");
+        projector->get_param_tensors(tensors, projector_prefix);
     }
 
     void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) override {

src/core/ggml_extend.hpp

@@ -3470,7 +3470,17 @@ public:
         ggml_tensor* w            = params["weight"];
         ggml_tensor* weight_scale = has_weight_scale ? params["weight_scale"] : nullptr;
         if (w->type == GGML_TYPE_F8_E4M3 || w->type == GGML_TYPE_F8_E5M2) {
-            w = ggml_cast(ctx->ggml_ctx, w, GGML_TYPE_BF16);
+            bool supports_fp8_matmul = false;
+            if (ctx->backend != nullptr) {
+                ggml_tensor* fp8_matmul = ggml_mul_mat(ctx->ggml_ctx, w, x);
+                if (force_prec_f32) {
+                    ggml_mul_mat_set_prec(fp8_matmul, GGML_PREC_F32);
+                }
+                supports_fp8_matmul = ggml_backend_supports_op(ctx->backend, fp8_matmul);
+            }
+            if (!supports_fp8_matmul) {
+                w = ggml_cast(ctx->ggml_ctx, w, GGML_TYPE_BF16);
+            }
         }
         ggml_tensor* b = nullptr;
         if (bias) {

src/model/common/block.hpp

@@ -268,10 +268,11 @@ public:
                 int64_t dim_out,
                 int64_t mult          = 4,
                 Activation activation = Activation::GEGLU,
-                bool precision_fix    = false) {
+                bool precision_fix    = false,
+                bool bias             = true) {
         int64_t inner_dim = dim * mult;
         if (activation == Activation::GELU) {
-            blocks["net.0"] = std::shared_ptr<GGMLBlock>(new GELU(dim, inner_dim));
+            blocks["net.0"] = std::shared_ptr<GGMLBlock>(new GELU(dim, inner_dim, bias));
         } else {
             blocks["net.0"] = std::shared_ptr<GGMLBlock>(new GEGLU(dim, inner_dim));
         }
@@ -285,7 +286,7 @@ public:
         // The purpose of the scale here is to prevent NaN issues in certain situations.
         // For example, when using Vulkan without enabling force_prec_f32,
         // or when using CUDA but the weights are k-quants.
-        blocks["net.2"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, dim_out, true, false, force_prec_f32, scale));
+        blocks["net.2"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, dim_out, bias, false, force_prec_f32, scale));
     }
 
     ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {

src/model/diffusion/ltxv.hpp

@@ -129,6 +129,10 @@ namespace LTXV {
         bool self_attention_gated  = false;
         bool cross_attention_gated = false;
 
+        bool ff_bias                         = true;
+        bool audio_ff_bias                   = true;
+        bool use_keyframes_abs_pos_embedding = false;
+
         static std::pair<int64_t, int64_t> infer_attention_layout(int64_t hidden_size,
                                                                   int64_t preferred_heads = -1) {
             if (preferred_heads > 0 && hidden_size % preferred_heads == 0) {
@@ -207,6 +211,19 @@ namespace LTXV {
                 tensor_storage_map.find(prefix + ".transformer_blocks.0.audio_attn2.to_gate_logits.weight") != tensor_storage_map.end()) {
                 config.cross_attention_gated = true;
             }
+            // LTX 2.5 sets ff_bias=false but leaves audio_ff_bias at its default, so the two
+            // branches must be detected separately; older checkpoints ship both sets of biases.
+            if (tensor_storage_map.find(prefix + ".transformer_blocks.0.ff.net.0.proj.bias") == tensor_storage_map.end() &&
+                tensor_storage_map.find(prefix + ".transformer_blocks.0.ff.net.2.bias") == tensor_storage_map.end()) {
+                config.ff_bias = false;
+            }
+            if (tensor_storage_map.find(prefix + ".transformer_blocks.0.audio_ff.net.0.proj.bias") == tensor_storage_map.end() &&
+                tensor_storage_map.find(prefix + ".transformer_blocks.0.audio_ff.net.2.bias") == tensor_storage_map.end()) {
+                config.audio_ff_bias = false;
+            }
+            if (tensor_storage_map.find(prefix + ".keyframes_abs_pos_embedding") != tensor_storage_map.end()) {
+                config.use_keyframes_abs_pos_embedding = true;
+            }
             if (tensor_storage_map.find(prefix + ".caption_projection.linear_1.weight") == tensor_storage_map.end() &&
                 tensor_storage_map.find(prefix + ".caption_projection.linear_2.weight") == tensor_storage_map.end()) {
                 config.use_caption_projection = false;
@@ -874,8 +891,7 @@ namespace LTXV {
                          const String2TensorStorage& tensor_storage_map = {},
                          const std::string prefix                       = "") override {
             if (num_learnable_registers > 0) {
-                ggml_type wtype               = get_type(prefix + "learnable_registers", tensor_storage_map, GGML_TYPE_F32);
-                params["learnable_registers"] = ggml_new_tensor_2d(ctx, wtype, hidden_size, num_learnable_registers);
+                params["learnable_registers"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, num_learnable_registers);
             }
         }
 
@@ -1130,7 +1146,9 @@ namespace LTXV {
                                 int64_t a_context_dim,
                                 bool apply_gated_attention,
                                 bool cross_attention_adaln,
-                                bool video_rope_interleaved)
+                                bool video_rope_interleaved,
+                                bool ff_bias       = true,
+                                bool audio_ff_bias = true)
             : v_dim(v_dim),
               a_dim(a_dim),
               cross_attention_adaln(cross_attention_adaln) {
@@ -1140,8 +1158,8 @@ namespace LTXV {
             blocks["audio_attn2"]         = std::make_shared<CrossAttention>(a_dim, a_context_dim, a_heads, ad_head, apply_gated_attention, false);
             blocks["audio_to_video_attn"] = std::make_shared<CrossAttention>(v_dim, a_dim, a_heads, ad_head, apply_gated_attention, false);
             blocks["video_to_audio_attn"] = std::make_shared<CrossAttention>(a_dim, v_dim, a_heads, ad_head, apply_gated_attention, false);
-            blocks["ff"]                  = std::make_shared<FeedForward>(v_dim, v_dim, 4, FeedForward::Activation::GELU);
-            blocks["audio_ff"]            = std::make_shared<FeedForward>(a_dim, a_dim, 4, FeedForward::Activation::GELU);
+            blocks["ff"]                  = std::make_shared<FeedForward>(v_dim, v_dim, 4, FeedForward::Activation::GELU, false, ff_bias);
+            blocks["audio_ff"]            = std::make_shared<FeedForward>(a_dim, a_dim, 4, FeedForward::Activation::GELU, false, audio_ff_bias);
         }
 
         std::vector<ggml_tensor*> get_ada_values(GGMLRunnerContext* ctx,
@@ -1320,6 +1338,12 @@ namespace LTXV {
                                                                    get_type(prefix + "audio_scale_shift_table", tensor_storage_map, GGML_TYPE_F32),
                                                                    config.audio_hidden_size,
                                                                    2);
+            if (config.use_keyframes_abs_pos_embedding) {
+                params["keyframes_abs_pos_embedding"] = ggml_new_tensor_2d(ctx,
+                                                                           get_type(prefix + "keyframes_abs_pos_embedding", tensor_storage_map, GGML_TYPE_F32),
+                                                                           config.hidden_size,
+                                                                           1);
+            }
         }
 
         LTXAVModelBlock(const LTXAVConfig& config)
@@ -1386,7 +1410,9 @@ namespace LTXV {
                                                                                                               config.audio_cross_attention_dim,
                                                                                                               config.self_attention_gated || config.cross_attention_gated,
                                                                                                               config.cross_attention_adaln,
-                                                                                                              config.video_rope_interleaved);
+                                                                                                              config.video_rope_interleaved,
+                                                                                                              config.ff_bias,
+                                                                                                              config.audio_ff_bias);
             }
 
             blocks["norm_out"]       = std::make_shared<LayerNorm>(config.hidden_size, 1e-6f, false);
@@ -1534,6 +1560,38 @@ namespace LTXV {
             return {v_context, a_context};
         }
 
+        // The video encoder is causal, so the first latent frame covers a single pixel frame while
+        // every later one covers temporal_scale_factor. LTX 2.5 marks that token class with a
+        // learned embedding added right after patchify_proj.
+        ggml_tensor* apply_keyframes_abs_pos_embedding(GGMLRunnerContext* ctx,
+                                                       ggml_tensor* vx,
+                                                       int64_t tokens_per_latent_frame) {
+            if (!config.use_keyframes_abs_pos_embedding || params.count("keyframes_abs_pos_embedding") == 0) {
+                return vx;
+            }
+            int64_t tokens = vx->ne[1];
+            if (tokens_per_latent_frame <= 0 || tokens_per_latent_frame > tokens) {
+                return vx;
+            }
+            auto embedding = params["keyframes_abs_pos_embedding"];
+            auto first     = ggml_cont(ctx->ggml_ctx,
+                                       ggml_view_3d(ctx->ggml_ctx, vx, vx->ne[0], tokens_per_latent_frame, vx->ne[2], vx->nb[1], vx->nb[2], 0));
+            first          = ggml_add(ctx->ggml_ctx, first, embedding);
+            if (tokens_per_latent_frame == tokens) {
+                return first;
+            }
+            auto rest = ggml_cont(ctx->ggml_ctx,
+                                  ggml_view_3d(ctx->ggml_ctx,
+                                               vx,
+                                               vx->ne[0],
+                                               tokens - tokens_per_latent_frame,
+                                               vx->ne[2],
+                                               vx->nb[1],
+                                               vx->nb[2],
+                                               tokens_per_latent_frame * vx->nb[1]));
+            return ggml_concat(ctx->ggml_ctx, first, rest, 1);
+        }
+
         std::vector<ggml_tensor*> get_output_scale_shift(GGMLRunnerContext* ctx,
                                                          ggml_tensor* table,
                                                          ggml_tensor* embedded_timestep,
@@ -1575,6 +1633,7 @@ namespace LTXV {
 
             vx = patchify_video(ctx, vx, n);
             vx = patchify_proj->forward(ctx, vx);
+            vx = apply_keyframes_abs_pos_embedding(ctx, vx, width * height);
             if (ax != nullptr && ggml_nelements(ax) > 0 && audio_time > 0) {
                 ax = patchify_audio(ctx, ax);
                 ax = audio_patchify_proj->forward(ctx, ax);

src/model/diffusion/minimax_h3.hpp

@@ -123,13 +123,6 @@ namespace MiniMaxH3 {
         return to_shift * base / (1.f + (to_shift - 1.f) * base);
     }
 
-    static float time_shift_slope(float sigma, float from_shift, float to_shift) {
-        float base = sigma / (from_shift + sigma * (1.f - from_shift));
-        float a    = 1.f + (from_shift - 1.f) * base;
-        float b    = 1.f + (to_shift - 1.f) * base;
-        return to_shift * a * a / (from_shift * b * b);
-    }
-
     struct TimeEmbedder : public GGMLBlock {
         TimeEmbedder(int64_t input_dim, int64_t hidden_dim, int64_t output_dim) {
             blocks["proj_in"]  = std::make_shared<Linear>(input_dim, hidden_dim, true, true);
@@ -594,8 +587,7 @@ namespace MiniMaxH3 {
                                                       const std::vector<TokenModulationSpan>& segments,
                                                       const std::vector<SequenceSegment>& sequence_segments,
                                                       const TokenModulationSpan& video_segment,
-                                                      const TokenModulationSpan& audio_segment,
-                                                      float audio_slope) {
+                                                      const TokenModulationSpan& audio_segment) {
             auto video_proj = std::dynamic_pointer_cast<Linear>(blocks["video_patch_proj"]);
             auto audio_proj = std::dynamic_pointer_cast<Linear>(blocks["audio_patch_proj"]);
 
@@ -715,7 +707,7 @@ namespace MiniMaxH3 {
                                                audio->ne[2]);
             audio_out        = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, audio_out, 1, 2, 0, 3));
             video_out        = ggml_ext_scale(ctx->ggml_ctx, video_out, -1.f);
-            audio_out        = ggml_ext_scale(ctx->ggml_ctx, audio_out, -audio_slope);
+            audio_out        = ggml_ext_scale(ctx->ggml_ctx, audio_out, -1.f);
             return {video_out, audio_out};
         }
     };
@@ -1040,9 +1032,9 @@ namespace MiniMaxH3 {
             GGML_ASSERT(!audio_input_cache.empty());
             GGML_ASSERT(!context_tensor.empty());
 
-            auto video   = make_input(video_input_cache);
-            auto audio   = make_input(audio_input_cache);
-            auto context = make_input(context_tensor);
+            auto video         = make_input(video_input_cache);
+            auto audio_carrier = make_input(audio_input_cache);
+            auto context       = make_input(context_tensor);
             std::vector<ggml_tensor*> condition_inputs;
             condition_inputs.reserve(condition_videos.size());
             for (const auto& condition : condition_videos) {
@@ -1054,21 +1046,26 @@ namespace MiniMaxH3 {
                 audio_condition_inputs.push_back(make_input(condition));
             }
 
-            float sigma_v = std::clamp(timestep[0] / 1000.f, 1e-6f, 1.f);
-            float t_v     = 1.f - sigma_v;
-            float t_a     = 1.f - time_shift_sigma(sigma_v, video_shift, audio_shift);
-            auto layout   = build_layout(context_tensor.shape()[1],
-                                         video_input_cache.shape()[2],
-                                         video_input_cache.shape()[1],
-                                         video_input_cache.shape()[0],
-                                         audio_length,
-                                         condition_videos,
-                                         condition_audios,
-                                         keyframe_indices,
-                                         reference_blocks,
-                                         text_tags,
-                                         t_v,
-                                         t_a);
+            float sigma_v     = std::clamp(timestep[0] / 1000.f, 1e-6f, 1.f);
+            float sigma_a     = time_shift_sigma(sigma_v, video_shift, audio_shift);
+            float audio_scale = video_shift / audio_shift;
+            float t_v         = 1.f - sigma_v;
+            float t_a         = 1.f - sigma_a;
+            // The sampler carries c_a = (sigma_v / sigma_a) * x_a so the packed
+            // latent follows one sigma schedule. Restore x_a for the H3 network.
+            auto audio  = ggml_ext_scale(compute_ctx, audio_carrier, sigma_a / sigma_v);
+            auto layout = build_layout(context_tensor.shape()[1],
+                                       video_input_cache.shape()[2],
+                                       video_input_cache.shape()[1],
+                                       video_input_cache.shape()[0],
+                                       audio_length,
+                                       condition_videos,
+                                       condition_audios,
+                                       keyframe_indices,
+                                       reference_blocks,
+                                       text_tags,
+                                       t_v,
+                                       t_a);
 
             position_input_cache = sd::Tensor<float>(
                 {3, static_cast<int64_t>(layout.positions.size() / 3)},
@@ -1129,10 +1126,15 @@ namespace MiniMaxH3 {
                                             layout.segments,
                                             layout.sequence_segments,
                                             layout.video_segment,
-                                            layout.audio_segment,
-                                            time_shift_slope(sigma_v, video_shift, audio_shift));
-            auto merged     = merge_av_latents(compute_ctx, output.first, output.second);
-            auto graph      = new_graph_custom(H3_GRAPH_SIZE);
+                                            layout.audio_segment);
+            // Convert the model's audio velocity to d(c_a) / d(sigma_v).
+            output.second = ggml_add(compute_ctx,
+                                     ggml_ext_scale(compute_ctx, audio, 1.f - audio_scale),
+                                     ggml_ext_scale(compute_ctx,
+                                                    output.second,
+                                                    1.f + (audio_scale - 1.f) * sigma_a));
+            auto merged   = merge_av_latents(compute_ctx, output.first, output.second);
+            auto graph    = new_graph_custom(H3_GRAPH_SIZE);
             ggml_build_forward_expand(graph, merged);
             return graph;
         }

src/model/te/llm.hpp

@@ -40,6 +40,7 @@ namespace LLM {
         MINISTRAL_3_3B,
         GEMMA3_12B,
         GEMMA2_2B,
+        GEMMA4_12B,
         GPT_OSS_20B,
         ARCH_COUNT,
     };
@@ -52,6 +53,7 @@ namespace LLM {
         "ministral3.3b",
         "gemma3_12b",
         "gemma2_2b",
+        "gemma4_12b",
         "gpt_oss_20b",
     };
 
@@ -120,6 +122,15 @@ namespace LLM {
         bool have_vision_weight = false;
         bool llama_cpp_style    = false;
 
+        // gemma4 config
+        int global_head_dim         = 0;
+        int num_global_kv_heads     = 0;
+        float global_partial_rotary = 1.f;
+        bool global_k_eq_v          = false;
+        bool v_norm                 = false;
+        bool layer_scalar           = false;
+        bool unscaled_attention     = false;
+
         static LLMConfig detect_from_weights(const String2TensorStorage& tensor_storage_map,
                                              const std::string& prefix,
                                              LLMArch arch) {
@@ -157,6 +168,27 @@ namespace LLM {
                 config.rope_thetas             = {1000000.f, 10000.f};
                 config.rope_scales             = {8.f, 1.f};
                 config.sliding_attention       = {1024, 1024, 1024, 1024, 1024, 0};
+            } else if (arch == LLMArch::GEMMA4_12B) {
+                config.head_dim                = 256;
+                config.num_heads               = 16;
+                config.num_kv_heads            = 8;
+                config.global_head_dim         = 512;
+                config.num_global_kv_heads     = 1;
+                config.global_partial_rotary   = 0.25f;
+                config.global_k_eq_v           = true;
+                config.v_norm                  = true;
+                config.layer_scalar            = true;
+                config.unscaled_attention      = true;
+                config.qkv_bias                = false;
+                config.qk_norm                 = true;
+                config.rms_norm_eps            = 1e-6f;
+                config.rms_norm_add            = false;
+                config.normalize_input         = true;
+                config.max_position_embeddings = 262144;
+                config.mlp_activation          = MLPActivation::GELU_TANH;
+                config.rope_thetas             = {1000000.f, 10000.f};
+                config.rope_scales             = {1.f, 1.f};
+                config.sliding_attention       = {1024, 1024, 1024, 1024, 1024, 0};
             } else if (arch == LLMArch::GEMMA2_2B) {
                 config.head_dim                = 256;
                 config.num_heads               = 8;
@@ -232,12 +264,12 @@ namespace LLM {
                             }
                         }
                     }
-                    if (contains(name, "visual.blocks.0.mlp.linear_fc1.weight") ||
-                        contains(name, "visual.blocks.0.mlp.gate_proj.weight")) {
+                    if (ends_with(name, "visual.blocks.0.mlp.linear_fc1.weight") ||
+                        ends_with(name, "visual.blocks.0.mlp.gate_proj.weight")) {
                         config.vision.intermediate_size = tensor_storage.ne[1];
                     }
-                    if (contains(name, "visual.merger.linear_fc2.weight") ||
-                        contains(name, "visual.merger.mlp.2.weight")) {
+                    if (ends_with(name, "visual.merger.linear_fc2.weight") ||
+                        ends_with(name, "visual.merger.mlp.2.weight")) {
                         config.vision.out_hidden_size = tensor_storage.ne[1];
                     }
                     continue;
@@ -256,13 +288,13 @@ namespace LLM {
                     config.hidden_size = tensor_storage.ne[0];
                     config.vocab_size  = tensor_storage.ne[1];
                 }
-                if (contains(name, "layers.0.mlp.gate_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.gate_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1];
                 }
-                if (contains(name, "layers.0.mlp.experts.gate_up_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.experts.gate_up_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1] / 2;
                 }
-                if (contains(name, "layers.0.mlp.experts.gate_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.experts.gate_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1];
                 }
             }
@@ -1063,6 +1095,11 @@ namespace LLM {
         std::vector<float> rope_thetas;
         std::vector<float> rope_scales;
         bool has_attention_sinks;
+        bool k_eq_v;
+        bool v_norm;
+        bool unscaled_attention;
+        float rms_norm_eps;
+        int rope_pairs;
 
         void init_params(ggml_context* ctx,
                          const String2TensorStorage& tensor_storage_map = {},
@@ -1073,24 +1110,48 @@ namespace LLM {
         }
 
     public:
-        Attention(const LLMConfig& config)
+        Attention(const LLMConfig& config, bool global_layer = false)
             : arch(config.arch),
               num_heads(config.num_heads),
-              num_kv_heads(config.num_kv_heads),
-              head_dim(config.head_dim),
+              num_kv_heads(global_layer && config.num_global_kv_heads > 0 ? config.num_global_kv_heads : config.num_kv_heads),
+              head_dim(global_layer && config.global_head_dim > 0 ? config.global_head_dim : config.head_dim),
               qk_norm(config.qk_norm),
               max_position_embeddings(config.max_position_embeddings),
               rope_thetas(config.rope_thetas),
               rope_scales(config.rope_scales),
-              has_attention_sinks(config.arch == LLMArch::GPT_OSS_20B) {
+              has_attention_sinks(config.arch == LLMArch::GPT_OSS_20B),
+              k_eq_v(global_layer && config.global_k_eq_v),
+              v_norm(config.v_norm),
+              unscaled_attention(config.unscaled_attention),
+              rms_norm_eps(config.rms_norm_eps),
+              rope_pairs(0) {
             blocks["q_proj"] = std::make_shared<Linear>(config.hidden_size, num_heads * head_dim, config.qkv_bias);
             blocks["k_proj"] = std::make_shared<Linear>(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias);
-            blocks["v_proj"] = std::make_shared<Linear>(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias);
+            if (!k_eq_v) {
+                blocks["v_proj"] = std::make_shared<Linear>(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias);
+            }
             blocks["o_proj"] = std::make_shared<Linear>(num_heads * head_dim, config.hidden_size, config.attention_out_bias);
             if (config.qk_norm) {
                 blocks["q_norm"] = std::make_shared<LLMRMSNorm>(head_dim, config.rms_norm_eps, config.rms_norm_add);
                 blocks["k_norm"] = std::make_shared<LLMRMSNorm>(head_dim, config.rms_norm_eps, config.rms_norm_add);
             }
+            // Proportional RoPE rotates only the leading `rope_pairs` dimension pairs of the head;
+            // the rest are left unrotated through freq_factors (see rope_freq_factors()).
+            float partial = global_layer ? config.global_partial_rotary : 1.f;
+            rope_pairs    = static_cast<int>(partial * head_dim / 2.f);
+        }
+
+        // ggml applies theta_i / freq_factors[i], so a huge factor collapses the angle to zero and
+        // leaves that pair unrotated. This reproduces transformers' "proportional" RoPE, whose
+        // inv_freq is zero-padded past `rope_pairs`, without reordering the head.
+        ggml_tensor* rope_freq_factors(ggml_context* ctx) const {
+            int pairs = head_dim / 2;
+            if (rope_pairs >= pairs) {
+                return nullptr;
+            }
+            auto rotated   = ggml_ext_ones(ctx, rope_pairs, 1, 1, 1);
+            auto unrotated = ggml_ext_full(ctx, 1e30f, pairs - rope_pairs, 1, 1, 1);
+            return ggml_concat(ctx, rotated, unrotated, 0);
         }
 
         ggml_tensor* forward(GGMLRunnerContext* ctx,
@@ -1103,12 +1164,12 @@ namespace LLM {
             int64_t N       = x->ne[2];
             auto q_proj     = std::dynamic_pointer_cast<Linear>(blocks["q_proj"]);
             auto k_proj     = std::dynamic_pointer_cast<Linear>(blocks["k_proj"]);
-            auto v_proj     = std::dynamic_pointer_cast<Linear>(blocks["v_proj"]);
+            auto v_proj     = k_eq_v ? nullptr : std::dynamic_pointer_cast<Linear>(blocks["v_proj"]);
             auto out_proj   = std::dynamic_pointer_cast<Linear>(blocks["o_proj"]);
 
-            auto q = q_proj->forward(ctx, x);  // [N, n_token, num_heads*head_dim]
-            auto k = k_proj->forward(ctx, x);  // [N, n_token, num_kv_heads*head_dim]
-            auto v = v_proj->forward(ctx, x);  // [N, n_token, num_kv_heads*head_dim]
+            auto q = q_proj->forward(ctx, x);               // [N, n_token, num_heads*head_dim]
+            auto k = k_proj->forward(ctx, x);               // [N, n_token, num_kv_heads*head_dim]
+            auto v = k_eq_v ? k : v_proj->forward(ctx, x);  // [N, n_token, num_kv_heads*head_dim]
 
             q = ggml_reshape_4d(ctx->ggml_ctx, q, head_dim, num_heads, n_token, N);     // [N, n_token, num_heads, head_dim]
             k = ggml_reshape_4d(ctx->ggml_ctx, k, head_dim, num_kv_heads, n_token, N);  // [N, n_token, num_kv_heads, head_dim]
@@ -1121,6 +1182,10 @@ namespace LLM {
                 q = q_norm->forward(ctx, q);
                 k = k_norm->forward(ctx, k);
             }
+            if (v_norm) {
+                // Gemma 4 normalizes V with a weightless RMS norm, and never rotates it.
+                v = ggml_rms_norm(ctx->ggml_ctx, v, rms_norm_eps);
+            }
 
             if (arch == LLMArch::MISTRAL_SMALL_3_2) {
                 q = ggml_rope_ext(ctx->ggml_ctx, q, input_pos, nullptr, 128, GGML_ROPE_TYPE_NORMAL, 8192, 1000000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
@@ -1191,6 +1256,35 @@ namespace LLM {
                                                  1.f,
                                                  32.f,
                                                  1.f);
+            } else if (arch == LLMArch::GEMMA4_12B) {
+                float rope_theta  = (rope_index == 1 ? 10000.0f : 1000000.0f);
+                auto freq_factors = rope_freq_factors(ctx->ggml_ctx);
+                q                 = ggml_rope_ext(ctx->ggml_ctx,
+                                                  q,
+                                                  input_pos,
+                                                  freq_factors,
+                                                  head_dim,
+                                                  GGML_ROPE_TYPE_NEOX,
+                                                  static_cast<int>(max_position_embeddings),
+                                                  rope_theta,
+                                                  1.f,
+                                                  0.f,
+                                                  1.f,
+                                                  32.f,
+                                                  1.f);
+                k                 = ggml_rope_ext(ctx->ggml_ctx,
+                                                  k,
+                                                  input_pos,
+                                                  freq_factors,
+                                                  head_dim,
+                                                  GGML_ROPE_TYPE_NEOX,
+                                                  static_cast<int>(max_position_embeddings),
+                                                  rope_theta,
+                                                  1.f,
+                                                  0.f,
+                                                  1.f,
+                                                  32.f,
+                                                  1.f);
             } else if (arch == LLMArch::GEMMA2_2B) {
                 q = ggml_rope_ext(ctx->ggml_ctx,
                                   q,
@@ -1228,6 +1322,11 @@ namespace LLM {
                 k               = ggml_rope_multi(ctx->ggml_ctx, k, input_pos, nullptr, head_dim, sections, GGML_ROPE_TYPE_MROPE, 128000, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
             }
 
+            if (unscaled_attention) {
+                // Gemma 4 attends with scaling=1.0; undo the helper's own 1/sqrt(head_dim).
+                q = ggml_ext_scale(ctx->ggml_ctx, q, std::sqrt(static_cast<float>(head_dim)));
+            }
+
             q = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, q, 0, 2, 1, 3));  // [N, num_heads, n_token, head_dim]
             q = ggml_reshape_3d(ctx->ggml_ctx, q, q->ne[0], q->ne[1], q->ne[2] * q->ne[3]);      // [N*num_heads, n_token, head_dim]
 
@@ -1266,15 +1365,30 @@ namespace LLM {
     protected:
         LLMArch arch;
         int sliding_attention;
+        bool has_layer_scalar;
         std::string post_attention_norm_name;
         std::string pre_ffw_norm_name;
         std::string post_ffw_norm_name;
 
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         std::string prefix                             = "") override {
+            GGMLBlock::init_params(ctx, tensor_storage_map, prefix);
+            if (has_layer_scalar) {
+                params["layer_scalar"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+            }
+        }
+
     public:
         TransformerBlock(const LLMConfig& config, int layer_index)
             : arch(config.arch),
-              sliding_attention(0) {
-            if (config.arch == LLMArch::GEMMA3_12B) {
+              sliding_attention(0),
+              has_layer_scalar(config.layer_scalar) {
+            if (config.arch == LLMArch::GEMMA4_12B) {
+                post_attention_norm_name = "post_attention_layernorm";
+                pre_ffw_norm_name        = "pre_feedforward_layernorm";
+                post_ffw_norm_name       = "post_feedforward_layernorm";
+            } else if (config.arch == LLMArch::GEMMA3_12B || config.arch == LLMArch::GEMMA4_12B) {
                 post_attention_norm_name = "post_attention_norm";       // attn_post_norm
                 pre_ffw_norm_name        = "post_attention_layernorm";  // ffn_norm
                 post_ffw_norm_name       = "post_ffw_norm";             // ffn_post_norm
@@ -1288,7 +1402,10 @@ namespace LLM {
                 pre_ffw_norm_name = "post_attention_layernorm";  // ffn_norm
             }
 
-            blocks["self_attn"] = std::make_shared<Attention>(config);
+            if (!config.sliding_attention.empty()) {
+                sliding_attention = config.sliding_attention[layer_index % config.sliding_attention.size()];
+            }
+            blocks["self_attn"] = std::make_shared<Attention>(config, sliding_attention == 0);
             if (config.arch == LLMArch::GPT_OSS_20B) {
                 blocks["mlp"] = std::make_shared<GPTOSSMLP>(config);
             } else {
@@ -1305,9 +1422,6 @@ namespace LLM {
             if (!post_ffw_norm_name.empty()) {
                 blocks[post_ffw_norm_name] = std::make_shared<LLMRMSNorm>(config.hidden_size, config.rms_norm_eps, config.rms_norm_add);
             }
-            if (!config.sliding_attention.empty()) {
-                sliding_attention = config.sliding_attention[layer_index % config.sliding_attention.size()];
-            }
         }
 
         ggml_tensor* forward(GGMLRunnerContext* ctx,
@@ -1329,7 +1443,7 @@ namespace LLM {
             }
             ggml_tensor* block_attention_mask = attention_mask;
             int rope_index                    = 0;
-            if ((arch == LLMArch::GEMMA3_12B || arch == LLMArch::GPT_OSS_20B) && sliding_attention > 0) {
+            if ((arch == LLMArch::GEMMA3_12B || arch == LLMArch::GEMMA4_12B || arch == LLMArch::GPT_OSS_20B) && sliding_attention > 0) {
                 block_attention_mask = sliding_attention_mask;
                 rope_index           = 1;
             }
@@ -1356,6 +1470,10 @@ namespace LLM {
             }
             x = ggml_add_inplace(ctx->ggml_ctx, x, residual);
 
+            if (has_layer_scalar) {
+                x = ggml_mul(ctx->ggml_ctx, x, params["layer_scalar"]);
+            }
+
             return x;
         }
     };
@@ -1850,6 +1968,7 @@ namespace LLM {
                 config.arch == LLMArch::MINISTRAL_3_3B ||
                 config.arch == LLMArch::QWEN3 ||
                 config.arch == LLMArch::GEMMA3_12B ||
+                config.arch == LLMArch::GEMMA4_12B ||
                 config.arch == LLMArch::GEMMA2_2B ||
                 config.arch == LLMArch::GPT_OSS_20B) {
                 input_pos_vec.resize(n_tokens);
@@ -1914,7 +2033,7 @@ namespace LLM {
                 set_backend_tensor_data(attention_mask, attention_mask_vec.data());
             }
 
-            if (config.arch == LLMArch::GEMMA3_12B || config.arch == LLMArch::GPT_OSS_20B) {
+            if (config.arch == LLMArch::GEMMA3_12B || config.arch == LLMArch::GEMMA4_12B || config.arch == LLMArch::GPT_OSS_20B) {
                 int sliding_window = 0;
                 for (int window : config.sliding_attention) {
                     sliding_window = std::max(sliding_window, window);

src/model/vae/minimax_h3_audio_vae.hpp

@@ -154,8 +154,9 @@ namespace MiniMaxH3 {
                          const String2TensorStorage& tensor_storage_map = {},
                          const std::string prefix                       = "") override {
             GGMLBlock::init_params(ctx, tensor_storage_map, prefix);
-            params["q_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
-            params["v_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
+            params["q_bias"]      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
+            params["zero_k_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
+            params["v_bias"]      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
         }
 
         ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
@@ -166,7 +167,7 @@ namespace MiniMaxH3 {
                 return ggml_reshape_4d(ctx->ggml_ctx, bias, bias->ne[0], 1, 1, 1);
             };
             auto q = ggml_add(ctx->ggml_ctx, qkv[0], bias_shape(params["q_bias"]));
-            auto k = qkv[1];
+            auto k = ggml_add(ctx->ggml_ctx, qkv[1], bias_shape(params["zero_k_bias"]));
             auto v = ggml_add(ctx->ggml_ctx, qkv[2], bias_shape(params["v_bias"]));
 
             int64_t sequence = x->ne[1];
@@ -358,22 +359,38 @@ namespace MiniMaxH3 {
         }
 
         ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* waveform) {
-            GGML_ASSERT(waveform->ne[1] == 2);
+            GGML_ASSERT(waveform->ne[1] * waveform->ne[2] * waveform->ne[3] == 2);
             auto encoder   = std::dynamic_pointer_cast<AudioEncoder>(blocks["encoder"]);
             auto pre       = std::dynamic_pointer_cast<AudioAttentionProjection>(blocks["pre_block"]);
             auto mean_proj = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["mean_proj"]);
 
-            waveform = ggml_reshape_3d(ctx->ggml_ctx, waveform, waveform->ne[0], 1, waveform->ne[1]);
-            auto x   = encoder->forward(ctx, waveform);  // [B*S, 2048, T]
-            x        = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
-            x        = pre->forward(ctx, x);
-            x        = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
-            auto z   = mean_proj->forward(ctx, x);
+            // GGML's batched conv1d storage interleaves the stream dimension
+            // with output channels. Subsequent layers then read stereo samples
+            // as adjacent feature channels. Run each mono stream independently,
+            // matching PyTorch's reshape(B*S, 1, samples), and concatenate only
+            // the completed normalized latents.
+            const int64_t streams = waveform->ne[2] * waveform->ne[3];
+            waveform              = ggml_reshape_3d(ctx->ggml_ctx,
+                                                    waveform,
+                                                    waveform->ne[0],
+                                                    1,
+                                                    streams);
+            ggml_tensor* stereo_z = nullptr;
+            for (int64_t stream = 0; stream < streams; ++stream) {
+                auto mono = ggml_ext_slice(ctx->ggml_ctx, waveform, 2, stream, stream + 1);
+                auto x    = encoder->forward(ctx, mono);
+                x         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
+                x         = pre->forward(ctx, x);
+                x         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
+                auto z    = mean_proj->forward(ctx, x);
 
-            auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
-            auto std  = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
-            z         = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
-            return ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
+                auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
+                auto std  = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
+                z         = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
+                z         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
+                stereo_z  = stereo_z == nullptr ? z : ggml_concat(ctx->ggml_ctx, stereo_z, z, 1);
+            }
+            return stereo_z;
         }
 
         ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* latent) {

src/name_conversion.cpp

@@ -149,6 +149,7 @@ std::string convert_cond_stage_model_name(std::string name, std::string prefix)
         {"ffn_up.", "mlp.up_proj."},
         {"ffn_post_norm.", "post_ffw_norm."},
         {"ffn_norm.", "post_attention_layernorm."},
+        {"layer_output_scale.weight", "layer_scalar"},
         {"output_norm.", "model.norm."},
     };
 
@@ -1459,6 +1460,7 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
         {"unet.", "model.diffusion_model."},
         {"transformer.", "model.diffusion_model."},  // dit
         {"vae.", "first_stage_model."},
+        {"text_encoders.llm.text_embedding_projection.", "text_embedding_projection."},
         {"text_encoder.", "cond_stage_model.transformer."},
         {"te.", "cond_stage_model.transformer."},
         {"text_encoder.2.", "cond_stage_model.1.transformer."},

src/runtime/denoiser.hpp

@@ -1043,6 +1043,16 @@ struct Denoiser {
                                                     const sd::Tensor<float>& latent) = 0;
     virtual float noise_level_to_sigma(float noise_level)                            = 0;
 
+    virtual sd::Tensor<float> process_latent_in(const sd::Tensor<float>& latent) {
+        // An empty result means the original latent can be used unchanged.
+        SD_UNUSED(latent);
+        return {};
+    }
+
+    virtual sd::Tensor<float> process_latent_out(sd::Tensor<float> latent) {
+        return latent;
+    }
+
     virtual std::vector<float> get_sigmas(uint32_t n, int image_seq_len, scheduler_t scheduler_type, SDVersion version, const char* extra_sample_args = nullptr) {
         auto bound_t_to_sigma = std::bind(&Denoiser::t_to_sigma, this, std::placeholders::_1);
         std::shared_ptr<SigmaScheduler> scheduler;
@@ -1286,6 +1296,40 @@ struct DiscreteFlowDenoiser : public Denoiser {
     }
 };
 
+struct H3AVFlowDenoiser : public DiscreteFlowDenoiser {
+    int64_t video_channels;
+    float audio_shift;
+
+    H3AVFlowDenoiser(float shift, float audio_shift, int64_t video_channels)
+        : DiscreteFlowDenoiser(shift),
+          video_channels(video_channels),
+          audio_shift(audio_shift) {
+        GGML_ASSERT(shift > 0.f && audio_shift > 0.f && video_channels > 0);
+    }
+
+    sd::Tensor<float> process_latent_in(const sd::Tensor<float>& latent) override {
+        return scale_audio(latent, shift / audio_shift);
+    }
+
+    sd::Tensor<float> process_latent_out(sd::Tensor<float> latent) override {
+        auto transformed = scale_audio(latent, audio_shift / shift);
+        if (transformed.empty()) {
+            return latent;
+        }
+        return transformed;
+    }
+
+private:
+    sd::Tensor<float> scale_audio(const sd::Tensor<float>& latent, float scale) const {
+        if (scale == 1.f || latent.dim() < 4 || latent.shape()[3] <= video_channels) {
+            return {};
+        }
+        auto video = sd::ops::slice(latent, 3, 0, video_channels);
+        auto audio = sd::ops::slice(latent, 3, video_channels, latent.shape()[3]) * scale;
+        return sd::ops::concat(video, audio, 3);
+    }
+};
+
 struct FluxFlowDenoiser : public DiscreteFlowDenoiser {
     FluxFlowDenoiser() = default;
 

src/runtime/preview_interval.h

@@ -0,0 +1,45 @@
+#ifndef __SD_RUNTIME_PREVIEW_INTERVAL_H__
+#define __SD_RUNTIME_PREVIEW_INTERVAL_H__
+
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+
+namespace sd::preview {
+
+    constexpr std::uint64_t logical_sample_step(int step) {
+        return step < 0 ? static_cast<std::uint64_t>(-static_cast<std::int64_t>(step))
+                        : static_cast<std::uint64_t>(step);
+    }
+
+    constexpr bool sample_step_is_complete(int step,
+                                           std::size_t total_steps,
+                                           bool terminal_sigma_is_zero) {
+        return step > 0 ||
+               (terminal_sigma_is_zero &&
+                step < 0 &&
+                logical_sample_step(step) == static_cast<std::uint64_t>(total_steps));
+    }
+
+    constexpr bool should_preview_sample_step(int step,
+                                              std::size_t total_steps,
+                                              bool terminal_sigma_is_zero,
+                                              int interval,
+                                              bool preview_final_step) {
+        if (interval > 0) {
+            return step % interval == 0;
+        }
+        if (!sample_step_is_complete(step, total_steps, terminal_sigma_is_zero)) {
+            return false;
+        }
+
+        std::uint64_t logical_step = logical_sample_step(step);
+        if (interval < 0) {
+            std::uint64_t requested_step = static_cast<std::uint64_t>(-static_cast<std::int64_t>(interval));
+            return logical_step == requested_step;
+        }
+        return preview_final_step && logical_step == static_cast<std::uint64_t>(total_steps);
+    }
+}  // namespace sd::preview
+
+#endif  // __SD_RUNTIME_PREVIEW_INTERVAL_H__

src/stable-diffusion.cpp

@@ -61,6 +61,7 @@
 #include "model/vae/wan_vae.hpp"
 #include "runtime/denoiser.hpp"
 #include "runtime/guidance.h"
+#include "runtime/preview_interval.h"
 #include "runtime/sample-cache.h"
 #include "upscaler.h"
 
@@ -1860,6 +1861,9 @@ public:
                     if (sd_version_is_ltxav(version)) {
                         LOG_INFO("running in LTXAV FLOW mode");
                         denoiser = std::make_shared<FluxFlowDenoiser>();
+                    } else if (sd_version_is_minimax_h3(version)) {
+                        LOG_INFO("running in MiniMax H3 AV FLOW mode");
+                        denoiser = std::make_shared<H3AVFlowDenoiser>(default_flow_shift, 3.f, get_latent_channel());
                     } else {
                         LOG_INFO("running in FLOW mode");
                         denoiser = std::make_shared<DiscreteFlowDenoiser>();
@@ -2467,8 +2471,11 @@ public:
                                     sd_get_preview_mode()};
     }
 
-    void report_sample_progress(int step, size_t total_steps, int64_t* last_progress_us) {
-        if (step > 0 || step == -(int)total_steps) {
+    void report_sample_progress(int step,
+                                size_t total_steps,
+                                bool terminal_sigma_is_zero,
+                                int64_t* last_progress_us) {
+        if (sd::preview::sample_step_is_complete(step, total_steps, terminal_sigma_is_zero)) {
             int64_t now        = ggml_time_us();
             int showstep       = std::abs(step);
             float step_seconds = last_progress_us != nullptr && *last_progress_us > 0
@@ -2530,6 +2537,7 @@ public:
                              int audio_length,
                              float frame_rate,
                              const sd_cache_params_t* cache_params,
+                             bool preview_final_step,
                              const sd::Tensor<float>& video_positions = {}) {
         struct RunnerDoneOnExit {
             GGMLRunner* runner = nullptr;
@@ -2589,8 +2597,9 @@ public:
             }
         }
 
-        size_t steps       = sigmas.size() - 1;
-        bool has_skiplayer = (slg_scale != 0.0f || slg_uncond) && !skip_layers.empty();
+        size_t steps                = sigmas.size() - 1;
+        bool terminal_sigma_is_zero = sigmas.back() == 0.f;
+        bool has_skiplayer          = (slg_scale != 0.0f || slg_uncond) && !skip_layers.empty();
         if (has_skiplayer && !sd_version_is_dit(version)) {
             has_skiplayer = false;
             LOG_WARN("SLG is incompatible with this model type");
@@ -2617,10 +2626,14 @@ public:
         int64_t last_progress_us     = ggml_time_us();
         SamplePreviewContext preview = prepare_sample_preview_context();
 
-        sd::Tensor<float> x_t      = !noise.empty()
-                                         ? denoiser->noise_scaling(sigmas[0], noise, init_latent)
-                                         : init_latent;
-        sd::Tensor<float> denoised = x_t;
+        sd::Tensor<float> processed_init_latent       = denoiser->process_latent_in(init_latent);
+        const sd::Tensor<float>& sampling_init_latent = processed_init_latent.empty()
+                                                            ? init_latent
+                                                            : processed_init_latent;
+        sd::Tensor<float> x_t                         = !noise.empty()
+                                                            ? denoiser->noise_scaling(sigmas[0], noise, sampling_init_latent)
+                                                            : sampling_init_latent;
+        sd::Tensor<float> denoised                    = x_t;
 
         auto denoise = [&](const sd::Tensor<float>& x, float sigma, int step) -> sd::guidance::GuiderOutput {
             if (get_cancel_flag() == SD_CANCEL_ALL) {
@@ -2639,14 +2652,21 @@ public:
             float c_out  = scaling[1];
             float c_in   = scaling[2];
 
+            bool preview_needed = preview.callback != nullptr &&
+                                  sd::preview::should_preview_sample_step(step,
+                                                                          steps,
+                                                                          terminal_sigma_is_zero,
+                                                                          sd_get_preview_interval(),
+                                                                          preview_final_step);
+
             std::vector<float> base_timesteps_vec = prepare_sample_timesteps(sigma, shifted_timestep);
             std::vector<float> timesteps_vec      = base_timesteps_vec;
             sd::Tensor<float> audio_timesteps_tensor;
             if (sd_version_is_ltxav(version) && !denoise_mask.empty()) {
-                timesteps_vec          = process_ltxav_video_timesteps(base_timesteps_vec, init_latent, denoise_mask);
+                timesteps_vec          = process_ltxav_video_timesteps(base_timesteps_vec, sampling_init_latent, denoise_mask);
                 audio_timesteps_tensor = sd::Tensor<float>({static_cast<int64_t>(base_timesteps_vec.size())}, base_timesteps_vec);
             } else {
-                timesteps_vec = process_timesteps(timesteps_vec, init_latent, denoise_mask, step);
+                timesteps_vec = process_timesteps(timesteps_vec, sampling_init_latent, denoise_mask, step);
             }
             const std::vector<float>& scaling_timesteps_vec = (sd_version_is_ltxav(version) && !denoise_mask.empty())
                                                                   ? base_timesteps_vec
@@ -2661,29 +2681,25 @@ public:
             }
             sd::Tensor<float> noised_input = x * c_in;
             if (!denoise_mask.empty() && (version == VERSION_WAN2_2_TI2V || sd_version_is_ltxav(version) || sd_version_is_lingbot_video(version))) {
-                noised_input = noised_input * denoise_mask + init_latent * (1.0f - denoise_mask);
+                noised_input = noised_input * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
             }
 
             if (cache_runtime.spectrum_enabled && cache_runtime.spectrum.should_predict()) {
                 cache_runtime.spectrum.predict(&denoised);
                 if (!denoise_mask.empty()) {
-                    denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask);
+                    denoised = denoised * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
                 }
-                if (sd_should_preview_denoised() && preview.callback != nullptr) {
-                    if (step % sd_get_preview_interval() == 0) {
-                        preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
-                    }
+                if (preview_needed && sd_should_preview_denoised()) {
+                    preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
                 }
-                report_sample_progress(step, steps, &last_progress_us);
+                report_sample_progress(step, steps, terminal_sigma_is_zero, &last_progress_us);
                 sd::guidance::GuiderOutput output;
                 output.pred = denoised;
                 return output;
             }
 
-            if (sd_should_preview_noisy() && preview.callback != nullptr) {
-                if (step % sd_get_preview_interval() == 0) {
-                    preview_image(step, noised_input, version, preview.mode, preview.callback, preview.data, true);
-                }
+            if (preview_needed && sd_should_preview_noisy()) {
+                preview_image(step, noised_input, version, preview.mode, preview.callback, preview.data, true);
             }
 
             sd::Tensor<float> cond_out;
@@ -2890,14 +2906,12 @@ public:
                 cache_runtime.spectrum.update(denoised);
             }
             if (!denoise_mask.empty()) {
-                denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask);
+                denoised = denoised * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
             }
-            if (sd_should_preview_denoised() && preview.callback != nullptr) {
-                if (step % sd_get_preview_interval() == 0) {
-                    preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
-                }
+            if (preview_needed && sd_should_preview_denoised()) {
+                preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
             }
-            report_sample_progress(step, steps, &last_progress_us);
+            report_sample_progress(step, steps, terminal_sigma_is_zero, &last_progress_us);
             output.pred = denoised;
             return output;
         };
@@ -2920,6 +2934,7 @@ public:
         if (inverse_noise_scaling) {
             x0 = denoiser->inverse_noise_scaling(sigmas[sigmas.size() - 1], x0);
         }
+        x0 = denoiser->process_latent_out(std::move(x0));
 
         if (control_net) {
             control_net->free_control_ctx();
@@ -4743,7 +4758,11 @@ static sd::Tensor<float> prepare_minimax_h3_reference_waveform(const sd_audio_t&
         static_cast<long double>(audio.sample_count) * target_sample_rate / audio.sample_rate));
     output_samples          = std::max<uint64_t>(1, output_samples);
     uint64_t padded_samples = (output_samples + 799) / 800 * 800;
-    sd::Tensor<float> waveform({static_cast<int64_t>(padded_samples), 2, 1, 1});
+    // Keep stereo streams planar for the mono-per-stream audio encoder:
+    // [samples, 1, stereo, batch]. This avoids flattening interleaved L/R
+    // storage into alternating samples when the encoder folds streams into
+    // its batch dimension.
+    sd::Tensor<float> waveform({static_cast<int64_t>(padded_samples), 1, 2, 1});
 
     for (uint64_t i = 0; i < output_samples; ++i) {
         long double source_pos = static_cast<long double>(i) * audio.sample_rate / target_sample_rate;
@@ -4754,7 +4773,7 @@ static sd::Tensor<float> prepare_minimax_h3_reference_waveform(const sd_audio_t&
             uint32_t source_channel = audio.channels == 1 ? 0 : std::min<uint32_t>(channel, audio.channels - 1);
             float a                 = audio.data[source0 * audio.channels + source_channel];
             float b                 = audio.data[source1 * audio.channels + source_channel];
-            waveform.index(static_cast<int64_t>(i), channel, 0, 0) =
+            waveform.index(static_cast<int64_t>(i), 0, channel, 0) =
                 std::clamp(a + (b - a) * fraction, -1.f, 1.f);
         }
     }
@@ -5711,7 +5730,8 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx,
                                                    1.f,
                                                    0,
                                                    static_cast<float>(request.fps),
-                                                   request.cache_params);
+                                                   request.cache_params,
+                                                   true);
         int64_t sampling_end  = ggml_time_ms();
         if (!x_0.empty()) {
             LOG_INFO("sampling completed, taking %.2fs", (sampling_end - sampling_start) * 1.0f / 1000);
@@ -5832,7 +5852,8 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx,
                                                             1.f,
                                                             0,
                                                             static_cast<float>(request.fps),
-                                                            request.cache_params);
+                                                            request.cache_params,
+                                                            false);
             int64_t hires_sample_end   = ggml_time_ms();
             if (!x_0.empty()) {
                 LOG_INFO("hires sampling %d/%d completed, taking %.2fs",
@@ -6966,6 +6987,7 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx,
                                                            latents.audio_length,
                                                            static_cast<float>(request.fps),
                                                            request.cache_params,
+                                                           true,
                                                            latents.video_positions);
         int64_t sampling_end          = ggml_time_ms();
         if (x_t_sampled.empty()) {
@@ -7008,6 +7030,7 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx,
                                                         latents.audio_length,
                                                         static_cast<float>(request.fps),
                                                         request.cache_params,
+                                                        plan.high_noise_sample_steps <= 0,
                                                         latents.video_positions);
 
     int64_t sampling_end = ggml_time_ms();
@@ -7146,6 +7169,7 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx,
                                             latents.audio_length,
                                             static_cast<float>(hires_request.fps),
                                             hires_request.cache_params,
+                                            false,
                                             hires_video_positions);
         sampling_end   = ggml_time_ms();
         if (final_latent.empty()) {