feat: add minimax-h3 support (#1854)

examples/cli/main.cpp

@@ -754,6 +754,18 @@ int main(int argc, const char* argv[]) {
         return true;
     };
 
+    auto load_audio = [&](const std::string& path, SDAudioOwner& audio) -> bool {
+        std::vector<float> samples;
+        uint32_t sample_rate = 0;
+        uint32_t channels    = 0;
+        if (!load_wav_from_file(path, samples, sample_rate, channels)) {
+            LOG_ERROR("load WAV audio from '%s' failed", path.c_str());
+            return false;
+        }
+        audio.reset(std::move(samples), sample_rate, channels);
+        return true;
+    };
+
     if (gen_params.init_image_path.size() > 0) {
         if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image)) {
             return 1;
@@ -777,6 +789,37 @@ int main(int argc, const char* argv[]) {
         }
     }
 
+    if (!gen_params.ref_video_paths.empty()) {
+        gen_params.ref_videos.clear();
+        gen_params.ref_videos.reserve(gen_params.ref_video_paths.size());
+        for (const auto& path : gen_params.ref_video_paths) {
+            std::vector<SDImageOwner> frames;
+            if (!load_images_from_dir(path, frames, 0, 0, 0, cli_params.verbose) || frames.empty()) {
+                LOG_ERROR("load reference video frames from '%s' failed", path.c_str());
+                return 1;
+            }
+            gen_params.ref_videos.push_back(std::move(frames));
+        }
+
+        gen_params.ref_video_audios.clear();
+        gen_params.ref_video_audios.resize(gen_params.ref_videos.size());
+        for (size_t i = 0; i < gen_params.ref_video_audio_paths.size(); ++i) {
+            if (!load_audio(gen_params.ref_video_audio_paths[i], gen_params.ref_video_audios[i])) {
+                return 1;
+            }
+        }
+    }
+
+    if (!gen_params.ref_audio_paths.empty()) {
+        gen_params.ref_audios.clear();
+        gen_params.ref_audios.resize(gen_params.ref_audio_paths.size());
+        for (size_t i = 0; i < gen_params.ref_audio_paths.size(); ++i) {
+            if (!load_audio(gen_params.ref_audio_paths[i], gen_params.ref_audios[i])) {
+                return 1;
+            }
+        }
+    }
+
     if (gen_params.mask_image_path.size() > 0) {
         if (!load_sd_image_from_file(gen_params.mask_image.put(),
                                      gen_params.mask_image_path.c_str(),

examples/common/common.cpp

@@ -1404,6 +1404,30 @@ ArgOptions SDGenerationParams::get_options() {
         return 1;
     };
 
+    auto on_ref_video_arg = [&](int argc, const char** argv, int index) {
+        if (++index >= argc) {
+            return -1;
+        }
+        ref_video_paths.push_back(argv[index]);
+        return 1;
+    };
+
+    auto on_ref_video_audio_arg = [&](int argc, const char** argv, int index) {
+        if (++index >= argc) {
+            return -1;
+        }
+        ref_video_audio_paths.push_back(argv[index]);
+        return 1;
+    };
+
+    auto on_ref_audio_arg = [&](int argc, const char** argv, int index) {
+        if (++index >= argc) {
+            return -1;
+        }
+        ref_audio_paths.push_back(argv[index]);
+        return 1;
+    };
+
     auto on_cache_mode_arg = [&](int argc, const char** argv, int index) {
         if (++index >= argc) {
             return -1;
@@ -1568,8 +1592,20 @@ ArgOptions SDGenerationParams::get_options() {
          on_high_noise_skip_layers_arg},
         {"-r",
          "--ref-image",
-         "reference image for Flux Kontext models (can be used multiple times)",
+         "reference image for Flux Kontext or MiniMax-H3 Ref2VA (can be used multiple times)",
          on_ref_image_arg},
+        {"",
+         "--ref-video",
+         "MiniMax-H3 Ref2VA reference video frame directory at 24 fps (can be used multiple times)",
+         on_ref_video_arg},
+        {"",
+         "--ref-video-audio",
+         "WAV soundtrack paired by index with --ref-video (can be used multiple times)",
+         on_ref_video_audio_arg},
+        {"",
+         "--ref-audio",
+         "standalone WAV reference for MiniMax-H3 Ref2VA (can be used multiple times)",
+         on_ref_audio_arg},
         {"",
          "--cache-mode",
          "caching method: 'easycache' (DiT), 'ucache' (UNET), 'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT Chebyshev+Taylor forecasting)",
@@ -2366,6 +2402,16 @@ bool SDGenerationParams::validate(SDMode mode) {
         return false;
     }
 
+    if (ref_video_audio_paths.size() > ref_video_paths.size()) {
+        LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video");
+        return false;
+    }
+
+    if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty())) {
+        LOG_ERROR("error: reference video and audio inputs require vid_gen mode");
+        return false;
+    }
+
     if (sample_params.shifted_timestep < 0 || sample_params.shifted_timestep > 1000) {
         LOG_ERROR("error: shifted_timestep must be in range [0, 1000]");
         return false;
@@ -2560,6 +2606,35 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
         control_frame_views.push_back(frame.get());
     }
 
+    ref_image_views.clear();
+    ref_image_views.reserve(ref_images.size());
+    for (auto& image : ref_images) {
+        ref_image_views.push_back(image.get());
+    }
+
+    ref_video_frame_views.clear();
+    ref_video_frame_views.resize(ref_videos.size());
+    ref_video_views.clear();
+    ref_video_views.reserve(ref_videos.size());
+    for (size_t i = 0; i < ref_videos.size(); ++i) {
+        auto& frame_views = ref_video_frame_views[i];
+        frame_views.reserve(ref_videos[i].size());
+        for (auto& frame : ref_videos[i]) {
+            frame_views.push_back(frame.get());
+        }
+        sd_audio_t audio = i < ref_video_audios.size() ? ref_video_audios[i].get() : sd_audio_t{};
+        ref_video_views.push_back({frame_views.empty() ? nullptr : frame_views.data(),
+                                   static_cast<int>(frame_views.size()),
+                                   24,
+                                   audio});
+    }
+
+    ref_audio_views.clear();
+    ref_audio_views.reserve(ref_audios.size());
+    for (auto& audio : ref_audios) {
+        ref_audio_views.push_back(audio.get());
+    }
+
     sample_params.guidance.slg.layers                 = skip_layers.empty() ? nullptr : skip_layers.data();
     sample_params.guidance.slg.layer_count            = skip_layers.size();
     high_noise_sample_params.guidance.slg.layers      = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data();
@@ -2578,6 +2653,12 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
     params.clip_skip                 = clip_skip;
     params.init_image                = init_image.get();
     params.end_image                 = end_image.get();
+    params.ref_images                = ref_image_views.empty() ? nullptr : ref_image_views.data();
+    params.ref_images_count          = static_cast<int>(ref_image_views.size());
+    params.ref_videos                = ref_video_views.empty() ? nullptr : ref_video_views.data();
+    params.ref_videos_count          = static_cast<int>(ref_video_views.size());
+    params.ref_audios                = ref_audio_views.empty() ? nullptr : ref_audio_views.data();
+    params.ref_audios_count          = static_cast<int>(ref_audio_views.size());
     params.control_frames            = control_frame_views.empty() ? nullptr : control_frame_views.data();
     params.control_frames_size       = static_cast<int>(control_frame_views.size());
     params.width                     = get_resolved_width();
@@ -2657,6 +2738,9 @@ std::string SDGenerationParams::to_string() const {
         << "  mask_image_path: \"" << mask_image_path << "\",\n"
         << "  control_image_path: \"" << control_image_path << "\",\n"
         << "  ref_image_paths: " << vec_str_to_string(ref_image_paths) << ",\n"
+        << "  ref_video_paths: " << vec_str_to_string(ref_video_paths) << ",\n"
+        << "  ref_video_audio_paths: " << vec_str_to_string(ref_video_audio_paths) << ",\n"
+        << "  ref_audio_paths: " << vec_str_to_string(ref_audio_paths) << ",\n"
         << "  control_video_path: \"" << control_video_path << "\",\n"
         << "  auto_resize_ref_image: " << (auto_resize_ref_image ? "true" : "false") << ",\n"
         << "  increase_ref_index: " << (increase_ref_index ? "true" : "false") << ",\n"

examples/common/common.h

@@ -212,6 +212,9 @@ struct SDGenerationParams {
     std::string control_image_path;
     std::string ip_adapter_image_path;
     std::vector<std::string> ref_image_paths;
+    std::vector<std::string> ref_video_paths;
+    std::vector<std::string> ref_video_audio_paths;
+    std::vector<std::string> ref_audio_paths;
     std::string control_video_path;
 
     sd_sample_params_t sample_params;
@@ -275,6 +278,9 @@ struct SDGenerationParams {
     SDImageOwner init_image;
     SDImageOwner end_image;
     std::vector<SDImageOwner> ref_images;
+    std::vector<std::vector<SDImageOwner>> ref_videos;
+    std::vector<SDAudioOwner> ref_video_audios;
+    std::vector<SDAudioOwner> ref_audios;
     SDImageOwner mask_image;
     SDImageOwner control_image;
     SDImageOwner ip_adapter_image;
@@ -283,6 +289,9 @@ struct SDGenerationParams {
 
     // Backing storage for sd_img_gen_params_t view fields.
     std::vector<sd_image_t> ref_image_views;
+    std::vector<std::vector<sd_image_t>> ref_video_frame_views;
+    std::vector<sd_ref_video_t> ref_video_views;
+    std::vector<sd_audio_t> ref_audio_views;
     std::vector<sd_image_t> pm_id_image_views;
     std::vector<sd_image_t> control_frame_views;
 

examples/common/media_io.cpp

@@ -1374,3 +1374,132 @@ bool write_wav_to_file(const std::string& path,
     file.write(reinterpret_cast<const char*>(pcm.data()), static_cast<std::streamsize>(pcm.size() * sizeof(int16_t)));
     return file.good();
 }
+
+static uint16_t read_le16(const uint8_t* data) {
+    return static_cast<uint16_t>(data[0]) |
+           (static_cast<uint16_t>(data[1]) << 8);
+}
+
+static uint32_t read_le32(const uint8_t* data) {
+    return static_cast<uint32_t>(data[0]) |
+           (static_cast<uint32_t>(data[1]) << 8) |
+           (static_cast<uint32_t>(data[2]) << 16) |
+           (static_cast<uint32_t>(data[3]) << 24);
+}
+
+bool load_wav_from_file(const std::string& path,
+                        std::vector<float>& interleaved_samples,
+                        uint32_t& sample_rate,
+                        uint32_t& channels) {
+    interleaved_samples.clear();
+    sample_rate = 0;
+    channels    = 0;
+
+    std::ifstream file(path, std::ios::binary);
+    uint8_t riff_header[12];
+    if (!file.read(reinterpret_cast<char*>(riff_header), sizeof(riff_header)) ||
+        std::memcmp(riff_header, "RIFF", 4) != 0 ||
+        std::memcmp(riff_header + 8, "WAVE", 4) != 0) {
+        return false;
+    }
+
+    uint16_t audio_format    = 0;
+    uint16_t bits_per_sample = 0;
+    uint16_t block_align     = 0;
+    std::streampos data_pos  = std::streampos(-1);
+    uint32_t data_size       = 0;
+
+    while (file.good()) {
+        uint8_t chunk_header[8];
+        if (!file.read(reinterpret_cast<char*>(chunk_header), sizeof(chunk_header))) {
+            break;
+        }
+        uint32_t chunk_size           = read_le32(chunk_header + 4);
+        std::streampos chunk_data_pos = file.tellg();
+
+        if (std::memcmp(chunk_header, "fmt ", 4) == 0) {
+            if (chunk_size < 16) {
+                return false;
+            }
+            std::vector<uint8_t> fmt(chunk_size);
+            if (!file.read(reinterpret_cast<char*>(fmt.data()), chunk_size)) {
+                return false;
+            }
+            audio_format    = read_le16(fmt.data());
+            channels        = read_le16(fmt.data() + 2);
+            sample_rate     = read_le32(fmt.data() + 4);
+            block_align     = read_le16(fmt.data() + 12);
+            bits_per_sample = read_le16(fmt.data() + 14);
+            if (audio_format == 0xfffe && chunk_size >= 40) {
+                audio_format = read_le16(fmt.data() + 24);
+            }
+        } else if (std::memcmp(chunk_header, "data", 4) == 0) {
+            data_pos  = chunk_data_pos;
+            data_size = chunk_size;
+            file.seekg(chunk_size, std::ios::cur);
+        } else {
+            file.seekg(chunk_size, std::ios::cur);
+        }
+
+        if (!file.good()) {
+            break;
+        }
+        if ((chunk_size & 1) != 0) {
+            file.seekg(1, std::ios::cur);
+        }
+    }
+
+    const uint32_t bytes_per_sample = (bits_per_sample + 7) / 8;
+    if (data_pos == std::streampos(-1) || data_size == 0 || channels == 0 || sample_rate == 0 ||
+        block_align == 0 || bytes_per_sample == 0 || block_align < channels * bytes_per_sample ||
+        (audio_format != 1 && audio_format != 3)) {
+        return false;
+    }
+
+    const uint64_t frame_count = data_size / block_align;
+    if (frame_count == 0 || frame_count > SIZE_MAX / channels) {
+        return false;
+    }
+    std::vector<uint8_t> pcm(data_size);
+    file.clear();
+    file.seekg(data_pos);
+    if (!file.read(reinterpret_cast<char*>(pcm.data()), data_size)) {
+        return false;
+    }
+
+    interleaved_samples.resize(static_cast<size_t>(frame_count * channels));
+    for (uint64_t frame = 0; frame < frame_count; ++frame) {
+        const uint8_t* frame_data = pcm.data() + frame * block_align;
+        for (uint32_t channel = 0; channel < channels; ++channel) {
+            const uint8_t* sample_data = frame_data + channel * bytes_per_sample;
+            float sample               = 0.0f;
+            if (audio_format == 3 && bits_per_sample == 32) {
+                std::memcpy(&sample, sample_data, sizeof(sample));
+            } else if (audio_format == 3 && bits_per_sample == 64) {
+                double value;
+                std::memcpy(&value, sample_data, sizeof(value));
+                sample = static_cast<float>(value);
+            } else if (audio_format == 1 && bits_per_sample == 8) {
+                sample = (static_cast<int>(sample_data[0]) - 128) / 128.0f;
+            } else if (audio_format == 1 && bits_per_sample == 16) {
+                sample = static_cast<int16_t>(read_le16(sample_data)) / 32768.0f;
+            } else if (audio_format == 1 && bits_per_sample == 24) {
+                int32_t value = static_cast<int32_t>(sample_data[0]) |
+                                (static_cast<int32_t>(sample_data[1]) << 8) |
+                                (static_cast<int32_t>(sample_data[2]) << 16);
+                if ((value & 0x800000) != 0) {
+                    value |= ~0xffffff;
+                }
+                sample = value / 8388608.0f;
+            } else if (audio_format == 1 && bits_per_sample == 32) {
+                int32_t value = static_cast<int32_t>(read_le32(sample_data));
+                sample        = value / 2147483648.0f;
+            } else {
+                interleaved_samples.clear();
+                return false;
+            }
+            interleaved_samples[static_cast<size_t>(frame * channels + channel)] = sample;
+        }
+    }
+    return true;
+}

examples/common/media_io.h

@@ -110,4 +110,9 @@ bool write_wav_to_file(const std::string& path,
                        uint32_t channels,
                        uint32_t sample_rate);
 
+bool load_wav_from_file(const std::string& path,
+                        std::vector<float>& interleaved_samples,
+                        uint32_t& sample_rate,
+                        uint32_t& channels);
+
 #endif  // __MEDIA_IO_H__

examples/common/resource_owners.hpp

@@ -141,6 +141,37 @@ public:
     }
 };
 
+class SDAudioOwner {
+private:
+    uint32_t sample_rate_ = 0;
+    uint32_t channels_    = 0;
+    std::vector<float> samples_;
+
+public:
+    SDAudioOwner() = default;
+
+    void reset(std::vector<float> samples = {}, uint32_t sample_rate = 0, uint32_t channels = 0) {
+        samples_     = std::move(samples);
+        sample_rate_ = sample_rate;
+        channels_    = channels;
+    }
+
+    bool empty() const {
+        return samples_.empty();
+    }
+
+    sd_audio_t get() {
+        return {sample_rate_,
+                channels_,
+                channels_ == 0 ? 0 : static_cast<uint64_t>(samples_.size() / channels_),
+                samples_.empty() ? nullptr : samples_.data()};
+    }
+
+    const std::vector<float>& samples() const {
+        return samples_;
+    }
+};
+
 class SDImageVec {
 private:
     std::vector<sd_image_t> images_;