DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 03:24:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add Wan2.2 S2V (audio+img-to-video) support (#1925)

Browse Source 
Co-authored-by: leejet <leejet714@gmail.com>
This commit is contained in:
Georgeandleejet authored and GitHub committed 2026-09-13 23:33:50 +08:00
1 parent 4a7da26b73
commit 0bd72f075a
32 files changed
+1533 -57
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

examples/cli/main.cpp
+4 -4
View File
@@ -419,7 +419,8 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy,
LOG_ERROR("save preview image to '%s' failed", path.string().c_str());
}
} else {
if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps, cli_params->compression_quality) != 0) {
int fps = cli_params->preview_method == PREVIEW_PROJ ? cli_params->preview_fps / 4 : cli_params->preview_fps;
if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, fps, cli_params->compression_quality) != 0) {
LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str());
}
}
@@ -687,8 +688,6 @@ int main(int argc, const char* argv[]) {
}
}
cli_params.preview_fps = gen_params.fps;
if (cli_params.preview_method == PREVIEW_PROJ)
cli_params.preview_fps /= 4;
sd_set_preview_callback(step_callback,
cli_params.preview_method,
@@ -951,9 +950,10 @@ int main(int argc, const char* argv[]) {
} else if (cli_params.mode == VID_GEN) {
sd_vid_gen_params_t vid_gen_params = gen_params.to_sd_vid_gen_params_t();
sd_image_t* generated_video = nullptr;
if (!generate_video(sd_ctx.get(), &vid_gen_params, &generated_video, &num_results, &generated_audio)) {
if (!generate_video(sd_ctx.get(), &vid_gen_params, &generated_video, &num_results, &generated_audio, &cli_params.preview_fps)) {
generated_video = nullptr;
}
gen_params.fps = cli_params.preview_fps;
results.adopt(generated_video, num_results);
}
examples/common/common.cpp
+19
View File
@@ -460,6 +460,11 @@ ArgOptions SDContextParams::get_options() {
"path to standalone LTX audio vae model",
0,
&audio_vae_path},
{"",
"--audio-encoder",
"path to wav2vec2 audio encoder model (Wan2.2 S2V)",
0,
&audio_encoder_path},
{"",
"--taesd",
"path to taesd. Using Tiny AutoEncoder for fast decoding (low quality)",
@@ -898,6 +903,7 @@ std::string SDContextParams::to_string() const {
<< " vae_path: \"" << vae_path << "\",\n"
<< " vae_format: \"" << vae_format << "\",\n"
<< " audio_vae_path: \"" << audio_vae_path << "\",\n"
<< " audio_encoder_path: \"" << audio_encoder_path << "\",\n"
<< " taesd_path: \"" << taesd_path << "\",\n"
<< " esrgan_path: \"" << esrgan_path << "\",\n"
<< " control_net_path: \"" << control_net_path << "\",\n"
@@ -963,6 +969,7 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) {
sd_ctx_params.embeddings_connectors_path = embeddings_connectors_path.c_str();
sd_ctx_params.vae_path = vae_path.c_str();
sd_ctx_params.audio_vae_path = audio_vae_path.c_str();
sd_ctx_params.audio_encoder_path = audio_encoder_path.c_str();
sd_ctx_params.taesd_path = taesd_path.c_str();
sd_ctx_params.control_net_path = control_net_path.c_str();
sd_ctx_params.ip_adapter_path = ip_adapter_path.c_str();
@@ -1515,6 +1522,14 @@ ArgOptions SDGenerationParams::get_options() {
return 1;
};
auto on_audio_arg = [&](int argc, const char** argv, int index) {
if (++index >= argc) {
return -1;
}
ref_audio_paths.push_back(argv[index]);
return 1;
};
auto on_cache_mode_arg = [&](int argc, const char** argv, int index) {
if (++index >= argc) {
return -1;
@@ -1704,6 +1719,10 @@ ArgOptions SDGenerationParams::get_options() {
"--ref-audio",
"standalone WAV reference for MiniMax-H3 Ref2VA (can be used multiple times)",
on_ref_audio_arg},
{"",
"--audio",
"driving audio track (Wan2.2 S2V; can be used once)",
on_audio_arg},
{"",
"--cache-mode",
"caching method: 'easycache' (DiT), 'ucache' (UNET), 'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT Chebyshev+Taylor forecasting)",
examples/common/common.h
+1
View File
@@ -131,6 +131,7 @@ struct SDContextParams {
std::string vae_path;
std::string vae_format = "auto";
std::string audio_vae_path;
std::string audio_encoder_path;
std::string taesd_path;
std::string esrgan_path;
std::string control_net_path;
examples/server/async_jobs.cpp
+2 -3
View File
@@ -245,7 +245,7 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
{
std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
sd_image_t* raw_results = nullptr;
if (!generate_video(runtime.sd_ctx, &params, &raw_results, &num_results, &generated_audio)) {
if (!generate_video(runtime.sd_ctx, &params, &raw_results, &num_results, &generated_audio, &output_fps)) {
raw_results = nullptr;
}
results.adopt(raw_results, num_results);
@@ -261,7 +261,7 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
std::vector<uint8_t> video_bytes = create_video_from_sd_images_to_vector(job.vid_gen.output_format,
results.data(),
num_results,
job.vid_gen.gen_params.fps,
output_fps,
job.vid_gen.output_compression,
generated_audio);
free_sd_audio(generated_audio);
@@ -273,7 +273,6 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
output_media_b64 = base64_encode(video_bytes);
output_media_mime_type = video_mime_type(job.vid_gen.output_format);
output_frame_count = num_results;
output_fps = job.vid_gen.gen_params.fps;
return true;
}