mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-08-10 06:07:58 +00:00
feat(animatediff): support img2video via --init-img (#1789)
This commit is contained in:
parent
fafe8e606c
commit
7717e82c00
BIN
assets/animatediff/img2video_demo.gif
Normal file
BIN
assets/animatediff/img2video_demo.gif
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.7 MiB |
@ -134,6 +134,25 @@ sd-cli -M vid_gen --model realisticVisionV60B1.safetensors \
|
|||||||
-p "close up photo of a rabbit ...<lora:v3_sd15_adapter:1.0>" ...
|
-p "close up photo of a rabbit ...<lora:v3_sd15_adapter:1.0>" ...
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## img2video
|
||||||
|
|
||||||
|
Pass a pre-rendered image via `-i / --init-img` to animate FROM it. All N output frames start from the encoded init latent, then per-frame noise is added at `--strength`. Character identity, composition, and quality are anchored by the init image; the motion module adds subtle motion on top.
|
||||||
|
|
||||||
|
Left: init image rendered with `-M img_gen`. Right: 8-frame vid_gen output.
|
||||||
|
|
||||||
|
<img src="../assets/animatediff/img2video_demo.gif" width="512"/>
|
||||||
|
|
||||||
|
```
|
||||||
|
sd-cli -M img_gen ... -o init.png # any high-quality still
|
||||||
|
sd-cli -M vid_gen --motion-module mm_sd15_v3.safetensors \
|
||||||
|
-i init.png --strength 0.75 \
|
||||||
|
--cfg-scale 7.0 --sampling-method euler --scheduler karras \
|
||||||
|
-H 512 -W 512 --video-frames 8 --steps 25 -s 42 \
|
||||||
|
-p "..." -o out.avi
|
||||||
|
```
|
||||||
|
|
||||||
|
`--strength` controls how far the motion module is allowed to deviate from the init image (higher = more motion, lower = more static).
|
||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
|
|
||||||
- The motion module was trained at `video_length=16`. Running with
|
- The motion module was trained at `video_length=16`. Running with
|
||||||
|
|||||||
@ -4687,8 +4687,16 @@ static std::optional<ImageGenerationLatents> prepare_image_generation_latents(sd
|
|||||||
init_latent.dim() >= 4 && init_latent.shape()[3] == 1) {
|
init_latent.dim() >= 4 && init_latent.shape()[3] == 1) {
|
||||||
int n_frames = sd_ctx->sd->animatediff_num_frames;
|
int n_frames = sd_ctx->sd->animatediff_num_frames;
|
||||||
std::vector<int64_t> shape(init_latent.shape().begin(), init_latent.shape().end());
|
std::vector<int64_t> shape(init_latent.shape().begin(), init_latent.shape().end());
|
||||||
shape[3] = n_frames;
|
shape[3] = n_frames;
|
||||||
init_latent = sd::Tensor<float>(std::move(shape)); // zero-filled batch of N frames; per-frame noise is generated later via randn_like.
|
if (!init_image_tensor.empty()) {
|
||||||
|
sd::Tensor<float> replicated(shape);
|
||||||
|
for (int f = 0; f < n_frames; ++f) {
|
||||||
|
sd::ops::slice_assign(&replicated, 3, f, f + 1, init_latent);
|
||||||
|
}
|
||||||
|
init_latent = std::move(replicated);
|
||||||
|
} else {
|
||||||
|
init_latent = sd::Tensor<float>(std::move(shape));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!control_image_tensor.empty()) {
|
if (!control_image_tensor.empty()) {
|
||||||
@ -6132,6 +6140,7 @@ static bool generate_animatediff_video(sd_ctx_t* sd_ctx,
|
|||||||
img_gen_params.height = sd_vid_gen_params->height;
|
img_gen_params.height = sd_vid_gen_params->height;
|
||||||
img_gen_params.sample_params = sd_vid_gen_params->sample_params;
|
img_gen_params.sample_params = sd_vid_gen_params->sample_params;
|
||||||
img_gen_params.strength = sd_vid_gen_params->strength;
|
img_gen_params.strength = sd_vid_gen_params->strength;
|
||||||
|
img_gen_params.init_image = sd_vid_gen_params->init_image;
|
||||||
img_gen_params.seed = sd_vid_gen_params->seed;
|
img_gen_params.seed = sd_vid_gen_params->seed;
|
||||||
img_gen_params.batch_count = 1;
|
img_gen_params.batch_count = 1;
|
||||||
img_gen_params.control_strength = 1.0f;
|
img_gen_params.control_strength = 1.0f;
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user