diff --git a/assets/wan/Wan2.2_A14B_vace_r2v.mp4 b/assets/wan/Wan2.2_A14B_vace_r2v.mp4 new file mode 100644 index 00000000..caf19c21 Binary files /dev/null and b/assets/wan/Wan2.2_A14B_vace_r2v.mp4 differ diff --git a/docs/wan.md b/docs/wan.md index 13c26a20..1b38bcbc 100644 --- a/docs/wan.md +++ b/docs/wan.md @@ -40,6 +40,9 @@ Wan models require `-M vid_gen`, including single-frame generation. `--video-fra - safetensors: https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/tree/main/split_files/diffusion_models - gguf: https://huggingface.co/QuantStack/Wan2.2-S2V-14B-GGUF/tree/main - int8_convrot safetensors: https://huggingface.co/noctrex/Wan2.2-S2V-14B-int8_convrot + - Wan2.2 VACE-Fun A14B + - safetensors: https://huggingface.co/alibaba-pai/Wan2.2-VACE-Fun-A14B + - gguf: https://huggingface.co/QuantStack/Wan2.2-VACE-Fun-A14B-GGUF/tree/main - Download vae - wan_2.1_vae (for all the wan model except Wan2.2 TI2V 5B) - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/vae/wan_2.1_vae.safetensors @@ -256,3 +259,25 @@ ffmpeg -i ..\..\ComfyUI\input\post+depth.mp4 -qscale:v 1 -vf fps=8 post+depth\fr ``` + +### Wan2.2 VACE-Fun A14B + +VACE-Fun runs as a MoE pair: `--diffusion-model` takes the low-noise expert and +`--high-noise-diffusion-model` the high-noise one. Reference-to-video uses `-i` +for the reference image, same as Wan2.1 VACE. + +#### R2V + +``` +.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.2-VACE-Fun-A14B-low-noise-Q8_0.gguf --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-VACE-Fun-A14B-high-noise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu +``` + + + +#### T2V + +Same command without `-i` (VACE context is synthesized from an empty control +video, like Wan2.1 VACE t2v). + +> On GPUs with ~12 GB VRAM, VACE also needs `--vae-tiling` — the control-video +> encode can exceed the budget otherwise. diff --git a/src/model_io/gguf_io.cpp b/src/model_io/gguf_io.cpp index 081fcf97..3e574487 100644 --- a/src/model_io/gguf_io.cpp +++ b/src/model_io/gguf_io.cpp @@ -47,10 +47,14 @@ bool read_gguf_file(const std::string& file_path, gguf_context* ctx_gguf_ = nullptr; ggml_context* ctx_meta_ = nullptr; - ctx_gguf_ = gguf_init_from_file(file_path.c_str(), {true, &ctx_meta_}); + GGUFReader gguf_reader; + bool probe_ok = gguf_reader.load(file_path); + + if (!probe_ok || !gguf_reader.has_tensors_beyond_ggml_limits()) { + ctx_gguf_ = gguf_init_from_file(file_path.c_str(), {true, &ctx_meta_}); + } if (!ctx_gguf_) { - GGUFReader gguf_reader; - if (!gguf_reader.load(file_path)) { + if (!probe_ok && !gguf_reader.load(file_path)) { set_error(error, "failed to open '" + file_path + "' with GGUFReader"); return false; } diff --git a/src/model_io/gguf_reader_ext.h b/src/model_io/gguf_reader_ext.h index 7b372e4f..b55d49f1 100644 --- a/src/model_io/gguf_reader_ext.h +++ b/src/model_io/gguf_reader_ext.h @@ -35,23 +35,72 @@ enum class GGUFMetadataType : uint32_t { class GGUFReader { private: std::vector tensors_; + bool has_wide_tensors_ = false; + uint64_t remaining_bytes_ = 0; size_t data_offset_; size_t alignment_ = 32; // default alignment is 32 template bool safe_read(std::ifstream& fin, T& value) { - fin.read(reinterpret_cast(&value), sizeof(T)); - return fin.good(); + return safe_read(fin, reinterpret_cast(&value), sizeof(T)); } bool safe_read(std::ifstream& fin, char* buffer, size_t size) { + if (size > remaining_bytes_) + return false; fin.read(buffer, size); - return fin.good(); + if (!fin.good()) + return false; + remaining_bytes_ -= size; + return true; } - bool safe_seek(std::ifstream& fin, std::streamoff offset, std::ios::seekdir dir) { - fin.seekg(offset, dir); - return fin.good(); + bool safe_skip(std::ifstream& fin, uint64_t count, uint64_t element_size = 1) { + if (count > remaining_bytes_ / element_size) + return false; + uint64_t size = count * element_size; + fin.seekg(static_cast(size), std::ios::cur); + if (!fin.good()) + return false; + remaining_bytes_ -= size; + return true; + } + + bool skip_metadata_values(std::ifstream& fin, GGUFMetadataType type, uint64_t count) { + switch (type) { + case GGUFMetadataType::UINT8: + case GGUFMetadataType::INT8: + case GGUFMetadataType::BOOL: + return safe_skip(fin, count); + + case GGUFMetadataType::UINT16: + case GGUFMetadataType::INT16: + return safe_skip(fin, count, 2); + + case GGUFMetadataType::UINT32: + case GGUFMetadataType::INT32: + case GGUFMetadataType::FLOAT32: + return safe_skip(fin, count, 4); + + case GGUFMetadataType::UINT64: + case GGUFMetadataType::INT64: + case GGUFMetadataType::FLOAT64: + return safe_skip(fin, count, 8); + + case GGUFMetadataType::STRING: + if (count > remaining_bytes_ / sizeof(uint64_t)) + return false; + for (uint64_t i = 0; i < count; i++) { + uint64_t len = 0; + if (!safe_read(fin, len) || !safe_skip(fin, len)) + return false; + } + return true; + + default: + LOG_ERROR("Unknown metadata type=%u", static_cast(type)); + return false; + } } bool read_metadata(std::ifstream& fin) { @@ -84,52 +133,12 @@ private: return true; } - switch (static_cast(type)) { - case GGUFMetadataType::UINT8: - case GGUFMetadataType::INT8: - case GGUFMetadataType::BOOL: - return safe_seek(fin, 1, std::ios::cur); - - case GGUFMetadataType::UINT16: - case GGUFMetadataType::INT16: - return safe_seek(fin, 2, std::ios::cur); - - case GGUFMetadataType::UINT32: - case GGUFMetadataType::INT32: - case GGUFMetadataType::FLOAT32: - return safe_seek(fin, 4, std::ios::cur); - - case GGUFMetadataType::UINT64: - case GGUFMetadataType::INT64: - case GGUFMetadataType::FLOAT64: - return safe_seek(fin, 8, std::ios::cur); - - case GGUFMetadataType::STRING: { - uint64_t len = 0; - if (!safe_read(fin, len)) - return false; - return safe_seek(fin, len, std::ios::cur); - } - - case GGUFMetadataType::ARRAY: { - uint32_t elem_type = 0; - uint64_t len = 0; - if (!safe_read(fin, elem_type)) - return false; - if (!safe_read(fin, len)) - return false; - - for (uint64_t i = 0; i < len; i++) { - if (!read_metadata(fin)) - return false; - } - return true; - } - - default: - LOG_ERROR("Unknown metadata type=%u", type); + uint64_t count = 1; + if (type == static_cast(GGUFMetadataType::ARRAY)) { + if (!safe_read(fin, type) || !safe_read(fin, count)) return false; } + return skip_metadata_values(fin, static_cast(type), count); } GGUFTensorInfo read_tensor_info(std::ifstream& fin) { @@ -154,6 +163,7 @@ private: } if (n_dims > GGML_MAX_DIMS) { + has_wide_tensors_ = true; for (uint32_t i = GGML_MAX_DIMS; i < n_dims; i++) { info.shape[GGML_MAX_DIMS - 1] *= info.shape[i]; // stack to last dim; } @@ -174,12 +184,20 @@ private: public: bool load(const std::string& file_path) { - std::ifstream fin(file_path, std::ios::binary); + std::ifstream fin(file_path, std::ios::binary | std::ios::ate); if (!fin) { LOG_ERROR("failed to open '%s'", file_path.c_str()); return false; } + std::streamoff file_size = fin.tellg(); + if (file_size < 0) + return false; + remaining_bytes_ = static_cast(file_size); + fin.seekg(0, std::ios::beg); + if (!fin.good()) + return false; + // --- Header --- char magic[4]; if (!safe_read(fin, magic, 4) || strncmp(magic, "GGUF", 4) != 0) { @@ -228,6 +246,8 @@ public: } const std::vector& tensors() const { return tensors_; } + + bool has_tensors_beyond_ggml_limits() const { return has_wide_tensors_; } size_t data_offset() const { return data_offset_; } };