DarthAffe/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-09-25 12:40:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add generation parameters into video metadata (#1901)

Browse Source 
Co-authored-by: leejet <leejet714@gmail.com>
This commit is contained in:
Санька Четвёртый 2026-09-13 23:01:09 +07:00 committed by GitHub
parent 5a5400bf0c
commit 42d6c0ab92
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 83 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
examples/cli/main.cpp
View File

@ -541,12 +541,16 @@ bool save_results(const SDCliParams& cli_params,
if (cli_params.mode == VID_GEN && num_results > 1) {
if (ext_lower != ".avi" && ext_lower != ".webp" && ext_lower != ".webm")
ext = ".avi";
std::string params = gen_params.embed_image_metadata
? get_image_params(ctx_params, gen_params, gen_params.seed, cli_params.mode)
: "";
fs::path video_path = base_path;
video_path += ext;
std::string final_ext_lower = ext.string();
std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower);
const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm");
if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr) == 0) {
if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) {
LOG_INFO("save result video to '%s'", video_path.string().c_str());
if (generated_audio != nullptr && !mux_audio) {
fs::path wav_path = video_path;

64
examples/common/media_io.cpp
View File

@ -810,7 +810,31 @@ uint8_t* load_image_from_memory(const char* image_bytes,
return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel);
}
std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) {
static void append_avi_metadata(std::vector<uint8_t>& data, const std::string& parameters) {
if (parameters.empty()) {
return;
}
std::vector<uint8_t> info_content;
write_fourcc(info_content, "INFO");
const size_t comment_size = parameters.size() + 1;
write_fourcc(info_content, "ICMT");
write_u32_le(info_content, static_cast<uint32_t>(comment_size));
info_content.insert(info_content.end(), parameters.begin(), parameters.end());
info_content.push_back(0);
if (comment_size & 1u) {
info_content.push_back(0);
}
write_fourcc(data, "LIST");
write_u32_le(data, static_cast<uint32_t>(info_content.size()));
data.insert(data.end(), info_content.begin(), info_content.end());
size_t start_pos = data.size();
}
std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
if (num_images == 0) {
fprintf(stderr, "Error: Image array is empty.\n");
return {};
@ -1000,6 +1024,8 @@ std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images
const size_t movi_size = avi_data.size() - movi_size_pos - 4;
patch_u32_le(avi_data, movi_size_pos, static_cast<uint32_t>(movi_size));
append_avi_metadata(avi_data, parameters);
write_fourcc(avi_data, "idx1");
write_u32_le(avi_data, static_cast<uint32_t>(index.size() * 16));
for (const auto& entry : index) {
@ -1015,8 +1041,8 @@ std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images
return avi_data;
}
int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) {
std::vector<uint8_t> avi_data = create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio);
int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
std::vector<uint8_t> avi_data = create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
if (avi_data.empty()) {
return -1;
}
@ -1146,7 +1172,7 @@ int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images
#endif
#ifdef SD_USE_WEBM
std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) {
std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
if (num_images == 0) {
fprintf(stderr, "Error: Image array is empty.\n");
return {};
@ -1213,6 +1239,21 @@ std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, in
segment.GetSegmentInfo()->set_writing_app("stable-diffusion.cpp");
segment.GetSegmentInfo()->set_muxing_app("stable-diffusion.cpp");
LOG_DEBUG("Embedding parameters to metadata: %s", parameters.c_str());
if (!parameters.empty()) {
mkvmuxer::Tag* tag = segment.AddTag();
if (tag) {
if (!tag->add_simple_tag("COMMENT", parameters.c_str())) {
LOG_WARN("Failed to add COMMENT simple tag.");
}
} else {
LOG_WARN("Failed to add tag to segment.");
}
} else {
LOG_INFO("Paramaters is empty, COMMENT tag not embedded.\n");
}
const uint64_t frame_duration_ns = std::max<uint64_t>(
1, static_cast<uint64_t>(std::llround(1000000000.0 / static_cast<double>(fps))));
uint64_t timestamp_ns = 0;
@ -1271,8 +1312,8 @@ std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, in
return writer.data();
}
int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) {
std::vector<uint8_t> webm_data = create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio);
int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
std::vector<uint8_t> webm_data = create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
if (webm_data.empty()) {
return -1;
}
@ -1289,7 +1330,8 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou
int num_images,
int fps,
int quality,
const sd_audio_t* audio) {
const sd_audio_t* audio,
const std::string& parameters) {
std::string format = output_format;
std::transform(format.begin(), format.end(), format.begin(),
[](unsigned char c) { return static_cast<char>(tolower(c)); });
@ -1299,7 +1341,7 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou
#ifdef SD_USE_WEBM
if (format == "webm") {
return create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio);
return create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
}
#endif
@ -1309,14 +1351,14 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou
}
#endif
return create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio);
return create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
}
int create_video_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) {
int create_video_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
std::string path = filename ? filename : "";
auto pos = path.find_last_of('.');
std::string ext = pos == std::string::npos ? "" : path.substr(pos);
std::vector<uint8_t> video_data = create_video_from_sd_images_to_vector(ext, images, num_images, fps, quality, audio);
std::vector<uint8_t> video_data = create_video_from_sd_images_to_vector(ext, images, num_images, fps, quality, audio, parameters);
if (video_data.empty()) {
return -1;
}

30
examples/common/media_io.h
View File

@ -57,13 +57,15 @@ int create_mjpg_avi_from_sd_images(const char* filename,
sd_image_t* images,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
int quality = 90,
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
int quality = 90,
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
#ifdef SD_USE_WEBP
int create_animated_webp_from_sd_images(const char* filename,
@ -82,27 +84,31 @@ int create_webm_from_sd_images(const char* filename,
sd_image_t* images,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
int quality = 90,
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
int quality = 90,
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
#endif
int create_video_from_sd_images(const char* filename,
sd_image_t* images,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
int quality = 90,
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& output_format,
sd_image_t* images,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
int quality = 90,
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
bool write_wav_to_file(const std::string& path,
const float* interleaved_samples,

6
examples/server/async_jobs.cpp
View File

@ -237,6 +237,9 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
int& output_fps,
std::string& error_message) {
sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t();
std::string str_params = job.vid_gen.gen_params.embed_image_metadata
? get_image_params(*runtime.ctx_params, job.vid_gen.gen_params, job.vid_gen.gen_params.seed, VID_GEN)
: "";
SDImageVec results;
int num_results = 0;
@ -263,7 +266,8 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
num_results,
output_fps,
job.vid_gen.output_compression,
generated_audio);
generated_audio,
str_params);
free_sd_audio(generated_audio);
if (video_bytes.empty()) {
error_message = "failed to encode generated video container";

3
src/model/te/llm.hpp
View File

@ -340,7 +340,8 @@ namespace LLM {
// The default would reject valid models, so only compare a detected dim.
if (enable_vision && out_hidden_size_detected &&
config.vision.out_hidden_size != config.hidden_size) {
LOG_ERROR("vision projector output size (%" PRId64 ") does not match LLM hidden size (%" PRId64 "), "
LOG_ERROR("vision projector output size (%" PRId64 ") does not match LLM hidden size (%" PRId64
"), "
"the vision weights (mmproj) likely belong to a different LLM variant, vision disabled",
config.vision.out_hidden_size,
config.hidden_size);