DarthAffe/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-09-25 04:32:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add generation parameters into video metadata (#1901)

Browse Source 
Co-authored-by: leejet <leejet714@gmail.com>
This commit is contained in:
Санька Четвёртый 2026-09-13 23:01:09 +07:00 committed by GitHub
parent 5a5400bf0c
commit 42d6c0ab92
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 83 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
examples/cli/main.cpp
View File

@ -541,12 +541,16 @@ bool save_results(const SDCliParams& cli_params,
if (cli_params.mode == VID_GEN && num_results > 1) { if (cli_params.mode == VID_GEN && num_results > 1) {
if (ext_lower != ".avi" && ext_lower != ".webp" && ext_lower != ".webm") if (ext_lower != ".avi" && ext_lower != ".webp" && ext_lower != ".webm")
ext = ".avi"; ext = ".avi";
std::string params = gen_params.embed_image_metadata
? get_image_params(ctx_params, gen_params, gen_params.seed, cli_params.mode)
: "";
fs::path video_path = base_path; fs::path video_path = base_path;
video_path += ext; video_path += ext;
std::string final_ext_lower = ext.string(); std::string final_ext_lower = ext.string();
std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower); std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower);
const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm"); const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm");
if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr) == 0) { if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) {
LOG_INFO("save result video to '%s'", video_path.string().c_str()); LOG_INFO("save result video to '%s'", video_path.string().c_str());
if (generated_audio != nullptr && !mux_audio) { if (generated_audio != nullptr && !mux_audio) {
fs::path wav_path = video_path; fs::path wav_path = video_path;

64
examples/common/media_io.cpp
View File

@ -810,7 +810,31 @@ uint8_t* load_image_from_memory(const char* image_bytes,
return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel); return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel);
} }
std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) { static void append_avi_metadata(std::vector<uint8_t>& data, const std::string& parameters) {
if (parameters.empty()) {
return;
}
std::vector<uint8_t> info_content;
write_fourcc(info_content, "INFO");
const size_t comment_size = parameters.size() + 1;
write_fourcc(info_content, "ICMT");
write_u32_le(info_content, static_cast<uint32_t>(comment_size));
info_content.insert(info_content.end(), parameters.begin(), parameters.end());
info_content.push_back(0);
if (comment_size & 1u) {
info_content.push_back(0);
}
write_fourcc(data, "LIST");
write_u32_le(data, static_cast<uint32_t>(info_content.size()));
data.insert(data.end(), info_content.begin(), info_content.end());
size_t start_pos = data.size();
}
std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
if (num_images == 0) { if (num_images == 0) {
fprintf(stderr, "Error: Image array is empty.\n"); fprintf(stderr, "Error: Image array is empty.\n");
return {}; return {};
@ -1000,6 +1024,8 @@ std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images
const size_t movi_size = avi_data.size() - movi_size_pos - 4; const size_t movi_size = avi_data.size() - movi_size_pos - 4;
patch_u32_le(avi_data, movi_size_pos, static_cast<uint32_t>(movi_size)); patch_u32_le(avi_data, movi_size_pos, static_cast<uint32_t>(movi_size));
append_avi_metadata(avi_data, parameters);
write_fourcc(avi_data, "idx1"); write_fourcc(avi_data, "idx1");
write_u32_le(avi_data, static_cast<uint32_t>(index.size() * 16)); write_u32_le(avi_data, static_cast<uint32_t>(index.size() * 16));
for (const auto& entry : index) { for (const auto& entry : index) {
@ -1015,8 +1041,8 @@ std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images
return avi_data; return avi_data;
} }
int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) { int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
std::vector<uint8_t> avi_data = create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio); std::vector<uint8_t> avi_data = create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
if (avi_data.empty()) { if (avi_data.empty()) {
return -1; return -1;
} }
@ -1146,7 +1172,7 @@ int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images
#endif #endif
#ifdef SD_USE_WEBM #ifdef SD_USE_WEBM
std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) { std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
if (num_images == 0) { if (num_images == 0) {
fprintf(stderr, "Error: Image array is empty.\n"); fprintf(stderr, "Error: Image array is empty.\n");
return {}; return {};
@ -1213,6 +1239,21 @@ std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, in
segment.GetSegmentInfo()->set_writing_app("stable-diffusion.cpp"); segment.GetSegmentInfo()->set_writing_app("stable-diffusion.cpp");
segment.GetSegmentInfo()->set_muxing_app("stable-diffusion.cpp"); segment.GetSegmentInfo()->set_muxing_app("stable-diffusion.cpp");
LOG_DEBUG("Embedding parameters to metadata: %s", parameters.c_str());
if (!parameters.empty()) {
mkvmuxer::Tag* tag = segment.AddTag();
if (tag) {
if (!tag->add_simple_tag("COMMENT", parameters.c_str())) {
LOG_WARN("Failed to add COMMENT simple tag.");
}
} else {
LOG_WARN("Failed to add tag to segment.");
}
} else {
LOG_INFO("Paramaters is empty, COMMENT tag not embedded.\n");
}
const uint64_t frame_duration_ns = std::max<uint64_t>( const uint64_t frame_duration_ns = std::max<uint64_t>(
1, static_cast<uint64_t>(std::llround(1000000000.0 / static_cast<double>(fps)))); 1, static_cast<uint64_t>(std::llround(1000000000.0 / static_cast<double>(fps))));
uint64_t timestamp_ns = 0; uint64_t timestamp_ns = 0;
@ -1271,8 +1312,8 @@ std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, in
return writer.data(); return writer.data();
} }
int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) { int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
std::vector<uint8_t> webm_data = create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio); std::vector<uint8_t> webm_data = create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
if (webm_data.empty()) { if (webm_data.empty()) {
return -1; return -1;
} }
@ -1289,7 +1330,8 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou
int num_images, int num_images,
int fps, int fps,
int quality, int quality,
const sd_audio_t* audio) { const sd_audio_t* audio,
const std::string& parameters) {
std::string format = output_format; std::string format = output_format;
std::transform(format.begin(), format.end(), format.begin(), std::transform(format.begin(), format.end(), format.begin(),
[](unsigned char c) { return static_cast<char>(tolower(c)); }); [](unsigned char c) { return static_cast<char>(tolower(c)); });
@ -1299,7 +1341,7 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou
#ifdef SD_USE_WEBM #ifdef SD_USE_WEBM
if (format == "webm") { if (format == "webm") {
return create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio); return create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
} }
#endif #endif
@ -1309,14 +1351,14 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou
} }
#endif #endif
return create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio); return create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
} }
int create_video_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) { int create_video_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
std::string path = filename ? filename : ""; std::string path = filename ? filename : "";
auto pos = path.find_last_of('.'); auto pos = path.find_last_of('.');
std::string ext = pos == std::string::npos ? "" : path.substr(pos); std::string ext = pos == std::string::npos ? "" : path.substr(pos);
std::vector<uint8_t> video_data = create_video_from_sd_images_to_vector(ext, images, num_images, fps, quality, audio); std::vector<uint8_t> video_data = create_video_from_sd_images_to_vector(ext, images, num_images, fps, quality, audio, parameters);
if (video_data.empty()) { if (video_data.empty()) {
return -1; return -1;
} }

18
examples/common/media_io.h
View File

@ -58,12 +58,14 @@ int create_mjpg_avi_from_sd_images(const char* filename,
int num_images, int num_images,
int fps, int fps,
int quality = 90, int quality = 90,
const sd_audio_t* audio = nullptr); const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images,
int num_images, int num_images,
int fps, int fps,
int quality = 90, int quality = 90,
const sd_audio_t* audio = nullptr); const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
#ifdef SD_USE_WEBP #ifdef SD_USE_WEBP
int create_animated_webp_from_sd_images(const char* filename, int create_animated_webp_from_sd_images(const char* filename,
@ -83,12 +85,14 @@ int create_webm_from_sd_images(const char* filename,
int num_images, int num_images,
int fps, int fps,
int quality = 90, int quality = 90,
const sd_audio_t* audio = nullptr); const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images,
int num_images, int num_images,
int fps, int fps,
int quality = 90, int quality = 90,
const sd_audio_t* audio = nullptr); const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
#endif #endif
int create_video_from_sd_images(const char* filename, int create_video_from_sd_images(const char* filename,
@ -96,13 +100,15 @@ int create_video_from_sd_images(const char* filename,
int num_images, int num_images,
int fps, int fps,
int quality = 90, int quality = 90,
const sd_audio_t* audio = nullptr); const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& output_format, std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& output_format,
sd_image_t* images, sd_image_t* images,
int num_images, int num_images,
int fps, int fps,
int quality = 90, int quality = 90,
const sd_audio_t* audio = nullptr); const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
bool write_wav_to_file(const std::string& path, bool write_wav_to_file(const std::string& path,
const float* interleaved_samples, const float* interleaved_samples,

6
examples/server/async_jobs.cpp
View File

@ -237,6 +237,9 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
int& output_fps, int& output_fps,
std::string& error_message) { std::string& error_message) {
sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t(); sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t();
std::string str_params = job.vid_gen.gen_params.embed_image_metadata
? get_image_params(*runtime.ctx_params, job.vid_gen.gen_params, job.vid_gen.gen_params.seed, VID_GEN)
: "";
SDImageVec results; SDImageVec results;
int num_results = 0; int num_results = 0;
@ -263,7 +266,8 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
num_results, num_results,
output_fps, output_fps,
job.vid_gen.output_compression, job.vid_gen.output_compression,
generated_audio); generated_audio,
str_params);
free_sd_audio(generated_audio); free_sd_audio(generated_audio);
if (video_bytes.empty()) { if (video_bytes.empty()) {
error_message = "failed to encode generated video container"; error_message = "failed to encode generated video container";

3
src/model/te/llm.hpp
View File

@ -340,7 +340,8 @@ namespace LLM {
// The default would reject valid models, so only compare a detected dim. // The default would reject valid models, so only compare a detected dim.
if (enable_vision && out_hidden_size_detected && if (enable_vision && out_hidden_size_detected &&
config.vision.out_hidden_size != config.hidden_size) { config.vision.out_hidden_size != config.hidden_size) {
LOG_ERROR("vision projector output size (%" PRId64 ") does not match LLM hidden size (%" PRId64 "), " LOG_ERROR("vision projector output size (%" PRId64 ") does not match LLM hidden size (%" PRId64
"), "
"the vision weights (mmproj) likely belong to a different LLM variant, vision disabled", "the vision weights (mmproj) likely belong to a different LLM variant, vision disabled",
config.vision.out_hidden_size, config.vision.out_hidden_size,
config.hidden_size); config.hidden_size);