feat: return generation metadata with core results (#2206)

src/pipeline/diffusion_engine.cpp

@@ -675,87 +675,93 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
     const auto* sd_ctx_params = &configuration.params;
     auto& use_tae             = configuration.use_tae;
     auto& use_audio_vae       = configuration.use_audio_vae;
+    const auto load_model     = [&](const char* name, const char* path, const std::string& prefix = "") {
+        if (!model_loader.init_from_file(path, prefix))
+            return false;
+        configuration.record_model(name, path);
+        return true;
+    };
     if (strlen(SAFE_STR(sd_ctx_params->model_path)) > 0) {
         LOG_INFO("loading model from '%s'", sd_ctx_params->model_path);
-        if (!model_loader.init_from_file(sd_ctx_params->model_path)) {
+        if (!load_model("model", sd_ctx_params->model_path)) {
             LOG_ERROR("init model loader from file failed: '%s'", sd_ctx_params->model_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->diffusion_model_path)) > 0) {
         LOG_INFO("loading diffusion model from '%s'", sd_ctx_params->diffusion_model_path);
-        if (!model_loader.init_from_file(sd_ctx_params->diffusion_model_path, "model.diffusion_model.")) {
+        if (!load_model("diffusion_model", sd_ctx_params->diffusion_model_path, "model.diffusion_model.")) {
             LOG_WARN("loading diffusion model from '%s' failed", sd_ctx_params->diffusion_model_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path)) > 0) {
         LOG_INFO("loading high noise diffusion model from '%s'", sd_ctx_params->high_noise_diffusion_model_path);
-        if (!model_loader.init_from_file(sd_ctx_params->high_noise_diffusion_model_path, "model.high_noise_diffusion_model.")) {
+        if (!load_model("high_noise_diffusion_model", sd_ctx_params->high_noise_diffusion_model_path, "model.high_noise_diffusion_model.")) {
             LOG_WARN("loading diffusion model from '%s' failed", sd_ctx_params->high_noise_diffusion_model_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->uncond_diffusion_model_path)) > 0) {
         LOG_INFO("loading unconditional diffusion model from '%s'", sd_ctx_params->uncond_diffusion_model_path);
-        if (!model_loader.init_from_file(sd_ctx_params->uncond_diffusion_model_path, "model.diffusion_model.uncond.")) {
+        if (!load_model("uncond_diffusion_model", sd_ctx_params->uncond_diffusion_model_path, "model.diffusion_model.uncond.")) {
             LOG_WARN("loading unconditional diffusion model from '%s' failed", sd_ctx_params->uncond_diffusion_model_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->clip_l_path)) > 0) {
         LOG_INFO("loading clip_l from '%s'", sd_ctx_params->clip_l_path);
-        if (!model_loader.init_from_file(sd_ctx_params->clip_l_path, "clip_l.")) {
+        if (!load_model("clip_l", sd_ctx_params->clip_l_path, "clip_l.")) {
             LOG_WARN("loading clip_l from '%s' failed", sd_ctx_params->clip_l_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->clip_g_path)) > 0) {
         LOG_INFO("loading clip_g from '%s'", sd_ctx_params->clip_g_path);
-        if (!model_loader.init_from_file(sd_ctx_params->clip_g_path, "clip_g.")) {
+        if (!load_model("clip_g", sd_ctx_params->clip_g_path, "clip_g.")) {
             LOG_WARN("loading clip_g from '%s' failed", sd_ctx_params->clip_g_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->clip_vision_path)) > 0) {
         LOG_INFO("loading clip_vision from '%s'", sd_ctx_params->clip_vision_path);
-        if (!model_loader.init_from_file(sd_ctx_params->clip_vision_path, "clip_vision.")) {
+        if (!load_model("clip_vision", sd_ctx_params->clip_vision_path, "clip_vision.")) {
             LOG_WARN("loading clip_vision from '%s' failed", sd_ctx_params->clip_vision_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->t5xxl_path)) > 0) {
         LOG_INFO("loading t5xxl from '%s'", sd_ctx_params->t5xxl_path);
-        if (!model_loader.init_from_file(sd_ctx_params->t5xxl_path, "text_encoders.t5xxl.transformer.")) {
+        if (!load_model("t5xxl", sd_ctx_params->t5xxl_path, "text_encoders.t5xxl.transformer.")) {
             LOG_WARN("loading t5xxl from '%s' failed", sd_ctx_params->t5xxl_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->pulid_weights_path)) > 0) {
         LOG_INFO("loading PuLID weights from '%s'", sd_ctx_params->pulid_weights_path);
-        if (!model_loader.init_from_file(sd_ctx_params->pulid_weights_path,
-                                         "model.diffusion_model.")) {
+        if (!load_model("pulid_weights", sd_ctx_params->pulid_weights_path,
+                        "model.diffusion_model.")) {
             LOG_WARN("loading PuLID weights from '%s' failed", sd_ctx_params->pulid_weights_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->llm_path)) > 0) {
         LOG_INFO("loading llm from '%s'", sd_ctx_params->llm_path);
-        if (!model_loader.init_from_file(sd_ctx_params->llm_path, "text_encoders.llm.")) {
+        if (!load_model("llm", sd_ctx_params->llm_path, "text_encoders.llm.")) {
             LOG_WARN("loading llm from '%s' failed", sd_ctx_params->llm_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->llm_vision_path)) > 0) {
         LOG_INFO("loading llm vision from '%s'", sd_ctx_params->llm_vision_path);
-        if (!model_loader.init_from_file(sd_ctx_params->llm_vision_path, "text_encoders.llm.visual.")) {
+        if (!load_model("llm_vision", sd_ctx_params->llm_vision_path, "text_encoders.llm.visual.")) {
             LOG_WARN("loading llm vision from '%s' failed", sd_ctx_params->llm_vision_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->vae_path)) > 0) {
         LOG_INFO("loading vae from '%s'", sd_ctx_params->vae_path);
-        if (!model_loader.init_from_file(sd_ctx_params->vae_path, "vae.")) {
+        if (!load_model("vae", sd_ctx_params->vae_path, "vae.")) {
             LOG_WARN("loading vae from '%s' failed", sd_ctx_params->vae_path);
             external_vae_is_invalid = true;
         }
@@ -763,7 +769,7 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
 
     if (strlen(SAFE_STR(sd_ctx_params->taesd_path)) > 0) {
         LOG_INFO("loading tae from '%s'", sd_ctx_params->taesd_path);
-        if (!model_loader.init_from_file(sd_ctx_params->taesd_path, "tae.")) {
+        if (!load_model("taesd", sd_ctx_params->taesd_path, "tae.")) {
             LOG_WARN("loading tae from '%s' failed", sd_ctx_params->taesd_path);
         } else {
             use_tae = true;
@@ -772,14 +778,14 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
 
     if (strlen(SAFE_STR(sd_ctx_params->embeddings_connectors_path)) > 0) {
         LOG_INFO("loading embeddings connectors from '%s'", sd_ctx_params->embeddings_connectors_path);
-        if (!model_loader.init_from_file(sd_ctx_params->embeddings_connectors_path)) {
+        if (!load_model("embeddings_connectors", sd_ctx_params->embeddings_connectors_path)) {
             LOG_WARN("loading embeddings connectors from '%s' failed", sd_ctx_params->embeddings_connectors_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->audio_vae_path)) > 0) {
         LOG_INFO("loading audio VAE from '%s'", sd_ctx_params->audio_vae_path);
-        if (!model_loader.init_from_file(sd_ctx_params->audio_vae_path)) {
+        if (!load_model("audio_vae", sd_ctx_params->audio_vae_path)) {
             LOG_WARN("loading audio VAE weights from '%s' failed", sd_ctx_params->audio_vae_path);
         } else {
             use_audio_vae = true;
@@ -788,15 +794,15 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
 
     if (strlen(SAFE_STR(sd_ctx_params->audio_encoder_path)) > 0) {
         LOG_INFO("loading audio encoder (wav2vec2) from '%s'", sd_ctx_params->audio_encoder_path);
-        if (!model_loader.init_from_file(sd_ctx_params->audio_encoder_path, "wav2vec2.")) {
+        if (!load_model("audio_encoder", sd_ctx_params->audio_encoder_path, "wav2vec2.")) {
             LOG_WARN("loading audio encoder weights from '%s' failed", sd_ctx_params->audio_encoder_path);
         }
     }
 
     if (strlen(SAFE_STR(sd_ctx_params->motion_module_path)) > 0) {
         LOG_INFO("loading motion module (AnimateDiff) from '%s'", sd_ctx_params->motion_module_path);
-        if (!model_loader.init_from_file(sd_ctx_params->motion_module_path,
-                                         "model.diffusion_model.motion_module.")) {
+        if (!load_model("motion_module", sd_ctx_params->motion_module_path,
+                        "model.diffusion_model.motion_module.")) {
             LOG_WARN("loading motion module from '%s' failed", sd_ctx_params->motion_module_path);
         } else {
             configuration.animatediff_loaded = true;
@@ -810,8 +816,11 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
         }
     }
 
+    if (configuration.control_net_file)
+        configuration.record_model("control_net", sd_ctx_params->control_net_path);
+
     if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0) {
-        if (!model_loader.init_from_file(sd_ctx_params->ip_adapter_path)) {
+        if (!load_model("ip_adapter", sd_ctx_params->ip_adapter_path)) {
             LOG_ERROR("init ip-adapter model loader from file failed: '%s'", sd_ctx_params->ip_adapter_path);
             return false;
         }
@@ -819,6 +828,8 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
 
     if (strlen(SAFE_STR(sd_ctx_params->photo_maker_path)) > 0) {
         configuration.photomaker_source_available = model_loader.add_file(sd_ctx_params->photo_maker_path, "pmid.");
+        if (configuration.photomaker_source_available)
+            configuration.record_model("photo_maker", sd_ctx_params->photo_maker_path);
         if (!configuration.photomaker_source_available) {
             LOG_WARN("loading stacked ID embedding from '%s' failed", sd_ctx_params->photo_maker_path);
         }

src/pipeline/diffusion_engine.h

@@ -123,6 +123,7 @@ public:
         sd_ctx_params_t params{};
         std::list<std::string> strings;
         std::vector<sd_embedding_t> embeddings;
+        std::map<std::string, std::string> models;
         ModelLoader::FileId control_net_file = 0;
         bool use_tae                         = false;
         bool use_audio_vae                   = false;
@@ -164,13 +165,24 @@ public:
             use_audio_vae               = other.use_audio_vae;
             photomaker_source_available = other.photomaker_source_available;
             animatediff_loaded          = other.animatediff_loaded;
+            models                      = other.models;
         }
         ModelConfig& operator=(const ModelConfig&) = delete;
 
+        void record_model(const std::string& name, const std::string& path) {
+            if (path.empty()) {
+                models.erase(name);
+                return;
+            }
+            const auto separator = path.find_last_of("/\\");
+            models[name]         = separator == std::string::npos ? path : path.substr(separator + 1);
+        }
+
         void set_control_net(ModelLoader::FileId id, const std::string& path) {
             control_net_file = id;
             strings.push_back(path);
             params.control_net_path = strings.back().c_str();
+            record_model("control_net", path);
         }
     };
 

src/pipeline/generation.h

@@ -2,6 +2,7 @@
 #define __SD_PIPELINE_GENERATION_H__
 
 #include "conditioning/conditioner.hpp"
+#include "json.hpp"
 #include "stable-diffusion.h"
 
 class StableDiffusionGGML;
@@ -54,14 +55,16 @@ namespace sd::pipeline {
     bool generate_image(StableDiffusionGGML* sd,
                         const sd_img_gen_params_t* sd_img_gen_params,
                         sd_image_t** images_out,
-                        int* num_images_out);
+                        int* num_images_out,
+                        std::vector<nlohmann::json>* infos_out = nullptr);
 
     bool generate_video(StableDiffusionGGML* sd,
                         const sd_vid_gen_params_t* sd_vid_gen_params,
                         sd_image_t** frames_out,
                         int* num_frames_out,
                         sd_audio_t** audio_out,
-                        int* fps_out);
+                        int* fps_out,
+                        nlohmann::json* info_out = nullptr);
 
     sd::Tensor<float> upscale_ltx_spatial_video_latent(StableDiffusionGGML* sd,
                                                        const char* model_path,

src/pipeline/image.cpp

@@ -1,4 +1,5 @@
 #include "generation.h"
+#include "generation_info.h"
 
 #include <algorithm>
 #include <cmath>
@@ -668,7 +669,7 @@ namespace sd::pipeline {
 
         int64_t t4 = ggml_time_ms();
         LOG_INFO("decode_first_stage completed, taking %.2fs", (t4 - t0) * 1.0f / 1000);
-        if (decoded_images.empty()) {
+        if (decoded_images.empty() || cancelled) {
             LOG_ERROR(cancelled ? "cancelled before any latent images were decoded" : "no decoded images");
             return nullptr;
         }
@@ -829,7 +830,10 @@ namespace sd::pipeline {
     bool generate_image(StableDiffusionGGML* sd,
                         const sd_img_gen_params_t* sd_img_gen_params,
                         sd_image_t** images_out,
-                        int* num_images_out) {
+                        int* num_images_out,
+                        std::vector<nlohmann::json>* infos_out) {
+        if (infos_out)
+            infos_out->clear();
         if (images_out != nullptr) {
             *images_out = nullptr;
         }
@@ -851,6 +855,7 @@ namespace sd::pipeline {
         int64_t t0            = ggml_time_ms();
         sd->vae_tiling_params = sd_img_gen_params->vae_tiling_params;
         GenerationRequest request(sd, sd_img_gen_params);
+        const auto info = sd::generation_info(request, *sd_img_gen_params);
         sd::ImagePreprocessor preprocessing(sd_img_gen_params->image_preprocess.rules);
         sd_img_gen_params_t processed_params = *sd_img_gen_params;
         if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
@@ -891,7 +896,6 @@ namespace sd::pipeline {
             return false;
         }
         ImageGenerationEmbeds embeds = std::move(*embeds_opt);
-
         std::vector<sd::Tensor<float>> final_latents;
         int64_t denoise_start = ggml_time_ms();
         for (int b = 0; b < request.batch_count; b++) {
@@ -908,7 +912,7 @@ namespace sd::pipeline {
             }
 
             int64_t sampling_start = ggml_time_ms();
-            int64_t cur_seed       = request.seed + b;
+            int64_t cur_seed       = request.seed_for_batch(b);
             LOG_INFO("generating image: %i/%i - seed %" PRId64, b + 1, request.batch_count, cur_seed);
 
             sd->rng->manual_seed(cur_seed);
@@ -994,7 +998,7 @@ namespace sd::pipeline {
             std::vector<float> hires_sigma_sched =
                 make_hires_sigma_schedule(sd,
                                           request.hires,
-                                          sd_img_gen_params->sample_params,
+                                          request.sample_params,
                                           plan.sample_method,
                                           plan.sample_steps,
                                           sd->get_image_seq_len(request.hires.target_height, request.hires.target_width),
@@ -1035,7 +1039,7 @@ namespace sd::pipeline {
                     LOG_ERROR("cancelling generation during hires fix");
                     return false;
                 }
-                int64_t cur_seed = request.seed + b;
+                int64_t cur_seed = request.seed_for_batch(b);
                 sd->rng->manual_seed(cur_seed);
                 sd->sampler_rng->manual_seed(cur_seed);
 
@@ -1113,6 +1117,19 @@ namespace sd::pipeline {
             return false;
         }
 
+        if (infos_out) {
+            try {
+                infos_out->assign(num_images, info);
+                // Layers and frames decoded from the same latent share its seed.
+                const int images_per_latent = num_images / static_cast<int>(final_latents.size());
+                for (int i = 0; i < num_images; ++i) {
+                    (*infos_out)[i]["seed"] = std::to_string(request.seed_for_batch(i / images_per_latent));
+                }
+            } catch (...) {
+                free_sd_images(result, num_images);
+                throw;
+            }
+        }
         sd->lora_stat();
 
         int64_t t1 = ggml_time_ms();

src/pipeline/request.cpp

@@ -1,4 +1,5 @@
 #include "request.h"
+#include "conditioning/conditioner.hpp"
 
 #include <algorithm>
 #include <cmath>
@@ -6,6 +7,8 @@
 #include <ctime>
 
 #include "diffusion_engine.h"
+#include "generation.h"
+#include "model/vae/vae.hpp"
 #include "model_definition.h"
 #include "runtime/denoiser.hpp"
 
@@ -66,7 +69,7 @@ namespace sd::pipeline {
         return DISCRETE_SCHEDULER;
     }
 
-    static int64_t resolve_seed(int64_t seed) {
+    int64_t resolve_seed(int64_t seed) {
         if (seed >= 0) {
             return seed;
         }
@@ -127,6 +130,7 @@ namespace sd::pipeline {
         seed                        = sd_img_gen_params->seed;
         batch_count                 = sd_img_gen_params->batch_count;
         qwen_image_layers           = std::max(0, sd_img_gen_params->qwen_image_layers);
+        is_layered_image            = sd->version == VERSION_QWEN_IMAGE_LAYERED;
         clip_skip                   = sd_img_gen_params->clip_skip;
         shifted_timestep            = sd_img_gen_params->sample_params.shifted_timestep;
         strength                    = sd_img_gen_params->strength;
@@ -137,8 +141,18 @@ namespace sd::pipeline {
         pm_params                   = sd_img_gen_params->pm_params;
         pulid_params                = sd_img_gen_params->pulid_params;
         hires                       = sd_img_gen_params->hires;
+        vae_tiling_params           = sd_img_gen_params->vae_tiling_params;
         cache_params                = &sd_img_gen_params->cache;
+        sample_params               = sd_img_gen_params->sample_params;
         resolve(sd);
+        if (!sd_img_gen_params->control_image.data)
+            models.erase("control_net");
+        if (!sd_img_gen_params->ip_adapter_image.data)
+            models.erase("ip_adapter");
+        if (!pm_params.id_images || pm_params.id_images_count <= 0)
+            models.erase("photo_maker");
+        if (!pulid_params.id_embedding_path || !*pulid_params.id_embedding_path)
+            models.erase("pulid_weights");
     }
 
     GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params) {
@@ -166,7 +180,20 @@ namespace sd::pipeline {
         guidance                    = sd_vid_gen_params->sample_params.guidance;
         high_noise_guidance         = sd_vid_gen_params->high_noise_sample_params.guidance;
         hires                       = sd_vid_gen_params->hires;
+        vae_tiling_params           = sd_vid_gen_params->vae_tiling_params;
+        sample_params               = sd_vid_gen_params->sample_params;
+        high_noise_sample_params    = sd_vid_gen_params->high_noise_sample_params;
+        use_high_noise              = sd->high_noise_diffusion_model != nullptr;
+        use_vace                    = sd->diffusion_model && (sd->diffusion_model->get_desc() == "Wan2.1-VACE-1.3B" ||
+                                                              sd->diffusion_model->get_desc() == "Wan2.x-VACE-14B");
+        shifted_timestep            = sample_params.shifted_timestep;
+        if (sd->config_ && sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd->version))
+            frames = std::min(requested_frames, 32);
         resolve(sd);
+        models.erase("control_net");
+        models.erase("ip_adapter");
+        models.erase("photo_maker");
+        models.erase("pulid_weights");
         if (frames != requested_frames) {
             LOG_WARN("align video frames from %d to %d for %s",
                      requested_frames,
@@ -201,6 +228,10 @@ namespace sd::pipeline {
                  spatial_multiple);
     }
 
+    int64_t GenerationRequest::seed_for_batch(int index) const {
+        return static_cast<int64_t>((static_cast<uint64_t>(seed) + index) & INT64_MAX);
+    }
+
     void GenerationRequest::resolve_hires() {
         if (!hires.enabled) {
             return;
@@ -307,6 +338,23 @@ namespace sd::pipeline {
     }
 
     void GenerationRequest::resolve(StableDiffusionGGML* sd) {
+        if (sd->config_) {
+            models = sd->config_->models;
+            if (!use_high_noise)
+                models.erase("high_noise_diffusion_model");
+            if (!sd->clip_vision)
+                models.erase("clip_vision");
+            if (!sd->audio_vae_model || frames <= 0)
+                models.erase("audio_vae");
+            if (!sd->audio_encoder || frames <= 0)
+                models.erase("audio_encoder");
+            if (!sd->config_->animatediff_loaded || !sd_version_supports_animatediff(sd->version) || frames <= 0)
+                models.erase("motion_module");
+            if (sd->uses_tae() && !sd->tae_preview_only())
+                models.erase("vae");
+            else
+                models.erase("taesd");
+        }
         if (sd_version_is_minit2i(sd->version) && (width != 512 || height != 512)) {
             width  = 512;
             height = 512;
@@ -314,7 +362,11 @@ namespace sd::pipeline {
         }
         align_generation_request_size();
         resolve_hires();
+        if (sd->first_stage_model)
+            vae_tiling_params = sd->first_stage_model->resolve_tiling_params(vae_tiling_params);
         seed = resolve_seed(seed);
+        if (sd->cond_stage_model)
+            clip_skip = sd->cond_stage_model->resolve_clip_skip(clip_skip);
 
         resolve_guidance(sd, &guidance, &use_uncond, &use_img_uncond, has_ref_images);
         if (sd->high_noise_diffusion_model) {
@@ -330,33 +382,44 @@ namespace sd::pipeline {
             LOG_WARN("timestep shifting is only supported for SDXL models!");
             shifted_timestep = 0;
         }
+        sample_params.guidance         = guidance;
+        sample_params.shifted_timestep = shifted_timestep;
+        sample_params.sample_method    = resolve_sample_method(sd, sample_params.sample_method);
+        sample_params.scheduler        = resolve_scheduler(sd, sample_params.scheduler, sample_params.sample_method);
+        sample_params.eta              = resolve_eta(sd, sample_params.eta, sample_params.sample_method);
+        if (frames > 0) {
+            high_noise_sample_params.guidance      = high_noise_guidance;
+            high_noise_sample_params.sample_method = resolve_sample_method(sd, high_noise_sample_params.sample_method);
+            high_noise_sample_params.scheduler     = resolve_scheduler(sd, high_noise_sample_params.scheduler, high_noise_sample_params.sample_method);
+            high_noise_sample_params.eta           = resolve_eta(sd, high_noise_sample_params.eta, high_noise_sample_params.sample_method);
+        }
     }
 
     SamplePlan::SamplePlan(StableDiffusionGGML* sd,
                            const sd_img_gen_params_t* sd_img_gen_params,
                            const GenerationRequest& request) {
-        sample_method     = sd_img_gen_params->sample_params.sample_method;
-        extra_sample_args = sd_img_gen_params->sample_params.extra_sample_args;
-        eta               = sd_img_gen_params->sample_params.eta;
-        sample_steps      = sd_img_gen_params->sample_params.sample_steps;
-        resolve(sd, &request, &sd_img_gen_params->sample_params);
+        sample_method     = request.sample_params.sample_method;
+        extra_sample_args = request.sample_params.extra_sample_args;
+        eta               = request.sample_params.eta;
+        sample_steps      = request.sample_params.sample_steps;
+        resolve(sd, &request, &request.sample_params);
     }
 
     SamplePlan::SamplePlan(StableDiffusionGGML* sd,
                            const sd_vid_gen_params_t* sd_vid_gen_params,
                            const GenerationRequest& request) {
-        sample_method     = sd_vid_gen_params->sample_params.sample_method;
-        extra_sample_args = sd_vid_gen_params->sample_params.extra_sample_args;
-        eta               = sd_vid_gen_params->sample_params.eta;
-        sample_steps      = sd_vid_gen_params->sample_params.sample_steps;
+        sample_method     = request.sample_params.sample_method;
+        extra_sample_args = request.sample_params.extra_sample_args;
+        eta               = request.sample_params.eta;
+        sample_steps      = request.sample_params.sample_steps;
         if (sd->high_noise_diffusion_model) {
-            high_noise_sample_steps      = sd_vid_gen_params->high_noise_sample_params.sample_steps;
-            high_noise_sample_method     = sd_vid_gen_params->high_noise_sample_params.sample_method;
-            high_noise_extra_sample_args = sd_vid_gen_params->high_noise_sample_params.extra_sample_args;
-            high_noise_eta               = sd_vid_gen_params->high_noise_sample_params.eta;
+            high_noise_sample_steps      = request.high_noise_sample_params.sample_steps;
+            high_noise_sample_method     = request.high_noise_sample_params.sample_method;
+            high_noise_extra_sample_args = request.high_noise_sample_params.extra_sample_args;
+            high_noise_eta               = request.high_noise_sample_params.eta;
         }
         moe_boundary = sd_vid_gen_params->moe_boundary;
-        resolve(sd, &request, &sd_vid_gen_params->sample_params);
+        resolve(sd, &request, &request.sample_params);
     }
 
     void SamplePlan::resolve(StableDiffusionGGML* sd,
@@ -382,10 +445,10 @@ namespace sd::pipeline {
                 }
             }
         } else {
-            scheduler_t scheduler = resolve_scheduler(sd,
-                                                      sample_params->scheduler,
-                                                      sample_method);
-            int sample_seq_len    = sd->get_image_seq_len(request->height, request->width);
+            scheduler          = resolve_scheduler(sd,
+                                                   sample_params->scheduler,
+                                                   sample_method);
+            int sample_seq_len = sd->get_image_seq_len(request->height, request->width);
             if (sd_version_is_ltxav(sd->version) && request->frames > 0) {
                 int latent_frames = ((request->frames - 1) / 8) + 1;
                 sample_seq_len *= latent_frames;

src/pipeline/request.h

@@ -1,6 +1,7 @@
 #ifndef __SD_PIPELINE_REQUEST_H__
 #define __SD_PIPELINE_REQUEST_H__
 
+#include <map>
 #include <string>
 #include <vector>
 
@@ -16,6 +17,8 @@ namespace sd::pipeline {
 
     enum scheduler_t default_scheduler(const StableDiffusionGGML* sd, enum sample_method_t sample_method);
 
+    int64_t resolve_seed(int64_t seed);
+
     float resolve_eta(StableDiffusionGGML* sd,
                       float eta,
                       enum sample_method_t sample_method);
@@ -23,38 +26,47 @@ namespace sd::pipeline {
     struct GenerationRequest {
         std::string prompt;
         std::string negative_prompt;
-        int width                                = -1;
-        int height                               = -1;
-        int clip_skip                            = -1;
-        int vae_scale_factor                     = -1;
-        int diffusion_model_down_factor          = -1;
-        int64_t seed                             = -1;
-        bool use_uncond                          = false;
-        bool use_img_uncond                      = false;
-        bool use_high_noise_uncond               = false;
-        bool use_high_noise_img_uncond           = false;
-        bool has_ref_images                      = false;
-        const sd_cache_params_t* cache_params    = nullptr;
-        int batch_count                          = 1;
-        int qwen_image_layers                    = 3;
-        int shifted_timestep                     = 0;
-        float strength                           = 1.f;
-        float control_strength                   = 0.f;
-        float eta                                = 0.f;
-        sd_guidance_params_t guidance            = {};
-        sd_guidance_params_t high_noise_guidance = {};
-        sd_pm_params_t pm_params                 = {};
-        sd_pulid_params_t pulid_params           = {};
-        sd_hires_params_t hires                  = {};
-        int frames                               = -1;
-        int requested_frames                     = -1;
-        int fps                                  = 16;
-        float vace_strength                      = 1.f;
+        std::map<std::string, std::string> models;
+        int width                                   = -1;
+        int height                                  = -1;
+        int clip_skip                               = -1;
+        int vae_scale_factor                        = -1;
+        int diffusion_model_down_factor             = -1;
+        int64_t seed                                = -1;
+        bool use_uncond                             = false;
+        bool use_img_uncond                         = false;
+        bool use_high_noise_uncond                  = false;
+        bool use_high_noise_img_uncond              = false;
+        bool has_ref_images                         = false;
+        bool is_layered_image                       = false;
+        bool use_high_noise                         = false;
+        bool use_vace                               = false;
+        const sd_cache_params_t* cache_params       = nullptr;
+        int batch_count                             = 1;
+        int qwen_image_layers                       = 3;
+        int shifted_timestep                        = 0;
+        float strength                              = 1.f;
+        float control_strength                      = 0.f;
+        float eta                                   = 0.f;
+        sd_sample_params_t sample_params            = {};
+        sd_sample_params_t high_noise_sample_params = {};
+        sd_guidance_params_t guidance               = {};
+        sd_guidance_params_t high_noise_guidance    = {};
+        sd_pm_params_t pm_params                    = {};
+        sd_pulid_params_t pulid_params              = {};
+        sd_hires_params_t hires                     = {};
+        sd_tiling_params_t vae_tiling_params        = {};
+        int frames                                  = -1;
+        int requested_frames                        = -1;
+        int fps                                     = 16;
+        float vace_strength                         = 1.f;
 
         GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params);
 
         GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params);
 
+        int64_t seed_for_batch(int index) const;
+
         void align_generation_request_size();
 
         void align_image_size(int* target_width, int* target_height, const char* label);
@@ -81,6 +93,7 @@ namespace sd::pipeline {
         int sample_steps                              = 0;
         int high_noise_sample_steps                   = 0;
         int total_steps                               = 0;
+        scheduler_t scheduler                         = SCHEDULER_COUNT;
         float moe_boundary                            = 0.f;
         std::vector<float> sigmas;
 

src/pipeline/video.cpp

@@ -1,4 +1,5 @@
 #include "generation.h"
+#include "generation_info.h"
 
 #include <algorithm>
 #include <cmath>
@@ -1575,7 +1576,13 @@ namespace sd::pipeline {
         img_gen_params.ref_image_args    = sd_vid_gen_params->ref_image_args;
 
         sd->animatediff_num_frames = n_frames;
-        bool ok                    = generate_image(sd, &img_gen_params, frames_out, num_frames_out);
+        bool ok                    = false;
+        try {
+            ok = generate_image(sd, &img_gen_params, frames_out, num_frames_out);
+        } catch (...) {
+            sd->animatediff_num_frames = 0;
+            throw;
+        }
         sd->animatediff_num_frames = 0;
         return ok;
     }
@@ -1585,14 +1592,24 @@ namespace sd::pipeline {
                         sd_image_t** frames_out,
                         int* num_frames_out,
                         sd_audio_t** audio_out,
-                        int* fps_out) {
+                        int* fps_out,
+                        nlohmann::json* info_out) {
+        if (info_out)
+            *info_out = nlohmann::json();
+        GenerationRequest request(sd, sd_vid_gen_params);
+        const auto info = sd::generation_info(request, *sd_vid_gen_params);
         if (sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd->version)) {
             LOG_INFO("AnimateDiff dispatch: %d frames, %dx%d",
                      sd_vid_gen_params->video_frames, sd_vid_gen_params->width, sd_vid_gen_params->height);
             if (fps_out != nullptr) {
                 *fps_out = std::max(1, sd_vid_gen_params->fps);
             }
-            return generate_animatediff_video(sd, sd_vid_gen_params, frames_out, num_frames_out);
+            auto params   = *sd_vid_gen_params;
+            params.seed   = request.seed;
+            const bool ok = generate_animatediff_video(sd, &params, frames_out, num_frames_out);
+            if (ok && info_out)
+                *info_out = info;
+            return ok;
         }
 
         sd->reset_cancel_flag();
@@ -1602,7 +1619,6 @@ namespace sd::pipeline {
         int64_t t0            = ggml_time_ms();
         sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
         sd->apply_circular_axes(sd_vid_gen_params->circular_x, sd_vid_gen_params->circular_y);
-        GenerationRequest request(sd, sd_vid_gen_params);
         sd::ImagePreprocessor preprocessing(sd_vid_gen_params->image_preprocess.rules);
         sd_vid_gen_params_t processed_params = *sd_vid_gen_params;
         if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
@@ -1734,9 +1750,9 @@ namespace sd::pipeline {
                                                     embeds.img_uncond,
                                                     sd::Tensor<float>(),
                                                     0.f,
-                                                    sd_vid_gen_params->sample_params.guidance,
+                                                    request.guidance,
                                                     plan.eta,
-                                                    sd_vid_gen_params->sample_params.shifted_timestep,
+                                                    request.shifted_timestep,
                                                     plan.sample_method,
                                                     sd->is_flow_denoiser(),
                                                     plan.extra_sample_args,
@@ -1846,7 +1862,7 @@ namespace sd::pipeline {
             std::vector<float> hires_sigma_sched =
                 make_hires_sigma_schedule(sd,
                                           request.hires,
-                                          sd_vid_gen_params->sample_params,
+                                          request.sample_params,
                                           hires_sample_method,
                                           plan.sample_steps,
                                           sd->get_image_seq_len(hires_request.height, hires_request.width) * T,
@@ -1873,9 +1889,9 @@ namespace sd::pipeline {
                                         embeds.img_uncond,
                                         sd::Tensor<float>(),
                                         0.f,
-                                        sd_vid_gen_params->sample_params.guidance,
+                                        request.guidance,
                                         hires_eta,
-                                        sd_vid_gen_params->sample_params.shifted_timestep,
+                                        request.shifted_timestep,
                                         hires_sample_method,
                                         sd->is_flow_denoiser(),
                                         plan.extra_sample_args,
@@ -2007,6 +2023,14 @@ namespace sd::pipeline {
                 generated_audio->sample_count = want_samples;
             }
         }
+        if (info_out) {
+            try {
+                *info_out = info;
+            } catch (...) {
+                free_sd_audio(generated_audio);
+                throw;
+            }
+        }
         if (audio_out != nullptr) {
             *audio_out = generated_audio;
         } else {