feat: return generation metadata with core results (#2206)

examples/server/CMakeLists.txt

@@ -64,8 +64,10 @@ endif()
 
 add_executable(${TARGET}
     ../common/common.cpp
+    ../common/generation_metadata.cpp
     ../common/log.cpp
     ../common/media_io.cpp
+    ../common/media_metadata.cpp
     main.cpp
     runtime.cpp
     model_catalog.cpp

examples/server/api.md

@@ -1776,3 +1776,14 @@ Example cancelled job:
 - `400 Bad Request` for an empty body, unsupported model mode, invalid JSON, invalid generation parameters, or an unsupported output format
 - `429 Too Many Requests` when the job queue is full
 - `500 Internal Server Error` for unexpected server exceptions during submission
+
+## Actual generation parameters
+
+Generation responses expose the core `sdcpp.generation/v1` record as
+`generation_info`, independently of `embed_image_metadata`. Native async image
+result items and OpenAI image items contain the parameter snapshot with each
+image's actual seed and without `batch_count`. Native video
+results contain the video request snapshot. SD API `info.generation_info` is an array
+aligned with `info.infotexts` and `info.all_seeds`; the outer `parameters` field
+remains the request echo. See [generation metadata](../../docs/generation_metadata.md)
+for field semantics, postprocessing and container storage.

examples/server/async_jobs.cpp

@@ -179,6 +179,7 @@ bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job) {
     job.status       = AsyncJobStatus::Cancelled;
     job.completed_at = unix_timestamp_now_ms();
     job.result_images_b64.clear();
+    job.result_generation_info = nullptr;
     job.result_media_b64.clear();
     job.result_media_mime_type.clear();
     job.result_frame_count = 0;
@@ -218,11 +219,12 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
                 {"fps", job.result_fps},
                 {"frame_count", job.result_frame_count},
                 {"b64_json", job.result_media_b64},
+                {"generation_info", job.result_generation_info},
             };
         } else {
             json images = json::array();
             for (size_t i = 0; i < job.result_images_b64.size(); ++i) {
-                images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}});
+                images.push_back({{"index", i}, {"b64_json", job.result_images_b64[i]}, {"generation_info", job.result_generation_info.at(i)}});
             }
             result["result"] = {
                 {"output_format", job.img_gen.output_format},
@@ -263,7 +265,9 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
 bool execute_img_gen_job(ServerRuntime& runtime,
                          AsyncGenerationJob& job,
                          std::vector<std::string>& output_images,
+                         json& output_info,
                          std::string& error_message) {
+    output_info                = json::array();
     sd_img_gen_params_t params = job.img_gen.to_sd_img_gen_params_t();
 
     SDImageVec results;
@@ -275,14 +279,10 @@ bool execute_img_gen_job(ServerRuntime& runtime,
         }
         JobCancellationMonitor cancellation(runtime.sd_ctx, job);
         set_preview_callback(job);
-        sd_image_t* raw_results = nullptr;
-        int num_results         = 0;
-        if (!generate_image(runtime.sd_ctx, &params, &raw_results, &num_results)) {
-            raw_results = nullptr;
-            num_results = 0;
-        }
+        sd_image_generation_result_t* generated = nullptr;
+        generate_image(runtime.sd_ctx, &params, &generated);
+        results.adopt(generated, runtime.ctx_params->lora_model_dir);
         clear_preview_callback();
-        results.adopt(raw_results, num_results);
         if (job.cancel_requested.load()) {
             return false;
         }
@@ -305,17 +305,13 @@ bool execute_img_gen_job(ServerRuntime& runtime,
         encoded_format = EncodedImageFormat::WEBP;
     }
 
-    int batch_count      = job.img_gen.gen_params.batch_count;
-    int images_per_batch = batch_count > 0 ? std::max(1, num_results / batch_count) : 1;
     for (int i = 0; i < num_results; ++i) {
         if (results[i].data == nullptr) {
             continue;
         }
 
         const std::string metadata = job.img_gen.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime.ctx_params,
-                                                            job.img_gen.gen_params,
-                                                            job.img_gen.gen_params.seed + i / images_per_batch)
+                                         ? results.generation_info(i)
                                          : "";
         auto image_bytes           = encode_image_to_vector(encoded_format,
                                                             results[i].data,
@@ -328,6 +324,7 @@ bool execute_img_gen_job(ServerRuntime& runtime,
             continue;
         }
         output_images.push_back(base64_encode(image_bytes));
+        output_info.push_back(json::parse(results.generation_info(i)));
     }
 
     if (output_images.empty()) {
@@ -344,11 +341,10 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
                          std::string& output_media_mime_type,
                          int& output_frame_count,
                          int& output_fps,
+                         json& output_info,
                          std::string& error_message) {
     sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t();
-    std::string str_params     = job.vid_gen.gen_params.embed_image_metadata
-                                     ? get_image_params(*runtime.ctx_params, job.vid_gen.gen_params, job.vid_gen.gen_params.seed, VID_GEN)
-                                     : "";
+    std::string str_params;
 
     SDImageVec results;
     int num_results             = 0;
@@ -361,12 +357,17 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
         }
         JobCancellationMonitor cancellation(runtime.sd_ctx, job);
         set_preview_callback(job);
-        sd_image_t* raw_results = nullptr;
-        if (!generate_video(runtime.sd_ctx, &params, &raw_results, &num_results, &generated_audio, &output_fps)) {
-            raw_results = nullptr;
+        sd_video_generation_result_t* generated = nullptr;
+        if (generate_video(runtime.sd_ctx, &params, &generated)) {
+            output_fps       = generated->fps;
+            generated_audio  = generated->audio;
+            generated->audio = nullptr;
+            results.adopt_video(generated, runtime.ctx_params->lora_model_dir);
+            output_info = json::parse(results.generation_info(0));
+            if (job.vid_gen.gen_params.embed_image_metadata)
+                str_params = results.generation_info(0);
         }
         clear_preview_callback();
-        results.adopt(raw_results, num_results);
     }
 
     num_results = results.count();
@@ -427,6 +428,7 @@ void async_job_worker(ServerRuntime& runtime) {
         }
 
         std::vector<std::string> output_images;
+        json output_info;
         std::string output_media_b64;
         std::string output_media_mime_type;
         int output_frame_count = 0;
@@ -435,7 +437,7 @@ void async_job_worker(ServerRuntime& runtime) {
         bool ok = false;
 
         if (job->kind == AsyncJobKind::ImgGen) {
-            ok = execute_img_gen_job(runtime, *job, output_images, error_message);
+            ok = execute_img_gen_job(runtime, *job, output_images, output_info, error_message);
         } else if (job->kind == AsyncJobKind::VidGen) {
             ok = execute_vid_gen_job(runtime,
                                      *job,
@@ -443,6 +445,7 @@ void async_job_worker(ServerRuntime& runtime) {
                                      output_media_mime_type,
                                      output_frame_count,
                                      output_fps,
+                                     output_info,
                                      error_message);
         } else {
             error_message = "unsupported job kind";
@@ -463,6 +466,7 @@ void async_job_worker(ServerRuntime& runtime) {
             } else if (ok) {
                 job->status                 = AsyncJobStatus::Completed;
                 job->result_images_b64      = std::move(output_images);
+                job->result_generation_info = std::move(output_info);
                 job->result_media_b64       = std::move(output_media_b64);
                 job->result_media_mime_type = std::move(output_media_mime_type);
                 job->result_frame_count     = output_frame_count;
@@ -474,6 +478,7 @@ void async_job_worker(ServerRuntime& runtime) {
                 job->error_code    = "generation_failed";
                 job->error_message = error_message.empty() ? "unknown generation error" : error_message;
                 job->result_images_b64.clear();
+                job->result_generation_info = nullptr;
                 job->result_media_b64.clear();
                 job->result_media_mime_type.clear();
                 job->result_frame_count = 0;

examples/server/async_jobs.h

@@ -40,6 +40,7 @@ struct AsyncGenerationJob {
     ImgGenJobRequest img_gen;
     VidGenJobRequest vid_gen;
     std::vector<std::string> result_images_b64;
+    json result_generation_info;
     std::string result_media_b64;
     std::string result_media_mime_type;
     int result_frame_count = 0;
@@ -74,6 +75,7 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
 bool execute_img_gen_job(ServerRuntime& runtime,
                          AsyncGenerationJob& job,
                          std::vector<std::string>& output_images,
+                         json& output_info,
                          std::string& error_message);
 bool execute_vid_gen_job(ServerRuntime& runtime,
                          AsyncGenerationJob& job,
@@ -81,5 +83,6 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
                          std::string& output_media_mime_type,
                          int& output_frame_count,
                          int& output_fps,
+                         json& output_info,
                          std::string& error_message);
 void async_job_worker(ServerRuntime& runtime);

examples/server/routes_openai.cpp

@@ -223,12 +223,10 @@ static bool execute_sync_img_gen_request(ServerRuntime& runtime,
 
     {
         std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
-        sd_image_t* raw_results = nullptr;
-        if (!generate_image(runtime.sd_ctx, &img_gen_params, &raw_results, &num_results)) {
-            raw_results = nullptr;
-            num_results = 0;
-        }
-        results.adopt(raw_results, num_results);
+        sd_image_generation_result_t* generated = nullptr;
+        generate_image(runtime.sd_ctx, &img_gen_params, &generated);
+        results.adopt(generated, runtime.ctx_params->lora_model_dir);
+        num_results = results.count();
         if (!results.empty() && !apply_server_adetailer(runtime, request, img_gen_params, results)) {
             error_message = "ADetailer failed";
             return false;
@@ -285,16 +283,14 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
             out["data"]          = json::array();
             out["output_format"] = request.output_format;
 
-            int result_count     = results.count();
-            int images_per_batch = request.gen_params.batch_count > 0 ? std::max(1, result_count / request.gen_params.batch_count) : 1;
+            int result_count = results.count();
+
             for (int i = 0; i < result_count; ++i) {
                 if (results[i].data == nullptr) {
                     continue;
                 }
                 std::string params = request.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime->ctx_params,
-                                                            request.gen_params,
-                                                            request.gen_params.seed + i / images_per_batch)
+                                         ? results.generation_info(i)
                                          : "";
                 auto image_bytes   = encode_image_to_vector(request.output_format == "jpeg"
                                                                 ? EncodedImageFormat::JPEG
@@ -313,7 +309,8 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                 }
 
                 json item;
-                item["b64_json"] = base64_encode(image_bytes);
+                item["b64_json"]        = base64_encode(image_bytes);
+                item["generation_info"] = json::parse(results.generation_info(i));
                 out["data"].push_back(item);
             }
 
@@ -359,16 +356,14 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
             out["data"]          = json::array();
             out["output_format"] = request.output_format;
 
-            int result_count     = results.count();
-            int images_per_batch = request.gen_params.batch_count > 0 ? std::max(1, result_count / request.gen_params.batch_count) : 1;
+            int result_count = results.count();
+
             for (int i = 0; i < result_count; ++i) {
                 if (results[i].data == nullptr) {
                     continue;
                 }
                 std::string params = request.gen_params.embed_image_metadata
-                                         ? get_image_params(*runtime->ctx_params,
-                                                            request.gen_params,
-                                                            request.gen_params.seed + i / images_per_batch)
+                                         ? results.generation_info(i)
                                          : "";
                 auto image_bytes   = encode_image_to_vector(request.output_format == "jpeg" ? EncodedImageFormat::JPEG : EncodedImageFormat::PNG,
                                                             results[i].data,
@@ -378,7 +373,8 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                                                             params,
                                                             request.output_compression);
                 json item;
-                item["b64_json"] = base64_encode(image_bytes);
+                item["b64_json"]        = base64_encode(image_bytes);
+                item["generation_info"] = json::parse(results.generation_info(i));
                 out["data"].push_back(item);
             }
 

examples/server/routes_sdapi.cpp

@@ -462,46 +462,27 @@ static bool build_sdapi_img_gen_request(const json& j,
            validate_server_img_gen_request(runtime, request, error_message);
 }
 
-static nlohmann::json prepare_info_field(const SDContextParams& ctx_params,
-                                         const SDGenerationParams& gen_params,
-                                         bool img2img) {
-    nlohmann::json jsoninfo = nlohmann::json::object();
-    jsoninfo["prompt"]      = gen_params.prompt;
-    if (!gen_params.negative_prompt.empty()) {
-        jsoninfo["negative_prompt"] = gen_params.negative_prompt;
-    }
-    jsoninfo["seed"]         = gen_params.seed;
-    jsoninfo["cfg_scale"]    = gen_params.sample_params.guidance.txt_cfg;
-    jsoninfo["width"]        = gen_params.get_resolved_width();
-    jsoninfo["height"]       = gen_params.get_resolved_height();
-    jsoninfo["steps"]        = gen_params.sample_params.sample_steps;
-    jsoninfo["sampler_name"] = sd_sample_method_name(gen_params.sample_params.sample_method);
-    if (gen_params.clip_skip != -1) {
-        jsoninfo["clip_skip"] = gen_params.clip_skip;
-    }
-    if (gen_params.sample_params.scheduler != scheduler_t::SCHEDULER_COUNT) {
-        jsoninfo["extra_generation_params"]                  = nlohmann::json::object();
-        jsoninfo["extra_generation_params"]["Schedule type"] = sd_scheduler_name(gen_params.sample_params.scheduler);
-    }
-    if (img2img) {
-        jsoninfo["denoising_strength"] = gen_params.strength;
-    }
-    // not clear what should happen if we have both model and diffusion_model
-    if (!ctx_params.diffusion_model_path.empty()) {
-        jsoninfo["sd_model_name"] = sd_basename(ctx_params.diffusion_model_path);
-    } else if (!ctx_params.model_path.empty()) {
-        jsoninfo["sd_model_name"] = sd_basename(ctx_params.model_path);
-    }
-    if (!ctx_params.vae_path.empty()) {
-        jsoninfo["sd_vae_name"] = sd_basename(ctx_params.vae_path);
-    }
-    jsoninfo["version"] = "stable-diffusion.cpp";
-
-    jsoninfo["infotexts"]            = nlohmann::json::array();
-    jsoninfo["all_prompts"]          = nlohmann::json::array();
-    jsoninfo["all_negative_prompts"] = nlohmann::json::array();
-    jsoninfo["all_seeds"]            = nlohmann::json::array();
-    return jsoninfo;
+static nlohmann::json prepare_info_field(const std::string& generation_info) {
+    const auto info       = nlohmann::json::parse(generation_info);
+    const auto& sampling  = info.at("sample_params");
+    nlohmann::json result = {{"prompt", info.at("prompt")},
+                             {"negative_prompt", info.at("negative_prompt")},
+                             {"seed", generation_seed(generation_info)},
+                             {"cfg_scale", sampling.at("guidance").at("txt_cfg")},
+                             {"width", info.at("width")},
+                             {"height", info.at("height")},
+                             {"steps", sampling.contains("custom_sigmas") ? json(sampling.at("custom_sigmas").size() - 1) : sampling.at("sample_steps")},
+                             {"sampler_name", sampling.at("sample_method")},
+                             {"version", "stable-diffusion.cpp"}};
+    if (info.contains("clip_skip"))
+        result["clip_skip"] = info.at("clip_skip");
+    if (info.contains("strength"))
+        result["denoising_strength"] = info.at("strength");
+    if (sampling.contains("scheduler"))
+        result["extra_generation_params"]["Schedule type"] = sampling.at("scheduler");
+    for (const auto* key : {"infotexts", "all_prompts", "all_negative_prompts", "all_seeds", "generation_info"})
+        result[key] = nlohmann::json::array();
+    return result;
 }
 
 struct SDAPIPreviewState {
@@ -621,12 +602,10 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                 std::lock_guard<std::mutex> lock(*runtime->sd_ctx_mutex);
                 const bool detailing = request.adetailer_enabled && !runtime->default_gen_params->ad_model_path.empty();
                 SDAPIPreviewGuard preview_guard(id_task, img_gen_params, img2img ? "img2img" : "txt2img", detailing);
-                sd_image_t* raw_results = nullptr;
-                if (!generate_image(runtime->sd_ctx, &img_gen_params, &raw_results, &num_results)) {
-                    raw_results = nullptr;
-                    num_results = 0;
-                }
-                results.adopt(raw_results, num_results);
+                sd_image_generation_result_t* generated = nullptr;
+                generate_image(runtime->sd_ctx, &img_gen_params, &generated);
+                results.adopt(generated, runtime->ctx_params->lora_model_dir);
+                num_results = results.count();
                 if (detailing && !results.empty())
                     preview_guard.begin_detailing();
                 if (!results.empty() && !apply_server_adetailer(*runtime, request, img_gen_params, results)) {
@@ -645,9 +624,8 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
             json out;
             out["images"]     = json::array();
             out["parameters"] = j;
-            json jsoninfo     = prepare_info_field(*runtime->ctx_params, request.gen_params, img2img);
+            json jsoninfo     = prepare_info_field(results.generation_info(0));
 
-            int images_per_batch = request.gen_params.batch_count > 0 ? std::max(1, num_results / request.gen_params.batch_count) : 1;
             for (int i = 0; i < num_results; ++i) {
                 if (results[i].data == nullptr) {
                     continue;
@@ -655,16 +633,14 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
 
                 bool embed_meta = request.gen_params.embed_image_metadata;
 
-                std::string params = get_image_params(*runtime->ctx_params,
-                                                      request.gen_params,
-                                                      request.gen_params.seed + i / images_per_batch);
+                std::string params = generation_info_to_webui(results.generation_info(i));
 
                 auto image_bytes = encode_image_to_vector(EncodedImageFormat::PNG,
                                                           results[i].data,
                                                           results[i].width,
                                                           results[i].height,
                                                           results[i].channel,
-                                                          embed_meta ? params : "");
+                                                          embed_meta ? results.generation_info(i) : "");
 
                 if (image_bytes.empty()) {
                     LOG_ERROR("write image to mem failed");
@@ -673,10 +649,12 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
 
                 out["images"].push_back(base64_encode(image_bytes));
 
-                jsoninfo["infotexts"][i]            = params;
-                jsoninfo["all_seeds"][i]            = request.gen_params.seed + i;
-                jsoninfo["all_prompts"][i]          = request.gen_params.prompt;
-                jsoninfo["all_negative_prompts"][i] = request.gen_params.negative_prompt;
+                const auto actual = json::parse(results.generation_info(i));
+                jsoninfo["infotexts"].push_back(params);
+                jsoninfo["all_seeds"].push_back(generation_seed(results.generation_info(i)));
+                jsoninfo["generation_info"].push_back(actual);
+                jsoninfo["all_prompts"].push_back(actual["prompt"]);
+                jsoninfo["all_negative_prompts"].push_back(actual["negative_prompt"]);
             }
 
             // not a mistake: it is supposed to be a string in json format

examples/server/routes_sdcpp.cpp

@@ -609,19 +609,20 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
             }
 
             SDImageOwner current(input.release());
+            std::string generation_info;
             for (int i = 0; i < repeats; ++i) {
-                sd_image_t* out_images = nullptr;
-                int out_count          = 0;
-                if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &out_images, &out_count) ||
-                    out_count <= 0 || out_images[0].data == nullptr) {
-                    free_sd_images(out_images, out_count);
+                sd_image_generation_result_t* generated = nullptr;
+                if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &generated) ||
+                    !generated || generated->count <= 0 || generated->items[0].image.data == nullptr) {
+                    free_sd_image_generation_result(generated);
                     res.status = 500;
                     res.set_content(R"({"error":"upscale failed"})", "application/json");
                     return;
                 }
-                sd_image_t produced = out_images[0];
-                out_images[0]       = {0, 0, 0, nullptr};
-                free_sd_images(out_images, out_count);
+                generation_info           = merge_generation_info(generation_info, generated->items[0].generation_info);
+                sd_image_t produced       = generated->items[0].image;
+                generated->items[0].image = {};
+                free_sd_image_generation_result(generated);
                 current.reset(produced);
             }
 
@@ -635,7 +636,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                                                                result.width,
                                                                result.height,
                                                                result.channel,
-                                                               "",
+                                                               generation_info,
                                                                compression);
             if (image_bytes.empty()) {
                 res.status = 500;
@@ -644,13 +645,14 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
             }
 
             json out;
-            out["upscaler"]      = used_name;
-            out["scale"]         = factor;
-            out["repeats"]       = repeats;
-            out["width"]         = result.width;
-            out["height"]        = result.height;
-            out["output_format"] = format;
-            json images          = json::array();
+            out["generation_info"] = json::parse(generation_info);
+            out["upscaler"]        = used_name;
+            out["scale"]           = factor;
+            out["repeats"]         = repeats;
+            out["width"]           = result.width;
+            out["height"]          = result.height;
+            out["output_format"]   = format;
+            json images            = json::array();
             images.push_back({{"index", 0}, {"b64_json", base64_encode(image_bytes)}});
             out["images"] = std::move(images);
             res.set_content(out.dump(), "application/json");

examples/server/runtime.cpp

@@ -136,7 +136,7 @@ bool apply_server_adetailer(ServerRuntime& runtime,
                 continue;
             }
             sd_img_gen_params_t ad_generation_params = params;
-            ad_generation_params.seed                = params.seed + i;
+            ad_generation_params.seed                = generation_seed(results.generation_info(i));
             ad_generation_params.width               = pass.width;
             ad_generation_params.height              = pass.height;
             ad_generation_params.strength            = pass.strength;
@@ -147,24 +147,23 @@ bool apply_server_adetailer(ServerRuntime& runtime,
                 ad_generation_params.sample_params.scheduler = SCHEDULER_COUNT;
             }
 
-            sd_image_t* detailed_images = nullptr;
-            int detailed_count          = 0;
+            sd_image_generation_result_t* detailed = nullptr;
             if (!adetail_image(ad_ctx.get(),
                                runtime.sd_ctx,
                                results[i],
                                &ad_params,
                                &ad_generation_params,
-                               &detailed_images,
-                               &detailed_count) ||
-                detailed_count <= 0 || detailed_images == nullptr || detailed_images[0].data == nullptr) {
-                free_sd_images(detailed_images, detailed_count);
+                               &detailed) ||
+                detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) {
+                free_sd_image_generation_result(detailed);
                 LOG_ERROR("ADetailer failed for image %d", i + 1);
                 return false;
             }
+            results.append_generation_info(i, detailed->items[0].generation_info, runtime.ctx_params->lora_model_dir);
             free(results[i].data);
-            results[i]         = detailed_images[0];
-            detailed_images[0] = {0, 0, 0, nullptr};
-            free_sd_images(detailed_images, detailed_count);
+            results[i]               = detailed->items[0].image;
+            detailed->items[0].image = {};
+            free_sd_image_generation_result(detailed);
         }
     }
     return true;