docs: performance tips markup (#1460)

docs/caching.md

@@ -131,8 +131,6 @@ sd-cli -m model.safetensors -p "a cat" --cache-mode spectrum
 | `warmup` | Steps to always compute before caching starts | 4 |
 | `stop` | Stop caching at this fraction of total steps | 0.9 |
 
-```
-
 ### Performance Tips
 
 - Start with default thresholds and adjust based on output quality

examples/cli/main.cpp

@@ -433,10 +433,11 @@ bool save_results(const SDCliParams& cli_params,
         if (!img.data)
             return false;
 
-        std::string params = gen_params.embed_image_metadata
-                                 ? get_image_params(ctx_params, gen_params, gen_params.seed + idx)
-                                 : "";
-        const bool ok      = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, 90);
+        const int64_t metadata_seed = cli_params.mode == VID_GEN ? gen_params.seed : gen_params.seed + idx;
+        std::string params          = gen_params.embed_image_metadata
+                                          ? get_image_params(ctx_params, gen_params, metadata_seed, cli_params.mode)
+                                          : "";
+        const bool ok               = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, 90);
         LOG_INFO("save result image %d to '%s' (%s)", idx, path.string().c_str(), ok ? "success" : "failure");
         return ok;
     };

examples/common/common.cpp

@@ -2281,7 +2281,192 @@ std::string version_string() {
     return std::string("stable-diffusion.cpp version ") + sd_version() + ", commit " + sd_commit();
 }
 
-std::string get_image_params(const SDContextParams& ctx_params, const SDGenerationParams& gen_params, int64_t seed) {
+static std::string safe_json_string(const char* value) {
+    return value ? value : "";
+}
+
+static void set_json_basename_if_not_empty(json& target, const char* key, const std::string& path) {
+    if (!path.empty()) {
+        target[key] = sd_basename(path);
+    }
+}
+
+static json build_sampling_metadata_json(const sd_sample_params_t& sample_params,
+                                         const std::vector<int>& skip_layers,
+                                         const std::vector<float>* custom_sigmas = nullptr) {
+    json sampling = {
+        {"steps", sample_params.sample_steps},
+        {"eta", sample_params.eta},
+        {"shifted_timestep", sample_params.shifted_timestep},
+        {"flow_shift", sample_params.flow_shift},
+        {"guidance",
+         {
+             {"txt_cfg", sample_params.guidance.txt_cfg},
+             {"img_cfg", sample_params.guidance.img_cfg},
+             {"distilled_guidance", sample_params.guidance.distilled_guidance},
+             {"slg",
+              {
+                  {"scale", sample_params.guidance.slg.scale},
+                  {"layers", skip_layers},
+                  {"start", sample_params.guidance.slg.layer_start},
+                  {"end", sample_params.guidance.slg.layer_end},
+              }},
+         }},
+    };
+    if (sample_params.sample_method != SAMPLE_METHOD_COUNT) {
+        sampling["method"] = safe_json_string(sd_sample_method_name(sample_params.sample_method));
+    }
+    if (sample_params.scheduler != SCHEDULER_COUNT) {
+        sampling["scheduler"] = safe_json_string(sd_scheduler_name(sample_params.scheduler));
+    }
+    if (custom_sigmas != nullptr) {
+        sampling["custom_sigmas"] = *custom_sigmas;
+    }
+    return sampling;
+}
+
+std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
+                                            const SDGenerationParams& gen_params,
+                                            int64_t seed,
+                                            SDMode mode) {
+    json root;
+    root["schema"]    = "sdcpp.image.params/v1";
+    root["mode"]      = mode == VID_GEN ? "vid_gen" : "img_gen";
+    root["generator"] = {
+        {"name", "stable-diffusion.cpp"},
+        {"version", safe_json_string(sd_version())},
+        {"commit", safe_json_string(sd_commit())},
+    };
+    root["seed"]   = seed;
+    root["width"]  = gen_params.get_resolved_width();
+    root["height"] = gen_params.get_resolved_height();
+
+    root["prompt"] = {
+        {"positive", gen_params.prompt},
+        {"negative", gen_params.negative_prompt},
+    };
+    root["sampling"] = build_sampling_metadata_json(gen_params.sample_params,
+                                                    gen_params.skip_layers,
+                                                    &gen_params.custom_sigmas);
+
+    json models;
+    set_json_basename_if_not_empty(models, "model", ctx_params.model_path);
+    set_json_basename_if_not_empty(models, "clip_l", ctx_params.clip_l_path);
+    set_json_basename_if_not_empty(models, "clip_g", ctx_params.clip_g_path);
+    set_json_basename_if_not_empty(models, "clip_vision", ctx_params.clip_vision_path);
+    set_json_basename_if_not_empty(models, "t5xxl", ctx_params.t5xxl_path);
+    set_json_basename_if_not_empty(models, "llm", ctx_params.llm_path);
+    set_json_basename_if_not_empty(models, "llm_vision", ctx_params.llm_vision_path);
+    set_json_basename_if_not_empty(models, "diffusion_model", ctx_params.diffusion_model_path);
+    set_json_basename_if_not_empty(models, "high_noise_diffusion_model", ctx_params.high_noise_diffusion_model_path);
+    set_json_basename_if_not_empty(models, "vae", ctx_params.vae_path);
+    set_json_basename_if_not_empty(models, "taesd", ctx_params.taesd_path);
+    set_json_basename_if_not_empty(models, "control_net", ctx_params.control_net_path);
+    root["models"] = std::move(models);
+
+    root["clip_skip"]             = gen_params.clip_skip;
+    root["strength"]              = gen_params.strength;
+    root["control_strength"]      = gen_params.control_strength;
+    root["auto_resize_ref_image"] = gen_params.auto_resize_ref_image;
+    root["increase_ref_index"]    = gen_params.increase_ref_index;
+    if (mode == VID_GEN) {
+        root["video"] = {
+            {"frame_count", gen_params.video_frames},
+            {"fps", gen_params.fps},
+        };
+        root["moe_boundary"]        = gen_params.moe_boundary;
+        root["vace_strength"]       = gen_params.vace_strength;
+        root["high_noise_sampling"] = build_sampling_metadata_json(gen_params.high_noise_sample_params,
+                                                                   gen_params.high_noise_skip_layers);
+    }
+
+    root["rng"] = safe_json_string(sd_rng_type_name(ctx_params.rng_type));
+    if (ctx_params.sampler_rng_type != RNG_TYPE_COUNT) {
+        root["sampler_rng"] = safe_json_string(sd_rng_type_name(ctx_params.sampler_rng_type));
+    }
+
+    json loras = json::array();
+    for (const auto& entry : gen_params.lora_map) {
+        loras.push_back({
+            {"name", sd_basename(entry.first)},
+            {"multiplier", entry.second},
+            {"is_high_noise", false},
+        });
+    }
+    for (const auto& entry : gen_params.high_noise_lora_map) {
+        loras.push_back({
+            {"name", sd_basename(entry.first)},
+            {"multiplier", entry.second},
+            {"is_high_noise", true},
+        });
+    }
+    if (!loras.empty()) {
+        root["loras"] = std::move(loras);
+    }
+
+    if (gen_params.hires_enabled) {
+        root["hires"] = {
+            {"enabled", gen_params.hires_enabled},
+            {"upscaler", gen_params.hires_upscaler},
+            {"model", gen_params.hires_upscaler_model_path.empty() ? "" : sd_basename(gen_params.hires_upscaler_model_path)},
+            {"scale", gen_params.hires_scale},
+            {"target_width", gen_params.hires_width},
+            {"target_height", gen_params.hires_height},
+            {"steps", gen_params.hires_steps},
+            {"denoising_strength", gen_params.hires_denoising_strength},
+            {"upscale_tile_size", gen_params.hires_upscale_tile_size},
+        };
+    }
+
+    if (gen_params.cache_params.mode != SD_CACHE_DISABLED) {
+        root["cache"] = {
+            {"requested_mode", gen_params.cache_mode},
+            {"requested_option", gen_params.cache_option},
+            {"mode", gen_params.cache_params.mode},
+            {"scm_mask", gen_params.scm_mask},
+            {"scm_policy_dynamic", gen_params.scm_policy_dynamic},
+            {"reuse_threshold", gen_params.cache_params.reuse_threshold},
+            {"start_percent", gen_params.cache_params.start_percent},
+            {"end_percent", gen_params.cache_params.end_percent},
+            {"error_decay_rate", gen_params.cache_params.error_decay_rate},
+            {"use_relative_threshold", gen_params.cache_params.use_relative_threshold},
+            {"reset_error_on_compute", gen_params.cache_params.reset_error_on_compute},
+            {"Fn_compute_blocks", gen_params.cache_params.Fn_compute_blocks},
+            {"Bn_compute_blocks", gen_params.cache_params.Bn_compute_blocks},
+            {"residual_diff_threshold", gen_params.cache_params.residual_diff_threshold},
+            {"max_warmup_steps", gen_params.cache_params.max_warmup_steps},
+            {"max_cached_steps", gen_params.cache_params.max_cached_steps},
+            {"max_continuous_cached_steps", gen_params.cache_params.max_continuous_cached_steps},
+            {"taylorseer_n_derivatives", gen_params.cache_params.taylorseer_n_derivatives},
+            {"taylorseer_skip_interval", gen_params.cache_params.taylorseer_skip_interval},
+            {"spectrum_w", gen_params.cache_params.spectrum_w},
+            {"spectrum_m", gen_params.cache_params.spectrum_m},
+            {"spectrum_lam", gen_params.cache_params.spectrum_lam},
+            {"spectrum_window_size", gen_params.cache_params.spectrum_window_size},
+            {"spectrum_flex_window", gen_params.cache_params.spectrum_flex_window},
+            {"spectrum_warmup_steps", gen_params.cache_params.spectrum_warmup_steps},
+            {"spectrum_stop_percent", gen_params.cache_params.spectrum_stop_percent},
+        };
+    }
+
+    if (gen_params.vae_tiling_params.enabled) {
+        root["vae_tiling"] = {
+            {"enabled", gen_params.vae_tiling_params.enabled},
+            {"tile_size_x", gen_params.vae_tiling_params.tile_size_x},
+            {"tile_size_y", gen_params.vae_tiling_params.tile_size_y},
+            {"target_overlap", gen_params.vae_tiling_params.target_overlap},
+            {"rel_size_x", gen_params.vae_tiling_params.rel_size_x},
+            {"rel_size_y", gen_params.vae_tiling_params.rel_size_y},
+        };
+    }
+
+    return root.dump();
+}
+
+std::string get_image_params(const SDContextParams& ctx_params,
+                             const SDGenerationParams& gen_params,
+                             int64_t seed,
+                             SDMode mode) {
     std::string parameter_string;
     if (gen_params.prompt_with_lora.size() != 0) {
         parameter_string += gen_params.prompt_with_lora + "\n";
@@ -2294,7 +2479,7 @@ std::string get_image_params(const SDContextParams& ctx_params, const SDGenerati
     parameter_string += "Steps: " + std::to_string(gen_params.sample_params.sample_steps) + ", ";
     parameter_string += "CFG scale: " + std::to_string(gen_params.sample_params.guidance.txt_cfg) + ", ";
     if (gen_params.sample_params.guidance.slg.scale != 0 && gen_params.skip_layers.size() != 0) {
-        parameter_string += "SLG scale: " + std::to_string(gen_params.sample_params.guidance.txt_cfg) + ", ";
+        parameter_string += "SLG scale: " + std::to_string(gen_params.sample_params.guidance.slg.scale) + ", ";
         parameter_string += "Skip layers: [";
         for (const auto& layer : gen_params.skip_layers) {
             parameter_string += std::to_string(layer) + ", ";
@@ -2347,5 +2532,6 @@ std::string get_image_params(const SDContextParams& ctx_params, const SDGenerati
         parameter_string += "Denoising strength: " + std::to_string(gen_params.hires_denoising_strength) + ", ";
     }
     parameter_string += "Version: stable-diffusion.cpp";
+    parameter_string += ", SDCPP: " + build_sdcpp_image_metadata_json(ctx_params, gen_params, seed, mode);
     return parameter_string;
 }

examples/common/common.h

@@ -249,6 +249,13 @@ struct SDGenerationParams {
 };
 
 std::string version_string();
-std::string get_image_params(const SDContextParams& ctx_params, const SDGenerationParams& gen_params, int64_t seed);
+std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
+                                            const SDGenerationParams& gen_params,
+                                            int64_t seed,
+                                            SDMode mode = IMG_GEN);
+std::string get_image_params(const SDContextParams& ctx_params,
+                             const SDGenerationParams& gen_params,
+                             int64_t seed,
+                             SDMode mode = IMG_GEN);
 
 #endif  // __EXAMPLES_COMMON_COMMON_H__

src/ggml_extend.hpp

@@ -2758,16 +2758,16 @@ __STATIC_INLINE__ ggml_tensor* ggml_ext_lokr_forward(
     bool is_conv,
     WeightAdapter::ForwardParams::conv2d_params_t conv_params,
     float scale) {
-    GGML_ASSERT((w1 != NULL || (w1a != NULL && w1b != NULL)));
-    GGML_ASSERT((w2 != NULL || (w2a != NULL && w2b != NULL)));
+    GGML_ASSERT((w1 != nullptr || (w1a != nullptr && w1b != nullptr)));
+    GGML_ASSERT((w2 != nullptr || (w2a != nullptr && w2b != nullptr)));
 
-    int uq = (w1 != NULL) ? (int)w1->ne[0] : (int)w1a->ne[0];
-    int up = (w1 != NULL) ? (int)w1->ne[1] : (int)w1b->ne[1];
+    int uq = (w1 != nullptr) ? (int)w1->ne[0] : (int)w1a->ne[0];
+    int up = (w1 != nullptr) ? (int)w1->ne[1] : (int)w1b->ne[1];
 
     int q_actual = is_conv ? (int)h->ne[2] : (int)h->ne[0];
     int vq       = q_actual / uq;
 
-    int vp = (w2 != NULL) ? (is_conv ? (int)w2->ne[3] : (int)w2->ne[1])
+    int vp = (w2 != nullptr) ? (is_conv ? (int)w2->ne[3] : (int)w2->ne[1])
                           : (int)w2a->ne[1];
     GGML_ASSERT(q_actual == (uq * vq) && "Input dimension mismatch for LoKR split");
 
@@ -2803,7 +2803,7 @@ __STATIC_INLINE__ ggml_tensor* ggml_ext_lokr_forward(
 #endif
 
         ggml_tensor* h_split = ggml_reshape_3d(ctx, h, vq, uq * merge_batch_uq, batch / merge_batch_uq);
-        if (w2 != NULL) {
+        if (w2 != nullptr) {
             hb = ggml_mul_mat(ctx, w2, h_split);
         } else {
             hb = ggml_mul_mat(ctx, w2b, ggml_mul_mat(ctx, w2a, h_split));
@@ -2816,7 +2816,7 @@ __STATIC_INLINE__ ggml_tensor* ggml_ext_lokr_forward(
         hb_t              = ggml_reshape_3d(ctx, hb_t, uq, vp * merge_batch_vp, batch / merge_batch_vp);
 
         ggml_tensor* hc_t;
-        if (w1 != NULL) {
+        if (w1 != nullptr) {
             hc_t = ggml_mul_mat(ctx, w1, hb_t);
         } else {
             hc_t = ggml_mul_mat(ctx, w1b, ggml_mul_mat(ctx, w1a, hb_t));
@@ -2834,7 +2834,7 @@ __STATIC_INLINE__ ggml_tensor* ggml_ext_lokr_forward(
         // 1. Reshape input: [W, H, vq*uq, batch] -> [W, H, vq, uq * batch]
         ggml_tensor* h_split = ggml_reshape_4d(ctx, h, h->ne[0], h->ne[1], vq, uq * batch);
 
-        if (w2 != NULL) {
+        if (w2 != nullptr) {
             hb = ggml_ext_conv_2d(ctx, h_split, w2, nullptr,
                                   conv_params.s0,
                                   conv_params.s1,
@@ -2902,7 +2902,7 @@ __STATIC_INLINE__ ggml_tensor* ggml_ext_lokr_forward(
         ggml_tensor* hb_merged = ggml_reshape_2d(ctx, hb, w_out * h_out * vp, uq * batch);
         ggml_tensor* hc_t;
         ggml_tensor* hb_merged_t = ggml_cont(ctx, ggml_transpose(ctx, hb_merged));
-        if (w1 != NULL) {
+        if (w1 != nullptr) {
             // Would be great to be able to transpose w1 instead to avoid transposing both hb and hc
             hc_t = ggml_mul_mat(ctx, w1, hb_merged_t);
         } else {

src/stable-diffusion.cpp

@@ -198,11 +198,11 @@ public:
                     device = 0;
                 }
                 if (device >= device_count) {
-                    LOG_WARN("Cannot find targeted vulkan device (%llu). Falling back to device 0.", device);
+                    LOG_WARN("Cannot find targeted vulkan device (%zu). Falling back to device 0.", device);
                     device = 0;
                 }
             }
-            LOG_INFO("Vulkan: Using device %llu", device);
+            LOG_INFO("Vulkan: Using device %zu", device);
             backend = ggml_backend_vk_init(device);
         }
         if (!backend) {
@@ -3233,7 +3233,7 @@ static sd_image_t* decode_image_outputs(sd_ctx_t* sd_ctx,
         }
         decoded_images.push_back(std::move(image));
         int64_t t2 = ggml_time_ms();
-        LOG_INFO("latent %" PRId64 " decoded, taking %.2fs", i + 1, (t2 - t1) * 1.0f / 1000);
+        LOG_INFO("latent %zu decoded, taking %.2fs", i + 1, (t2 - t1) * 1.0f / 1000);
     }
 
     int64_t t4 = ggml_time_ms();
@@ -3475,7 +3475,7 @@ SD_API sd_image_t* generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* s
         sd_ctx->sd->diffusion_model->free_params_buffer();
     }
     int64_t denoise_end = ggml_time_ms();
-    LOG_INFO("generating %" PRId64 " latent images completed, taking %.2fs",
+    LOG_INFO("generating %zu latent images completed, taking %.2fs",
              final_latents.size(),
              (denoise_end - denoise_start) * 1.0f / 1000);
 

src/tokenizers/clip_tokenizer.cpp

@@ -62,7 +62,7 @@ void CLIPTokenizer::load_from_merges(const std::string& merges_utf8_str) {
     }
     vocab.push_back(utf8_to_utf32("<|startoftext|>"));
     vocab.push_back(utf8_to_utf32("<|endoftext|>"));
-    LOG_DEBUG("vocab size: %llu", vocab.size());
+    LOG_DEBUG("vocab size: %zu", vocab.size());
     int i = 0;
     for (const auto& token : vocab) {
         encoder[token] = i;

src/tokenizers/mistral_tokenizer.cpp

@@ -28,7 +28,7 @@ void MistralTokenizer::load_from_merges(const std::string& merges_utf8_str, cons
         byte_decoder[pair.second] = pair.first;
     }
     std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
-    LOG_DEBUG("merges size %llu", merges.size());
+    LOG_DEBUG("merges size %zu", merges.size());
     std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
     for (const auto& merge : merges) {
         size_t space_pos = merge.find(' ');

src/tokenizers/qwen2_tokenizer.cpp

@@ -11,7 +11,7 @@ void Qwen2Tokenizer::load_from_merges(const std::string& merges_utf8_str) {
     }
 
     std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
-    LOG_DEBUG("merges size %llu", merges.size());
+    LOG_DEBUG("merges size %zu", merges.size());
     std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
     for (const auto& merge : merges) {
         size_t space_pos = merge.find(' ');

src/util.cpp

@@ -119,10 +119,10 @@ std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename) {
         filename.c_str(),
         GENERIC_READ,
         FILE_SHARE_READ,
-        NULL,
+        nullptr,
         OPEN_EXISTING,
         FILE_ATTRIBUTE_NORMAL,
-        NULL);
+        nullptr);
 
     if (file_handle == INVALID_HANDLE_VALUE) {
         return nullptr;
@@ -136,16 +136,16 @@ std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename) {
 
     file_size = static_cast<size_t>(size.QuadPart);
 
-    HANDLE mapping_handle = CreateFileMapping(file_handle, NULL, PAGE_READONLY, 0, 0, NULL);
+    HANDLE mapping_handle = CreateFileMapping(file_handle, nullptr, PAGE_READONLY, 0, 0, nullptr);
 
-    if (mapping_handle == NULL) {
+    if (mapping_handle == nullptr) {
         CloseHandle(file_handle);
         return nullptr;
     }
 
     mapped_data = MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_size);
 
-    if (mapped_data == NULL) {
+    if (mapped_data == nullptr) {
         CloseHandle(mapping_handle);
         CloseHandle(file_handle);
         return nullptr;
@@ -203,7 +203,7 @@ std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename) {
 
     size_t file_size = sb.st_size;
 
-    void* mapped_data = mmap(NULL, file_size, PROT_READ, mmap_flags, file_descriptor, 0);
+    void* mapped_data = mmap(nullptr, file_size, PROT_READ, mmap_flags, file_descriptor, 0);
 
     close(file_descriptor);