From fd43b542a63db62fe26960536a6848b1fb7f6d7a Mon Sep 17 00:00:00 2001 From: leejet Date: Tue, 6 Oct 2026 19:51:12 +0800 Subject: [PATCH] refactor: escape newlines in prompt logs and clarify source separators (#2106) --- examples/common/log.cpp | 2 +- src/conditioning/conditioner.hpp | 12 ++++++------ src/core/util.cpp | 17 ++++++++++++++++- src/core/util.h | 1 + src/tokenizers/bpe_tokenizer.cpp | 2 +- src/tokenizers/hf_tokenizer.cpp | 2 +- src/tokenizers/t5_unigram_tokenizer.cpp | 2 +- 7 files changed, 27 insertions(+), 11 deletions(-) diff --git a/examples/common/log.cpp b/examples/common/log.cpp index c2ce5a30..12d06832 100644 --- a/examples/common/log.cpp +++ b/examples/common/log.cpp @@ -139,7 +139,7 @@ void example_log_printf(sd_log_level_t level, const char* file, int line, const while (!message.empty() && (message.back() == '\n' || message.back() == '\r')) { message.pop_back(); } - message += " - " + sd_basename(file) + ":" + std::to_string(line) + '\n'; + message += " --- " + sd_basename(file) + ":" + std::to_string(line) + '\n'; log_print(level, message.c_str(), log_level, log_color); } diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index 5fe83c15..cf6226b5 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -399,7 +399,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { ss << "['" << item.first << "', " << item.second << "], "; } ss << "]"; - LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str()); + LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str()); } auto on_new_token_cb = [&](std::string& str, std::vector& bpe_tokens) -> bool { @@ -817,7 +817,7 @@ struct SD3CLIPEmbedder : public Conditioner { ss << "['" << item.first << "', " << item.second << "], "; } ss << "]"; - LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str()); + LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str()); } auto on_new_token_cb = [&](std::string& str, std::vector& bpe_tokens) -> bool { @@ -1207,7 +1207,7 @@ struct FluxCLIPEmbedder : public Conditioner { ss << "['" << item.first << "', " << item.second << "], "; } ss << "]"; - LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str()); + LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str()); } auto on_new_token_cb = [&](std::string& str, std::vector& bpe_tokens) -> bool { @@ -1484,7 +1484,7 @@ struct T5CLIPEmbedder : public Conditioner { ss << "['" << item.first << "', " << item.second << "], "; } ss << "]"; - LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str()); + LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str()); } auto on_new_token_cb = [&](std::string& str, std::vector& bpe_tokens) -> bool { @@ -1873,7 +1873,7 @@ struct AnimaConditioner : public Conditioner { ss << "['" << item.first << "', " << item.second << "], "; } ss << "]"; - LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str()); + LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str()); } std::vector qwen_tokens; @@ -2144,7 +2144,7 @@ struct LLMEmbedder : public Conditioner { ss << "['" << item.first << "', " << item.second << "], "; } ss << "]"; - LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str()); + LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str()); } std::vector tokens; diff --git a/src/core/util.cpp b/src/core/util.cpp index cbad4e48..5292c38b 100644 --- a/src/core/util.cpp +++ b/src/core/util.cpp @@ -62,6 +62,21 @@ void replace_all_chars(std::string& str, char target, char replacement) { } } +std::string escape_newlines(const std::string& text) { + std::string escaped; + escaped.reserve(text.size()); + for (char ch : text) { + if (ch == '\n') { + escaped += "\\n"; + } else if (ch == '\r') { + escaped += "\\r"; + } else { + escaped += ch; + } + } + return escaped; +} + static std::string sd_vformat(const char* fmt, va_list ap) { char small[128]; va_list ap2; @@ -652,7 +667,7 @@ static void sd_log_dispatch(sd_log_level_t level, const std::string& origin, con if (origin == "ggml") { message = origin + ": " + message + '\n'; } else { - message += " - " + origin + '\n'; + message += " --- " + origin + '\n'; } sd_log_cb(level, message.c_str(), sd_log_cb_data); } diff --git a/src/core/util.h b/src/core/util.h index abe41162..0929f60e 100644 --- a/src/core/util.h +++ b/src/core/util.h @@ -29,6 +29,7 @@ bool contains(const std::string& str, const std::string& substr); std::string sd_format(const char* fmt, ...); void replace_all_chars(std::string& str, char target, char replacement); +std::string escape_newlines(const std::string& text); int round_up_to(int value, int base); diff --git a/src/tokenizers/bpe_tokenizer.cpp b/src/tokenizers/bpe_tokenizer.cpp index 79b1029e..ca86d65d 100644 --- a/src/tokenizers/bpe_tokenizer.cpp +++ b/src/tokenizers/bpe_tokenizer.cpp @@ -242,7 +242,7 @@ bool BPETokenizer::encode(const std::string& text, std::vector& result, on_ ss << "\"" << token << "\", "; } ss << "]"; - LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), bpe_tokens.size(), ss.str().c_str()); + LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", escape_newlines(text).c_str(), bpe_tokens.size(), escape_newlines(ss.str()).c_str()); result = std::move(bpe_tokens); return true; } diff --git a/src/tokenizers/hf_tokenizer.cpp b/src/tokenizers/hf_tokenizer.cpp index 821d41d7..f8daf6d2 100644 --- a/src/tokenizers/hf_tokenizer.cpp +++ b/src/tokenizers/hf_tokenizer.cpp @@ -735,7 +735,7 @@ bool HFTokenizer::encode(const std::string& text, std::vector& tokens, on_n } } ss << "]"; - LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), result.size(), ss.str().c_str()); + LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", escape_newlines(text).c_str(), result.size(), escape_newlines(ss.str()).c_str()); tokens = std::move(result); return true; } diff --git a/src/tokenizers/t5_unigram_tokenizer.cpp b/src/tokenizers/t5_unigram_tokenizer.cpp index 3b39754f..342dca1a 100644 --- a/src/tokenizers/t5_unigram_tokenizer.cpp +++ b/src/tokenizers/t5_unigram_tokenizer.cpp @@ -343,7 +343,7 @@ bool T5UniGramTokenizer::encode(const std::string& input, std::vector& resu ss << "\"" << token_str << "\", "; } ss << "]"; - LOG_VERBOSE("split prompt \"%s\" to tokens %s", input.c_str(), ss.str().c_str()); + LOG_VERBOSE("split prompt \"%s\" to tokens %s", escape_newlines(input).c_str(), escape_newlines(ss.str()).c_str()); result = std::move(tokens); return true;