DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 19:36:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: escape newlines in prompt logs and clarify source separators (#2106)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-10-06 19:51:12 +08:00
1 parent 0963357214
commit fd43b542a6
7 files changed
+27 -11
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/tokenizers/bpe_tokenizer.cpp
+1 -1
View File
@@ -242,7 +242,7 @@ bool BPETokenizer::encode(const std::string& text, std::vector<int>& result, on_
ss << "\"" << token << "\", ";
}
ss << "]";
LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), bpe_tokens.size(), ss.str().c_str());
LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", escape_newlines(text).c_str(), bpe_tokens.size(), escape_newlines(ss.str()).c_str());
result = std::move(bpe_tokens);
return true;
}
src/tokenizers/hf_tokenizer.cpp
+1 -1
View File
@@ -735,7 +735,7 @@ bool HFTokenizer::encode(const std::string& text, std::vector<int>& tokens, on_n
}
}
ss << "]";
LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), result.size(), ss.str().c_str());
LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", escape_newlines(text).c_str(), result.size(), escape_newlines(ss.str()).c_str());
tokens = std::move(result);
return true;
}
src/tokenizers/t5_unigram_tokenizer.cpp
+1 -1
View File
@@ -343,7 +343,7 @@ bool T5UniGramTokenizer::encode(const std::string& input, std::vector<int>& resu
ss << "\"" << token_str << "\", ";
}
ss << "]";
LOG_VERBOSE("split prompt \"%s\" to tokens %s", input.c_str(), ss.str().c_str());
LOG_VERBOSE("split prompt \"%s\" to tokens %s", escape_newlines(input).c_str(), escape_newlines(ss.str()).c_str());
result = std::move(tokens);
return true;