DarthAffe/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-03-24 02:08:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: safely handle whitespace and consecutive newlines (#1288)

Browse Source
This commit is contained in:
leejet 2026-02-19 20:54:42 +08:00 committed by GitHub
parent c5eb1e4137
commit c9cd49701a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/tokenize_util.cpp
View File

@ -919,15 +919,21 @@ std::vector<std::string> token_split(const std::string& text) {
// `\s*[\r\n]+|\s+(?!\S)|\s+` // `\s*[\r\n]+|\s+(?!\S)|\s+`
if (is_space(cp)) { if (is_space(cp)) {
std::string token = codepoint_to_utf8(cp); std::string token;
++i; bool saw_new_line = false;
while (i < cps.size() && is_space(cps[i])) { while (i < cps.size() && is_space(cps[i])) {
token += codepoint_to_utf8(cps[i]); token += codepoint_to_utf8(cps[i]);
++i;
if (cps[i] == U'\r' || cps[i] == U'\n') { if (cps[i] == U'\r' || cps[i] == U'\n') {
break; saw_new_line = true;
} else {
if (saw_new_line) {
break;
}
} }
++i;
} }
tokens.push_back(token); tokens.push_back(token);