DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-12 11:56:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: use tokenizer-specific pre-tokenization rules (#1975)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-09-15 02:37:52 +08:00
1 parent 07a85c74cb
commit 59c23bce0d
9 files changed
+92 -1021
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/tokenizers/qwen2_tokenizer.cpp
+2 -1
View File
@@ -87,7 +87,8 @@ Qwen2Tokenizer::Qwen2Tokenizer(const std::string& merges_utf8_str)
}
Qwen2Tokenizer::Qwen2Tokenizer(const std::string& merges_utf8_str,
const std::vector<std::string>& special_tokens_override) {
const std::vector<std::string>& special_tokens_override)
: BPETokenizer(R"((?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+)") {
UNK_TOKEN = "<|endoftext|>";
EOS_TOKEN = "<|endoftext|>";
PAD_TOKEN = "<|endoftext|>";