DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-12 03:49:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: use tokenizer-specific pre-tokenization rules (#1975)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-09-15 02:37:52 +08:00
1 parent 07a85c74cb
commit 59c23bce0d
9 files changed
+92 -1021
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/tokenizers/gemma_tokenizer.cpp
+3 -1
View File
@@ -46,7 +46,9 @@ void GemmaTokenizer::load_from_merges(const std::string& merges_utf8_str, const
bpe_len = rank;
}
GemmaTokenizer::GemmaTokenizer(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
GemmaTokenizer::GemmaTokenizer(const std::string& merges_utf8_str, const std::string& vocab_utf8_str)
: BPETokenizer("") {
// Gemma replaces spaces with metaspace before its literal-space Split, so no regex boundaries apply.
byte_level_bpe = false;
byte_fallback = true;
add_bos_token = true;