mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-11-11 11:26:34 +00:00
feat: add PiD support (#1585)
This commit is contained in:
leejet authored and GitHub committed
1 parent
d2797b8667
commit
0982807139
23 files changed
+1387
-23
No files matched your search
@@ -182,7 +182,8 @@ std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t
|
||||
unsigned char b = utf8_token_str[i];
|
||||
char hex_buf[16];
|
||||
snprintf(hex_buf, sizeof(hex_buf), "<0x%02X>", b);
|
||||
iter = encoder.find(utf8_to_utf32(hex_buf));
|
||||
iter = encoder.find(utf8_to_utf32(hex_buf));
|
||||
token_id = iter != encoder.end() ? iter->second : UNK_TOKEN_ID;
|
||||
bpe_tokens.push_back(token_id);
|
||||
token_strs.push_back(hex_buf);
|
||||
}
|
||||
|
||||
@@ -189,3 +189,164 @@ GemmaTokenizer::GemmaTokenizer(const std::string& merges_utf8_str, const std::st
|
||||
load_from_merges(load_gemma_merges(), load_gemma_vocab_json());
|
||||
}
|
||||
}
|
||||
|
||||
std::string Gemma2Tokenizer::normalize(const std::string& text) const {
|
||||
std::string normalized = text;
|
||||
size_t pos = 0;
|
||||
while ((pos = normalized.find(' ', pos)) != std::string::npos) {
|
||||
normalized.replace(pos, 1, "\xE2\x96\x81");
|
||||
pos += 3;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
void Gemma2Tokenizer::load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
|
||||
nlohmann::json vocab;
|
||||
try {
|
||||
vocab = nlohmann::json::parse(vocab_utf8_str);
|
||||
} catch (const nlohmann::json::parse_error&) {
|
||||
GGML_ABORT("invalid vocab json str");
|
||||
}
|
||||
for (const auto& [key, value] : vocab.items()) {
|
||||
std::u32string token = utf8_to_utf32(key);
|
||||
int i = value;
|
||||
encoder[token] = i;
|
||||
decoder[i] = token;
|
||||
}
|
||||
encoder_len = static_cast<int>(vocab.size());
|
||||
LOG_DEBUG("vocab size: %d", encoder_len);
|
||||
|
||||
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
|
||||
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
|
||||
for (const auto& merge : merges) {
|
||||
size_t space_pos = merge.find(' ');
|
||||
merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
|
||||
}
|
||||
LOG_DEBUG("merges size %zu", merge_pairs.size());
|
||||
|
||||
int rank = 0;
|
||||
for (const auto& merge : merge_pairs) {
|
||||
bpe_ranks[merge] = rank++;
|
||||
}
|
||||
bpe_len = rank;
|
||||
}
|
||||
|
||||
Gemma2Tokenizer::Gemma2Tokenizer(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
|
||||
byte_level_bpe = false;
|
||||
byte_fallback = true;
|
||||
add_bos_token = true;
|
||||
PAD_TOKEN = "<pad>";
|
||||
EOS_TOKEN = "<eos>";
|
||||
BOS_TOKEN = "<bos>";
|
||||
UNK_TOKEN = "<unk>";
|
||||
|
||||
PAD_TOKEN_ID = 0;
|
||||
EOS_TOKEN_ID = 1;
|
||||
BOS_TOKEN_ID = 2;
|
||||
UNK_TOKEN_ID = 3;
|
||||
|
||||
std::vector<std::string> special_tokens_before_merge = {
|
||||
PAD_TOKEN,
|
||||
EOS_TOKEN,
|
||||
BOS_TOKEN,
|
||||
UNK_TOKEN,
|
||||
"<mask>",
|
||||
"<2mass>",
|
||||
"[@BOS@]",
|
||||
};
|
||||
for (int i = 0; i <= 98; i++) {
|
||||
special_tokens_before_merge.push_back("<unused" + std::to_string(i) + ">");
|
||||
}
|
||||
special_tokens_before_merge.push_back("<start_of_turn>");
|
||||
special_tokens_before_merge.push_back("<end_of_turn>");
|
||||
for (int i = 1; i <= 31; i++) {
|
||||
special_tokens_before_merge.push_back(std::string(i, '\n'));
|
||||
}
|
||||
for (int i = 2; i <= 31; i++) {
|
||||
std::string whitespace_token;
|
||||
for (int j = 0; j < i; j++) {
|
||||
whitespace_token += "\xE2\x96\x81";
|
||||
}
|
||||
special_tokens_before_merge.push_back(whitespace_token);
|
||||
}
|
||||
std::vector<std::string> html_tokens = {
|
||||
"<table>",
|
||||
"<caption>",
|
||||
"<thead>",
|
||||
"<tbody>",
|
||||
"<tfoot>",
|
||||
"<tr>",
|
||||
"<th>",
|
||||
"<td>",
|
||||
"</table>",
|
||||
"</caption>",
|
||||
"</thead>",
|
||||
"</tbody>",
|
||||
"</tfoot>",
|
||||
"</tr>",
|
||||
"</th>",
|
||||
"</td>",
|
||||
"<h1>",
|
||||
"<h2>",
|
||||
"<h3>",
|
||||
"<h4>",
|
||||
"<h5>",
|
||||
"<h6>",
|
||||
"<blockquote>",
|
||||
"</h1>",
|
||||
"</h2>",
|
||||
"</h3>",
|
||||
"</h4>",
|
||||
"</h5>",
|
||||
"</h6>",
|
||||
"</blockquote>",
|
||||
"<strong>",
|
||||
"<em>",
|
||||
"<b>",
|
||||
"<i>",
|
||||
"<u>",
|
||||
"<s>",
|
||||
"<sub>",
|
||||
"<sup>",
|
||||
"<code>",
|
||||
"</strong>",
|
||||
"</em>",
|
||||
"</b>",
|
||||
"</i>",
|
||||
"</u>",
|
||||
"</s>",
|
||||
"</sub>",
|
||||
"</sup>",
|
||||
"</code>",
|
||||
};
|
||||
special_tokens_before_merge.insert(special_tokens_before_merge.end(),
|
||||
html_tokens.begin(),
|
||||
html_tokens.end());
|
||||
for (int i = 0; i <= 0xFF; i++) {
|
||||
char hex_buf[16];
|
||||
snprintf(hex_buf, sizeof(hex_buf), "<0x%02X>", i);
|
||||
special_tokens_before_merge.push_back(hex_buf);
|
||||
}
|
||||
|
||||
std::vector<std::string> special_tokens_after_merge = {
|
||||
"[toxicity=0]",
|
||||
};
|
||||
for (int i = 1; i <= 31; i++) {
|
||||
special_tokens_after_merge.insert(special_tokens_after_merge.begin() + i - 1,
|
||||
std::string(i, '\t'));
|
||||
}
|
||||
for (int i = 99; i <= 99; i++) {
|
||||
special_tokens_after_merge.push_back("<unused" + std::to_string(i) + ">");
|
||||
}
|
||||
|
||||
special_tokens = special_tokens_before_merge;
|
||||
special_tokens.insert(special_tokens.end(),
|
||||
special_tokens_after_merge.begin(),
|
||||
special_tokens_after_merge.end());
|
||||
|
||||
if (merges_utf8_str.size() > 0 && vocab_utf8_str.size() > 0) {
|
||||
load_from_merges(merges_utf8_str, vocab_utf8_str);
|
||||
} else {
|
||||
load_from_merges(load_gemma2_merges(), load_gemma2_vocab_json());
|
||||
}
|
||||
}
|
||||
@@ -14,4 +14,13 @@ public:
|
||||
explicit GemmaTokenizer(const std::string& merges_utf8_str = "", const std::string& vocab_utf8_str = "");
|
||||
};
|
||||
|
||||
class Gemma2Tokenizer : public BPETokenizer {
|
||||
protected:
|
||||
void load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str);
|
||||
std::string normalize(const std::string& text) const override;
|
||||
|
||||
public:
|
||||
explicit Gemma2Tokenizer(const std::string& merges_utf8_str = "", const std::string& vocab_utf8_str = "");
|
||||
};
|
||||
|
||||
#endif // __SD_TOKENIZERS_GEMMA_TOKENIZER_H__
|
||||
@@ -0,0 +1,3 @@
|
||||
static const unsigned char gemma2_merges_utf8_c_str[] = {
|
||||
0x23,0x76,0x65,0x72,0x73,0x69,0x6f,0x6e,0x3a,0x20,0x30,0x2e,0x32,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81Line truncated
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
static const unsigned char gemma2_vocab_json_utf8_c_str[] = {
|
||||
0x7b,0x22,0x3c,0x70,0x61,0x64,0x3e,0x22,0x3a,0x30,0x2c,0x22,0x3c,0x65,0x6f,0x73,0x3e,0x22,0x3a,0x31,0x2c,0x22,0x3c,0x62,0x6f,0x73,0x3e,0x22,0x3a,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x6b,0x3e,0x22,0x3a,0x33,0x2c,0x22,0x3c,0x6d,0x61,0x73,0x6b,0x3e,0x22,0x3a,0x34,0x2c,0x22,0x3c,0x32,0x6d,0x61,0x73,0x73,0x3e,0x22,0x3a,0x35,0x2c,0x22,0x5b,0x40,0x42,0x4f,0x53,0x40,0x5d,0x22,0x3a,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x30,0x3e,0x22,0x3a,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x3e,0x22,0x3a,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x3e,0x22,0x3a,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x3e,0x22,0x3a,0x31,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x3e,0x22,0x3a,0x31,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x3e,0x22,0x3a,0x31,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x36,0x3e,0x22,0x3a,0x31,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x37,0x3e,0x22,0x3a,0x31,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x38,0x3e,0x22,0x3a,0x31,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x39,0x3e,0x22,0x3a,0x31,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x30,0x3e,0x22,0x3a,0x31,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x31,0x3e,0x22,0x3a,0x31,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x32,0x3e,0x22,0x3a,0x31,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x33,0x3e,0x22,0x3a,0x32,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x34,0x3e,0x22,0x3a,0x32,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x35,0x3e,0x22,0x3a,0x32,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x36,0x3e,0x22,0x3a,0x32,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x37,0x3e,0x22,0x3a,0x32,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x38,0x3e,0x22,0x3a,0x32,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x39,0x3e,0x22,0x3a,0x32,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x30,0x3e,0x22,0x3a,0x32,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x31,0x3e,0x22,0x3a,0x32,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x32,0x3e,0x22,0x3a,0x32,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x33,0x3e,0x22,0x3a,0x33,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x34,0x3e,0x22,0x3a,0x33,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x35,0x3e,0x22,0x3a,0x33,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x36,0x3e,0x22,0x3a,0x33,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x37,0x3e,0x22,0x3a,0x33,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x38,0x3e,0x22,0x3a,0x33,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x39,0x3e,0x22,0x3a,0x33,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x30,0x3e,0x22,0x3a,0x33,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x31,0x3e,0x22,0x3a,0x33,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x32,0x3e,0x22,0x3a,0x33,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x33,0x3e,0x22,0x3a,0x34,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x34,0x3e,0x22,0x3a,0x34,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x35,0x3e,0x22,0x3a,0x34,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x36,0x3e,0x22,0x3a,0x34,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x37,0x3e,0x22,0x3a,0x34,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x38,0x3e,0x22,0x3a,0x34,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x39,0x3e,0x22,0x3a,0x34,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x30,0x3e,0x22,0x3a,0x34,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x31,0x3e,0x22,0x3a,0x34,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x32,0x3e,0x22,0x3a,0x34,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x33,0x3e,0x22,0x3a,0x35,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x34,0x3e,0x22,0x3a,0x35,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x35,0x3e,0x22,0x3a,0x35,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x36,0x3e,0x22,0x3a,0x35,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x37,0x3e,0x22,0x3a,0x35,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x38,0x3e,0x22,0x3a,0x35,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x39,0x3e,0x22,0x3a,0x35,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x30,0x3e,0x22,0x3a,0x35,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x31,0x3e,0x22,0x3a,0x35,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x32,0x3e,0x22,0x3a,0x35,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x33,0x3e,0x22,0x3a,0x36,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x34,0x3e,0x22,0x3a,0x36,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x35,0x3e,0x22,0x3a,0x36,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x36,0x3e,0x22,0x3a,0x36,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x37,0x3e,0x22,0x3a,0x36,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35Line truncated
|
||||
};
|
||||
@@ -1,5 +1,7 @@
|
||||
#include "vocab.h"
|
||||
#include "clip_merges.hpp"
|
||||
#include "gemma2_merges.hpp"
|
||||
#include "gemma2_vocab.hpp"
|
||||
#include "gemma_merges.hpp"
|
||||
#include "gemma_vocab.hpp"
|
||||
#include "gpt_oss_merges.hpp"
|
||||
@@ -50,6 +52,16 @@ std::string load_gemma_vocab_json() {
|
||||
return json_str;
|
||||
}
|
||||
|
||||
std::string load_gemma2_merges() {
|
||||
std::string merges_utf8_str(reinterpret_cast<const char*>(gemma2_merges_utf8_c_str), sizeof(gemma2_merges_utf8_c_str));
|
||||
return merges_utf8_str;
|
||||
}
|
||||
|
||||
std::string load_gemma2_vocab_json() {
|
||||
std::string json_str(reinterpret_cast<const char*>(gemma2_vocab_json_utf8_c_str), sizeof(gemma2_vocab_json_utf8_c_str));
|
||||
return json_str;
|
||||
}
|
||||
|
||||
std::string load_gpt_oss_merges() {
|
||||
std::string merges_utf8_str(reinterpret_cast<const char*>(gpt_oss_merges_utf8_c_str), sizeof(gpt_oss_merges_utf8_c_str));
|
||||
return merges_utf8_str;
|
||||
|
||||
@@ -11,6 +11,8 @@ std::string load_t5_tokenizer_json();
|
||||
std::string load_umt5_tokenizer_json();
|
||||
std::string load_gemma_merges();
|
||||
std::string load_gemma_vocab_json();
|
||||
std::string load_gemma2_merges();
|
||||
std::string load_gemma2_vocab_json();
|
||||
std::string load_gpt_oss_merges();
|
||||
std::string load_gpt_oss_vocab_json();
|
||||
|
||||
|
||||
Reference in new issue
Block a user