mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-11-11 19:36:33 +00:00
feat: add ltx2.3 support (#1463)
* add GemmaTokenizer * add basic ltx2.3 support * change vocab file encoding * fix ci * fix ubuntu build * add temporal tiling support * add ltx audio support * update ggml submodule url * fix generate_video * add i2v support * minify bundled Gemma tokenizer vocab sources * pass video fps into temporal rope embeddings * fix av_ca_timestep_scale_multiplier * add LTX2Scheduler support * update docs * fix ci
This commit is contained in:
leejet authored and GitHub committed
1 parent
3b4d26f3d9
commit
67dda3f897
44 files changed
+6393
-240
No files matched your search
@@ -162,13 +162,37 @@ std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t
|
||||
|
||||
std::string token_str = token;
|
||||
std::u32string utf32_token;
|
||||
for (int i = 0; i < static_cast<int>(token_str.length()); i++) {
|
||||
unsigned char b = token_str[i];
|
||||
utf32_token += byte_encoder[b];
|
||||
if (byte_level_bpe) {
|
||||
for (int i = 0; i < token_str.length(); i++) {
|
||||
unsigned char b = token_str[i];
|
||||
utf32_token += byte_encoder[b];
|
||||
}
|
||||
} else {
|
||||
utf32_token = utf8_to_utf32(token_str);
|
||||
}
|
||||
auto bpe_strs = bpe(utf32_token);
|
||||
for (auto bpe_str : bpe_strs) {
|
||||
bpe_tokens.push_back(encoder[bpe_str]);
|
||||
int token_id;
|
||||
auto iter = encoder.find(bpe_str);
|
||||
if (iter != encoder.end()) {
|
||||
token_id = iter->second;
|
||||
} else {
|
||||
if (byte_fallback) {
|
||||
auto utf8_token_str = utf32_to_utf8(bpe_str);
|
||||
for (int i = 0; i < utf8_token_str.length(); i++) {
|
||||
unsigned char b = utf8_token_str[i];
|
||||
char hex_buf[16];
|
||||
snprintf(hex_buf, sizeof(hex_buf), "<0x%02X>", b);
|
||||
iter = encoder.find(utf8_to_utf32(hex_buf));
|
||||
bpe_tokens.push_back(token_id);
|
||||
token_strs.push_back(hex_buf);
|
||||
}
|
||||
continue;
|
||||
} else {
|
||||
token_id = UNK_TOKEN_ID;
|
||||
}
|
||||
}
|
||||
bpe_tokens.push_back(token_id);
|
||||
token_strs.push_back(utf32_to_utf8(bpe_str));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,8 +20,10 @@ protected:
|
||||
std::map<std::u32string, int> encoder;
|
||||
std::map<int, std::u32string> decoder;
|
||||
std::map<std::pair<std::u32string, std::u32string>, int> bpe_ranks;
|
||||
int encoder_len = 0;
|
||||
int bpe_len = 0;
|
||||
int encoder_len = 0;
|
||||
int bpe_len = 0;
|
||||
bool byte_level_bpe = true;
|
||||
bool byte_fallback = false;
|
||||
|
||||
protected:
|
||||
static std::vector<std::pair<int, std::u32string>> bytes_to_unicode();
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
#include "gemma_tokenizer.h"
|
||||
|
||||
#include "ggml.h"
|
||||
#include "json.hpp"
|
||||
#include "util.h"
|
||||
#include "vocab/vocab.h"
|
||||
|
||||
std::string GemmaTokenizer::normalize(const std::string& text) const {
|
||||
std::string normalized = text;
|
||||
size_t pos = 0;
|
||||
while ((pos = normalized.find(' ', pos)) != std::string::npos) {
|
||||
normalized.replace(pos, 1, "\xE2\x96\x81");
|
||||
pos += 3;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
void GemmaTokenizer::load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
|
||||
nlohmann::json vocab;
|
||||
try {
|
||||
vocab = nlohmann::json::parse(vocab_utf8_str);
|
||||
} catch (const nlohmann::json::parse_error&) {
|
||||
GGML_ABORT("invalid vocab json str");
|
||||
}
|
||||
for (const auto& [key, value] : vocab.items()) {
|
||||
std::u32string token = utf8_to_utf32(key);
|
||||
int i = value;
|
||||
encoder[token] = i;
|
||||
decoder[i] = token;
|
||||
}
|
||||
encoder_len = static_cast<int>(vocab.size());
|
||||
LOG_DEBUG("vocab size: %d", encoder_len);
|
||||
|
||||
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
|
||||
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
|
||||
for (const auto& merge : merges) {
|
||||
size_t space_pos = merge.find(' ');
|
||||
merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
|
||||
}
|
||||
LOG_DEBUG("merges size %zu", merge_pairs.size());
|
||||
|
||||
int rank = 0;
|
||||
for (const auto& merge : merge_pairs) {
|
||||
bpe_ranks[merge] = rank++;
|
||||
}
|
||||
bpe_len = rank;
|
||||
}
|
||||
|
||||
GemmaTokenizer::GemmaTokenizer(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
|
||||
byte_level_bpe = false;
|
||||
byte_fallback = true;
|
||||
add_bos_token = true;
|
||||
pad_left = true;
|
||||
PAD_TOKEN = "<pad>";
|
||||
EOS_TOKEN = "<eos>";
|
||||
BOS_TOKEN = "<bos>";
|
||||
UNK_TOKEN = "<unk>";
|
||||
|
||||
PAD_TOKEN_ID = 0;
|
||||
EOS_TOKEN_ID = 1;
|
||||
BOS_TOKEN_ID = 2;
|
||||
UNK_TOKEN_ID = 3;
|
||||
|
||||
std::vector<std::string> special_tokens_before_merge = {
|
||||
PAD_TOKEN,
|
||||
EOS_TOKEN,
|
||||
BOS_TOKEN,
|
||||
UNK_TOKEN,
|
||||
"<mask>",
|
||||
"[multimodal]",
|
||||
};
|
||||
for (int i = 0; i <= 98; i++) {
|
||||
special_tokens_before_merge.push_back("<unused" + std::to_string(i) + ">");
|
||||
}
|
||||
special_tokens_before_merge.push_back("<start_of_turn>");
|
||||
special_tokens_before_merge.push_back("<end_of_turn>");
|
||||
for (int i = 1; i <= 31; i++) {
|
||||
special_tokens_before_merge.push_back(std::string(i, '\n'));
|
||||
}
|
||||
for (int i = 2; i <= 31; i++) {
|
||||
std::string whitespace_token;
|
||||
for (int j = 0; j < i; j++) {
|
||||
whitespace_token += "\xE2\x96\x81";
|
||||
}
|
||||
special_tokens_before_merge.push_back(whitespace_token);
|
||||
}
|
||||
std::vector<std::string> html_tokens = {
|
||||
"<table>",
|
||||
"<caption>",
|
||||
"<thead>",
|
||||
"<tbody>",
|
||||
"<tfoot>",
|
||||
"<tr>",
|
||||
"<th>",
|
||||
"<td>",
|
||||
"</table>",
|
||||
"</caption>",
|
||||
"</thead>",
|
||||
"</tbody>",
|
||||
"</tfoot>",
|
||||
"</tr>",
|
||||
"</th>",
|
||||
"</td>",
|
||||
"<h1>",
|
||||
"<h2>",
|
||||
"<h3>",
|
||||
"<h4>",
|
||||
"<h5>",
|
||||
"<h6>",
|
||||
"<blockquote>",
|
||||
"</h1>",
|
||||
"</h2>",
|
||||
"</h3>",
|
||||
"</h4>",
|
||||
"</h5>",
|
||||
"</h6>",
|
||||
"</blockquote>",
|
||||
"<strong>",
|
||||
"<em>",
|
||||
"<b>",
|
||||
"<i>",
|
||||
"<u>",
|
||||
"<s>",
|
||||
"<sub>",
|
||||
"<sup>",
|
||||
"<code>",
|
||||
"</strong>",
|
||||
"</em>",
|
||||
"</b>",
|
||||
"</i>",
|
||||
"</u>",
|
||||
"</s>",
|
||||
"</sub>",
|
||||
"</sup>",
|
||||
"</code>",
|
||||
"<a>",
|
||||
"<html>",
|
||||
"<body>",
|
||||
"<img>",
|
||||
"<span>",
|
||||
"<bbox>",
|
||||
"<ul>",
|
||||
"<li>",
|
||||
"<div>",
|
||||
"<iframe>",
|
||||
"<footer>",
|
||||
"</a>",
|
||||
"</html>",
|
||||
"</body>",
|
||||
"</img>",
|
||||
"</span>",
|
||||
"</bbox>",
|
||||
"</ul>",
|
||||
"</li>",
|
||||
"</div>",
|
||||
"</iframe>",
|
||||
"</footer>",
|
||||
};
|
||||
special_tokens_before_merge.insert(special_tokens_before_merge.end(),
|
||||
html_tokens.begin(),
|
||||
html_tokens.end());
|
||||
for (int i = 0; i <= 0xFF; i++) {
|
||||
char hex_buf[16];
|
||||
snprintf(hex_buf, sizeof(hex_buf), "<0x%02X>", i);
|
||||
special_tokens_before_merge.push_back(hex_buf);
|
||||
}
|
||||
|
||||
std::vector<std::string> special_tokens_after_merge = {
|
||||
"<start_of_image>",
|
||||
"<end_of_image>",
|
||||
};
|
||||
for (int i = 1; i <= 31; i++) {
|
||||
special_tokens_after_merge.insert(special_tokens_after_merge.begin() + i - 1,
|
||||
std::string(i, '\t'));
|
||||
}
|
||||
for (int i = 99; i <= 6241; i++) {
|
||||
special_tokens_after_merge.push_back("<unused" + std::to_string(i) + ">");
|
||||
}
|
||||
special_tokens_after_merge.push_back("<image_soft_token>");
|
||||
|
||||
special_tokens = special_tokens_before_merge;
|
||||
special_tokens.insert(special_tokens.end(),
|
||||
special_tokens_after_merge.begin(),
|
||||
special_tokens_after_merge.end());
|
||||
|
||||
if (merges_utf8_str.size() > 0 && vocab_utf8_str.size() > 0) {
|
||||
load_from_merges(merges_utf8_str, vocab_utf8_str);
|
||||
} else {
|
||||
load_from_merges(load_gemma_merges(), load_gemma_vocab_json());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
#ifndef __SD_TOKENIZERS_GEMMA_TOKENIZER_H__
|
||||
#define __SD_TOKENIZERS_GEMMA_TOKENIZER_H__
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "bpe_tokenizer.h"
|
||||
|
||||
class GemmaTokenizer : public BPETokenizer {
|
||||
protected:
|
||||
void load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str);
|
||||
std::string normalize(const std::string& text) const override;
|
||||
|
||||
public:
|
||||
explicit GemmaTokenizer(const std::string& merges_utf8_str = "", const std::string& vocab_utf8_str = "");
|
||||
};
|
||||
|
||||
#endif // __SD_TOKENIZERS_GEMMA_TOKENIZER_H__
|
||||
@@ -0,0 +1,3 @@
|
||||
static const unsigned char gemma_merges_utf8_c_str[] = {
|
||||
0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96Line truncated
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
static const unsigned char gemma_vocab_json_utf8_c_str[] = {
|
||||
0x7b,0x22,0x3c,0x70,0x61,0x64,0x3e,0x22,0x3a,0x30,0x2c,0x22,0x3c,0x65,0x6f,0x73,0x3e,0x22,0x3a,0x31,0x2c,0x22,0x3c,0x62,0x6f,0x73,0x3e,0x22,0x3a,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x6b,0x3e,0x22,0x3a,0x33,0x2c,0x22,0x3c,0x6d,0x61,0x73,0x6b,0x3e,0x22,0x3a,0x34,0x2c,0x22,0x5b,0x6d,0x75,0x6c,0x74,0x69,0x6d,0x6f,0x64,0x61,0x6c,0x5d,0x22,0x3a,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x30,0x3e,0x22,0x3a,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x3e,0x22,0x3a,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x3e,0x22,0x3a,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x3e,0x22,0x3a,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x3e,0x22,0x3a,0x31,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x3e,0x22,0x3a,0x31,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x36,0x3e,0x22,0x3a,0x31,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x37,0x3e,0x22,0x3a,0x31,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x38,0x3e,0x22,0x3a,0x31,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x39,0x3e,0x22,0x3a,0x31,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x30,0x3e,0x22,0x3a,0x31,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x31,0x3e,0x22,0x3a,0x31,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x32,0x3e,0x22,0x3a,0x31,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x33,0x3e,0x22,0x3a,0x31,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x34,0x3e,0x22,0x3a,0x32,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x35,0x3e,0x22,0x3a,0x32,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x36,0x3e,0x22,0x3a,0x32,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x37,0x3e,0x22,0x3a,0x32,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x38,0x3e,0x22,0x3a,0x32,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x39,0x3e,0x22,0x3a,0x32,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x30,0x3e,0x22,0x3a,0x32,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x31,0x3e,0x22,0x3a,0x32,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x32,0x3e,0x22,0x3a,0x32,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x33,0x3e,0x22,0x3a,0x32,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x34,0x3e,0x22,0x3a,0x33,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x35,0x3e,0x22,0x3a,0x33,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x36,0x3e,0x22,0x3a,0x33,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x37,0x3e,0x22,0x3a,0x33,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x38,0x3e,0x22,0x3a,0x33,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x39,0x3e,0x22,0x3a,0x33,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x30,0x3e,0x22,0x3a,0x33,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x31,0x3e,0x22,0x3a,0x33,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x32,0x3e,0x22,0x3a,0x33,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x33,0x3e,0x22,0x3a,0x33,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x34,0x3e,0x22,0x3a,0x34,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x35,0x3e,0x22,0x3a,0x34,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x36,0x3e,0x22,0x3a,0x34,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x37,0x3e,0x22,0x3a,0x34,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x38,0x3e,0x22,0x3a,0x34,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x39,0x3e,0x22,0x3a,0x34,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x30,0x3e,0x22,0x3a,0x34,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x31,0x3e,0x22,0x3a,0x34,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x32,0x3e,0x22,0x3a,0x34,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x33,0x3e,0x22,0x3a,0x34,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x34,0x3e,0x22,0x3a,0x35,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x35,0x3e,0x22,0x3a,0x35,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x36,0x3e,0x22,0x3a,0x35,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x37,0x3e,0x22,0x3a,0x35,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x38,0x3e,0x22,0x3a,0x35,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x39,0x3e,0x22,0x3a,0x35,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x30,0x3e,0x22,0x3a,0x35,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x31,0x3e,0x22,0x3a,0x35,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x32,0x3e,0x22,0x3a,0x35,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x33,0x3e,0x22,0x3a,0x35,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x34,0x3e,0x22,0x3a,0x36,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x35,0x3e,0x22,0x3a,0x36,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x36,0x3e,0x22,0x3a,0x36,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x37,0x3e,0x22,0x3a,0x36,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x38,0x3e,0x22,0x3a,0x36,0x34,0x2c,0x22Line truncated
|
||||
};
|
||||
@@ -1,5 +1,7 @@
|
||||
#include "vocab.h"
|
||||
#include "clip_t5.hpp"
|
||||
#include "gemma_merges.hpp"
|
||||
#include "gemma_vocab.hpp"
|
||||
#include "mistral.hpp"
|
||||
#include "qwen.hpp"
|
||||
#include "umt5.hpp"
|
||||
@@ -32,4 +34,14 @@ std::string load_t5_tokenizer_json() {
|
||||
std::string load_umt5_tokenizer_json() {
|
||||
std::string json_str(reinterpret_cast<const char*>(umt5_tokenizer_json_str), sizeof(umt5_tokenizer_json_str));
|
||||
return json_str;
|
||||
}
|
||||
|
||||
std::string load_gemma_merges() {
|
||||
std::string merges_utf8_str(reinterpret_cast<const char*>(gemma_merges_utf8_c_str), sizeof(gemma_merges_utf8_c_str));
|
||||
return merges_utf8_str;
|
||||
}
|
||||
|
||||
std::string load_gemma_vocab_json() {
|
||||
std::string json_str(reinterpret_cast<const char*>(gemma_vocab_json_utf8_c_str), sizeof(gemma_vocab_json_utf8_c_str));
|
||||
return json_str;
|
||||
}
|
||||
@@ -9,5 +9,7 @@ std::string load_mistral_merges();
|
||||
std::string load_mistral_vocab_json();
|
||||
std::string load_t5_tokenizer_json();
|
||||
std::string load_umt5_tokenizer_json();
|
||||
std::string load_gemma_merges();
|
||||
std::string load_gemma_vocab_json();
|
||||
|
||||
#endif // __SD_TOKENIZERS_VOCAB_VOCAB_H__
|
||||
Reference in new issue
Block a user