mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-11-11 03:24:30 +00:00
refactor: require external Gemma 2 and GPT-OSS tokenizers (#1974)
This commit is contained in:
leejet authored and GitHub committed
1 parent
4964abdfc5
commit
f9ddc0f388
19 files changed
+73
-337
No files matched your search
@@ -7,6 +7,7 @@
|
||||
#include <limits>
|
||||
#include <optional>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include "core/ggml_tensor_utils.h"
|
||||
|
||||
#include "core/tensor_ggml.hpp"
|
||||
@@ -1960,6 +1961,14 @@ struct LLMEmbedder : public Conditioner {
|
||||
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
|
||||
const TokenizerConfig& tokenizers = {})
|
||||
: version(version) {
|
||||
if (!tokenizers.has(TokenizerConfig::MAIN)) {
|
||||
if (sd_version_is_lens(version)) {
|
||||
throw std::runtime_error("Lens requires an external GPT-OSS tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer");
|
||||
}
|
||||
if (sd_version_is_pid(version)) {
|
||||
throw std::runtime_error("PiD requires an external Gemma 2 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer");
|
||||
}
|
||||
}
|
||||
LLM::LLMArch arch = LLM::LLMArch::QWEN2_5_VL;
|
||||
if (version == VERSION_FLUX2) {
|
||||
arch = LLM::LLMArch::MISTRAL_SMALL_3_2;
|
||||
@@ -1998,10 +2007,6 @@ struct LLMEmbedder : public Conditioner {
|
||||
if (!tokenizer) {
|
||||
if (arch == LLM::LLMArch::MISTRAL_SMALL_3_2 || arch == LLM::LLMArch::MINISTRAL_3_3B) {
|
||||
tokenizer = std::make_shared<MistralTokenizer>();
|
||||
} else if (arch == LLM::LLMArch::GPT_OSS_20B) {
|
||||
tokenizer = std::make_shared<GPTOSSTokenizer>();
|
||||
} else if (arch == LLM::LLMArch::GEMMA2_2B) {
|
||||
tokenizer = std::make_shared<Gemma2Tokenizer>();
|
||||
} else {
|
||||
tokenizer = std::make_shared<Qwen2Tokenizer>();
|
||||
}
|
||||
|
||||
+21
-7
@@ -15,6 +15,7 @@
|
||||
#include <regex>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
@@ -31,9 +32,9 @@
|
||||
#include "model_manager.h"
|
||||
#include "tokenizers/bpe_tokenizer.h"
|
||||
#include "tokenizers/gemma_tokenizer.h"
|
||||
#include "tokenizers/gpt_oss_tokenizer.h"
|
||||
#include "tokenizers/mistral_tokenizer.h"
|
||||
#include "tokenizers/qwen2_tokenizer.h"
|
||||
#include "tokenizers/tokenizer_config.h"
|
||||
|
||||
namespace LLM {
|
||||
constexpr int LLM_GRAPH_SIZE = 65536;
|
||||
@@ -2351,7 +2352,7 @@ namespace LLM {
|
||||
};
|
||||
|
||||
struct LLMEmbedder {
|
||||
std::shared_ptr<BPETokenizer> tokenizer;
|
||||
std::shared_ptr<Tokenizer> tokenizer;
|
||||
LLMRunner model;
|
||||
|
||||
LLMEmbedder(LLMArch arch,
|
||||
@@ -2359,14 +2360,27 @@ namespace LLM {
|
||||
const String2TensorStorage& tensor_storage_map = {},
|
||||
const std::string prefix = "",
|
||||
bool enable_vision = false,
|
||||
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
|
||||
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
|
||||
const TokenizerConfig& tokenizers = {})
|
||||
: model(arch, backend, tensor_storage_map, prefix, enable_vision, weight_manager) {
|
||||
int pad_id = 151643;
|
||||
if (arch == LLMArch::MISTRAL_SMALL_3_2 || arch == LLMArch::MINISTRAL_3_3B) {
|
||||
tokenizer = std::make_shared<MistralTokenizer>();
|
||||
pad_id = 11;
|
||||
} else if (arch == LLMArch::GPT_OSS_20B) {
|
||||
tokenizer = std::make_shared<GPTOSSTokenizer>();
|
||||
} else {
|
||||
tokenizer = std::make_shared<Qwen2Tokenizer>();
|
||||
pad_id = 199999;
|
||||
} else if (arch == LLMArch::GEMMA2_2B) {
|
||||
pad_id = 0;
|
||||
}
|
||||
tokenizer = tokenizers.create(TokenizerConfig::MAIN, model.config.vocab_size, pad_id);
|
||||
if (!tokenizer) {
|
||||
if (arch == LLMArch::GPT_OSS_20B || arch == LLMArch::GEMMA2_2B) {
|
||||
throw std::runtime_error("GPT-OSS and Gemma 2 require an external tokenizer.json in the main tokenizer slot");
|
||||
}
|
||||
if (arch == LLMArch::MISTRAL_SMALL_3_2 || arch == LLMArch::MINISTRAL_3_3B) {
|
||||
tokenizer = std::make_shared<MistralTokenizer>();
|
||||
} else {
|
||||
tokenizer = std::make_shared<Qwen2Tokenizer>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -189,164 +189,3 @@ GemmaTokenizer::GemmaTokenizer(const std::string& merges_utf8_str, const std::st
|
||||
load_from_merges(load_gemma_merges(), load_gemma_vocab_json());
|
||||
}
|
||||
}
|
||||
|
||||
std::string Gemma2Tokenizer::normalize(const std::string& text) const {
|
||||
std::string normalized = text;
|
||||
size_t pos = 0;
|
||||
while ((pos = normalized.find(' ', pos)) != std::string::npos) {
|
||||
normalized.replace(pos, 1, "\xE2\x96\x81");
|
||||
pos += 3;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
void Gemma2Tokenizer::load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
|
||||
nlohmann::json vocab;
|
||||
try {
|
||||
vocab = nlohmann::json::parse(vocab_utf8_str);
|
||||
} catch (const nlohmann::json::parse_error&) {
|
||||
GGML_ABORT("invalid vocab json str");
|
||||
}
|
||||
for (const auto& [key, value] : vocab.items()) {
|
||||
std::u32string token = utf8_to_utf32(key);
|
||||
int i = value;
|
||||
encoder[token] = i;
|
||||
decoder[i] = token;
|
||||
}
|
||||
encoder_len = static_cast<int>(vocab.size());
|
||||
LOG_VERBOSE("vocab size: %d", encoder_len);
|
||||
|
||||
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
|
||||
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
|
||||
for (const auto& merge : merges) {
|
||||
size_t space_pos = merge.find(' ');
|
||||
merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
|
||||
}
|
||||
LOG_VERBOSE("merges size %zu", merge_pairs.size());
|
||||
|
||||
int rank = 0;
|
||||
for (const auto& merge : merge_pairs) {
|
||||
bpe_ranks[merge] = rank++;
|
||||
}
|
||||
bpe_len = rank;
|
||||
}
|
||||
|
||||
Gemma2Tokenizer::Gemma2Tokenizer(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
|
||||
byte_level_bpe = false;
|
||||
byte_fallback = true;
|
||||
add_bos_token = true;
|
||||
PAD_TOKEN = "<pad>";
|
||||
EOS_TOKEN = "<eos>";
|
||||
BOS_TOKEN = "<bos>";
|
||||
UNK_TOKEN = "<unk>";
|
||||
|
||||
PAD_TOKEN_ID = 0;
|
||||
EOS_TOKEN_ID = 1;
|
||||
BOS_TOKEN_ID = 2;
|
||||
UNK_TOKEN_ID = 3;
|
||||
|
||||
std::vector<std::string> special_tokens_before_merge = {
|
||||
PAD_TOKEN,
|
||||
EOS_TOKEN,
|
||||
BOS_TOKEN,
|
||||
UNK_TOKEN,
|
||||
"<mask>",
|
||||
"<2mass>",
|
||||
"[@BOS@]",
|
||||
};
|
||||
for (int i = 0; i <= 98; i++) {
|
||||
special_tokens_before_merge.push_back("<unused" + std::to_string(i) + ">");
|
||||
}
|
||||
special_tokens_before_merge.push_back("<start_of_turn>");
|
||||
special_tokens_before_merge.push_back("<end_of_turn>");
|
||||
for (int i = 1; i <= 31; i++) {
|
||||
special_tokens_before_merge.push_back(std::string(i, '\n'));
|
||||
}
|
||||
for (int i = 2; i <= 31; i++) {
|
||||
std::string whitespace_token;
|
||||
for (int j = 0; j < i; j++) {
|
||||
whitespace_token += "\xE2\x96\x81";
|
||||
}
|
||||
special_tokens_before_merge.push_back(whitespace_token);
|
||||
}
|
||||
std::vector<std::string> html_tokens = {
|
||||
"<table>",
|
||||
"<caption>",
|
||||
"<thead>",
|
||||
"<tbody>",
|
||||
"<tfoot>",
|
||||
"<tr>",
|
||||
"<th>",
|
||||
"<td>",
|
||||
"</table>",
|
||||
"</caption>",
|
||||
"</thead>",
|
||||
"</tbody>",
|
||||
"</tfoot>",
|
||||
"</tr>",
|
||||
"</th>",
|
||||
"</td>",
|
||||
"<h1>",
|
||||
"<h2>",
|
||||
"<h3>",
|
||||
"<h4>",
|
||||
"<h5>",
|
||||
"<h6>",
|
||||
"<blockquote>",
|
||||
"</h1>",
|
||||
"</h2>",
|
||||
"</h3>",
|
||||
"</h4>",
|
||||
"</h5>",
|
||||
"</h6>",
|
||||
"</blockquote>",
|
||||
"<strong>",
|
||||
"<em>",
|
||||
"<b>",
|
||||
"<i>",
|
||||
"<u>",
|
||||
"<s>",
|
||||
"<sub>",
|
||||
"<sup>",
|
||||
"<code>",
|
||||
"</strong>",
|
||||
"</em>",
|
||||
"</b>",
|
||||
"</i>",
|
||||
"</u>",
|
||||
"</s>",
|
||||
"</sub>",
|
||||
"</sup>",
|
||||
"</code>",
|
||||
};
|
||||
special_tokens_before_merge.insert(special_tokens_before_merge.end(),
|
||||
html_tokens.begin(),
|
||||
html_tokens.end());
|
||||
for (int i = 0; i <= 0xFF; i++) {
|
||||
char hex_buf[16];
|
||||
snprintf(hex_buf, sizeof(hex_buf), "<0x%02X>", i);
|
||||
special_tokens_before_merge.push_back(hex_buf);
|
||||
}
|
||||
|
||||
std::vector<std::string> special_tokens_after_merge = {
|
||||
"[toxicity=0]",
|
||||
};
|
||||
for (int i = 1; i <= 31; i++) {
|
||||
special_tokens_after_merge.insert(special_tokens_after_merge.begin() + i - 1,
|
||||
std::string(i, '\t'));
|
||||
}
|
||||
for (int i = 99; i <= 99; i++) {
|
||||
special_tokens_after_merge.push_back("<unused" + std::to_string(i) + ">");
|
||||
}
|
||||
|
||||
special_tokens = special_tokens_before_merge;
|
||||
special_tokens.insert(special_tokens.end(),
|
||||
special_tokens_after_merge.begin(),
|
||||
special_tokens_after_merge.end());
|
||||
|
||||
if (merges_utf8_str.size() > 0 && vocab_utf8_str.size() > 0) {
|
||||
load_from_merges(merges_utf8_str, vocab_utf8_str);
|
||||
} else {
|
||||
load_from_merges(load_gemma2_merges(), load_gemma2_vocab_json());
|
||||
}
|
||||
}
|
||||
@@ -14,13 +14,4 @@ public:
|
||||
explicit GemmaTokenizer(const std::string& merges_utf8_str = "", const std::string& vocab_utf8_str = "");
|
||||
};
|
||||
|
||||
class Gemma2Tokenizer : public BPETokenizer {
|
||||
protected:
|
||||
void load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str);
|
||||
std::string normalize(const std::string& text) const override;
|
||||
|
||||
public:
|
||||
explicit Gemma2Tokenizer(const std::string& merges_utf8_str = "", const std::string& vocab_utf8_str = "");
|
||||
};
|
||||
|
||||
#endif // __SD_TOKENIZERS_GEMMA_TOKENIZER_H__
|
||||
@@ -1,91 +0,0 @@
|
||||
#include "gpt_oss_tokenizer.h"
|
||||
|
||||
#include "core/util.h"
|
||||
#include "json.hpp"
|
||||
#include "vocab/vocab.h"
|
||||
|
||||
void GPTOSSTokenizer::load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
|
||||
auto byte_unicode_pairs = bytes_to_unicode();
|
||||
byte_encoder = std::map<int, std::u32string>(byte_unicode_pairs.begin(), byte_unicode_pairs.end());
|
||||
for (auto& pair : byte_unicode_pairs) {
|
||||
byte_decoder[pair.second] = pair.first;
|
||||
}
|
||||
|
||||
nlohmann::json vocab;
|
||||
try {
|
||||
vocab = nlohmann::json::parse(vocab_utf8_str);
|
||||
} catch (const nlohmann::json::parse_error&) {
|
||||
GGML_ABORT("invalid vocab json str");
|
||||
}
|
||||
for (const auto& [key, value] : vocab.items()) {
|
||||
std::u32string token = utf8_to_utf32(key);
|
||||
int i = value;
|
||||
encoder[token] = i;
|
||||
decoder[i] = token;
|
||||
}
|
||||
encoder_len = static_cast<int>(encoder.size());
|
||||
for (auto& special_token : special_tokens) {
|
||||
auto token = utf8_to_utf32(special_token);
|
||||
encoder[token] = encoder_len;
|
||||
decoder[encoder_len] = token;
|
||||
encoder_len++;
|
||||
}
|
||||
encoder_len = static_cast<int>(encoder.size());
|
||||
LOG_VERBOSE("vocab size: %d", encoder_len);
|
||||
|
||||
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
|
||||
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
|
||||
for (const auto& merge : merges) {
|
||||
size_t space_pos = merge.find(' ');
|
||||
merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
|
||||
}
|
||||
LOG_VERBOSE("merges size %zu", merge_pairs.size());
|
||||
|
||||
int rank = 0;
|
||||
for (const auto& merge : merge_pairs) {
|
||||
bpe_ranks[merge] = rank++;
|
||||
}
|
||||
bpe_len = rank;
|
||||
}
|
||||
|
||||
GPTOSSTokenizer::GPTOSSTokenizer(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
|
||||
BOS_TOKEN = "<|startoftext|>";
|
||||
UNK_TOKEN = "<|endoftext|>";
|
||||
EOS_TOKEN = "<|endoftext|>";
|
||||
PAD_TOKEN = "<|endoftext|>";
|
||||
|
||||
BOS_TOKEN_ID = 199998;
|
||||
EOS_TOKEN_ID = 199999;
|
||||
UNK_TOKEN_ID = 199999;
|
||||
PAD_TOKEN_ID = 199999;
|
||||
|
||||
special_tokens = {
|
||||
"<|startoftext|>",
|
||||
"<|endoftext|>",
|
||||
"<|reserved_200000|>",
|
||||
"<|reserved_200001|>",
|
||||
"<|return|>",
|
||||
"<|constrain|>",
|
||||
"<|reserved_200004|>",
|
||||
"<|channel|>",
|
||||
"<|start|>",
|
||||
"<|end|>",
|
||||
"<|message|>",
|
||||
"<|reserved_200009|>",
|
||||
"<|reserved_200010|>",
|
||||
"<|reserved_200011|>",
|
||||
"<|call|>",
|
||||
"<|reserved_200013|>",
|
||||
"<|reserved_200014|>",
|
||||
"<|reserved_200015|>",
|
||||
"<|reserved_200016|>",
|
||||
"<|reserved_200017|>",
|
||||
"<|endofprompt|>",
|
||||
};
|
||||
|
||||
if (merges_utf8_str.size() > 0) {
|
||||
load_from_merges(merges_utf8_str, vocab_utf8_str);
|
||||
} else {
|
||||
load_from_merges(load_gpt_oss_merges(), load_gpt_oss_vocab_json());
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
#ifndef __SD_TOKENIZERS_GPT_OSS_TOKENIZER_H__
|
||||
#define __SD_TOKENIZERS_GPT_OSS_TOKENIZER_H__
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "bpe_tokenizer.h"
|
||||
|
||||
class GPTOSSTokenizer : public BPETokenizer {
|
||||
protected:
|
||||
void load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str);
|
||||
|
||||
public:
|
||||
explicit GPTOSSTokenizer(const std::string& merges_utf8_str = "", const std::string& vocab_utf8_str = "");
|
||||
};
|
||||
|
||||
#endif // __SD_TOKENIZERS_GPT_OSS_TOKENIZER_H__
|
||||
@@ -1,3 +0,0 @@
|
||||
static const unsigned char gemma2_merges_utf8_c_str[] = {
|
||||
0x23,0x76,0x65,0x72,0x73,0x69,0x6f,0x6e,0x3a,0x20,0x30,0x2e,0x32,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x20,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0x0a,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x20,0x5c,0x74,0x5c,0x74,0x5c,0x74,0x0a,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x5c,0x6e,0x20,0x5c,0x6e,0x5c,0x6e,0x0a,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81,0xe2,0x96,0x81Line truncated
|
||||
};
|
||||
@@ -1,3 +0,0 @@
|
||||
static const unsigned char gemma2_vocab_json_utf8_c_str[] = {
|
||||
0x7b,0x22,0x3c,0x70,0x61,0x64,0x3e,0x22,0x3a,0x30,0x2c,0x22,0x3c,0x65,0x6f,0x73,0x3e,0x22,0x3a,0x31,0x2c,0x22,0x3c,0x62,0x6f,0x73,0x3e,0x22,0x3a,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x6b,0x3e,0x22,0x3a,0x33,0x2c,0x22,0x3c,0x6d,0x61,0x73,0x6b,0x3e,0x22,0x3a,0x34,0x2c,0x22,0x3c,0x32,0x6d,0x61,0x73,0x73,0x3e,0x22,0x3a,0x35,0x2c,0x22,0x5b,0x40,0x42,0x4f,0x53,0x40,0x5d,0x22,0x3a,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x30,0x3e,0x22,0x3a,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x3e,0x22,0x3a,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x3e,0x22,0x3a,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x3e,0x22,0x3a,0x31,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x3e,0x22,0x3a,0x31,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x3e,0x22,0x3a,0x31,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x36,0x3e,0x22,0x3a,0x31,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x37,0x3e,0x22,0x3a,0x31,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x38,0x3e,0x22,0x3a,0x31,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x39,0x3e,0x22,0x3a,0x31,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x30,0x3e,0x22,0x3a,0x31,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x31,0x3e,0x22,0x3a,0x31,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x32,0x3e,0x22,0x3a,0x31,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x33,0x3e,0x22,0x3a,0x32,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x34,0x3e,0x22,0x3a,0x32,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x35,0x3e,0x22,0x3a,0x32,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x36,0x3e,0x22,0x3a,0x32,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x37,0x3e,0x22,0x3a,0x32,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x38,0x3e,0x22,0x3a,0x32,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x31,0x39,0x3e,0x22,0x3a,0x32,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x30,0x3e,0x22,0x3a,0x32,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x31,0x3e,0x22,0x3a,0x32,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x32,0x3e,0x22,0x3a,0x32,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x33,0x3e,0x22,0x3a,0x33,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x34,0x3e,0x22,0x3a,0x33,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x35,0x3e,0x22,0x3a,0x33,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x36,0x3e,0x22,0x3a,0x33,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x37,0x3e,0x22,0x3a,0x33,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x38,0x3e,0x22,0x3a,0x33,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x32,0x39,0x3e,0x22,0x3a,0x33,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x30,0x3e,0x22,0x3a,0x33,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x31,0x3e,0x22,0x3a,0x33,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x32,0x3e,0x22,0x3a,0x33,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x33,0x3e,0x22,0x3a,0x34,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x34,0x3e,0x22,0x3a,0x34,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x35,0x3e,0x22,0x3a,0x34,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x36,0x3e,0x22,0x3a,0x34,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x37,0x3e,0x22,0x3a,0x34,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x38,0x3e,0x22,0x3a,0x34,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x33,0x39,0x3e,0x22,0x3a,0x34,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x30,0x3e,0x22,0x3a,0x34,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x31,0x3e,0x22,0x3a,0x34,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x32,0x3e,0x22,0x3a,0x34,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x33,0x3e,0x22,0x3a,0x35,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x34,0x3e,0x22,0x3a,0x35,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x35,0x3e,0x22,0x3a,0x35,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x36,0x3e,0x22,0x3a,0x35,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x37,0x3e,0x22,0x3a,0x35,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x38,0x3e,0x22,0x3a,0x35,0x35,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x34,0x39,0x3e,0x22,0x3a,0x35,0x36,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x30,0x3e,0x22,0x3a,0x35,0x37,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x31,0x3e,0x22,0x3a,0x35,0x38,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x32,0x3e,0x22,0x3a,0x35,0x39,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x33,0x3e,0x22,0x3a,0x36,0x30,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x34,0x3e,0x22,0x3a,0x36,0x31,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x35,0x3e,0x22,0x3a,0x36,0x32,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x36,0x3e,0x22,0x3a,0x36,0x33,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35,0x37,0x3e,0x22,0x3a,0x36,0x34,0x2c,0x22,0x3c,0x75,0x6e,0x75,0x73,0x65,0x64,0x35Line truncated
|
||||
};
|
||||
@@ -1,3 +0,0 @@
|
||||
static const unsigned char gpt_oss_merges_utf8_c_str[] = {
|
||||
0xc4,0xa0,0x20,0xc4,0xa0,0x0a,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0x0a,0x69,0x20,0x6e,0x0a,0x65,0x20,0x72,0x0a,0xc4,0xa0,0x20,0x74,0x0a,0xc4,0xa0,0x20,0x61,0x0a,0x65,0x20,0x6e,0x0a,0x6f,0x20,0x6e,0x0a,0x72,0x20,0x65,0x0a,0xc4,0xa0,0x20,0x73,0x0a,0x61,0x20,0x74,0x0a,0x6f,0x20,0x72,0x0a,0x65,0x20,0x73,0x0a,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0x0a,0x61,0x20,0x6e,0x0a,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0x0a,0xc4,0xa0,0x20,0x64,0x0a,0x68,0x20,0x65,0x0a,0xc4,0xa0,0x20,0x63,0x0a,0xc4,0xa0,0x20,0x70,0x0a,0x69,0x20,0x73,0x0a,0x61,0x20,0x72,0x0a,0x69,0x20,0x74,0x0a,0xc4,0x8a,0x20,0xc4,0x8a,0x0a,0x61,0x20,0x6c,0x0a,0xc3,0xa0,0x20,0xc2,0xa4,0x0a,0x6c,0x20,0x65,0x0a,0x6f,0x20,0x75,0x0a,0xc4,0xa0,0x20,0x6d,0x0a,0xc4,0xa0,0x20,0x66,0x0a,0xc4,0xa0,0x20,0x77,0x0a,0xc4,0xa0,0x20,0x62,0x0a,0x61,0x20,0x73,0x0a,0x69,0x20,0x6e,0x67,0x0a,0x69,0x6e,0x20,0x67,0x0a,0xc4,0xa0,0x20,0x74,0x68,0x65,0x0a,0xc4,0xa0,0x74,0x20,0x68,0x65,0x0a,0xc4,0xa0,0x74,0x68,0x20,0x65,0x0a,0x69,0x20,0x63,0x0a,0x65,0x20,0x74,0x0a,0xc4,0xa0,0x20,0x6f,0x0a,0x69,0x20,0x6f,0x6e,0x0a,0x69,0x6f,0x20,0x6e,0x0a,0x65,0x20,0x64,0x0a,0x65,0x20,0x6c,0x0a,0xc4,0xa0,0x20,0x6e,0x0a,0x72,0x20,0x6f,0x0a,0x65,0x20,0x6e,0x74,0x0a,0x65,0x6e,0x20,0x74,0x0a,0xc4,0xa0,0x20,0xc3,0x90,0x0a,0x6e,0x20,0x64,0x0a,0x73,0x20,0x74,0x0a,0xc3,0xa1,0x20,0xc4,0xa5,0x0a,0xc3,0x90,0x20,0xc2,0xb0,0x0a,0xc4,0xa0,0x20,0x6c,0x0a,0xc4,0xa0,0x20,0x69,0x6e,0x0a,0xc4,0xa0,0x69,0x20,0x6e,0x0a,0x3b,0x20,0xc4,0x8a,0x0a,0x63,0x20,0x74,0x0a,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0x0a,0x6f,0x20,0x6d,0x0a,0x69,0x20,0x6c,0x0a,0xc4,0xa0,0x20,0x68,0x0a,0x61,0x20,0x6d,0x0a,0xc4,0xa0,0x20,0x3d,0x0a,0x69,0x20,0x64,0x0a,0xc4,0xa0,0x20,0x74,0x6f,0x0a,0xc4,0xa0,0x74,0x20,0x6f,0x0a,0xc3,0x90,0x20,0xc2,0xbe,0x0a,0xc3,0xa2,0x20,0xc4,0xa2,0x0a,0xc4,0xa0,0x20,0x65,0x0a,0xc3,0x98,0x20,0xc2,0xa7,0x0a,0x69,0x20,0x6d,0x0a,0xc4,0xa0,0x20,0x72,0x65,0x0a,0xc4,0xa0,0x72,0x20,0x65,0x0a,0xc4,0xa0,0x20,0x76,0x0a,0x61,0x20,0x64,0x0a,0xc4,0xa0,0x20,0x74,0x68,0x0a,0xc4,0xa0,0x74,0x20,0x68,0x0a,0xc4,0xa0,0x20,0x61,0x6e,0x64,0x0a,0xc4,0xa0,0x61,0x20,0x6e,0x64,0x0a,0xc4,0xa0,0x61,0x6e,0x20,0x64,0x0a,0xc3,0x90,0x20,0xc2,0xb5,0x0a,0xc4,0xa0,0x20,0x6f,0x66,0x0a,0xc4,0xa0,0x6f,0x20,0x66,0x0a,0xc4,0xa0,0x20,0x67,0x0a,0x75,0x20,0x72,0x0a,0xc3,0x90,0x20,0xc2,0xb8,0x0a,0x63,0x20,0x68,0x0a,0xc4,0xa0,0x20,0xc3,0xa0,0x0a,0xc4,0xa0,0x20,0x64,0x65,0x0a,0xc4,0xa0,0x64,0x20,0x65,0x0a,0xc4,0x89,0x20,0xc4,0x89,0x0a,0xc4,0xa0,0x20,0x53,0x0a,0xc4,0xa0,0x20,0x75,0x0a,0xc3,0x91,0x20,0xc4,0xa4,0x0a,0x75,0x20,0x74,0x0a,0x6f,0x20,0x6c,0x0a,0xc3,0x90,0x20,0xc2,0xbd,0x0a,0xc4,0xa0,0x20,0x79,0x0a,0x69,0x20,0x67,0x0a,0x73,0x20,0x65,0x0a,0xc3,0x91,0x20,0xc4,0xa2,0x0a,0x6f,0x20,0x74,0x0a,0x65,0x20,0x6d,0x0a,0x61,0x20,0x67,0x0a,0x69,0x20,0x76,0x0a,0xc4,0xa0,0x20,0x28,0x0a,0x71,0x20,0x75,0x0a,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0Line truncated
|
||||
};
|
||||
@@ -1,3 +0,0 @@
|
||||
static const unsigned char gpt_oss_vocab_json_utf8_c_str[] = {
|
||||
0x7b,0x22,0x21,0x22,0x3a,0x30,0x2c,0x22,0x5c,0x22,0x22,0x3a,0x31,0x2c,0x22,0x23,0x22,0x3a,0x32,0x2c,0x22,0x24,0x22,0x3a,0x33,0x2c,0x22,0x25,0x22,0x3a,0x34,0x2c,0x22,0x26,0x22,0x3a,0x35,0x2c,0x22,0x27,0x22,0x3a,0x36,0x2c,0x22,0x28,0x22,0x3a,0x37,0x2c,0x22,0x29,0x22,0x3a,0x38,0x2c,0x22,0x2a,0x22,0x3a,0x39,0x2c,0x22,0x2b,0x22,0x3a,0x31,0x30,0x2c,0x22,0x2c,0x22,0x3a,0x31,0x31,0x2c,0x22,0x2d,0x22,0x3a,0x31,0x32,0x2c,0x22,0x2e,0x22,0x3a,0x31,0x33,0x2c,0x22,0x2f,0x22,0x3a,0x31,0x34,0x2c,0x22,0x30,0x22,0x3a,0x31,0x35,0x2c,0x22,0x31,0x22,0x3a,0x31,0x36,0x2c,0x22,0x32,0x22,0x3a,0x31,0x37,0x2c,0x22,0x33,0x22,0x3a,0x31,0x38,0x2c,0x22,0x34,0x22,0x3a,0x31,0x39,0x2c,0x22,0x35,0x22,0x3a,0x32,0x30,0x2c,0x22,0x36,0x22,0x3a,0x32,0x31,0x2c,0x22,0x37,0x22,0x3a,0x32,0x32,0x2c,0x22,0x38,0x22,0x3a,0x32,0x33,0x2c,0x22,0x39,0x22,0x3a,0x32,0x34,0x2c,0x22,0x3a,0x22,0x3a,0x32,0x35,0x2c,0x22,0x3b,0x22,0x3a,0x32,0x36,0x2c,0x22,0x3c,0x22,0x3a,0x32,0x37,0x2c,0x22,0x3d,0x22,0x3a,0x32,0x38,0x2c,0x22,0x3e,0x22,0x3a,0x32,0x39,0x2c,0x22,0x3f,0x22,0x3a,0x33,0x30,0x2c,0x22,0x40,0x22,0x3a,0x33,0x31,0x2c,0x22,0x41,0x22,0x3a,0x33,0x32,0x2c,0x22,0x42,0x22,0x3a,0x33,0x33,0x2c,0x22,0x43,0x22,0x3a,0x33,0x34,0x2c,0x22,0x44,0x22,0x3a,0x33,0x35,0x2c,0x22,0x45,0x22,0x3a,0x33,0x36,0x2c,0x22,0x46,0x22,0x3a,0x33,0x37,0x2c,0x22,0x47,0x22,0x3a,0x33,0x38,0x2c,0x22,0x48,0x22,0x3a,0x33,0x39,0x2c,0x22,0x49,0x22,0x3a,0x34,0x30,0x2c,0x22,0x4a,0x22,0x3a,0x34,0x31,0x2c,0x22,0x4b,0x22,0x3a,0x34,0x32,0x2c,0x22,0x4c,0x22,0x3a,0x34,0x33,0x2c,0x22,0x4d,0x22,0x3a,0x34,0x34,0x2c,0x22,0x4e,0x22,0x3a,0x34,0x35,0x2c,0x22,0x4f,0x22,0x3a,0x34,0x36,0x2c,0x22,0x50,0x22,0x3a,0x34,0x37,0x2c,0x22,0x51,0x22,0x3a,0x34,0x38,0x2c,0x22,0x52,0x22,0x3a,0x34,0x39,0x2c,0x22,0x53,0x22,0x3a,0x35,0x30,0x2c,0x22,0x54,0x22,0x3a,0x35,0x31,0x2c,0x22,0x55,0x22,0x3a,0x35,0x32,0x2c,0x22,0x56,0x22,0x3a,0x35,0x33,0x2c,0x22,0x57,0x22,0x3a,0x35,0x34,0x2c,0x22,0x58,0x22,0x3a,0x35,0x35,0x2c,0x22,0x59,0x22,0x3a,0x35,0x36,0x2c,0x22,0x5a,0x22,0x3a,0x35,0x37,0x2c,0x22,0x5b,0x22,0x3a,0x35,0x38,0x2c,0x22,0x5c,0x5c,0x22,0x3a,0x35,0x39,0x2c,0x22,0x5d,0x22,0x3a,0x36,0x30,0x2c,0x22,0x5e,0x22,0x3a,0x36,0x31,0x2c,0x22,0x5f,0x22,0x3a,0x36,0x32,0x2c,0x22,0x60,0x22,0x3a,0x36,0x33,0x2c,0x22,0x61,0x22,0x3a,0x36,0x34,0x2c,0x22,0x62,0x22,0x3a,0x36,0x35,0x2c,0x22,0x63,0x22,0x3a,0x36,0x36,0x2c,0x22,0x64,0x22,0x3a,0x36,0x37,0x2c,0x22,0x65,0x22,0x3a,0x36,0x38,0x2c,0x22,0x66,0x22,0x3a,0x36,0x39,0x2c,0x22,0x67,0x22,0x3a,0x37,0x30,0x2c,0x22,0x68,0x22,0x3a,0x37,0x31,0x2c,0x22,0x69,0x22,0x3a,0x37,0x32,0x2c,0x22,0x6a,0x22,0x3a,0x37,0x33,0x2c,0x22,0x6b,0x22,0x3a,0x37,0x34,0x2c,0x22,0x6c,0x22,0x3a,0x37,0x35,0x2c,0x22,0x6d,0x22,0x3a,0x37,0x36,0x2c,0x22,0x6e,0x22,0x3a,0x37,0x37,0x2c,0x22,0x6f,0x22,0x3a,0x37,0x38,0x2c,0x22,0x70,0x22,0x3a,0x37,0x39,0x2c,0x22,0x71,0x22,0x3a,0x38,0x30,0x2c,0x22,0x72,0x22,0x3a,0x38,0x31,0x2c,0x22,0x73,0x22,0x3a,0x38,0x32,0x2c,0x22,0x74,0x22,0x3a,0x38,0x33,0x2c,0x22,0x75,0x22,0x3a,0x38,0x34,0x2c,0x22,0x76,0x22,0x3a,0x38,0x35,0x2c,0x22,0x77,0x22,0x3a,0x38,0x36,0x2c,0x22,0x78,0x22,0x3a,0x38,0x37,0x2c,0x22,0x79,0x22,0x3a,0x38,0x38,0x2c,0x22,0x7a,0x22,0x3a,0x38,0x39,0x2c,0x22,0x7b,0x22,0x3a,0x39,0x30,0x2c,0x22,0x7c,0x22,0x3a,0x39,0x31,0x2c,0x22,0x7d,0x22,0x3a,0x39,0x32,0x2c,0x22,0x7e,0x22,0x3a,0x39,0x33,0x2c,0x22,0xc2,0xa1,0x22,0x3a,0x39,0x34,0x2c,0x22,0xc2,0xa2,0x22,0x3a,0x39,0x35,0x2c,0x22,0xc2,0xa3,0x22,0x3a,0x39,0x36,0x2c,0x22,0xc2,0xa4,0x22,0x3a,0x39,0x37,0x2c,0x22,0xc2,0xa5,0x22,0x3a,0x39,0x38,0x2c,0x22,0xc2,0xa6,0x22,0x3a,0x39,0x39,0x2c,0x22,0xc2,0xa7,0x22,0x3a,0x31,0x30,0x30,0x2c,0x22,0xc2,0xa8,0x22,0x3a,0x31,0x30,0x31,0x2c,0x22,0xc2,0xa9,0x22,0x3a,0x31,0x30,0x32,0x2c,0x22,0xc2,0xaa,0x22,0x3a,0x31,0x30,0x33,0x2c,0x22,0xc2,0xab,0x22,0x3a,0x31,0x30,0x34,0x2c,0x22,0xc2,0xac,0x22,0x3a,0x31,0x30,0x35,0x2c,0x22,0xc2,0xae,0x22,0x3a,0x31,0x30,0x36,0x2c,0x22,0xc2,0xaf,0x22,0x3a,0x31,0x30,0x37,0x2c,0x22,0xc2,0xb0,0x22,0x3a,0x31,0x30,0x38,0x2c,0x22,0xc2,0xb1,0x22,0x3a,0x31,0x30,0x39,0x2c,0x22,0xc2,0xb2,0x22,0x3a,0x31,0x31,0x30,0x2c,0x22,0xc2,0xb3,0x22,0x3a,0x31,0x31,0x31,0x2c,0x22,0xc2,0xb4,0x22,0x3a,0x31,0x31,0x32,0x2c,0x22,0xc2,0xb5,0x22,0x3a,0x31,0x31,0x33,0x2c,0x22,0xc2,0xb6,0x22,0x3a,0x31,0x31,0x34,0x2c,0x22,0xc2,0xb7,0x22,0x3a,0x31,0x31,0x35,0x2c,0x22,0xc2,0xb8,0x22,0x3a,0x31,0x31,0x36,0x2c,0x22,0xc2,0xb9,0x22,0x3a,0x31,0x31,0x37,0x2c,0x22,0xc2,0xba,0x22,0x3a,0x31,0x31,0x38,0x2c,0x22,0xc2,0xbb,0x22,0x3a,0x31,0x31,0x39,0x2c,0x22,0xc2,0xbc,0x22,0x3a,0x31,0x32,0x30,0x2c,0x22,0xc2,0xbd,0x22,0x3a,0x31,0x32,0x31,0x2c,0x22,0xc2,0xbe,0x22,0x3a,0x31,0x32,0x32,0x2c,0x22,0xc2,0xbf,0x22,0x3a,0x31,0x32,0x33,0x2c,0x22,0xc3,0x80,0x22,0x3a,0x31,0x32,0x34,0x2c,0x22,0xc3,0x81,0x22,0x3a,0x31,0x32,0x35,0x2c,0x22,0xc3,0x82,0x22,0x3a,0x31,0x32,0x36,0x2c,0x22,0xc3,0x83,0x22,0x3a,0x31,0x32,0x37,0x2c,0x22,0xc3,0x84,0x22,0x3a,0x31,0x32,0x38,0x2c,0x22,0xc3,0x85,0x22,0x3a,0x31,0x32,0x39,0x2c,0x22,0xc3,0x86,0x22,0x3a,0x31,0x33,0x30,0x2c,0x22,0xc3,0x87,0x22,0x3a,0x31,0x33,0x31,0x2c,0x22,0xc3,0x88,0x22,0x3a,0x31,0x33,0x32,0x2c,0x22,0xc3,0x89,0x22Line truncated
|
||||
};
|
||||
@@ -1,11 +1,7 @@
|
||||
#include "vocab.h"
|
||||
#include "clip_merges.hpp"
|
||||
#include "gemma2_merges.hpp"
|
||||
#include "gemma2_vocab.hpp"
|
||||
#include "gemma_merges.hpp"
|
||||
#include "gemma_vocab.hpp"
|
||||
#include "gpt_oss_merges.hpp"
|
||||
#include "gpt_oss_vocab.hpp"
|
||||
#include "mistral_merges.hpp"
|
||||
#include "mistral_vocab.hpp"
|
||||
#include "qwen_merges.hpp"
|
||||
@@ -51,23 +47,3 @@ std::string load_gemma_vocab_json() {
|
||||
std::string json_str(reinterpret_cast<const char*>(gemma_vocab_json_utf8_c_str), sizeof(gemma_vocab_json_utf8_c_str));
|
||||
return json_str;
|
||||
}
|
||||
|
||||
std::string load_gemma2_merges() {
|
||||
std::string merges_utf8_str(reinterpret_cast<const char*>(gemma2_merges_utf8_c_str), sizeof(gemma2_merges_utf8_c_str));
|
||||
return merges_utf8_str;
|
||||
}
|
||||
|
||||
std::string load_gemma2_vocab_json() {
|
||||
std::string json_str(reinterpret_cast<const char*>(gemma2_vocab_json_utf8_c_str), sizeof(gemma2_vocab_json_utf8_c_str));
|
||||
return json_str;
|
||||
}
|
||||
|
||||
std::string load_gpt_oss_merges() {
|
||||
std::string merges_utf8_str(reinterpret_cast<const char*>(gpt_oss_merges_utf8_c_str), sizeof(gpt_oss_merges_utf8_c_str));
|
||||
return merges_utf8_str;
|
||||
}
|
||||
|
||||
std::string load_gpt_oss_vocab_json() {
|
||||
std::string json_str(reinterpret_cast<const char*>(gpt_oss_vocab_json_utf8_c_str), sizeof(gpt_oss_vocab_json_utf8_c_str));
|
||||
return json_str;
|
||||
}
|
||||
@@ -11,9 +11,5 @@ std::string load_t5_tokenizer_json();
|
||||
std::string load_umt5_tokenizer_json();
|
||||
std::string load_gemma_merges();
|
||||
std::string load_gemma_vocab_json();
|
||||
std::string load_gemma2_merges();
|
||||
std::string load_gemma2_vocab_json();
|
||||
std::string load_gpt_oss_merges();
|
||||
std::string load_gpt_oss_vocab_json();
|
||||
|
||||
#endif // __SD_TOKENIZERS_VOCAB_VOCAB_H__
|
||||
Reference in new issue
Block a user