mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-11-11 11:26:34 +00:00
feat: add microsoft lens support (#1560)
This commit is contained in:
leejet authored and GitHub committed
1 parent
07b2b18e70
commit
92dc7268fc
19 files changed
+1042
-20
No files matched your search
@@ -0,0 +1,91 @@
|
||||
#include "gpt_oss_tokenizer.h"
|
||||
|
||||
#include "json.hpp"
|
||||
#include "util.h"
|
||||
#include "vocab/vocab.h"
|
||||
|
||||
void GPTOSSTokenizer::load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
|
||||
auto byte_unicode_pairs = bytes_to_unicode();
|
||||
byte_encoder = std::map<int, std::u32string>(byte_unicode_pairs.begin(), byte_unicode_pairs.end());
|
||||
for (auto& pair : byte_unicode_pairs) {
|
||||
byte_decoder[pair.second] = pair.first;
|
||||
}
|
||||
|
||||
nlohmann::json vocab;
|
||||
try {
|
||||
vocab = nlohmann::json::parse(vocab_utf8_str);
|
||||
} catch (const nlohmann::json::parse_error&) {
|
||||
GGML_ABORT("invalid vocab json str");
|
||||
}
|
||||
for (const auto& [key, value] : vocab.items()) {
|
||||
std::u32string token = utf8_to_utf32(key);
|
||||
int i = value;
|
||||
encoder[token] = i;
|
||||
decoder[i] = token;
|
||||
}
|
||||
encoder_len = static_cast<int>(encoder.size());
|
||||
for (auto& special_token : special_tokens) {
|
||||
auto token = utf8_to_utf32(special_token);
|
||||
encoder[token] = encoder_len;
|
||||
decoder[encoder_len] = token;
|
||||
encoder_len++;
|
||||
}
|
||||
encoder_len = static_cast<int>(encoder.size());
|
||||
LOG_DEBUG("vocab size: %d", encoder_len);
|
||||
|
||||
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
|
||||
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
|
||||
for (const auto& merge : merges) {
|
||||
size_t space_pos = merge.find(' ');
|
||||
merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
|
||||
}
|
||||
LOG_DEBUG("merges size %zu", merge_pairs.size());
|
||||
|
||||
int rank = 0;
|
||||
for (const auto& merge : merge_pairs) {
|
||||
bpe_ranks[merge] = rank++;
|
||||
}
|
||||
bpe_len = rank;
|
||||
}
|
||||
|
||||
GPTOSSTokenizer::GPTOSSTokenizer(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
|
||||
BOS_TOKEN = "<|startoftext|>";
|
||||
UNK_TOKEN = "<|endoftext|>";
|
||||
EOS_TOKEN = "<|endoftext|>";
|
||||
PAD_TOKEN = "<|endoftext|>";
|
||||
|
||||
BOS_TOKEN_ID = 199998;
|
||||
EOS_TOKEN_ID = 199999;
|
||||
UNK_TOKEN_ID = 199999;
|
||||
PAD_TOKEN_ID = 199999;
|
||||
|
||||
special_tokens = {
|
||||
"<|startoftext|>",
|
||||
"<|endoftext|>",
|
||||
"<|reserved_200000|>",
|
||||
"<|reserved_200001|>",
|
||||
"<|return|>",
|
||||
"<|constrain|>",
|
||||
"<|reserved_200004|>",
|
||||
"<|channel|>",
|
||||
"<|start|>",
|
||||
"<|end|>",
|
||||
"<|message|>",
|
||||
"<|reserved_200009|>",
|
||||
"<|reserved_200010|>",
|
||||
"<|reserved_200011|>",
|
||||
"<|call|>",
|
||||
"<|reserved_200013|>",
|
||||
"<|reserved_200014|>",
|
||||
"<|reserved_200015|>",
|
||||
"<|reserved_200016|>",
|
||||
"<|reserved_200017|>",
|
||||
"<|endofprompt|>",
|
||||
};
|
||||
|
||||
if (merges_utf8_str.size() > 0) {
|
||||
load_from_merges(merges_utf8_str, vocab_utf8_str);
|
||||
} else {
|
||||
load_from_merges(load_gpt_oss_merges(), load_gpt_oss_vocab_json());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#ifndef __SD_TOKENIZERS_GPT_OSS_TOKENIZER_H__
|
||||
#define __SD_TOKENIZERS_GPT_OSS_TOKENIZER_H__
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "bpe_tokenizer.h"
|
||||
|
||||
class GPTOSSTokenizer : public BPETokenizer {
|
||||
protected:
|
||||
void load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str);
|
||||
|
||||
public:
|
||||
explicit GPTOSSTokenizer(const std::string& merges_utf8_str = "", const std::string& vocab_utf8_str = "");
|
||||
};
|
||||
|
||||
#endif // __SD_TOKENIZERS_GPT_OSS_TOKENIZER_H__
|
||||
@@ -0,0 +1,3 @@
|
||||
static const unsigned char gpt_oss_merges_utf8_c_str[] = {
|
||||
0xc4,0xa0,0x20,0xc4,0xa0,0x0a,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0x0a,0x69,0x20,0x6e,0x0a,0x65,0x20,0x72,0x0a,0xc4,0xa0,0x20,0x74,0x0a,0xc4,0xa0,0x20,0x61,0x0a,0x65,0x20,0x6e,0x0a,0x6f,0x20,0x6e,0x0a,0x72,0x20,0x65,0x0a,0xc4,0xa0,0x20,0x73,0x0a,0x61,0x20,0x74,0x0a,0x6f,0x20,0x72,0x0a,0x65,0x20,0x73,0x0a,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0x0a,0x61,0x20,0x6e,0x0a,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0x0a,0xc4,0xa0,0x20,0x64,0x0a,0x68,0x20,0x65,0x0a,0xc4,0xa0,0x20,0x63,0x0a,0xc4,0xa0,0x20,0x70,0x0a,0x69,0x20,0x73,0x0a,0x61,0x20,0x72,0x0a,0x69,0x20,0x74,0x0a,0xc4,0x8a,0x20,0xc4,0x8a,0x0a,0x61,0x20,0x6c,0x0a,0xc3,0xa0,0x20,0xc2,0xa4,0x0a,0x6c,0x20,0x65,0x0a,0x6f,0x20,0x75,0x0a,0xc4,0xa0,0x20,0x6d,0x0a,0xc4,0xa0,0x20,0x66,0x0a,0xc4,0xa0,0x20,0x77,0x0a,0xc4,0xa0,0x20,0x62,0x0a,0x61,0x20,0x73,0x0a,0x69,0x20,0x6e,0x67,0x0a,0x69,0x6e,0x20,0x67,0x0a,0xc4,0xa0,0x20,0x74,0x68,0x65,0x0a,0xc4,0xa0,0x74,0x20,0x68,0x65,0x0a,0xc4,0xa0,0x74,0x68,0x20,0x65,0x0a,0x69,0x20,0x63,0x0a,0x65,0x20,0x74,0x0a,0xc4,0xa0,0x20,0x6f,0x0a,0x69,0x20,0x6f,0x6e,0x0a,0x69,0x6f,0x20,0x6e,0x0a,0x65,0x20,0x64,0x0a,0x65,0x20,0x6c,0x0a,0xc4,0xa0,0x20,0x6e,0x0a,0x72,0x20,0x6f,0x0a,0x65,0x20,0x6e,0x74,0x0a,0x65,0x6e,0x20,0x74,0x0a,0xc4,0xa0,0x20,0xc3,0x90,0x0a,0x6e,0x20,0x64,0x0a,0x73,0x20,0x74,0x0a,0xc3,0xa1,0x20,0xc4,0xa5,0x0a,0xc3,0x90,0x20,0xc2,0xb0,0x0a,0xc4,0xa0,0x20,0x6c,0x0a,0xc4,0xa0,0x20,0x69,0x6e,0x0a,0xc4,0xa0,0x69,0x20,0x6e,0x0a,0x3b,0x20,0xc4,0x8a,0x0a,0x63,0x20,0x74,0x0a,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0x0a,0x6f,0x20,0x6d,0x0a,0x69,0x20,0x6c,0x0a,0xc4,0xa0,0x20,0x68,0x0a,0x61,0x20,0x6d,0x0a,0xc4,0xa0,0x20,0x3d,0x0a,0x69,0x20,0x64,0x0a,0xc4,0xa0,0x20,0x74,0x6f,0x0a,0xc4,0xa0,0x74,0x20,0x6f,0x0a,0xc3,0x90,0x20,0xc2,0xbe,0x0a,0xc3,0xa2,0x20,0xc4,0xa2,0x0a,0xc4,0xa0,0x20,0x65,0x0a,0xc3,0x98,0x20,0xc2,0xa7,0x0a,0x69,0x20,0x6d,0x0a,0xc4,0xa0,0x20,0x72,0x65,0x0a,0xc4,0xa0,0x72,0x20,0x65,0x0a,0xc4,0xa0,0x20,0x76,0x0a,0x61,0x20,0x64,0x0a,0xc4,0xa0,0x20,0x74,0x68,0x0a,0xc4,0xa0,0x74,0x20,0x68,0x0a,0xc4,0xa0,0x20,0x61,0x6e,0x64,0x0a,0xc4,0xa0,0x61,0x20,0x6e,0x64,0x0a,0xc4,0xa0,0x61,0x6e,0x20,0x64,0x0a,0xc3,0x90,0x20,0xc2,0xb5,0x0a,0xc4,0xa0,0x20,0x6f,0x66,0x0a,0xc4,0xa0,0x6f,0x20,0x66,0x0a,0xc4,0xa0,0x20,0x67,0x0a,0x75,0x20,0x72,0x0a,0xc3,0x90,0x20,0xc2,0xb8,0x0a,0x63,0x20,0x68,0x0a,0xc4,0xa0,0x20,0xc3,0xa0,0x0a,0xc4,0xa0,0x20,0x64,0x65,0x0a,0xc4,0xa0,0x64,0x20,0x65,0x0a,0xc4,0x89,0x20,0xc4,0x89,0x0a,0xc4,0xa0,0x20,0x53,0x0a,0xc4,0xa0,0x20,0x75,0x0a,0xc3,0x91,0x20,0xc4,0xa4,0x0a,0x75,0x20,0x74,0x0a,0x6f,0x20,0x6c,0x0a,0xc3,0x90,0x20,0xc2,0xbd,0x0a,0xc4,0xa0,0x20,0x79,0x0a,0x69,0x20,0x67,0x0a,0x73,0x20,0x65,0x0a,0xc3,0x91,0x20,0xc4,0xa2,0x0a,0x6f,0x20,0x74,0x0a,0x65,0x20,0x6d,0x0a,0x61,0x20,0x67,0x0a,0x69,0x20,0x76,0x0a,0xc4,0xa0,0x20,0x28,0x0a,0x71,0x20,0x75,0x0a,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0,0x0a,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0xc4,0xa0,0x20,0xc4,0xa0,0xc4,0xa0Line truncated
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
static const unsigned char gpt_oss_vocab_json_utf8_c_str[] = {
|
||||
0x7b,0x22,0x21,0x22,0x3a,0x30,0x2c,0x22,0x5c,0x22,0x22,0x3a,0x31,0x2c,0x22,0x23,0x22,0x3a,0x32,0x2c,0x22,0x24,0x22,0x3a,0x33,0x2c,0x22,0x25,0x22,0x3a,0x34,0x2c,0x22,0x26,0x22,0x3a,0x35,0x2c,0x22,0x27,0x22,0x3a,0x36,0x2c,0x22,0x28,0x22,0x3a,0x37,0x2c,0x22,0x29,0x22,0x3a,0x38,0x2c,0x22,0x2a,0x22,0x3a,0x39,0x2c,0x22,0x2b,0x22,0x3a,0x31,0x30,0x2c,0x22,0x2c,0x22,0x3a,0x31,0x31,0x2c,0x22,0x2d,0x22,0x3a,0x31,0x32,0x2c,0x22,0x2e,0x22,0x3a,0x31,0x33,0x2c,0x22,0x2f,0x22,0x3a,0x31,0x34,0x2c,0x22,0x30,0x22,0x3a,0x31,0x35,0x2c,0x22,0x31,0x22,0x3a,0x31,0x36,0x2c,0x22,0x32,0x22,0x3a,0x31,0x37,0x2c,0x22,0x33,0x22,0x3a,0x31,0x38,0x2c,0x22,0x34,0x22,0x3a,0x31,0x39,0x2c,0x22,0x35,0x22,0x3a,0x32,0x30,0x2c,0x22,0x36,0x22,0x3a,0x32,0x31,0x2c,0x22,0x37,0x22,0x3a,0x32,0x32,0x2c,0x22,0x38,0x22,0x3a,0x32,0x33,0x2c,0x22,0x39,0x22,0x3a,0x32,0x34,0x2c,0x22,0x3a,0x22,0x3a,0x32,0x35,0x2c,0x22,0x3b,0x22,0x3a,0x32,0x36,0x2c,0x22,0x3c,0x22,0x3a,0x32,0x37,0x2c,0x22,0x3d,0x22,0x3a,0x32,0x38,0x2c,0x22,0x3e,0x22,0x3a,0x32,0x39,0x2c,0x22,0x3f,0x22,0x3a,0x33,0x30,0x2c,0x22,0x40,0x22,0x3a,0x33,0x31,0x2c,0x22,0x41,0x22,0x3a,0x33,0x32,0x2c,0x22,0x42,0x22,0x3a,0x33,0x33,0x2c,0x22,0x43,0x22,0x3a,0x33,0x34,0x2c,0x22,0x44,0x22,0x3a,0x33,0x35,0x2c,0x22,0x45,0x22,0x3a,0x33,0x36,0x2c,0x22,0x46,0x22,0x3a,0x33,0x37,0x2c,0x22,0x47,0x22,0x3a,0x33,0x38,0x2c,0x22,0x48,0x22,0x3a,0x33,0x39,0x2c,0x22,0x49,0x22,0x3a,0x34,0x30,0x2c,0x22,0x4a,0x22,0x3a,0x34,0x31,0x2c,0x22,0x4b,0x22,0x3a,0x34,0x32,0x2c,0x22,0x4c,0x22,0x3a,0x34,0x33,0x2c,0x22,0x4d,0x22,0x3a,0x34,0x34,0x2c,0x22,0x4e,0x22,0x3a,0x34,0x35,0x2c,0x22,0x4f,0x22,0x3a,0x34,0x36,0x2c,0x22,0x50,0x22,0x3a,0x34,0x37,0x2c,0x22,0x51,0x22,0x3a,0x34,0x38,0x2c,0x22,0x52,0x22,0x3a,0x34,0x39,0x2c,0x22,0x53,0x22,0x3a,0x35,0x30,0x2c,0x22,0x54,0x22,0x3a,0x35,0x31,0x2c,0x22,0x55,0x22,0x3a,0x35,0x32,0x2c,0x22,0x56,0x22,0x3a,0x35,0x33,0x2c,0x22,0x57,0x22,0x3a,0x35,0x34,0x2c,0x22,0x58,0x22,0x3a,0x35,0x35,0x2c,0x22,0x59,0x22,0x3a,0x35,0x36,0x2c,0x22,0x5a,0x22,0x3a,0x35,0x37,0x2c,0x22,0x5b,0x22,0x3a,0x35,0x38,0x2c,0x22,0x5c,0x5c,0x22,0x3a,0x35,0x39,0x2c,0x22,0x5d,0x22,0x3a,0x36,0x30,0x2c,0x22,0x5e,0x22,0x3a,0x36,0x31,0x2c,0x22,0x5f,0x22,0x3a,0x36,0x32,0x2c,0x22,0x60,0x22,0x3a,0x36,0x33,0x2c,0x22,0x61,0x22,0x3a,0x36,0x34,0x2c,0x22,0x62,0x22,0x3a,0x36,0x35,0x2c,0x22,0x63,0x22,0x3a,0x36,0x36,0x2c,0x22,0x64,0x22,0x3a,0x36,0x37,0x2c,0x22,0x65,0x22,0x3a,0x36,0x38,0x2c,0x22,0x66,0x22,0x3a,0x36,0x39,0x2c,0x22,0x67,0x22,0x3a,0x37,0x30,0x2c,0x22,0x68,0x22,0x3a,0x37,0x31,0x2c,0x22,0x69,0x22,0x3a,0x37,0x32,0x2c,0x22,0x6a,0x22,0x3a,0x37,0x33,0x2c,0x22,0x6b,0x22,0x3a,0x37,0x34,0x2c,0x22,0x6c,0x22,0x3a,0x37,0x35,0x2c,0x22,0x6d,0x22,0x3a,0x37,0x36,0x2c,0x22,0x6e,0x22,0x3a,0x37,0x37,0x2c,0x22,0x6f,0x22,0x3a,0x37,0x38,0x2c,0x22,0x70,0x22,0x3a,0x37,0x39,0x2c,0x22,0x71,0x22,0x3a,0x38,0x30,0x2c,0x22,0x72,0x22,0x3a,0x38,0x31,0x2c,0x22,0x73,0x22,0x3a,0x38,0x32,0x2c,0x22,0x74,0x22,0x3a,0x38,0x33,0x2c,0x22,0x75,0x22,0x3a,0x38,0x34,0x2c,0x22,0x76,0x22,0x3a,0x38,0x35,0x2c,0x22,0x77,0x22,0x3a,0x38,0x36,0x2c,0x22,0x78,0x22,0x3a,0x38,0x37,0x2c,0x22,0x79,0x22,0x3a,0x38,0x38,0x2c,0x22,0x7a,0x22,0x3a,0x38,0x39,0x2c,0x22,0x7b,0x22,0x3a,0x39,0x30,0x2c,0x22,0x7c,0x22,0x3a,0x39,0x31,0x2c,0x22,0x7d,0x22,0x3a,0x39,0x32,0x2c,0x22,0x7e,0x22,0x3a,0x39,0x33,0x2c,0x22,0xc2,0xa1,0x22,0x3a,0x39,0x34,0x2c,0x22,0xc2,0xa2,0x22,0x3a,0x39,0x35,0x2c,0x22,0xc2,0xa3,0x22,0x3a,0x39,0x36,0x2c,0x22,0xc2,0xa4,0x22,0x3a,0x39,0x37,0x2c,0x22,0xc2,0xa5,0x22,0x3a,0x39,0x38,0x2c,0x22,0xc2,0xa6,0x22,0x3a,0x39,0x39,0x2c,0x22,0xc2,0xa7,0x22,0x3a,0x31,0x30,0x30,0x2c,0x22,0xc2,0xa8,0x22,0x3a,0x31,0x30,0x31,0x2c,0x22,0xc2,0xa9,0x22,0x3a,0x31,0x30,0x32,0x2c,0x22,0xc2,0xaa,0x22,0x3a,0x31,0x30,0x33,0x2c,0x22,0xc2,0xab,0x22,0x3a,0x31,0x30,0x34,0x2c,0x22,0xc2,0xac,0x22,0x3a,0x31,0x30,0x35,0x2c,0x22,0xc2,0xae,0x22,0x3a,0x31,0x30,0x36,0x2c,0x22,0xc2,0xaf,0x22,0x3a,0x31,0x30,0x37,0x2c,0x22,0xc2,0xb0,0x22,0x3a,0x31,0x30,0x38,0x2c,0x22,0xc2,0xb1,0x22,0x3a,0x31,0x30,0x39,0x2c,0x22,0xc2,0xb2,0x22,0x3a,0x31,0x31,0x30,0x2c,0x22,0xc2,0xb3,0x22,0x3a,0x31,0x31,0x31,0x2c,0x22,0xc2,0xb4,0x22,0x3a,0x31,0x31,0x32,0x2c,0x22,0xc2,0xb5,0x22,0x3a,0x31,0x31,0x33,0x2c,0x22,0xc2,0xb6,0x22,0x3a,0x31,0x31,0x34,0x2c,0x22,0xc2,0xb7,0x22,0x3a,0x31,0x31,0x35,0x2c,0x22,0xc2,0xb8,0x22,0x3a,0x31,0x31,0x36,0x2c,0x22,0xc2,0xb9,0x22,0x3a,0x31,0x31,0x37,0x2c,0x22,0xc2,0xba,0x22,0x3a,0x31,0x31,0x38,0x2c,0x22,0xc2,0xbb,0x22,0x3a,0x31,0x31,0x39,0x2c,0x22,0xc2,0xbc,0x22,0x3a,0x31,0x32,0x30,0x2c,0x22,0xc2,0xbd,0x22,0x3a,0x31,0x32,0x31,0x2c,0x22,0xc2,0xbe,0x22,0x3a,0x31,0x32,0x32,0x2c,0x22,0xc2,0xbf,0x22,0x3a,0x31,0x32,0x33,0x2c,0x22,0xc3,0x80,0x22,0x3a,0x31,0x32,0x34,0x2c,0x22,0xc3,0x81,0x22,0x3a,0x31,0x32,0x35,0x2c,0x22,0xc3,0x82,0x22,0x3a,0x31,0x32,0x36,0x2c,0x22,0xc3,0x83,0x22,0x3a,0x31,0x32,0x37,0x2c,0x22,0xc3,0x84,0x22,0x3a,0x31,0x32,0x38,0x2c,0x22,0xc3,0x85,0x22,0x3a,0x31,0x32,0x39,0x2c,0x22,0xc3,0x86,0x22,0x3a,0x31,0x33,0x30,0x2c,0x22,0xc3,0x87,0x22,0x3a,0x31,0x33,0x31,0x2c,0x22,0xc3,0x88,0x22,0x3a,0x31,0x33,0x32,0x2c,0x22,0xc3,0x89,0x22Line truncated
|
||||
};
|
||||
@@ -2,6 +2,8 @@
|
||||
#include "clip_merges.hpp"
|
||||
#include "gemma_merges.hpp"
|
||||
#include "gemma_vocab.hpp"
|
||||
#include "gpt_oss_merges.hpp"
|
||||
#include "gpt_oss_vocab.hpp"
|
||||
#include "mistral_merges.hpp"
|
||||
#include "mistral_vocab.hpp"
|
||||
#include "qwen_merges.hpp"
|
||||
@@ -46,4 +48,14 @@ std::string load_gemma_merges() {
|
||||
std::string load_gemma_vocab_json() {
|
||||
std::string json_str(reinterpret_cast<const char*>(gemma_vocab_json_utf8_c_str), sizeof(gemma_vocab_json_utf8_c_str));
|
||||
return json_str;
|
||||
}
|
||||
|
||||
std::string load_gpt_oss_merges() {
|
||||
std::string merges_utf8_str(reinterpret_cast<const char*>(gpt_oss_merges_utf8_c_str), sizeof(gpt_oss_merges_utf8_c_str));
|
||||
return merges_utf8_str;
|
||||
}
|
||||
|
||||
std::string load_gpt_oss_vocab_json() {
|
||||
std::string json_str(reinterpret_cast<const char*>(gpt_oss_vocab_json_utf8_c_str), sizeof(gpt_oss_vocab_json_utf8_c_str));
|
||||
return json_str;
|
||||
}
|
||||
@@ -11,5 +11,7 @@ std::string load_t5_tokenizer_json();
|
||||
std::string load_umt5_tokenizer_json();
|
||||
std::string load_gemma_merges();
|
||||
std::string load_gemma_vocab_json();
|
||||
std::string load_gpt_oss_merges();
|
||||
std::string load_gpt_oss_vocab_json();
|
||||
|
||||
#endif // __SD_TOKENIZERS_VOCAB_VOCAB_H__
|
||||
Reference in new issue
Block a user