DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 03:24:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support external Hugging Face tokenizer JSON files (#1973)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-09-15 00:27:39 +08:00
1 parent 42d6c0ab92
commit 4964abdfc5
58 files changed
+97390 -180
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/conditioning/conditioner.hpp
+222 -80
View File
@@ -17,6 +17,7 @@
#include "model/te/t5.hpp"
#include "model_loader.h"
#include "tokenizers/sensenova_u1_tokenizer.h"
#include "tokenizers/tokenizer_config.h"
struct SDCondition {
sd::Tensor<float> c_crossattn;
@@ -159,7 +160,7 @@ public:
// Ref: https://github.com/AUTOMATIC1111/stable-diffusion-webui/blob/cad87bf4e3e0b0a759afa94e933527c3123d59bc/modules/sd_hijack_clip.py#L283
struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
SDVersion version = VERSION_SD1;
CLIPTokenizer tokenizer;
std::shared_ptr<Tokenizer> tokenizer;
std::shared_ptr<CLIPTextModelRunner> text_model;
std::shared_ptr<CLIPTextModelRunner> text_model2;
@@ -173,12 +174,18 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
const String2TensorStorage& tensor_storage_map,
const std::map<std::string, std::string>& orig_embedding_map,
SDVersion version = VERSION_SD1,
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
: version(version), tokenizer(sd_version_is_sd2(version) ? 0 : 49407) {
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
const TokenizerConfig& tokenizers = {})
: version(version) {
const int pad_id = sd_version_is_sd2(version) ? 0 : 49407;
tokenizer = tokenizers.create(TokenizerConfig::MAIN, 49408, pad_id, false, true);
if (!tokenizer) {
tokenizer = std::make_shared<CLIPTokenizer>(pad_id);
}
for (const auto& kv : orig_embedding_map) {
std::string name = normalize_embedding_name(kv.first);
embedding_map[name] = kv.second;
tokenizer.add_special_token(name);
tokenizer->add_special_token(name);
}
bool force_clip_f32 = !embedding_map.empty();
if (sd_version_is_sd1(version)) {
@@ -365,16 +372,15 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
return load_embedding(name, iter->second, bpe_tokens);
}
std::vector<int> convert_token_to_id(std::string text) {
bool convert_token_to_id(const std::string& text, std::vector<int>& tokens) {
auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
return append_embedding_tokens(str, bpe_tokens);
};
std::vector<int> curr_tokens = tokenizer.encode(text, on_new_token_cb);
return curr_tokens;
return tokenizer->encode(text, tokens, on_new_token_cb);
}
std::string decode(const std::vector<int>& tokens) {
return tokenizer.decode(tokens);
bool decode(const std::vector<int>& tokens, std::string& text) {
return tokenizer->decode(tokens, text);
}
std::pair<std::vector<int>, std::vector<float>> tokenize(std::string text,
@@ -412,18 +418,21 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
if (padding_size > 0) {
LOG_VERBOSE("BREAK token encountered, padding current chunk by %zu tokens.", padding_size);
tokens.insert(tokens.end(), padding_size, tokenizer.EOS_TOKEN_ID);
tokens.insert(tokens.end(), padding_size, tokenizer->EOS_TOKEN_ID);
weights.insert(weights.end(), padding_size, 1.0f);
}
continue; // Skip to the next item after handling BREAK
}
std::vector<int> curr_tokens = tokenizer.encode(curr_text, on_new_token_cb);
std::vector<int> curr_tokens;
if (!tokenizer->encode(curr_text, curr_tokens, on_new_token_cb)) {
return {};
}
tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
weights.insert(weights.end(), curr_tokens.size(), curr_weight);
}
tokenizer.pad_tokens(tokens, &weights, nullptr, min_length, max_length, allow_overflow_expand);
tokenizer->pad_tokens(tokens, &weights, nullptr, min_length, max_length, allow_overflow_expand);
// for (int i = 0; i < tokens.size(); i++) {
// std::cout << tokens[i] << ":" << weights[i] << ", ";
@@ -460,7 +469,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
sd::Tensor<int32_t> input_ids2;
size_t max_token_idx = 0;
if (sd_version_is_sdxl(version)) {
auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), tokenizer.EOS_TOKEN_ID);
auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), tokenizer->EOS_TOKEN_ID);
if (it != chunk_tokens.end()) {
std::fill(std::next(it), chunk_tokens.end(), 0);
}
@@ -561,7 +570,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
SDCondition get_learned_condition(int n_threads,
const ConditionerParams& conditioner_params) override {
auto tokens_and_weights = tokenize(conditioner_params.text, text_model->model.n_token, text_model->model.n_token, true);
auto tokens_and_weights = tokenize(conditioner_params.text, text_model->model.n_token, text_model->model.n_token, true);
if (tokens_and_weights.first.empty()) {
return {};
}
std::vector<int>& tokens = tokens_and_weights.first;
std::vector<float>& weights = tokens_and_weights.second;
return get_learned_condition_common(n_threads,
@@ -629,8 +641,8 @@ struct FrozenCLIPVisionEmbedder : public GGMLRunner {
};
struct SD3CLIPEmbedder : public Conditioner {
CLIPTokenizer clip_l_tokenizer;
CLIPTokenizer clip_g_tokenizer;
std::shared_ptr<Tokenizer> clip_l_tokenizer;
std::shared_ptr<Tokenizer> clip_g_tokenizer;
T5UniGramTokenizer t5_tokenizer;
std::shared_ptr<CLIPTextModelRunner> clip_l;
std::shared_ptr<CLIPTextModelRunner> clip_g;
@@ -638,8 +650,8 @@ struct SD3CLIPEmbedder : public Conditioner {
SD3CLIPEmbedder(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
: clip_g_tokenizer(0) {
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
const TokenizerConfig& tokenizers = {}) {
bool use_clip_l = false;
bool use_clip_g = false;
bool use_t5 = false;
@@ -657,9 +669,17 @@ struct SD3CLIPEmbedder : public Conditioner {
return;
}
if (use_clip_l) {
clip_l_tokenizer = tokenizers.create(TokenizerConfig::CLIP_L, 49408, 49407, false, true);
if (!clip_l_tokenizer) {
clip_l_tokenizer = std::make_shared<CLIPTokenizer>();
}
clip_l = std::make_shared<CLIPTextModelRunner>(backend, tensor_storage_map, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, false, false, weight_manager);
}
if (use_clip_g) {
clip_g_tokenizer = tokenizers.create(TokenizerConfig::CLIP_G, 49408, 0, false, true);
if (!clip_g_tokenizer) {
clip_g_tokenizer = std::make_shared<CLIPTokenizer>(0);
}
clip_g = std::make_shared<CLIPTextModelRunner>(backend, tensor_storage_map, "text_encoders.clip_g.transformer.text_model", OPEN_CLIP_VIT_BIGG_14, false, false, weight_manager);
}
if (use_t5) {
@@ -811,27 +831,36 @@ struct SD3CLIPEmbedder : public Conditioner {
const std::string& curr_text = item.first;
float curr_weight = item.second;
if (clip_l) {
std::vector<int> curr_tokens = clip_l_tokenizer.encode(curr_text, on_new_token_cb);
std::vector<int> curr_tokens;
if (!clip_l_tokenizer->encode(curr_text, curr_tokens, on_new_token_cb)) {
return {};
}
clip_l_tokens.insert(clip_l_tokens.end(), curr_tokens.begin(), curr_tokens.end());
clip_l_weights.insert(clip_l_weights.end(), curr_tokens.size(), curr_weight);
}
if (clip_g) {
std::vector<int> curr_tokens = clip_g_tokenizer.encode(curr_text, on_new_token_cb);
std::vector<int> curr_tokens;
if (!clip_g_tokenizer->encode(curr_text, curr_tokens, on_new_token_cb)) {
return {};
}
clip_g_tokens.insert(clip_g_tokens.end(), curr_tokens.begin(), curr_tokens.end());
clip_g_weights.insert(clip_g_weights.end(), curr_tokens.size(), curr_weight);
}
if (t5) {
std::vector<int> curr_tokens = t5_tokenizer.encode(curr_text);
std::vector<int> curr_tokens;
if (!t5_tokenizer.encode(curr_text, curr_tokens)) {
return {};
}
t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
}
}
if (clip_l) {
clip_l_tokenizer.pad_tokens(clip_l_tokens, &clip_l_weights, nullptr, min_length, max_length, allow_overflow_expand);
clip_l_tokenizer->pad_tokens(clip_l_tokens, &clip_l_weights, nullptr, min_length, max_length, allow_overflow_expand);
}
if (clip_g) {
clip_g_tokenizer.pad_tokens(clip_g_tokens, &clip_g_weights, nullptr, min_length, max_length, allow_overflow_expand);
clip_g_tokenizer->pad_tokens(clip_g_tokens, &clip_g_weights, nullptr, min_length, max_length, allow_overflow_expand);
}
if (t5) {
t5_tokenizer.pad_tokens(t5_tokens, &t5_weights, nullptr, min_length, max_length, true);
@@ -902,7 +931,7 @@ struct SD3CLIPEmbedder : public Conditioner {
chunk_hidden_states_l = ::apply_token_weights(std::move(chunk_hidden_states_l), chunk_weights);
if (chunk_idx == 0) {
auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_l_tokenizer.EOS_TOKEN_ID);
auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_l_tokenizer->EOS_TOKEN_ID);
max_token_idx = std::min<size_t>(std::distance(chunk_tokens.begin(), it), chunk_tokens.size() - 1);
pooled_l = clip_l->compute(n_threads,
input_ids,
@@ -945,7 +974,7 @@ struct SD3CLIPEmbedder : public Conditioner {
chunk_hidden_states_g = ::apply_token_weights(std::move(chunk_hidden_states_g), chunk_weights);
if (chunk_idx == 0) {
auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_g_tokenizer.EOS_TOKEN_ID);
auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_g_tokenizer->EOS_TOKEN_ID);
max_token_idx = std::min<size_t>(std::distance(chunk_tokens.begin(), it), chunk_tokens.size() - 1);
pooled_g = clip_g->compute(n_threads,
input_ids,
@@ -1023,6 +1052,9 @@ struct SD3CLIPEmbedder : public Conditioner {
SDCondition get_learned_condition(int n_threads,
const ConditionerParams& conditioner_params) override {
auto tokens_and_weights = tokenize(conditioner_params.text, 77, 77, true);
if (tokens_and_weights.empty()) {
return {};
}
return get_learned_condition_common(n_threads,
tokens_and_weights,
conditioner_params.clip_skip,
@@ -1031,7 +1063,7 @@ struct SD3CLIPEmbedder : public Conditioner {
};
struct FluxCLIPEmbedder : public Conditioner {
CLIPTokenizer clip_l_tokenizer;
std::shared_ptr<Tokenizer> clip_l_tokenizer;
T5UniGramTokenizer t5_tokenizer;
std::shared_ptr<CLIPTextModelRunner> clip_l;
std::shared_ptr<T5Runner> t5;
@@ -1039,7 +1071,8 @@ struct FluxCLIPEmbedder : public Conditioner {
FluxCLIPEmbedder(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr) {
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
const TokenizerConfig& tokenizers = {}) {
bool use_clip_l = false;
bool use_t5 = false;
for (auto pair : tensor_storage_map) {
@@ -1056,6 +1089,11 @@ struct FluxCLIPEmbedder : public Conditioner {
}
if (use_clip_l) {
auto slot = tokenizers.has(TokenizerConfig::CLIP_L) ? TokenizerConfig::CLIP_L : TokenizerConfig::MAIN;
clip_l_tokenizer = tokenizers.create(slot, 49408, 49407, false, true);
if (!clip_l_tokenizer) {
clip_l_tokenizer = std::make_shared<CLIPTokenizer>();
}
clip_l = std::make_shared<CLIPTextModelRunner>(backend, tensor_storage_map, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, true, false, weight_manager);
} else {
LOG_WARN("clip_l text encoder not found! Prompt adherence might be degraded.");
@@ -1181,19 +1219,25 @@ struct FluxCLIPEmbedder : public Conditioner {
const std::string& curr_text = item.first;
float curr_weight = item.second;
if (clip_l) {
std::vector<int> curr_tokens = clip_l_tokenizer.encode(curr_text, on_new_token_cb);
std::vector<int> curr_tokens;
if (!clip_l_tokenizer->encode(curr_text, curr_tokens, on_new_token_cb)) {
return {};
}
clip_l_tokens.insert(clip_l_tokens.end(), curr_tokens.begin(), curr_tokens.end());
clip_l_weights.insert(clip_l_weights.end(), curr_tokens.size(), curr_weight);
}
if (t5) {
std::vector<int> curr_tokens = t5_tokenizer.encode(curr_text);
std::vector<int> curr_tokens;
if (!t5_tokenizer.encode(curr_text, curr_tokens)) {
return {};
}
t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
}
}
if (clip_l) {
clip_l_tokenizer.pad_tokens(clip_l_tokens, &clip_l_weights, nullptr, 77, 77, true);
clip_l_tokenizer->pad_tokens(clip_l_tokens, &clip_l_weights, nullptr, 77, 77, true);
}
if (t5) {
t5_tokenizer.pad_tokens(t5_tokens, &t5_weights, nullptr, min_length, max_length, true);
@@ -1243,7 +1287,7 @@ struct FluxCLIPEmbedder : public Conditioner {
sd::Tensor<int32_t> input_ids({static_cast<int64_t>(chunk_tokens.size())}, chunk_tokens);
size_t max_token_idx = 0;
auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_l_tokenizer.EOS_TOKEN_ID);
auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_l_tokenizer->EOS_TOKEN_ID);
max_token_idx = std::min<size_t>(std::distance(chunk_tokens.begin(), it), chunk_tokens.size() - 1);
pooled = clip_l->compute(n_threads,
@@ -1306,6 +1350,9 @@ struct FluxCLIPEmbedder : public Conditioner {
SDCondition get_learned_condition(int n_threads,
const ConditionerParams& conditioner_params) override {
auto tokens_and_weights = tokenize(conditioner_params.text, chunk_len, chunk_len);
if (tokens_and_weights.empty()) {
return {};
}
return get_learned_condition_common(n_threads,
tokens_and_weights,
conditioner_params.clip_skip,
@@ -1449,7 +1496,10 @@ struct T5CLIPEmbedder : public Conditioner {
const std::string& curr_text = item.first;
float curr_weight = item.second;
std::vector<int> curr_tokens = t5_tokenizer.encode(curr_text);
std::vector<int> curr_tokens;
if (!t5_tokenizer.encode(curr_text, curr_tokens)) {
return {};
}
t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
}
@@ -1547,6 +1597,9 @@ struct T5CLIPEmbedder : public Conditioner {
SDCondition get_learned_condition(int n_threads,
const ConditionerParams& conditioner_params) override {
auto tokens_and_weights = tokenize(conditioner_params.text, chunk_len, chunk_len);
if (std::get<0>(tokens_and_weights).empty()) {
return {};
}
return get_learned_condition_common(n_threads,
tokens_and_weights,
conditioner_params.clip_skip,
@@ -1645,7 +1698,10 @@ struct MiniT2IConditioner : public Conditioner {
return result;
}
std::vector<int> tokens = tokenizer.encode(conditioner_params.text);
std::vector<int> tokens;
if (!tokenizer.encode(conditioner_params.text, tokens)) {
return {};
}
if (tokens.size() > prompt_length) {
tokens.resize(prompt_length);
}
@@ -1712,7 +1768,10 @@ struct SenseNovaU1Conditioner : public Conditioner {
}
SDCondition tokenize_condition(const std::string& text, bool is_negative) {
auto tokens = tokenizer.encode(build_query(text, is_negative));
std::vector<int> tokens;
if (!tokenizer.encode(build_query(text, is_negative), tokens)) {
return {};
}
if (tokens.empty() || tokens.size() > kMaxPromptTokens) {
LOG_ERROR("SenseNova U1.5 prompt token count %zu is outside [1, %zu]",
tokens.size(),
@@ -1737,20 +1796,24 @@ struct SenseNovaU1Conditioner : public Conditioner {
};
struct AnimaConditioner : public Conditioner {
std::shared_ptr<BPETokenizer> qwen_tokenizer;
std::shared_ptr<Tokenizer> qwen_tokenizer;
T5UniGramTokenizer t5_tokenizer;
std::shared_ptr<LLM::LLMRunner> llm;
AnimaConditioner(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr) {
qwen_tokenizer = std::make_shared<Qwen2Tokenizer>();
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
const TokenizerConfig& tokenizers = {}) {
llm = std::make_shared<LLM::LLMRunner>(LLM::LLMArch::QWEN3,
backend,
tensor_storage_map,
"text_encoders.llm",
false,
weight_manager);
qwen_tokenizer = tokenizers.create(TokenizerConfig::MAIN, llm->config.vocab_size, 151643);
if (!qwen_tokenizer) {
qwen_tokenizer = std::make_shared<Qwen2Tokenizer>();
}
}
void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
@@ -1817,7 +1880,10 @@ struct AnimaConditioner : public Conditioner {
for (const auto& item : parsed_attention) {
const std::string& curr_text = item.first;
std::vector<int> curr_tokens = qwen_tokenizer->tokenize(curr_text, nullptr);
std::vector<int> curr_tokens;
if (!qwen_tokenizer->tokenize(curr_text, curr_tokens, nullptr)) {
return {};
}
qwen_tokens.insert(qwen_tokens.end(), curr_tokens.begin(), curr_tokens.end());
// Anima uses uniform Qwen token weights.
qwen_weights.insert(qwen_weights.end(), curr_tokens.size(), 1.f);
@@ -1830,7 +1896,10 @@ struct AnimaConditioner : public Conditioner {
for (const auto& item : parsed_attention) {
const std::string& curr_text = item.first;
float curr_weight = item.second;
std::vector<int> curr_tokens = t5_tokenizer.encode(curr_text);
std::vector<int> curr_tokens;
if (!t5_tokenizer.encode(curr_text, curr_tokens)) {
return {};
}
t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
}
@@ -1849,6 +1918,10 @@ struct AnimaConditioner : public Conditioner {
auto& t5_tokens = std::get<2>(tokenized);
auto& t5_weights = std::get<3>(tokenized);
if (qwen_tokens.empty()) {
return {};
}
sd::Tensor<int32_t> input_ids({static_cast<int64_t>(qwen_tokens.size()), 1}, qwen_tokens);
auto hidden_states = llm->compute(n_threads,
input_ids,
@@ -1875,7 +1948,7 @@ struct AnimaConditioner : public Conditioner {
struct LLMEmbedder : public Conditioner {
SDVersion version;
std::shared_ptr<BPETokenizer> tokenizer;
std::shared_ptr<Tokenizer> tokenizer;
std::shared_ptr<LLM::LLMRunner> llm;
std::shared_ptr<T5Runner> byt5;
@@ -1884,7 +1957,8 @@ struct LLMEmbedder : public Conditioner {
SDVersion version = VERSION_QWEN_IMAGE,
const std::string prefix = "",
bool enable_vision = false,
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
const TokenizerConfig& tokenizers = {})
: version(version) {
LLM::LLMArch arch = LLM::LLMArch::QWEN2_5_VL;
if (version == VERSION_FLUX2) {
@@ -1906,21 +1980,32 @@ struct LLMEmbedder : public Conditioner {
} else if (sd_version_is_z_image(version) || version == VERSION_OVIS_IMAGE || version == VERSION_FLUX2_KLEIN) {
arch = LLM::LLMArch::QWEN3;
}
if (arch == LLM::LLMArch::MISTRAL_SMALL_3_2 || arch == LLM::LLMArch::MINISTRAL_3_3B) {
tokenizer = std::make_shared<MistralTokenizer>();
} else if (arch == LLM::LLMArch::GPT_OSS_20B) {
tokenizer = std::make_shared<GPTOSSTokenizer>();
} else if (arch == LLM::LLMArch::GEMMA2_2B) {
tokenizer = std::make_shared<Gemma2Tokenizer>();
} else {
tokenizer = std::make_shared<Qwen2Tokenizer>();
}
llm = std::make_shared<LLM::LLMRunner>(arch,
llm = std::make_shared<LLM::LLMRunner>(arch,
backend,
tensor_storage_map,
"text_encoders.llm",
enable_vision,
weight_manager);
int pad_id = 151643;
if (arch == LLM::LLMArch::MISTRAL_SMALL_3_2 || arch == LLM::LLMArch::MINISTRAL_3_3B) {
pad_id = 11;
} else if (arch == LLM::LLMArch::GPT_OSS_20B) {
pad_id = 199999;
} else if (arch == LLM::LLMArch::GEMMA2_2B) {
pad_id = 0;
}
tokenizer = tokenizers.create(TokenizerConfig::MAIN, llm->config.vocab_size, pad_id);
if (!tokenizer) {
if (arch == LLM::LLMArch::MISTRAL_SMALL_3_2 || arch == LLM::LLMArch::MINISTRAL_3_3B) {
tokenizer = std::make_shared<MistralTokenizer>();
} else if (arch == LLM::LLMArch::GPT_OSS_20B) {
tokenizer = std::make_shared<GPTOSSTokenizer>();
} else if (arch == LLM::LLMArch::GEMMA2_2B) {
tokenizer = std::make_shared<Gemma2Tokenizer>();
} else {
tokenizer = std::make_shared<Qwen2Tokenizer>();
}
}
if (sd_version_is_hunyuan_video(version)) {
const std::string byt5_prefix = "text_encoders.t5xxl.transformer";
for (const auto& [name, _] : tensor_storage_map) {
@@ -2059,7 +2144,10 @@ struct LLMEmbedder : public Conditioner {
for (const auto& item : parsed_attention) {
const std::string& curr_text = item.first;
float curr_weight = item.second;
std::vector<int> curr_tokens = tokenizer->encode(curr_text, nullptr);
std::vector<int> curr_tokens;
if (!tokenizer->encode(curr_text, curr_tokens, nullptr)) {
return {};
}
tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
weights.insert(weights.end(), curr_tokens.size(), curr_weight);
}
@@ -2092,6 +2180,10 @@ struct LLMEmbedder : public Conditioner {
auto& weights = std::get<1>(tokens_weights_mask);
auto& mask = std::get<2>(tokens_weights_mask);
if (tokens.empty()) {
return {};
}
sd::Tensor<int32_t> input_ids({static_cast<int64_t>(tokens.size())}, tokens);
sd::Tensor<float> attention_mask;
if (!mask.empty()) {
@@ -2251,7 +2343,11 @@ struct LLMEmbedder : public Conditioner {
GGML_ASSERT(image_outputs.size() == 4);
auto image_embed = std::move(image_outputs[0]);
prompt += "<|vision_start|>";
int image_embed_idx = static_cast<int>(tokenizer->encode(prompt, nullptr).size());
std::vector<int> prefix_tokens;
if (!tokenizer->encode(prompt, prefix_tokens, nullptr)) {
return false;
}
int image_embed_idx = static_cast<int>(prefix_tokens.size());
image_embeds.emplace_back(image_embed_idx, image_embed);
if (deepstack_image_embeds.empty()) {
deepstack_image_embeds.resize(image_outputs.size() - 1);
@@ -2267,6 +2363,7 @@ struct LLMEmbedder : public Conditioner {
prompt += placeholder;
}
prompt += "<|vision_end|>";
return true;
};
const auto* references = conditioner_params.minimax_h3_references;
@@ -2283,11 +2380,13 @@ struct LLMEmbedder : public Conditioner {
GGML_ASSERT(item.frames.size() == 1);
auto resized = resize_for_vision(item.frames[0]);
prompt += "<Picture " + std::to_string(++picture_index) + ">: ";
add_vision_outputs(llm->encode_image_outputs(n_threads,
resized,
false),
static_cast<int>(resized.shape()[1]) / patch_size,
static_cast<int>(resized.shape()[0]) / patch_size);
if (!add_vision_outputs(llm->encode_image_outputs(n_threads,
resized,
false),
static_cast<int>(resized.shape()[1]) / patch_size,
static_cast<int>(resized.shape()[0]) / patch_size)) {
return {};
}
continue;
}
@@ -2311,22 +2410,26 @@ struct LLMEmbedder : public Conditioner {
second.shape()[3]});
}
auto pair = sd::ops::concat(first.unsqueeze(2), second.unsqueeze(2), 2);
add_vision_outputs(llm->encode_video_block_outputs(n_threads,
pair,
false),
static_cast<int>(first.shape()[1]) / patch_size,
static_cast<int>(first.shape()[0]) / patch_size);
if (!add_vision_outputs(llm->encode_video_block_outputs(n_threads,
pair,
false),
static_cast<int>(first.shape()[1]) / patch_size,
static_cast<int>(first.shape()[0]) / patch_size)) {
return {};
}
}
}
} else if (conditioner_params.ref_images != nullptr) {
for (size_t i = 0; i < conditioner_params.ref_images->size(); ++i) {
auto resized = resize_for_vision((*conditioner_params.ref_images)[i]);
prompt += "<Picture " + std::to_string(i + 1) + ">: ";
add_vision_outputs(llm->encode_image_outputs(n_threads,
resized,
false),
static_cast<int>(resized.shape()[1]) / patch_size,
static_cast<int>(resized.shape()[0]) / patch_size);
if (!add_vision_outputs(llm->encode_image_outputs(n_threads,
resized,
false),
static_cast<int>(resized.shape()[1]) / patch_size,
static_cast<int>(resized.shape()[0]) / patch_size)) {
return {};
}
}
}
}
@@ -2362,7 +2465,10 @@ struct LLMEmbedder : public Conditioner {
"enhanced description for the prompt below and avoid including any additional "
"commentary or evaluations:<|im_end|>\n<|im_start|>user\n";
auto prefix_tokens = tokenizer->encode(prompt_prefix, nullptr);
std::vector<int> prefix_tokens;
if (!tokenizer->encode(prompt_prefix, prefix_tokens, nullptr)) {
return {};
}
prompt_template_encode_start_idx = 0;
for (int token : prefix_tokens) {
if (token != pad_token) {
@@ -2415,7 +2521,11 @@ struct LLMEmbedder : public Conditioner {
GGML_ASSERT(!image_embed.empty());
std::string image_prefix = prompt + img_prompt + "<|vision_start|>";
int image_embed_idx = static_cast<int>(tokenizer->encode(image_prefix, nullptr).size());
std::vector<int> prefix_tokens;
if (!tokenizer->encode(image_prefix, prefix_tokens, nullptr)) {
return {};
}
int image_embed_idx = static_cast<int>(prefix_tokens.size());
image_embeds.emplace_back(image_embed_idx, image_embed);
img_prompt += "<|vision_start|>";
@@ -2557,7 +2667,11 @@ struct LLMEmbedder : public Conditioner {
GGML_ASSERT(!image_embed.empty());
std::string image_prefix = prompt_prefix + img_prompt + "<|vision_start|>";
int image_embed_idx = static_cast<int>(tokenizer->encode(image_prefix, nullptr).size());
std::vector<int> prefix_tokens;
if (!tokenizer->encode(image_prefix, prefix_tokens, nullptr)) {
return {};
}
int image_embed_idx = static_cast<int>(prefix_tokens.size());
image_embeds.emplace_back(image_embed_idx, image_embed);
img_prompt += "<|vision_start|>";
@@ -2625,7 +2739,11 @@ struct LLMEmbedder : public Conditioner {
GGML_ASSERT(!image_embed.empty());
std::string image_prefix = prompt + img_prompt + "Picture " + std::to_string(i + 1) + ": <|vision_start|>";
int image_embed_idx = static_cast<int>(tokenizer->encode(image_prefix, nullptr).size());
std::vector<int> prefix_tokens;
if (!tokenizer->encode(image_prefix, prefix_tokens, nullptr)) {
return {};
}
int image_embed_idx = static_cast<int>(prefix_tokens.size());
image_embeds.emplace_back(image_embed_idx, image_embed);
img_prompt += "Picture " + std::to_string(i + 1) + ": <|vision_start|>";
@@ -2830,7 +2948,10 @@ struct LLMEmbedder : public Conditioner {
"- User Prompt: A busy city street -> Enhanced: A bustling city street scene at dusk, featuring glowing street lamps, a diverse crowd of people in colorful clothing, and a double-decker bus passing by towering glass skyscrapers.\n"
"Please generate only the enhanced description for the prompt below and avoid including any additional commentary or evaluations:\n"
"User Prompt: ";
auto chi_tokens = std::get<0>(tokenize(chi_prompt, {0, 0}));
auto chi_tokens = std::get<0>(tokenize(chi_prompt, {0, 0}));
if (chi_tokens.empty()) {
return {};
}
size_t num_chi_tokens = chi_tokens.size();
max_length = (int)num_chi_tokens + pixeldit_max_length - 2;
min_length = max_length;
@@ -2849,7 +2970,9 @@ struct LLMEmbedder : public Conditioner {
0,
false,
max_length);
GGML_ASSERT(!hidden_states.empty());
if (hidden_states.empty()) {
return {};
}
if (hidden_states.shape()[1] > pixeldit_max_length) {
auto bos = sd::ops::slice(hidden_states, 1, 0, 1);
@@ -2882,6 +3005,9 @@ struct LLMEmbedder : public Conditioner {
max_length,
deepstack_image_embeds,
image_grids);
if (hidden_states.empty()) {
return {};
}
std::vector<sd::Tensor<float>> extra_hidden_states_vec;
if (sd_version_is_hunyuan_video(version) && byt5) {
std::vector<std::string> quoted_texts;
@@ -2932,6 +3058,9 @@ struct LLMEmbedder : public Conditioner {
prompt_template_encode_start_idx,
spell_quotes,
max_length);
if (extra_hidden_states.empty()) {
return {};
}
extra_hidden_states_vec.push_back(std::move(extra_hidden_states));
}
@@ -3030,7 +3159,7 @@ struct LTXAVEmbedder : public Conditioner {
static constexpr int64_t kNumStates = 49;
static constexpr int64_t kMinLength = 1024;
std::shared_ptr<GemmaTokenizer> tokenizer;
std::shared_ptr<Tokenizer> tokenizer;
std::shared_ptr<LLM::LLMRunner> llm;
std::shared_ptr<LTXAVTextProjectionRunner> projector;
std::string projector_prefix;
@@ -3057,17 +3186,21 @@ struct LTXAVEmbedder : public Conditioner {
const String2TensorStorage& tensor_storage_map = {},
const std::string& llm_prefix = "text_encoders.llm",
const std::string& projector_prefix = "text_embedding_projection",
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
const TokenizerConfig& tokenizers = {})
: projector_prefix(projector_prefix) {
LLM::LLMArch arch = detect_gemma_arch(tensor_storage_map, llm_prefix);
LOG_INFO("ltxav text encoder: %s", arch == LLM::LLMArch::GEMMA4_12B ? "gemma 4" : "gemma 3");
tokenizer = std::make_shared<GemmaTokenizer>();
llm = std::make_shared<LLM::LLMRunner>(arch,
llm = std::make_shared<LLM::LLMRunner>(arch,
backend,
tensor_storage_map,
llm_prefix,
false,
weight_manager);
tokenizer = tokenizers.create(TokenizerConfig::MAIN, llm->config.vocab_size, 0, true);
if (!tokenizer) {
tokenizer = std::make_shared<GemmaTokenizer>();
}
dual_projection = tensor_storage_map.find(projector_prefix + ".video_aggregate_embed.weight") != tensor_storage_map.end();
projector = std::make_shared<LTXAVTextProjectionRunner>(backend,
tensor_storage_map,
@@ -3146,7 +3279,10 @@ struct LTXAVEmbedder : public Conditioner {
std::vector<int> tokens;
std::vector<float> weights;
for (const auto& item : parsed_attention) {
auto curr_tokens = tokenizer->encode(item.first, nullptr);
std::vector<int> curr_tokens;
if (!tokenizer->encode(item.first, curr_tokens, nullptr)) {
return {};
}
tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
weights.insert(weights.end(), curr_tokens.size(), item.second);
}
@@ -3164,6 +3300,10 @@ struct LTXAVEmbedder : public Conditioner {
auto& weights = std::get<1>(tokens_weights_mask);
auto& mask = std::get<2>(tokens_weights_mask);
if (tokens.empty()) {
return {};
}
sd::Tensor<int32_t> input_ids({static_cast<int64_t>(tokens.size())}, std::vector<int32_t>(tokens.begin(), tokens.end()));
sd::Tensor<float> attention_mask;
if (!mask.empty()) {
@@ -3262,7 +3402,9 @@ struct LTXAVEmbedder : public Conditioner {
prompt_attn_range.second = static_cast<int>(prompt.size());
auto hidden_states = encode_prompt(n_threads, prompt, prompt_attn_range);
GGML_ASSERT(!hidden_states.empty());
if (hidden_states.empty()) {
return {};
}
int64_t t1 = ggml_time_ms();
LOG_VERBOSE("computing LTXAV condition graph completed, taking %" PRId64 " ms", t1 - t0);
src/core/regex.cpp
+171
View File
@@ -0,0 +1,171 @@
#include "regex.h"
#include <cstdint>
#include <limits>
#include <mutex>
#define ONIG_ESCAPE_UCHAR_COLLISION
#define ONIG_ESCAPE_REGEX_T_COLLISION
#include <oniguruma.h>
namespace sd {
struct Regex::Impl {
OnigRegex regex = nullptr;
~Impl() {
if (regex) {
onig_free(regex);
}
}
};
struct RegexRegionDeleter {
void operator()(OnigRegion* region) const {
onig_region_free(region, 1);
}
};
static bool regex_error(std::string* error, const std::string& message) {
if (error) {
*error = message;
}
return false;
}
static bool regex_onig_error(std::string* error, int code, OnigErrorInfo* info = nullptr) {
OnigUChar buffer[ONIG_MAX_ERROR_MESSAGE_LEN];
onig_error_code_to_str(buffer, code, info);
return regex_error(error, reinterpret_cast<const char*>(buffer));
}
static int regex_initialize() {
static std::once_flag once;
static int result = ONIG_NORMAL;
std::call_once(once, [] {
OnigEncoding encodings[] = {ONIG_ENCODING_UTF8};
result = onig_initialize(encodings, 1);
});
// onig_end() would invalidate expressions held by other Regex instances.
return result;
}
// Rust str excludes overlong encodings, surrogates and extended UTF-8 accepted by Oniguruma.
static bool regex_valid_utf8(const std::string& text) {
size_t position = 0;
while (position < text.size()) {
const auto lead = static_cast<unsigned char>(text[position++]);
if (lead < 0x80) {
continue;
}
int count = 0;
if (lead >= 0xC2 && lead <= 0xDF) {
count = 1;
} else if (lead >= 0xE0 && lead <= 0xEF) {
count = 2;
} else if (lead >= 0xF0 && lead <= 0xF4) {
count = 3;
}
if (count == 0 || text.size() - position < static_cast<size_t>(count)) {
return false;
}
uint32_t codepoint = lead & (0x7F >> count);
for (int i = 0; i < count; ++i) {
const auto byte = static_cast<unsigned char>(text[position++]);
if ((byte & 0xC0) != 0x80) {
return false;
}
codepoint = (codepoint << 6) | (byte & 0x3F);
}
constexpr uint32_t minimum[] = {0, 0x80, 0x800, 0x10000};
if (codepoint < minimum[count] || codepoint > 0x10FFFF || (codepoint >= 0xD800 && codepoint <= 0xDFFF)) {
return false;
}
}
return true;
}
Regex::Regex() = default;
Regex::~Regex() = default;
Regex::Regex(Regex&&) noexcept = default;
Regex& Regex::operator=(Regex&&) noexcept = default;
bool Regex::compile(const std::string& pattern, std::string* error) {
if (error) {
error->clear();
}
const int initialized = regex_initialize();
if (initialized != ONIG_NORMAL) {
return regex_onig_error(error, initialized);
}
if (pattern.size() > static_cast<size_t>(std::numeric_limits<int>::max())) {
return regex_error(error, "regex pattern exceeds Oniguruma's offset range");
}
const auto* begin = reinterpret_cast<const OnigUChar*>(pattern.data());
const auto* end = begin + pattern.size();
if (!regex_valid_utf8(pattern)) {
return regex_error(error, "regex pattern is not valid UTF-8");
}
auto next = std::make_unique<Impl>();
OnigErrorInfo info{};
static std::mutex compile_mutex;
std::lock_guard<std::mutex> lock(compile_mutex);
const int result = onig_new(&next->regex, begin, end, ONIG_OPTION_NONE,
ONIG_ENCODING_UTF8, ONIG_SYNTAX_ONIGURUMA, &info);
if (result != ONIG_NORMAL) {
return regex_onig_error(error, result, &info);
}
impl_ = std::move(next);
return true;
}
bool Regex::find_matches(const std::string& text, std::vector<Match>& matches, std::string* error) const {
matches.clear();
if (error) {
error->clear();
}
if (!impl_) {
return regex_error(error, "regex has not been compiled");
}
if (text.size() > static_cast<size_t>(std::numeric_limits<int>::max())) {
return regex_error(error, "regex input exceeds Oniguruma's offset range");
}
const auto* begin = reinterpret_cast<const OnigUChar*>(text.data());
const auto* end = begin + text.size();
if (!regex_valid_utf8(text)) {
return regex_error(error, "regex input is not valid UTF-8");
}
std::unique_ptr<OnigRegion, RegexRegionDeleter> region(onig_region_new());
if (!region) {
return regex_error(error, "failed to allocate regex match region");
}
size_t position = 0;
while (position <= text.size()) {
const int result = onig_search(impl_->regex, begin, end, begin + position, end,
region.get(), ONIG_OPTION_NONE);
if (result == ONIG_MISMATCH) {
break;
}
if (result < 0) {
matches.clear();
return regex_onig_error(error, result);
}
const size_t match_begin = static_cast<size_t>(region->beg[0]);
const size_t match_end = static_cast<size_t>(region->end[0]);
// Match rust-onig's find_iter: suppress an empty match at the previous match's end.
if (match_begin == match_end && !matches.empty() && matches.back().second == match_end) {
if (position == text.size()) {
break;
}
position += static_cast<size_t>(ONIGENC_MBC_ENC_LEN(ONIG_ENCODING_UTF8, begin + position));
continue;
}
matches.emplace_back(match_begin, match_end);
position = match_end;
}
return true;
}
} // namespace sd
src/core/regex.h
+32
View File
@@ -0,0 +1,32 @@
#ifndef __SD_CORE_REGEX_H__
#define __SD_CORE_REGEX_H__
#include <cstddef>
#include <memory>
#include <string>
#include <utility>
#include <vector>
namespace sd {
class Regex {
struct Impl;
std::unique_ptr<Impl> impl_;
public:
using Match = std::pair<size_t, size_t>;
Regex();
~Regex();
Regex(Regex&&) noexcept;
Regex& operator=(Regex&&) noexcept;
// Failed compilation leaves the previous expression intact.
bool compile(const std::string& pattern, std::string* error = nullptr);
// Matches are non-overlapping UTF-8 byte ranges; each call owns its search state.
bool find_matches(const std::string& text, std::vector<Match>& matches, std::string* error = nullptr) const;
};
} // namespace sd
#endif // __SD_CORE_REGEX_H__
src/extensions/photomaker_extension.cpp
+64 -30
View File
@@ -18,12 +18,15 @@ tokenize_photomaker_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
auto tokens_and_weights = clip_conditioner.tokenize(text);
std::vector<int> source_tokens = std::move(tokens_and_weights.first);
std::vector<float> source_weights = std::move(tokens_and_weights.second);
if (source_tokens.empty()) {
return {};
}
if (!source_tokens.empty() && source_tokens.front() == clip_conditioner.tokenizer.BOS_TOKEN_ID) {
if (!source_tokens.empty() && source_tokens.front() == clip_conditioner.tokenizer->BOS_TOKEN_ID) {
source_tokens.erase(source_tokens.begin());
source_weights.erase(source_weights.begin());
}
if (!source_tokens.empty() && source_tokens.back() == clip_conditioner.tokenizer.EOS_TOKEN_ID) {
if (!source_tokens.empty() && source_tokens.back() == clip_conditioner.tokenizer->EOS_TOKEN_ID) {
source_tokens.pop_back();
source_weights.pop_back();
}
@@ -49,12 +52,12 @@ tokenize_photomaker_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
weights.push_back(source_weights[i]);
}
clip_conditioner.tokenizer.pad_tokens(tokens,
&weights,
nullptr,
clip_conditioner.text_model->model.n_token,
clip_conditioner.text_model->model.n_token,
true);
clip_conditioner.tokenizer->pad_tokens(tokens,
&weights,
nullptr,
clip_conditioner.text_model->model.n_token,
clip_conditioner.text_model->model.n_token,
true);
std::vector<bool> class_token_mask;
for (int i = 0; i < tokens.size(); i++) {
class_token_mask.push_back(class_idx >= 0 && class_idx + 1 <= i && i < class_idx + 1 + trigger_token_count);
@@ -69,8 +72,14 @@ get_photomaker_condition_with_trigger(FrozenCLIPEmbedderWithCustomWords& clip_co
const ConditionerParams& conditioner_params,
const std::string& trigger_word,
int trigger_token_count) {
auto image_tokens = clip_conditioner.convert_token_to_id(trigger_word);
GGML_ASSERT(image_tokens.size() == 1);
std::vector<int> image_tokens;
if (!clip_conditioner.convert_token_to_id(trigger_word, image_tokens)) {
return {};
}
if (image_tokens.size() != 1) {
LOG_ERROR("PhotoMaker trigger word must encode to one token");
return {};
}
auto tokens_and_weights = tokenize_photomaker_trigger(clip_conditioner,
conditioner_params.text,
trigger_token_count,
@@ -78,27 +87,43 @@ get_photomaker_condition_with_trigger(FrozenCLIPEmbedderWithCustomWords& clip_co
std::vector<int>& tokens = std::get<0>(tokens_and_weights);
std::vector<float>& weights = std::get<1>(tokens_and_weights);
std::vector<bool>& trigger_mask = std::get<2>(tokens_and_weights);
auto cond = clip_conditioner.get_learned_condition_common(n_threads,
tokens,
weights,
conditioner_params.clip_skip,
conditioner_params.width,
conditioner_params.height,
conditioner_params.zero_out_masked);
if (tokens.empty()) {
return {};
}
auto cond = clip_conditioner.get_learned_condition_common(n_threads,
tokens,
weights,
conditioner_params.clip_skip,
conditioner_params.width,
conditioner_params.height,
conditioner_params.zero_out_masked);
return std::make_tuple(std::move(cond), trigger_mask);
}
static std::string remove_photomaker_trigger_from_prompt(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
const std::string& prompt,
const std::string& trigger_word) {
auto image_tokens = clip_conditioner.convert_token_to_id(trigger_word);
GGML_ASSERT(image_tokens.size() == 1);
static bool remove_photomaker_trigger_from_prompt(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
const std::string& prompt,
const std::string& trigger_word,
std::string& result) {
std::vector<int> image_tokens;
if (!clip_conditioner.convert_token_to_id(trigger_word, image_tokens)) {
return false;
}
if (image_tokens.size() != 1) {
LOG_ERROR("PhotoMaker trigger word must encode to one token");
return false;
}
auto tokens_and_weights = clip_conditioner.tokenize(prompt);
std::vector<int>& tokens = tokens_and_weights.first;
auto it = std::find(tokens.begin(), tokens.end(), image_tokens[0]);
GGML_ASSERT(it != tokens.end());
if (tokens.empty()) {
return false;
}
auto it = std::find(tokens.begin(), tokens.end(), image_tokens[0]);
if (it == tokens.end()) {
LOG_ERROR("PhotoMaker trigger word was not found in tokenized prompt");
return false;
}
tokens.erase(it);
return clip_conditioner.decode(tokens);
return clip_conditioner.decode(tokens, result);
}
struct PhotoMakerExtension : public GenerationExtension {
@@ -223,6 +248,10 @@ struct PhotoMakerExtension : public GenerationExtension {
trigger_token_count);
SDCondition prepared_id_condition = std::get<0>(cond_tup);
auto class_tokens_mask = std::get<1>(cond_tup);
if (prepared_id_condition.empty()) {
LOG_ERROR("failed to encode PhotoMaker prompt");
return false;
}
if (std::find(class_tokens_mask.begin(), class_tokens_mask.end(), true) == class_tokens_mask.end()) {
LOG_WARN("PhotoMaker trigger word '%s' was not found in prompt", trigger_word.c_str());
LOG_WARN("Turn off PhotoMaker for this request");
@@ -263,11 +292,16 @@ struct PhotoMakerExtension : public GenerationExtension {
prepared_id_condition.c_crossattn = std::move(res);
int64_t t1 = ggml_time_ms();
id_condition = std::move(prepared_id_condition);
start_merge_step = int(ctx.pm_params.style_strength / 100.f * ctx.total_steps);
ctx.condition_params.text = remove_photomaker_trigger_from_prompt(*clip_conditioner,
ctx.condition_params.text,
trigger_word);
std::string prompt;
if (!remove_photomaker_trigger_from_prompt(*clip_conditioner,
ctx.condition_params.text,
trigger_word,
prompt)) {
return false;
}
id_condition = std::move(prepared_id_condition);
start_merge_step = int(ctx.pm_params.style_strength / 100.f * ctx.total_steps);
ctx.condition_params.text = std::move(prompt);
LOG_INFO("Photomaker ID Stacking, taking %" PRId64 " ms", t1 - t0);
LOG_INFO("PHOTOMAKER: start_merge_step: %d", start_merge_step);
src/model/diffusion/hidream_o1.hpp
+22 -6
View File
@@ -484,13 +484,19 @@ namespace HiDreamO1 {
};
struct HiDreamO1Conditioner : public Conditioner {
Qwen2Tokenizer tokenizer;
std::shared_ptr<Tokenizer> tokenizer;
std::shared_ptr<HiDreamO1VisionRunner> vision_runner;
HiDreamO1Conditioner(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
: vision_runner(std::make_shared<HiDreamO1VisionRunner>(backend, tensor_storage_map, "model.visual", weight_manager)) {}
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
const TokenizerConfig& tokenizers = {})
: vision_runner(std::make_shared<HiDreamO1VisionRunner>(backend, tensor_storage_map, "model.visual", weight_manager)) {
tokenizer = tokenizers.create(TokenizerConfig::MAIN, HiDreamO1Config::detect_from_weights(tensor_storage_map, "").llm.vocab_size, 151643);
if (!tokenizer) {
tokenizer = std::make_shared<Qwen2Tokenizer>();
}
}
void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
vision_runner->get_param_tensors(tensors);
@@ -538,7 +544,10 @@ namespace HiDreamO1 {
if (ref_images.empty()) {
prompt += conditioner_params.text;
prompt += "<|im_end|>\n<|im_start|>assistant\n<|boi_token|><|tms_token|>";
auto input_ids = tokenizer.encode(prompt, nullptr);
std::vector<int> input_ids;
if (!tokenizer->encode(prompt, input_ids, nullptr)) {
return {};
}
std::vector<int32_t> input_ids_pad = input_ids;
input_ids_pad.push_back(VISION_START_TOKEN_ID);
@@ -612,7 +621,11 @@ namespace HiDreamO1 {
auto patch_img = resized_ref * 2.0f - 1.0f;
result.c_ref_images.push_back(std::move(patch_img));
int64_t prompt_start = static_cast<int64_t>(tokenizer.encode(prompt + "<|vision_start|>", nullptr).size());
std::vector<int> prefix_tokens;
if (!tokenizer->encode(prompt + "<|vision_start|>", prefix_tokens, nullptr)) {
return {};
}
int64_t prompt_start = static_cast<int64_t>(prefix_tokens.size());
prompt += "<|vision_start|>";
prompt += repeat_special_token("<|image_pad|>", image_tokens);
prompt += "<|vision_end|>";
@@ -623,7 +636,10 @@ namespace HiDreamO1 {
prompt += conditioner_params.text;
prompt += "<|im_end|>\n<|im_start|>assistant\n<|boi_token|><|tms_token|>";
auto input_ids = tokenizer.encode(prompt, nullptr);
std::vector<int> input_ids;
if (!tokenizer->encode(prompt, input_ids, nullptr)) {
return {};
}
std::vector<int32_t> input_ids_pad = input_ids;
input_ids_pad.push_back(VISION_START_TOKEN_ID);
src/model/te/llm.hpp
+4 -1
View File
@@ -2402,7 +2402,10 @@ namespace LLM {
for (const auto& item : parsed_attention) {
const std::string& curr_text = item.first;
float curr_weight = item.second;
std::vector<int> curr_tokens = tokenizer->tokenize(curr_text, nullptr);
std::vector<int> curr_tokens;
if (!tokenizer->tokenize(curr_text, curr_tokens, nullptr)) {
return {};
}
tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
weights.insert(weights.end(), curr_tokens.size(), curr_weight);
}
src/model/te/t5.hpp
+4 -1
View File
@@ -567,7 +567,10 @@ struct T5Embedder {
for (const auto& item : parsed_attention) {
const std::string& curr_text = item.first;
float curr_weight = item.second;
std::vector<int> curr_tokens = tokenizer.encode(curr_text);
std::vector<int> curr_tokens;
if (!tokenizer.encode(curr_text, curr_tokens)) {
return {};
}
tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
weights.insert(weights.end(), curr_tokens.size(), curr_weight);
}
src/pipeline/diffusion_engine.h
+1
View File
@@ -128,6 +128,7 @@ public:
&sd_ctx_params_t::clip_g_path, &sd_ctx_params_t::clip_vision_path,
&sd_ctx_params_t::t5xxl_path, &sd_ctx_params_t::llm_path,
&sd_ctx_params_t::llm_vision_path, &sd_ctx_params_t::diffusion_model_path,
&sd_ctx_params_t::tokenizer,
&sd_ctx_params_t::high_noise_diffusion_model_path, &sd_ctx_params_t::uncond_diffusion_model_path,
&sd_ctx_params_t::embeddings_connectors_path, &sd_ctx_params_t::vae_path,
&sd_ctx_params_t::audio_vae_path, &sd_ctx_params_t::taesd_path,
src/pipeline/model_builders.cpp
+48 -22
View File
@@ -73,12 +73,13 @@ namespace sd::model_builders {
}
}
bool build_core_runners(const Context& ctx, CoreRunners& runners) {
bool build_core_runners(const Context& ctx, CoreRunners& runners) try {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
CoreRunners result;
TokenizerConfig tokenizers(sd_ctx_params->tokenizer);
if (!ensure_backend_pair(ctx.backends, SDBackendModule::TE) ||
!ensure_backend_pair(ctx.backends, SDBackendModule::DIFFUSION)) {
return false;
@@ -87,7 +88,8 @@ namespace sd::model_builders {
if (sd_version_is_sd3(version)) {
result.conditioner = std::make_shared<SD3CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<MMDiTRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -98,7 +100,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Pid::PiDRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model.net",
@@ -109,7 +112,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Ideogram4::Ideogram4Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -120,7 +124,8 @@ namespace sd::model_builders {
version,
"",
true,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Krea2::Krea2Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -147,11 +152,13 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
} else {
result.conditioner = std::make_shared<FluxCLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager);
weight_manager,
tokenizers);
}
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
@@ -166,7 +173,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -178,7 +186,8 @@ namespace sd::model_builders {
tensor_storage_map,
"text_encoders.llm",
"text_embedding_projection",
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<LTXV::LTXAVRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -189,7 +198,8 @@ namespace sd::model_builders {
version,
"",
true,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<MiniMaxH3::MiniMaxH3Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -200,7 +210,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Hunyuan::HunyuanVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -258,7 +269,8 @@ namespace sd::model_builders {
version,
"",
enable_vision,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<LingBotVideo::LingBotVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -271,7 +283,8 @@ namespace sd::model_builders {
version,
"",
enable_vision,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Qwen::QwenImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -284,7 +297,8 @@ namespace sd::model_builders {
version,
"",
true,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<MageFlow::MageFlowRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -295,7 +309,8 @@ namespace sd::model_builders {
version,
"",
true,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -305,7 +320,8 @@ namespace sd::model_builders {
} else if (version == VERSION_HIDREAM_O1) {
result.conditioner = std::make_shared<HiDreamO1::HiDreamO1Conditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<HiDreamO1::HiDreamO1Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model",
@@ -327,7 +343,8 @@ namespace sd::model_builders {
} else if (sd_version_is_anima(version)) {
result.conditioner = std::make_shared<AnimaConditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Anima::AnimaRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -338,7 +355,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<ZImage::ZImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -350,7 +368,8 @@ namespace sd::model_builders {
version,
"",
true,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Boogu::BooguImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -362,7 +381,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<ErnieImage::ErnieImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -373,7 +393,8 @@ namespace sd::model_builders {
version,
"",
false,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<Lens::LensRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -387,7 +408,8 @@ namespace sd::model_builders {
tensor_storage_map,
embbeding_map,
version,
weight_manager);
weight_manager,
tokenizers);
result.diffusion = std::make_shared<UNetModelRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
@@ -429,8 +451,12 @@ namespace sd::model_builders {
if (result.ip_adapter) {
result.ip_adapter->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
}
tokenizers.validate_usage();
runners = std::move(result);
return true;
} catch (const std::exception& error) {
LOG_ERROR("failed to build model runners: %s", error.what());
return false;
}
bool build_vae_runners(const Context& ctx, const VAEOptions& options, VAERunners& runners) {
src/pipeline/video.cpp
+25 -13
View File
@@ -1139,10 +1139,10 @@ namespace sd::pipeline {
return latents;
}
static ImageGenerationEmbeds prepare_video_generation_embeds(StableDiffusionGGML* sd,
const sd_vid_gen_params_t* sd_vid_gen_params,
const GenerationRequest& request,
const ImageGenerationLatents& latents) {
static std::optional<ImageGenerationEmbeds> prepare_video_generation_embeds(StableDiffusionGGML* sd,
const sd_vid_gen_params_t* sd_vid_gen_params,
const GenerationRequest& request,
const ImageGenerationLatents& latents) {
ConditionerRunnerEndOnExit conditioner_runner_end{sd->cond_stage_model.get()};
ImageGenerationEmbeds embeds;
@@ -1159,8 +1159,12 @@ namespace sd::pipeline {
int64_t prepare_start_ms = ggml_time_ms();
embeds.cond = sd->cond_stage_model->get_learned_condition(sd->n_threads,
condition_params);
embeds.cond.c_concat = latents.concat_latent;
embeds.cond.c_vector = latents.clip_vision_output;
if (embeds.cond.empty()) {
LOG_ERROR("failed to encode video prompt");
return std::nullopt;
}
embeds.cond.c_concat = latents.concat_latent;
embeds.cond.c_vector = latents.clip_vision_output;
if (sd_version_is_minimax_h3(sd->version)) {
embeds.cond.c_ref_images = latents.ref_latents;
embeds.cond.c_ref_audios = latents.reference_audio_latents;
@@ -1178,9 +1182,13 @@ namespace sd::pipeline {
}
}
if (request.use_uncond) {
condition_params.text = request.negative_prompt;
embeds.uncond = sd->cond_stage_model->get_learned_condition(sd->n_threads,
condition_params);
condition_params.text = request.negative_prompt;
embeds.uncond = sd->cond_stage_model->get_learned_condition(sd->n_threads,
condition_params);
if (embeds.uncond.empty()) {
LOG_ERROR("failed to encode negative video prompt");
return std::nullopt;
}
embeds.uncond.c_concat = latents.concat_latent;
embeds.uncond.c_vector = latents.clip_vision_output;
if (sd_version_is_minimax_h3(sd->version)) {
@@ -1574,10 +1582,14 @@ namespace sd::pipeline {
}
ImageGenerationLatents latents = std::move(*latent_inputs_opt);
ImageGenerationEmbeds embeds = prepare_video_generation_embeds(sd,
sd_vid_gen_params,
request,
latents);
auto embeds_opt = prepare_video_generation_embeds(sd,
sd_vid_gen_params,
request,
latents);
if (!embeds_opt) {
return false;
}
ImageGenerationEmbeds embeds = std::move(*embeds_opt);
if (latent_upscale_enabled) {
LOG_INFO("generate_video %dx%dx%d -> LTX latent spatial upscale",
request.width,
src/stable-diffusion.cpp
+2
View File
@@ -349,6 +349,7 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) {
"t5xxl_path: %s\n"
"llm_path: %s\n"
"llm_vision_path: %s\n"
"tokenizer: %s\n"
"diffusion_model_path: %s\n"
"high_noise_diffusion_model_path: %s\n"
"uncond_diffusion_model_path: %s\n"
@@ -387,6 +388,7 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) {
SAFE_STR(sd_ctx_params->t5xxl_path),
SAFE_STR(sd_ctx_params->llm_path),
SAFE_STR(sd_ctx_params->llm_vision_path),
SAFE_STR(sd_ctx_params->tokenizer),
SAFE_STR(sd_ctx_params->diffusion_model_path),
SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path),
SAFE_STR(sd_ctx_params->uncond_diffusion_model_path),
src/tokenizers/bpe_tokenizer.cpp
+7 -2
View File
@@ -130,7 +130,11 @@ std::vector<std::u32string> BPETokenizer::bpe(const std::u32string& token) const
return word;
}
std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t on_new_token_cb) {
bool BPETokenizer::encode(const std::string& text, std::vector<int>& result, on_new_token_cb_t on_new_token_cb, std::string* error) {
result.clear();
if (error) {
error->clear();
}
std::vector<int32_t> bpe_tokens;
std::vector<std::string> token_strs;
@@ -206,7 +210,8 @@ std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t
}
ss << "]";
LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), bpe_tokens.size(), ss.str().c_str());
return bpe_tokens;
result = std::move(bpe_tokens);
return true;
}
std::string BPETokenizer::decode_token(int token_id) const {
src/tokenizers/bpe_tokenizer.h
+1 -1
View File
@@ -36,7 +36,7 @@ public:
BPETokenizer() = default;
virtual ~BPETokenizer() = default;
std::vector<int> encode(const std::string& text, on_new_token_cb_t on_new_token_cb = nullptr) override;
bool encode(const std::string& text, std::vector<int>& tokens, on_new_token_cb_t on_new_token_cb = nullptr, std::string* error = nullptr) override;
};
#endif // __SD_TOKENIZERS_BPE_TOKENIZER_H__
src/tokenizers/hf_tokenizer.cpp
+856
View File
@@ -0,0 +1,856 @@
#include "hf_tokenizer.h"
#include <algorithm>
#include <array>
#include <climits>
#include <cstdlib>
#include <fstream>
#include <queue>
#include <sstream>
#include <stdexcept>
#include <unordered_map>
#include <unordered_set>
#include "core/regex.h"
#include "core/util.h"
#include "json.hpp"
#include "utf8proc.h"
using TokenizerJSON = nlohmann::json;
static void tokenizer_require(bool condition, const std::string& message) {
if (!condition) {
throw std::runtime_error("tokenizer.json: " + message);
}
}
static std::string tokenizer_utf8(int32_t codepoint) {
utf8proc_uint8_t bytes[4];
auto count = utf8proc_encode_char(codepoint, bytes);
return std::string(reinterpret_cast<const char*>(bytes), count);
}
static bool tokenizer_error(std::string* error, const std::string& message) {
if (error) {
*error = "tokenizer.json: " + message;
} else {
LOG_ERROR("tokenizer.json: %s", message.c_str());
}
return false;
}
static bool tokenizer_next(const std::string& text, size_t& offset, int32_t& codepoint, std::string* error) {
auto size = utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(text.data() + offset), text.size() - offset, &codepoint);
if (size <= 0) {
return tokenizer_error(error, "invalid UTF-8 input");
}
offset += size;
return true;
}
static int tokenizer_id(const TokenizerJSON& value) {
tokenizer_require(value.is_number_integer(), "token ID must be an integer");
auto id = value.get<int64_t>();
tokenizer_require(id >= 0 && id <= INT_MAX, "token ID outside int32 range");
return static_cast<int>(id);
}
static uint64_t tokenizer_pair(int left, int right) {
return (static_cast<uint64_t>(left) << 32) | static_cast<uint32_t>(right);
}
struct HFTokenizer::Impl {
struct Pattern {
std::string literal;
std::shared_ptr<sd::Regex> regex;
explicit Pattern(const TokenizerJSON& config) {
tokenizer_require(config.is_object() && config.size() == 1, "invalid String/Regex pattern");
if (config.contains("String")) {
literal = config.at("String").get<std::string>();
} else {
tokenizer_require(config.contains("Regex"), "unsupported pattern");
regex = std::make_shared<sd::Regex>();
std::string error;
bool ok = regex->compile(config.at("Regex").get<std::string>(), &error);
tokenizer_require(ok, "invalid regex: " + error);
}
}
bool matches(const std::string& text, std::vector<sd::Regex::Match>& result, std::string* error) const {
result.clear();
if (regex) {
std::string regex_error;
if (!regex->find_matches(text, result, &regex_error)) {
return tokenizer_error(error, "regex search failed: " + regex_error);
}
} else if (literal.empty()) {
size_t offset = 0;
for (;;) {
result.emplace_back(offset, offset);
if (offset == text.size()) {
break;
}
int32_t cp;
if (!tokenizer_next(text, offset, cp, error)) {
result.clear();
return false;
}
}
} else {
size_t offset = 0;
while ((offset = text.find(literal, offset)) != std::string::npos) {
result.emplace_back(offset, offset + literal.size());
offset += literal.size();
}
}
return true;
}
bool replace(const std::string& text, const std::string& replacement, std::string& result, std::string* error) const {
result.clear();
size_t offset = 0;
std::vector<sd::Regex::Match> found;
if (!matches(text, found, error)) {
return false;
}
for (const auto& match : found) {
result.append(text, offset, match.first - offset);
result += replacement;
offset = match.second;
}
result.append(text, offset, std::string::npos);
return true;
}
bool split(const std::string& text, const std::string& behavior, bool invert, std::vector<std::string>& result, std::string* error) const {
result.clear();
struct Part {
size_t start, end;
bool matched;
};
std::vector<Part> parts;
size_t offset = 0;
std::vector<sd::Regex::Match> found;
if (!matches(text, found, error)) {
return false;
}
for (const auto& match : found) {
if (match.first > offset) {
parts.push_back({offset, match.first, invert});
}
parts.push_back({match.first, match.second, !invert});
offset = match.second;
}
if (offset < text.size()) {
parts.push_back({offset, text.size(), invert});
}
if (behavior == "MergedWithNext") {
std::reverse(parts.begin(), parts.end());
}
std::vector<Part> merged;
bool previous = false;
for (const auto& part : parts) {
bool join = (behavior == "Contiguous" && part.matched == previous) ||
((behavior == "MergedWithPrevious" || behavior == "MergedWithNext") && part.matched && !previous);
if (join && !merged.empty()) {
merged.back().start = std::min(merged.back().start, part.start);
merged.back().end = std::max(merged.back().end, part.end);
} else if (behavior != "Removed" || !part.matched) {
merged.push_back(part);
}
previous = part.matched;
}
if (behavior == "MergedWithNext") {
std::reverse(merged.begin(), merged.end());
}
for (const auto& part : merged) {
if (part.start != part.end) {
result.push_back(text.substr(part.start, part.end - part.start));
}
}
return true;
}
};
struct Step {
std::string type, content, behavior;
std::shared_ptr<Pattern> pattern;
bool invert = false, prefix_space = false;
};
struct Trie {
struct Node {
std::unordered_map<unsigned char, size_t> children;
int id = -1;
};
std::vector<Node> nodes{1};
void add(const std::string& text, int id) {
size_t index = 0;
for (unsigned char c : text) {
auto found = nodes[index].children.find(c);
if (found == nodes[index].children.end()) {
size_t next = nodes.size();
nodes[index].children.emplace(c, next);
nodes.emplace_back();
index = next;
} else {
index = found->second;
}
}
nodes[index].id = id;
}
std::pair<size_t, int> match(const std::string& text, size_t start) const {
size_t index = 0;
std::pair<size_t, int> result{start, -1};
for (size_t end = start; end < text.size(); ++end) {
auto found = nodes[index].children.find(static_cast<unsigned char>(text[end]));
if (found == nodes[index].children.end()) {
break;
}
index = found->second;
if (nodes[index].id >= 0) {
result = {end + 1, nodes[index].id};
}
}
return result;
}
};
struct Merge {
size_t rank;
int id;
};
std::unordered_map<std::string, int> vocab;
std::unordered_map<std::string, int> added_vocab;
std::unordered_map<int, std::string> tokens;
std::unordered_map<uint64_t, Merge> merges;
std::unordered_set<int> special_ids;
std::vector<std::string> custom_tokens;
std::vector<Step> normalizers, pre_tokenizers, decoders;
Trie raw_added, normalized_added;
std::array<std::string, 256> byte_encoder;
std::unordered_map<int32_t, unsigned char> byte_decoder;
std::string suffix;
int unk = -1;
bool fuse_unk = false, byte_fallback = false, ignore_merges = false, has_decoder = false;
Impl() {
int extra = 256;
for (int byte = 0; byte < 256; ++byte) {
int cp = ((byte >= 33 && byte <= 126) || (byte >= 161 && byte <= 172) || byte >= 174) ? byte : extra++;
byte_encoder[byte] = tokenizer_utf8(cp);
byte_decoder[cp] = static_cast<unsigned char>(byte);
}
}
static void parse_steps(const TokenizerJSON& config, const std::string& stage, std::vector<Step>& out, int depth = 0) {
tokenizer_require(depth < 32, stage + " nesting is too deep");
if (config.is_null()) {
return;
}
Step step;
step.type = config.at("type").get<std::string>();
if (step.type == "Sequence") {
const char* key = stage == "normalizer" ? "normalizers" : stage == "pre_tokenizer" ? "pretokenizers"
: "decoders";
for (const auto& child : config.at(key)) {
parse_steps(child, stage, out, depth + 1);
}
return;
}
if ((stage == "normalizer" || stage == "decoder") && step.type == "Replace") {
step.pattern = std::make_shared<Pattern>(config.at("pattern"));
step.content = config.at("content").get<std::string>();
} else if (stage == "normalizer" && (step.type == "NFC" || step.type == "Lowercase")) {
} else if (stage == "pre_tokenizer" && step.type == "Split") {
step.pattern = std::make_shared<Pattern>(config.at("pattern"));
step.behavior = config.at("behavior").get<std::string>();
tokenizer_require(step.behavior == "Removed" || step.behavior == "Isolated" || step.behavior == "Contiguous" || step.behavior == "MergedWithPrevious" || step.behavior == "MergedWithNext", "unsupported Split behavior: " + step.behavior);
step.invert = config.value("invert", false);
} else if ((stage == "pre_tokenizer" || stage == "decoder") && step.type == "ByteLevel") {
step.prefix_space = config.value("add_prefix_space", true);
if (stage == "pre_tokenizer" && config.value("use_regex", true)) {
step.pattern = std::make_shared<Pattern>(TokenizerJSON{{"Regex", R"('s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+)"}});
}
} else if (stage == "decoder" && (step.type == "ByteFallback" || step.type == "Fuse")) {
} else {
tokenizer_require(false, "unsupported " + stage + ": " + step.type);
}
out.push_back(std::move(step));
}
bool normalize(std::string text, std::string& result, std::string* error) const {
result.clear();
for (const auto& step : normalizers) {
if (step.type == "Replace") {
std::string replaced;
if (!step.pattern->replace(text, step.content, replaced, error)) {
return false;
}
text = std::move(replaced);
} else if (step.type == "NFC") {
utf8proc_uint8_t* output = nullptr;
auto size = utf8proc_map(reinterpret_cast<const utf8proc_uint8_t*>(text.data()), text.size(), &output, static_cast<utf8proc_option_t>(UTF8PROC_STABLE | UTF8PROC_COMPOSE));
std::unique_ptr<utf8proc_uint8_t, decltype(&std::free)> buffer(output, &std::free);
if (size < 0) {
return tokenizer_error(error, std::string("NFC normalization failed: ") + utf8proc_errmsg(size));
}
text.assign(reinterpret_cast<const char*>(output), size);
} else {
std::string lower;
for (size_t i = 0; i < text.size();) {
int32_t cp;
if (!tokenizer_next(text, i, cp, error)) {
return false;
}
// Rust char::to_lowercase uses full, context-free lowercase. U+0130 expands.
lower += cp == 0x130 ? "i\xcc\x87" : tokenizer_utf8(utf8proc_tolower(cp));
}
text = std::move(lower);
}
}
result = std::move(text);
return true;
}
bool pre_tokenize(const std::string& text, std::vector<std::string>& result, std::string* error) const {
result.clear();
std::vector<std::string> pieces{text};
for (const auto& step : pre_tokenizers) {
std::vector<std::string> next;
for (auto piece : pieces) {
if (piece.empty()) {
continue;
}
std::vector<std::string> split;
if (step.type == "Split") {
if (!step.pattern->split(piece, step.behavior, step.invert, split, error)) {
return false;
}
next.insert(next.end(), split.begin(), split.end());
} else {
if (step.prefix_space && piece.front() != ' ') {
piece.insert(piece.begin(), ' ');
}
if (step.pattern) {
if (!step.pattern->split(piece, "Isolated", false, split, error)) {
return false;
}
} else {
split.push_back(piece);
}
for (const auto& part : split) {
std::string encoded;
for (unsigned char byte : part) {
encoded += byte_encoder[byte];
}
next.push_back(std::move(encoded));
}
}
}
pieces = std::move(next);
}
result = std::move(pieces);
return true;
}
bool bpe(const std::string& text, std::vector<int>& ids, std::string* error) const {
ids.clear();
if (ignore_merges) {
auto found = vocab.find(text);
if (found != vocab.end()) {
ids.push_back(found->second);
return true;
}
}
bool pending_unk = false;
for (size_t i = 0; i < text.size();) {
int32_t cp;
size_t end = i;
if (!tokenizer_next(text, end, cp, error)) {
ids.clear();
return false;
}
std::string symbol = text.substr(i, end - i);
if (end == text.size()) {
symbol += suffix;
}
i = end;
auto found = vocab.find(symbol);
if (found != vocab.end()) {
if (pending_unk) {
ids.push_back(unk);
pending_unk = false;
}
ids.push_back(found->second);
continue;
}
if (byte_fallback) {
std::vector<int> bytes;
for (unsigned char byte : symbol) {
const char* hex = "0123456789ABCDEF";
std::string token = "<0x00>";
token[3] = hex[byte >> 4];
token[4] = hex[byte & 15];
auto fallback = vocab.find(token);
if (fallback == vocab.end()) {
break;
}
bytes.push_back(fallback->second);
}
if (bytes.size() == symbol.size()) {
ids.insert(ids.end(), bytes.begin(), bytes.end());
continue;
}
}
if (unk >= 0) {
if (pending_unk && !fuse_unk) {
ids.push_back(unk);
}
pending_unk = true;
}
}
if (pending_unk) {
ids.push_back(unk);
}
struct Symbol {
int id;
size_t prev, next, generation = 0;
bool alive = true;
};
struct Candidate {
size_t rank, left, right, left_generation, right_generation;
int id;
bool operator<(const Candidate& other) const {
return rank != other.rank ? rank > other.rank : left > other.left;
}
};
const size_t none = ids.size();
std::vector<Symbol> symbols;
for (size_t i = 0; i < ids.size(); ++i) {
symbols.push_back({ids[i], i == 0 ? none : i - 1, i + 1});
}
std::priority_queue<Candidate> queue;
auto push = [&](size_t left) {
if (left == none || symbols[left].next == none) {
return;
}
size_t right = symbols[left].next;
auto found = merges.find(tokenizer_pair(symbols[left].id, symbols[right].id));
if (found != merges.end()) {
queue.push({found->second.rank, left, right, symbols[left].generation, symbols[right].generation, found->second.id});
}
};
for (size_t i = 0; i < symbols.size(); ++i) {
push(i);
}
while (!queue.empty()) {
Candidate item = queue.top();
queue.pop();
auto& left = symbols[item.left];
auto& right = symbols[item.right];
if (!left.alive || !right.alive || left.next != item.right || left.generation != item.left_generation || right.generation != item.right_generation) {
continue;
}
left.id = item.id;
left.next = right.next;
++left.generation;
right.alive = false;
if (left.next != none) {
symbols[left.next].prev = item.left;
}
push(left.prev);
push(item.left);
}
ids.clear();
for (const auto& symbol : symbols) {
if (symbol.alive) {
ids.push_back(symbol.id);
}
}
return true;
}
int lookup(const std::string& token) const {
auto added = added_vocab.find(token);
if (added != added_vocab.end()) {
return added->second;
}
auto found = vocab.find(token);
return found == vocab.end() ? -1 : found->second;
}
void add_token(const std::string& token, int id, bool added = false) {
auto old = tokens.find(id);
tokenizer_require(old == tokens.end() || old->second == token, "conflicting token ID " + std::to_string(id));
int old_id = lookup(token);
tokenizer_require(old_id < 0 || old_id == id, "conflicting ID for token " + token);
tokens[id] = token;
(added ? added_vocab : vocab)[token] = id;
}
};
HFTokenizer::HFTokenizer(const std::string& path)
: impl_(new Impl) {
std::ifstream stream(path, std::ios::binary);
tokenizer_require(stream.good(), "cannot open " + path);
TokenizerJSON config;
stream >> config;
tokenizer_require(config.value("version", std::string("1.0")) == "1.0", "unsupported version");
tokenizer_require(config.value("padding", TokenizerJSON()).is_null(), "JSON padding is unsupported; padding is controlled by the text encoder");
tokenizer_require(config.value("truncation", TokenizerJSON()).is_null(), "JSON truncation is unsupported; truncation is controlled by the text encoder");
const auto& model = config.at("model");
tokenizer_require(model.at("type") == "BPE", "only BPE models are supported");
tokenizer_require(model.value("dropout", TokenizerJSON()).is_null() || model.at("dropout") == 0, "BPE dropout is unsupported");
const auto& prefix = model.value("continuing_subword_prefix", TokenizerJSON());
tokenizer_require(prefix.is_null() || prefix == "", "nonempty continuing_subword_prefix is unsupported");
const auto& suffix = model.value("end_of_word_suffix", TokenizerJSON());
impl_->suffix = suffix.is_null() ? "" : suffix.get<std::string>();
impl_->fuse_unk = model.value("fuse_unk", false);
impl_->byte_fallback = model.value("byte_fallback", false);
impl_->ignore_merges = model.value("ignore_merges", false);
tokenizer_require(model.at("vocab").is_object(), "BPE vocab must be an object");
impl_->vocab.reserve(model.at("vocab").size());
impl_->tokens.reserve(model.at("vocab").size());
for (const auto& entry : model.at("vocab").items()) {
impl_->add_token(entry.key(), tokenizer_id(entry.value()));
}
if (!model.value("unk_token", TokenizerJSON()).is_null()) {
UNK_TOKEN = model.at("unk_token").get<std::string>();
impl_->unk = impl_->lookup(UNK_TOKEN);
tokenizer_require(impl_->unk >= 0, "unk_token is absent from vocab");
}
UNK_TOKEN_ID = impl_->unk;
tokenizer_require(model.at("merges").is_array(), "BPE merges must be an array");
impl_->merges.reserve(model.at("merges").size());
size_t rank = 0;
for (const auto& merge : model.at("merges")) {
std::string left, right;
if (merge.is_string()) {
auto value = merge.get<std::string>();
auto space = value.find(' ');
tokenizer_require(space != std::string::npos && value.find(' ', space + 1) == std::string::npos, "invalid legacy BPE merge");
left = value.substr(0, space);
right = value.substr(space + 1);
} else {
tokenizer_require(merge.is_array() && merge.size() == 2, "BPE merge must contain two tokens");
left = merge.at(0).get<std::string>();
right = merge.at(1).get<std::string>();
}
int a = impl_->lookup(left), b = impl_->lookup(right), id = impl_->lookup(left + right);
tokenizer_require(a >= 0 && b >= 0 && id >= 0, "BPE merge references a missing vocab token");
impl_->merges[tokenizer_pair(a, b)] = {rank++, id};
}
Impl::parse_steps(config.value("normalizer", TokenizerJSON()), "normalizer", impl_->normalizers);
Impl::parse_steps(config.value("pre_tokenizer", TokenizerJSON()), "pre_tokenizer", impl_->pre_tokenizers);
impl_->has_decoder = !config.value("decoder", TokenizerJSON()).is_null();
Impl::parse_steps(config.value("decoder", TokenizerJSON()), "decoder", impl_->decoders);
size_t next_added_id = impl_->vocab.size();
for (const auto& token : config.value("added_tokens", TokenizerJSON::array())) {
for (const char* flag : {"single_word", "lstrip", "rstrip"}) {
tokenizer_require(!token.value(flag, false), std::string("added_tokens.") + flag + "=true is unsupported");
}
auto content = token.at("content").get<std::string>();
tokenizer_require(!content.empty(), "empty added token is unsupported");
int id = tokenizer_id(token.at("id"));
if (impl_->lookup(content) < 0) {
tokenizer_require(static_cast<size_t>(id) == next_added_id++, "nonconsecutive added token IDs would be reassigned by Hugging Face tokenizers");
}
impl_->add_token(content, id, true);
if (token.value("special", false)) {
special_tokens.push_back(content);
impl_->special_ids.insert(id);
}
bool normalized = token.value("normalized", true);
std::string pattern = content;
if (normalized) {
std::string error;
bool ok = impl_->normalize(content, pattern, &error);
tokenizer_require(ok, error);
}
tokenizer_require(!pattern.empty(), "added token normalizes to an empty string");
(normalized ? impl_->normalized_added : impl_->raw_added).add(pattern, id);
}
const auto& processor = config.value("post_processor", TokenizerJSON());
BOS_TOKEN_ID = EOS_TOKEN_ID = -1;
if (!processor.is_null()) {
auto type = processor.at("type").get<std::string>();
auto special = [&](const TokenizerJSON& pair) {
tokenizer_require(pair.is_array() && pair.size() == 2, "invalid postprocessor special token");
int id = tokenizer_id(pair.at(1));
tokenizer_require(impl_->lookup(pair.at(0).get<std::string>()) == id, "postprocessor token/ID does not match vocab");
return id;
};
if (type == "RobertaProcessing") {
BOS_TOKEN_ID = special(processor.at("cls"));
EOS_TOKEN_ID = special(processor.at("sep"));
} else if (type == "TemplateProcessing") {
bool seen_sequence = false;
for (const auto& item : processor.at("single")) {
if (item.contains("Sequence")) {
tokenizer_require(!seen_sequence && item.at("Sequence").at("id") == "A", "single template must contain exactly one sequence A");
seen_sequence = true;
} else {
auto name = item.at("SpecialToken").at("id").get<std::string>();
const auto& token = processor.at("special_tokens").at(name);
tokenizer_require(token.at("ids").size() == 1 && token.at("tokens").size() == 1, "multi-ID template special tokens are unsupported");
int id = special(TokenizerJSON::array({token.at("tokens").at(0), token.at("ids").at(0)}));
int& target = seen_sequence ? EOS_TOKEN_ID : BOS_TOKEN_ID;
tokenizer_require(target < 0, "single template supports at most one prefix and one suffix token");
target = id;
}
}
tokenizer_require(seen_sequence, "single template has no sequence A");
} else {
tokenizer_require(type == "ByteLevel", "unsupported post_processor: " + type);
}
}
add_bos_token = BOS_TOKEN_ID >= 0;
add_eos_token = EOS_TOKEN_ID >= 0;
BOS_TOKEN = decode_token(BOS_TOKEN_ID);
EOS_TOKEN = decode_token(EOS_TOKEN_ID);
set_padding(0, false);
}
HFTokenizer::~HFTokenizer() = default;
void HFTokenizer::set_padding(int token_id, bool left) {
PAD_TOKEN_ID = token_id;
PAD_TOKEN = decode_token(token_id);
pad_left = left;
}
void HFTokenizer::validate_vocab_size(int64_t embedding_rows) const {
tokenizer_require(embedding_rows > 0, "text encoder has no token embedding rows");
for (const auto& token : impl_->tokens) {
tokenizer_require(token.first < embedding_rows, "token ID " + std::to_string(token.first) + " exceeds text encoder vocabulary (" + std::to_string(embedding_rows) + ")");
}
tokenizer_require(PAD_TOKEN_ID >= 0 && PAD_TOKEN_ID < embedding_rows, "padding ID exceeds text encoder vocabulary");
}
int HFTokenizer::token_to_id(const std::string& token) const {
return impl_->lookup(token);
}
void HFTokenizer::add_special_token(const std::string& token) {
Tokenizer::add_special_token(token);
if (!token.empty()) {
impl_->custom_tokens.push_back(token);
}
}
bool HFTokenizer::encode(const std::string& text, std::vector<int>& tokens, on_new_token_cb_t callback, std::string* error) {
tokens.clear();
if (error) {
error->clear();
}
for (size_t i = 0; i < text.size();) {
int32_t cp;
if (!tokenizer_next(text, i, cp, error)) {
return false;
}
}
std::vector<int> result;
Impl::Trie raw_custom, normalized_custom;
if (callback) {
for (size_t index = 0; index < impl_->custom_tokens.size(); ++index) {
const auto& token = impl_->custom_tokens[index];
raw_custom.add(token, static_cast<int>(index));
std::string normalized;
if (!impl_->normalize(token, normalized, error)) {
return false;
}
if (!normalized.empty()) {
normalized_custom.add(normalized, static_cast<int>(index));
}
}
}
auto encode_plain = [&](const std::string& value) {
std::vector<std::string> pieces;
if (!impl_->pre_tokenize(value, pieces, error)) {
return false;
}
for (auto& piece : pieces) {
if (callback && callback(piece, result)) {
continue;
}
std::vector<int> ids;
if (!impl_->bpe(piece, ids, error)) {
return false;
}
result.insert(result.end(), ids.begin(), ids.end());
}
return true;
};
auto extract = [&](const std::string& value, const Impl::Trie& added, const Impl::Trie& custom_tokens, const auto& encode_gap) {
size_t start = 0, i = 0;
while (i < value.size()) {
auto match = added.match(value, i);
auto custom = custom_tokens.match(value, i);
bool use_custom = custom.second >= 0 && custom.first >= match.first;
if (match.second < 0 && !use_custom) {
++i;
continue;
}
if (!encode_gap(value.substr(start, i - start))) {
return false;
}
if (use_custom) {
auto token = impl_->custom_tokens[custom.second];
if (!callback(token, result)) {
if (match.second >= 0 && match.first == custom.first) {
result.push_back(match.second);
} else if (!encode_gap(value.substr(i, custom.first - i))) {
return false;
}
}
} else {
result.push_back(match.second);
}
start = i = use_custom ? custom.first : match.first;
}
return encode_gap(value.substr(start));
};
auto encode_normalized = [&](const std::string& value) {
std::string normalized;
if (!impl_->normalize(value, normalized, error)) {
return false;
}
return extract(normalized, impl_->normalized_added, normalized_custom, encode_plain);
};
if (!extract(text, impl_->raw_added, raw_custom, encode_normalized)) {
return false;
}
std::stringstream ss;
ss << "[";
for (int id : result) {
auto token = impl_->tokens.find(id);
if (token != impl_->tokens.end()) {
ss << "\"" << token->second << "\", ";
} else {
ss << "\"<id:" << id << ">\", ";
}
}
ss << "]";
LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), result.size(), ss.str().c_str());
tokens = std::move(result);
return true;
}
std::string HFTokenizer::decode_token(int id) const {
auto found = impl_->tokens.find(id);
return found == impl_->tokens.end() ? "" : found->second;
}
static std::string tokenizer_lossy_utf8(const std::string& bytes, bool fallback) {
std::string result;
for (size_t i = 0; i < bytes.size();) {
int32_t cp;
auto count = utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(bytes.data() + i), bytes.size() - i, &cp);
if (count > 0) {
result.append(bytes, i, count);
i += count;
} else if (fallback) {
result.clear();
for (size_t j = 0; j < bytes.size(); ++j) {
result += "\xef\xbf\xbd";
}
return result;
} else {
result += "\xef\xbf\xbd";
unsigned char lead = bytes[i++];
size_t expected = lead >= 0xc2 && lead <= 0xdf ? 2 : lead >= 0xe0 && lead <= 0xef ? 3
: lead >= 0xf0 && lead <= 0xf4 ? 4
: 1;
for (size_t j = 1; j < expected && i < bytes.size(); ++j) {
unsigned char c = bytes[i];
if (c < 0x80 || c > 0xbf || (j == 1 && ((lead == 0xe0 && c < 0xa0) || (lead == 0xed && c > 0x9f) || (lead == 0xf0 && c < 0x90) || (lead == 0xf4 && c > 0x8f)))) {
break;
}
++i;
}
}
}
return result;
}
bool HFTokenizer::decode(const std::vector<int>& ids, std::string& text, std::string* error) const {
text.clear();
if (error) {
error->clear();
}
std::vector<std::string> pieces;
for (int id : ids) {
if (!impl_->special_ids.count(id) && impl_->tokens.count(id)) {
pieces.push_back(decode_token(id));
}
}
for (const auto& step : impl_->decoders) {
if (step.type == "Replace") {
for (auto& piece : pieces) {
std::string replaced;
if (!step.pattern->replace(piece, step.content, replaced, error)) {
return false;
}
piece = std::move(replaced);
}
} else if (step.type == "ByteLevel" || step.type == "Fuse") {
std::string joined;
for (const auto& piece : pieces) {
std::string bytes;
if (step.type == "ByteLevel") {
for (size_t i = 0; i < piece.size();) {
int32_t cp;
if (!tokenizer_next(piece, i, cp, error)) {
return false;
}
auto found = impl_->byte_decoder.find(cp);
if (found == impl_->byte_decoder.end()) {
bytes = piece;
break;
}
bytes += static_cast<char>(found->second);
}
} else {
bytes = piece;
}
joined += bytes;
}
pieces = {step.type == "ByteLevel" ? tokenizer_lossy_utf8(joined, false) : joined};
} else {
std::vector<std::string> decoded;
std::string bytes;
auto flush = [&] {
if (!bytes.empty()) {
decoded.push_back(tokenizer_lossy_utf8(bytes, true));
bytes.clear();
}
};
for (const auto& piece : pieces) {
auto hex = [](char c) { return c >= '0' && c <= '9' ? c - '0' : c >= 'A' && c <= 'F' ? c - 'A' + 10
: c >= 'a' && c <= 'f' ? c - 'a' + 10
: -1; };
if (piece.size() == 6 && piece.compare(0, 3, "<0x") == 0 && piece[5] == '>' && hex(piece[3]) >= 0 && hex(piece[4]) >= 0) {
bytes += static_cast<char>((hex(piece[3]) << 4) | hex(piece[4]));
} else {
flush();
decoded.push_back(piece);
}
}
flush();
pieces = std::move(decoded);
}
}
std::string result;
for (size_t i = 0; i < pieces.size(); ++i) {
if (i && !impl_->has_decoder) {
result += ' ';
}
result += pieces[i];
}
text = std::move(result);
return true;
}
src/tokenizers/hf_tokenizer.h
+26
View File
@@ -0,0 +1,26 @@
#ifndef __SD_TOKENIZERS_HF_TOKENIZER_H__
#define __SD_TOKENIZERS_HF_TOKENIZER_H__
#include <memory>
#include "tokenizer.h"
class HFTokenizer : public Tokenizer {
struct Impl;
std::unique_ptr<Impl> impl_;
std::string decode_token(int token_id) const override;
public:
explicit HFTokenizer(const std::string& path);
~HFTokenizer() override;
// Padding belongs to the encoder; tokenizer.json supplies the single-sequence template.
void set_padding(int token_id, bool left);
void validate_vocab_size(int64_t embedding_rows) const;
int token_to_id(const std::string& token) const;
void add_special_token(const std::string& token) override;
bool encode(const std::string& text, std::vector<int>& tokens, on_new_token_cb_t on_new_token_cb = nullptr, std::string* error = nullptr) override;
bool decode(const std::vector<int>& tokens, std::string& text, std::string* error = nullptr) const override;
};
#endif // __SD_TOKENIZERS_HF_TOKENIZER_H__
src/tokenizers/t5_unigram_tokenizer.cpp
+7 -2
View File
@@ -287,7 +287,11 @@ std::string T5UniGramTokenizer::normalize(const std::string& input) const {
return normalized;
}
std::vector<int> T5UniGramTokenizer::encode(const std::string& input, on_new_token_cb_t on_new_token_cb) {
bool T5UniGramTokenizer::encode(const std::string& input, std::vector<int>& result, on_new_token_cb_t on_new_token_cb, std::string* error) {
result.clear();
if (error) {
error->clear();
}
std::vector<int32_t> tokens;
std::vector<std::string> token_strs;
std::string normalized = normalize(input);
@@ -335,5 +339,6 @@ std::vector<int> T5UniGramTokenizer::encode(const std::string& input, on_new_tok
ss << "]";
LOG_VERBOSE("split prompt \"%s\" to tokens %s", input.c_str(), ss.str().c_str());
return tokens;
result = std::move(tokens);
return true;
}
src/tokenizers/t5_unigram_tokenizer.h
+1 -1
View File
@@ -64,7 +64,7 @@ public:
explicit T5UniGramTokenizer(bool is_umt5 = false);
~T5UniGramTokenizer();
std::vector<int> encode(const std::string& input, on_new_token_cb_t on_new_token_cb = nullptr) override;
bool encode(const std::string& input, std::vector<int>& tokens, on_new_token_cb_t on_new_token_cb = nullptr, std::string* error = nullptr) override;
};
#endif // __SD_TOKENIZERS_T5_UNIGRAM_TOKENIZER_H__
src/tokenizers/tokenizer.cpp
+20 -11
View File
@@ -23,17 +23,22 @@ std::string Tokenizer::normalize(const std::string& text) const {
return text;
}
std::vector<int> Tokenizer::tokenize(const std::string& text,
on_new_token_cb_t on_new_token_cb,
bool padding,
size_t min_length,
size_t max_length,
bool allow_overflow_expand) {
std::vector<int> tokens = encode(text, on_new_token_cb);
bool Tokenizer::tokenize(const std::string& text,
std::vector<int>& tokens,
on_new_token_cb_t on_new_token_cb,
bool padding,
size_t min_length,
size_t max_length,
bool allow_overflow_expand,
std::string* error) {
if (!encode(text, tokens, on_new_token_cb, error)) {
tokens.clear();
return false;
}
if (padding) {
pad_tokens(tokens, nullptr, nullptr, min_length, max_length, allow_overflow_expand);
}
return tokens;
return true;
}
void Tokenizer::pad_tokens(std::vector<int>& tokens,
@@ -200,8 +205,11 @@ static std::string clean_up_tokenization(std::string& text) {
return std::regex_replace(text, pattern, ",");
}
std::string Tokenizer::decode(const std::vector<int>& tokens) const {
std::string text;
bool Tokenizer::decode(const std::vector<int>& tokens, std::string& text, std::string* error) const {
text.clear();
if (error) {
error->clear();
}
for (int token_id : tokens) {
if (token_id == BOS_TOKEN_ID || token_id == EOS_TOKEN_ID || token_id == PAD_TOKEN_ID) {
@@ -218,5 +226,6 @@ std::string Tokenizer::decode(const std::vector<int>& tokens) const {
}
text = clean_up_tokenization(text);
return trim(text);
text = trim(text);
return true;
}
src/tokenizers/tokenizer.h
+12 -9
View File
@@ -33,22 +33,25 @@ public:
virtual ~Tokenizer() = default;
void add_special_token(const std::string& token);
virtual void add_special_token(const std::string& token);
bool is_special_token(const std::string& token) const;
virtual std::vector<int> encode(const std::string& text, on_new_token_cb_t on_new_token_cb = nullptr) = 0;
std::vector<int> tokenize(const std::string& text,
on_new_token_cb_t on_new_token_cb = nullptr,
bool padding = false,
size_t min_length = 0,
size_t max_length = 100000000,
bool allow_overflow_expand = false);
// An empty output may be valid; failures return false and clear the output.
virtual bool encode(const std::string& text, std::vector<int>& tokens, on_new_token_cb_t on_new_token_cb = nullptr, std::string* error = nullptr) = 0;
bool tokenize(const std::string& text,
std::vector<int>& tokens,
on_new_token_cb_t on_new_token_cb = nullptr,
bool padding = false,
size_t min_length = 0,
size_t max_length = 100000000,
bool allow_overflow_expand = false,
std::string* error = nullptr);
void pad_tokens(std::vector<int>& tokens,
std::vector<float>* weights,
std::vector<float>* mask,
size_t min_length = 0,
size_t max_length = 100000000,
bool allow_overflow_expand = false);
std::string decode(const std::vector<int>& tokens) const;
virtual bool decode(const std::vector<int>& tokens, std::string& text, std::string* error = nullptr) const;
};
#endif // __SD_TOKENIZERS_TOKENIZER_H__
src/tokenizers/tokenizer_config.cpp
+77
View File
@@ -0,0 +1,77 @@
#include "tokenizer_config.h"
#include <stdexcept>
#include "core/util.h"
#include "hf_tokenizer.h"
TokenizerConfig::TokenizerConfig(const char* config) {
if (!config || !*config) {
return;
}
const std::string value = config;
const bool assignments = value.find('=') != std::string::npos;
size_t begin = 0;
for (;;) {
const auto end = assignments ? value.find(',', begin) : std::string::npos;
const auto entry = value.substr(begin, end == std::string::npos ? end : end - begin);
const auto equal = entry.find('=');
if (assignments && equal == std::string::npos) {
throw std::runtime_error("invalid tokenizer entry '" + entry + "'; expected main=FILE,clip-l=FILE,clip-g=FILE");
}
const auto key = assignments ? entry.substr(0, equal) : "main";
const auto path = assignments ? entry.substr(equal + 1) : entry;
Slot slot = MAIN;
if (key == "clip-l") {
slot = CLIP_L;
} else if (key == "clip-g") {
slot = CLIP_G;
} else if (key != "main") {
throw std::runtime_error("unknown tokenizer slot '" + key + "'; expected main, clip-l or clip-g");
}
if (path.empty()) {
throw std::runtime_error("tokenizer slot '" + key + "' requires a nonempty path");
}
if (!paths_[slot].empty()) {
throw std::runtime_error("tokenizer slot '" + key + "' is specified more than once");
}
paths_[slot] = path;
if (end == std::string::npos) {
break;
}
begin = end + 1;
}
}
bool TokenizerConfig::has(Slot slot) const {
return !paths_[slot].empty();
}
std::shared_ptr<Tokenizer> TokenizerConfig::create(Slot slot, int64_t embedding_rows, int padding_id, bool pad_left, bool clip) const {
if (!has(slot)) {
return nullptr;
}
try {
auto tokenizer = std::make_shared<HFTokenizer>(paths_[slot]);
tokenizer->set_padding(padding_id, pad_left);
tokenizer->validate_vocab_size(embedding_rows);
if (clip && (tokenizer->BOS_TOKEN_ID < 0 || tokenizer->EOS_TOKEN_ID < 0)) {
throw std::runtime_error("CLIP requires a single BOS + A + EOS template");
}
used_[slot] = true;
const char* names[] = {"main", "clip-l", "clip-g"};
LOG_INFO("using external tokenizer (%s): %s", names[slot], paths_[slot].c_str());
return tokenizer;
} catch (const std::exception& error) {
throw std::runtime_error("failed to load tokenizer '" + paths_[slot] + "': " + error.what());
}
}
void TokenizerConfig::validate_usage() const {
const char* names[] = {"main", "clip-l", "clip-g"};
for (size_t i = 0; i < paths_.size(); ++i) {
if (!paths_[i].empty() && !used_[i]) {
throw std::runtime_error(std::string("tokenizer slot '") + names[i] + "' does not target an active, supported text encoder; SD3 uses the clip-l and clip-g slots");
}
}
}
src/tokenizers/tokenizer_config.h
+28
View File
@@ -0,0 +1,28 @@
#ifndef __SD_TOKENIZERS_TOKENIZER_CONFIG_H__
#define __SD_TOKENIZERS_TOKENIZER_CONFIG_H__
#include <array>
#include <memory>
#include <string>
#include "tokenizer.h"
class TokenizerConfig {
public:
enum Slot { MAIN,
CLIP_L,
CLIP_G };
private:
std::array<std::string, 3> paths_;
mutable std::array<bool, 3> used_{};
public:
TokenizerConfig() = default;
explicit TokenizerConfig(const char* config);
bool has(Slot slot) const;
std::shared_ptr<Tokenizer> create(Slot slot, int64_t embedding_rows, int padding_id, bool pad_left = false, bool clip = false) const;
void validate_usage() const;
};
#endif // __SD_TOKENIZERS_TOKENIZER_CONFIG_H__