DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 11:26:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support external Hugging Face tokenizer JSON files (#1973)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-09-15 00:27:39 +08:00
1 parent 42d6c0ab92
commit 4964abdfc5
58 files changed
+97390 -180
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/model/te/llm.hpp
+4 -1
View File
@@ -2402,7 +2402,10 @@ namespace LLM {
for (const auto& item : parsed_attention) {
const std::string& curr_text = item.first;
float curr_weight = item.second;
std::vector<int> curr_tokens = tokenizer->tokenize(curr_text, nullptr);
std::vector<int> curr_tokens;
if (!tokenizer->tokenize(curr_text, curr_tokens, nullptr)) {
return {};
}
tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
weights.insert(weights.end(), curr_tokens.size(), curr_weight);
}