mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-11-11 11:26:34 +00:00
feat: add PixArt model family support (#2047)
This commit is contained in:
1 parent
4c3cf7543d
commit
39ada0863b
10 files changed
+512
-5
No files matched your search
@@ -295,6 +295,12 @@ bool T5UniGramTokenizer::encode(const std::string& input, std::vector<int>& resu
|
||||
std::vector<int32_t> tokens;
|
||||
std::vector<std::string> token_strs;
|
||||
std::string normalized = normalize(input);
|
||||
if (normalized.empty()) {
|
||||
// HF reference tokenizers emit no pieces for empty input; pad_tokens
|
||||
// still appends EOS so the sequence becomes [EOS] + padding.
|
||||
result = std::move(tokens);
|
||||
return true;
|
||||
}
|
||||
auto splited_texts = split_with_special_tokens(normalized, special_tokens);
|
||||
if (splited_texts.empty()) {
|
||||
splited_texts.push_back(normalized); // for empty string
|
||||
|
||||
Reference in new issue
Block a user