DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-12 03:49:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support external Hugging Face tokenizer JSON files (#1973)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-09-15 00:27:39 +08:00
1 parent 42d6c0ab92
commit 4964abdfc5
58 files changed
+97390 -180
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/tokenizers/bpe_tokenizer.h
+1 -1
View File
@@ -36,7 +36,7 @@ public:
BPETokenizer() = default;
virtual ~BPETokenizer() = default;
std::vector<int> encode(const std::string& text, on_new_token_cb_t on_new_token_cb = nullptr) override;
bool encode(const std::string& text, std::vector<int>& tokens, on_new_token_cb_t on_new_token_cb = nullptr, std::string* error = nullptr) override;
};
#endif // __SD_TOKENIZERS_BPE_TOKENIZER_H__