fix: enhance the tokenizer's handing of Unicode (#120)

This commit is contained in:
leejet authored and GitHub committed 2023-12-21 00:22:03 +08:00
1 parent 9842a3f819
commit 8f6b4a39d6
6 files changed
+43830 -80117

No files matched your search

+1 -1
View File
@@ -115,7 +115,7 @@ public:
bool init_from_file(const std::string& file_path, const std::string& prefix = "");
SDVersion get_sd_version();
ggml_type get_sd_wtype();
bool load_vocab(on_new_token_cb_t on_new_token_cb);
std::string load_merges();
bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb);
int64_t cal_mem_size(ggml_backend_t backend);
~ModelLoader() = default;