feat: add SenseNova U1.5 support (#1935)

This commit is contained in:
Maphist0 authored and GitHub committed 2026-09-12 00:39:59 +08:00
1 parent e06b205384
commit 7f986a9d73
19 files changed
+1173 -15

No files matched your search

+3
View File
@@ -2,12 +2,15 @@
#define __SD_TOKENIZERS_QWEN2_TOKENIZER_H__
#include <string>
#include <vector>
#include "bpe_tokenizer.h"
class Qwen2Tokenizer : public BPETokenizer {
protected:
void load_from_merges(const std::string& merges_utf8_str);
Qwen2Tokenizer(const std::string& merges_utf8_str,
const std::vector<std::string>& special_tokens_override);
public:
explicit Qwen2Tokenizer(const std::string& merges_utf8_str = "");