DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 03:24:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: require external Gemma 2 and GPT-OSS tokenizers (#1974)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-09-15 01:25:13 +08:00
1 parent 4964abdfc5
commit f9ddc0f388
19 files changed
+73 -337
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/model/te/llm.hpp
+21 -7
View File
@@ -15,6 +15,7 @@
#include <regex>
#include <set>
#include <sstream>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
@@ -31,9 +32,9 @@
#include "model_manager.h"
#include "tokenizers/bpe_tokenizer.h"
#include "tokenizers/gemma_tokenizer.h"
#include "tokenizers/gpt_oss_tokenizer.h"
#include "tokenizers/mistral_tokenizer.h"
#include "tokenizers/qwen2_tokenizer.h"
#include "tokenizers/tokenizer_config.h"
namespace LLM {
constexpr int LLM_GRAPH_SIZE = 65536;
@@ -2351,7 +2352,7 @@ namespace LLM {
};
struct LLMEmbedder {
std::shared_ptr<BPETokenizer> tokenizer;
std::shared_ptr<Tokenizer> tokenizer;
LLMRunner model;
LLMEmbedder(LLMArch arch,
@@ -2359,14 +2360,27 @@ namespace LLM {
const String2TensorStorage& tensor_storage_map = {},
const std::string prefix = "",
bool enable_vision = false,
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
const TokenizerConfig& tokenizers = {})
: model(arch, backend, tensor_storage_map, prefix, enable_vision, weight_manager) {
int pad_id = 151643;
if (arch == LLMArch::MISTRAL_SMALL_3_2 || arch == LLMArch::MINISTRAL_3_3B) {
tokenizer = std::make_shared<MistralTokenizer>();
pad_id = 11;
} else if (arch == LLMArch::GPT_OSS_20B) {
tokenizer = std::make_shared<GPTOSSTokenizer>();
} else {
tokenizer = std::make_shared<Qwen2Tokenizer>();
pad_id = 199999;
} else if (arch == LLMArch::GEMMA2_2B) {
pad_id = 0;
}
tokenizer = tokenizers.create(TokenizerConfig::MAIN, model.config.vocab_size, pad_id);
if (!tokenizer) {
if (arch == LLMArch::GPT_OSS_20B || arch == LLMArch::GEMMA2_2B) {
throw std::runtime_error("GPT-OSS and Gemma 2 require an external tokenizer.json in the main tokenizer slot");
}
if (arch == LLMArch::MISTRAL_SMALL_3_2 || arch == LLMArch::MINISTRAL_3_3B) {
tokenizer = std::make_shared<MistralTokenizer>();
} else {
tokenizer = std::make_shared<Qwen2Tokenizer>();
}
}
}