mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-11-11 19:36:33 +00:00
47 lines
1.5 KiB
C++
47 lines
1.5 KiB
C++
#ifndef __SD_TOKENIZERS_BPE_TOKENIZER_H__
|
|
#define __SD_TOKENIZERS_BPE_TOKENIZER_H__
|
|
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
#include <functional>
|
|
#include <map>
|
|
#include <memory>
|
|
#include <set>
|
|
#include <string>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#include "core/regex.h"
|
|
#include "tokenizer.h"
|
|
|
|
class BPETokenizer : public Tokenizer {
|
|
private:
|
|
std::unique_ptr<sd::Regex> split_regex_;
|
|
|
|
protected:
|
|
std::map<int, std::u32string> byte_encoder;
|
|
std::map<std::u32string, int> byte_decoder;
|
|
std::map<std::u32string, int> encoder;
|
|
std::map<int, std::u32string> decoder;
|
|
std::map<std::pair<std::u32string, std::u32string>, int> bpe_ranks;
|
|
int encoder_len = 0;
|
|
int bpe_len = 0;
|
|
bool byte_level_bpe = true;
|
|
bool byte_fallback = false;
|
|
|
|
protected:
|
|
static std::vector<std::pair<int, std::u32string>> bytes_to_unicode();
|
|
static std::vector<std::u32string> split_utf32(const std::string& text, char32_t delimiter = U'\n');
|
|
bool token_split(const std::string& text, std::vector<std::string>& tokens, std::string* error = nullptr) const;
|
|
std::vector<std::u32string> bpe(const std::u32string& token) const;
|
|
std::string decode_token(int token_id) const override;
|
|
|
|
public:
|
|
explicit BPETokenizer(const std::string& pattern);
|
|
virtual ~BPETokenizer() = default;
|
|
|
|
bool encode(const std::string& text, std::vector<int>& tokens, on_new_token_cb_t on_new_token_cb = nullptr, std::string* error = nullptr) override;
|
|
};
|
|
|
|
#endif // __SD_TOKENIZERS_BPE_TOKENIZER_H__
|