mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-06-24 23:26:43 +00:00
67dda3f897
* add GemmaTokenizer * add basic ltx2.3 support * change vocab file encoding * fix ci * fix ubuntu build * add temporal tiling support * add ltx audio support * update ggml submodule url * fix generate_video * add i2v support * minify bundled Gemma tokenizer vocab sources * pass video fps into temporal rope embeddings * fix av_ca_timestep_scale_multiplier * add LTX2Scheduler support * update docs * fix ci
18 lines
527 B
C++
18 lines
527 B
C++
#ifndef __SD_TOKENIZERS_GEMMA_TOKENIZER_H__
|
|
#define __SD_TOKENIZERS_GEMMA_TOKENIZER_H__
|
|
|
|
#include <string>
|
|
|
|
#include "bpe_tokenizer.h"
|
|
|
|
class GemmaTokenizer : public BPETokenizer {
|
|
protected:
|
|
void load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str);
|
|
std::string normalize(const std::string& text) const override;
|
|
|
|
public:
|
|
explicit GemmaTokenizer(const std::string& merges_utf8_str = "", const std::string& vocab_utf8_str = "");
|
|
};
|
|
|
|
#endif // __SD_TOKENIZERS_GEMMA_TOKENIZER_H__
|