add qwen tokenizer

examples/cli/main.cpp

@@ -27,6 +27,8 @@
 
 #include "avi_writer.h"
 
+#include "qwen.hpp"
+
 #if defined(_WIN32)
 #define NOMINMAX
 #include <windows.h>
@@ -1138,6 +1140,20 @@ bool load_images_from_dir(const std::string dir,
 
 int main(int argc, const char* argv[]) {
     SDParams params;
+    params.verbose = true;
+    sd_set_log_callback(sd_log_cb, (void*)&params);
+    auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
+        return false;
+    };
+    // auto tokenizer = CLIPTokenizer();
+    auto tokenizer = Qwen::Qwen2Tokenizer();
+    std::string text("a lovely cat");
+    auto tokens = tokenizer.encode(text, on_new_token_cb);
+    for (auto token : tokens) {
+        std::cout << token << " ";
+    }
+    std::cout << std::endl;
+    exit(1);
     parse_args(argc, argv, params);
     params.sample_params.guidance.slg.layers                 = params.skip_layers.data();
     params.sample_params.guidance.slg.layer_count            = params.skip_layers.size();