add GemmaTokenizer

examples/cli/CMakeLists.txt

@@ -7,6 +7,10 @@ add_executable(${TARGET}
     image_metadata.cpp
     main.cpp
 )
+target_include_directories(${TARGET} PRIVATE
+    "${CMAKE_CURRENT_SOURCE_DIR}/.."
+    "${PROJECT_SOURCE_DIR}/src"
+)
 install(TARGETS ${TARGET} RUNTIME)
 target_link_libraries(${TARGET} PRIVATE stable-diffusion zip ${CMAKE_THREAD_LIBS_INIT})
 if(SD_WEBP)

examples/cli/main.cpp

@@ -19,6 +19,7 @@
 #include "common/media_io.h"
 #include "common/resource_owners.hpp"
 #include "image_metadata.h"
+#include "llm.hpp"
 
 namespace fs = std::filesystem;
 
@@ -498,6 +499,15 @@ int main(int argc, const char* argv[]) {
     SDContextParams ctx_params;
     SDGenerationParams gen_params;
 
+    cli_params.verbose = true;
+    sd_set_log_callback(sd_log_cb, (void*)&cli_params);
+    LLM::GemmaTokenizer tokenizer;
+    auto tokens = tokenizer.tokenize("<html> 一只可爱的小猫");
+    for (auto token : tokens) {
+        LOG_INFO("%d", token);
+    }
+    return 0;
+
     parse_args(argc, argv, cli_params, ctx_params, gen_params);
     sd_set_log_callback(sd_log_cb, (void*)&cli_params);
     log_verbose = cli_params.verbose;