feat: support external Hugging Face tokenizer JSON files (#1973)

CMakeLists.txt

@@ -342,6 +342,7 @@ add_subdirectory(thirdparty)
 
 target_sources(${SD_LIB} PRIVATE $<TARGET_OBJECTS:zip>)
 target_link_libraries(${SD_LIB} PUBLIC ggml)
+target_link_libraries(${SD_LIB} PRIVATE onig sd-utf8proc)
 target_include_directories(${SD_LIB} PUBLIC . src include)
 target_include_directories(${SD_LIB} PRIVATE src/core)
 target_include_directories(${SD_LIB} PUBLIC . thirdparty)

docs/tokenizers.md

@@ -0,0 +1,97 @@
+# JSON tokenizers
+
+Use a Hugging Face `tokenizer.json` to supply the tokenizer vocabulary, merges,
+added tokens, and processing stages. Without this option, sd.cpp keeps its
+embedded tokenizer for the selected model.
+
+```shell
+sd-cli --diffusion-model model.gguf --llm text_encoder.gguf \
+  --tokenizer tokenizer_gemma2.json --vae vae.safetensors -p "a cat"
+```
+
+Choose the JSON belonging to the text encoder checkpoint. Checking that IDs fit
+the embedding table does not establish that two vocabularies have the same
+meaning. The JSON file is loaded when the text encoder is created; its embedded
+vocabulary is not loaded in this case.
+
+| Option | Encoder |
+| --- | --- |
+| `--tokenizer FILE` | Main LLM/BPE encoder: Gemma 2, Gemma 3, Qwen 2/3, Mistral, GPT-OSS; also Anima and HiDream-O1 |
+| `--tokenizer FILE` | Shared CLIP tokenizer in SD1/SD2/SDXL, or CLIP-L in Flux |
+| `--tokenizer clip-l=FILE` | Separate CLIP-L in SD3 or Flux |
+| `--tokenizer clip-g=FILE` | Separate CLIP-G in SD3 |
+
+Use comma-separated assignments to configure multiple slots, for example
+`--tokenizer main=main.json,clip-l=clip_l.json,clip-g=clip_g.json`.
+A plain file path is equivalent to `main=FILE`. You may also repeat `--tokenizer`
+with explicit assignments, such as `--tokenizer main=main.json --tokenizer clip-l=clip.json`.
+Empty assignment paths, unknown keys, malformed assignments and
+duplicate slots are rejected. Commas separate entries in the assignment form;
+quote the complete argument when paths contain spaces.
+
+SD3 overrides must name the `clip-l` or `clip-g` slot. SDXL uses one shared
+tokenizer for both CLIP encoders. Do not supply both `main` and `clip-l` for Flux.
+A slot targeting an absent or unsupported encoder fails initialization.
+T5/SentencePiece Unigram tokenizers are outside this implementation's scope.
+
+For example, SD3 can load the same CLIP JSON into both slots:
+
+```shell
+sd-cli --diffusion-model sd3.gguf --clip_l clip_l.safetensors \
+  --clip_g clip_g.safetensors --t5xxl t5xxl.gguf --vae vae.safetensors \
+  --tokenizer clip-l=tokenizer_clip.json,clip-g=tokenizer_clip.json \
+  -p "a cat"
+```
+
+The C API accepts the same string in `sd_ctx_params_t::tokenizer`. A null or
+empty value keeps the embedded tokenizers. The CLI passes the string through;
+`TokenizerConfig` parses and validates it when text encoders are initialized.
+
+```c
+sd_ctx_params_t params;
+sd_ctx_params_init(&params);
+params.tokenizer = "clip-l=tokenizer_clip.json,clip-g=tokenizer_clip.json";
+```
+
+Rebuild applications against the updated public header when using the updated
+library.
+
+## Supported components
+
+| Stage | Supported configurations |
+| --- | --- |
+| Normalizer | `Sequence`, `NFC`, `Lowercase`, `Replace` with String/Regex patterns |
+| PreTokenizer | `Sequence`, `Split` with String/Regex patterns, all five delimiter behaviors and `invert`; `ByteLevel` with `add_prefix_space` and `use_regex` |
+| Model | Deterministic `BPE`, string or array-pair merges, `unk_token`, `fuse_unk`, `byte_fallback`, `ignore_merges`, `end_of_word_suffix` |
+| PostProcessor | Single-sequence `TemplateProcessing` with at most one prefix and one suffix token, `RobertaProcessing`, `ByteLevel` |
+| Decoder | `Sequence`, `Replace`, `ByteLevel`, `ByteFallback`, `Fuse` |
+| AddedToken | Special and ordinary added tokens, original IDs, raw or normalized matching, leftmost-longest matching |
+
+`ByteLevel.use_regex` defaults to true when omitted. ByteLevel postprocessing
+changes offsets only and adds no tokens. Added tokens with `single_word`,
+`lstrip`, or `rstrip` enabled, nonzero BPE dropout, nonempty
+`continuing_subword_prefix`, and unsupported component types fail loading.
+New added-token IDs must follow the model vocabulary consecutively; configurations
+whose IDs Hugging Face would reassign are rejected.
+JSON `padding` and `truncation` must be null. This API returns IDs, not offsets,
+type IDs, or paired-input encodings; the pair template is not used.
+
+The pipeline covers the CLIP, Gemma 2, Gemma 3, GPT-OSS, Mistral 3, Qwen 2 and
+Qwen 3 JSON configurations used by the differential test. It does not imply
+support for every tokenizer published under those model names.
+
+## Prompt integration
+
+Prompt attention parsing and model-specific chat/image templates remain in the
+conditioner. Raw `encode()` does not add BOS/EOS. The conditioner concatenates
+weighted prompt fragments, then the existing padding/chunking step applies the
+JSON single-sequence template once per sequence or CLIP chunk. Padding ID,
+direction, length limits and attention masks remain text encoder policies.
+CLIP requires both BOS and EOS because its chunking reserves those positions.
+
+The internal `encode()`, `tokenize()`, and `decode()` interfaces return a success
+flag and write to an output parameter. A successful result may be empty; a failed
+call clears its output. JSON tokenizer input, normalization, and regex failures
+return `false` with diagnostic information instead of throwing. Invalid
+JSON, unsupported stages, conflicting IDs and IDs outside the encoder embedding
+table fail initialization instead of falling back to the embedded tokenizer.

examples/common/common.cpp

@@ -410,6 +410,11 @@ ArgOptions SDContextParams::get_options() {
          "path to the llm text encoder. For example: (qwenvl2.5 for qwen-image, mistral-small3.2 for flux2, ...)",
          0,
          &llm_path},
+        {"",
+         "--tokenizer",
+         "tokenizer.json path, or comma-separated main=FILE,clip-l=FILE,clip-g=FILE assignments",
+         (int)',',
+         &tokenizer},
         {"",
          "--llm_vision",
          "path to the llm vit",
@@ -896,6 +901,7 @@ std::string SDContextParams::to_string() const {
         << "  t5xxl_path: \"" << t5xxl_path << "\",\n"
         << "  llm_path: \"" << llm_path << "\",\n"
         << "  llm_vision_path: \"" << llm_vision_path << "\",\n"
+        << "  tokenizer: \"" << tokenizer << "\",\n"
         << "  diffusion_model_path: \"" << diffusion_model_path << "\",\n"
         << "  high_noise_diffusion_model_path: \"" << high_noise_diffusion_model_path << "\",\n"
         << "  uncond_diffusion_model_path: \"" << uncond_diffusion_model_path << "\",\n"
@@ -963,6 +969,7 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) {
     sd_ctx_params.t5xxl_path                      = t5xxl_path.c_str();
     sd_ctx_params.llm_path                        = llm_path.c_str();
     sd_ctx_params.llm_vision_path                 = llm_vision_path.c_str();
+    sd_ctx_params.tokenizer                       = tokenizer.c_str();
     sd_ctx_params.diffusion_model_path            = diffusion_model_path.c_str();
     sd_ctx_params.high_noise_diffusion_model_path = high_noise_diffusion_model_path.c_str();
     sd_ctx_params.uncond_diffusion_model_path     = uncond_diffusion_model_path.c_str();

examples/common/common.h

@@ -124,6 +124,7 @@ struct SDContextParams {
     std::string t5xxl_path;
     std::string llm_path;
     std::string llm_vision_path;
+    std::string tokenizer;
     std::string diffusion_model_path;
     std::string high_noise_diffusion_model_path;
     std::string uncond_diffusion_model_path;

include/stable-diffusion.h

@@ -244,6 +244,7 @@ typedef struct {
     bool disable_segmented_compute;  // Force monolithic graph execution even when automatic graph cutting would fit memory better
     float linear_scale;              // Override linear input scaling; 0 keeps the model default
     float attn_scale;                // Override flash-attention K/V scaling; 0 keeps the model default
+    const char* tokenizer;           // Optional tokenizer.json path or main=FILE,clip-l=FILE,clip-g=FILE assignments
 } sd_ctx_params_t;
 
 typedef struct {

src/conditioning/conditioner.hpp

@@ -17,6 +17,7 @@
 #include "model/te/t5.hpp"
 #include "model_loader.h"
 #include "tokenizers/sensenova_u1_tokenizer.h"
+#include "tokenizers/tokenizer_config.h"
 
 struct SDCondition {
     sd::Tensor<float> c_crossattn;
@@ -159,7 +160,7 @@ public:
 // Ref: https://github.com/AUTOMATIC1111/stable-diffusion-webui/blob/cad87bf4e3e0b0a759afa94e933527c3123d59bc/modules/sd_hijack_clip.py#L283
 struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
     SDVersion version = VERSION_SD1;
-    CLIPTokenizer tokenizer;
+    std::shared_ptr<Tokenizer> tokenizer;
     std::shared_ptr<CLIPTextModelRunner> text_model;
     std::shared_ptr<CLIPTextModelRunner> text_model2;
 
@@ -173,12 +174,18 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
                                       const String2TensorStorage& tensor_storage_map,
                                       const std::map<std::string, std::string>& orig_embedding_map,
                                       SDVersion version                                   = VERSION_SD1,
-                                      std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+                                      std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
-        : version(version), tokenizer(sd_version_is_sd2(version) ? 0 : 49407) {
+                                      const TokenizerConfig& tokenizers                   = {})
+        : version(version) {
+        const int pad_id = sd_version_is_sd2(version) ? 0 : 49407;
+        tokenizer        = tokenizers.create(TokenizerConfig::MAIN, 49408, pad_id, false, true);
+        if (!tokenizer) {
+            tokenizer = std::make_shared<CLIPTokenizer>(pad_id);
+        }
         for (const auto& kv : orig_embedding_map) {
             std::string name    = normalize_embedding_name(kv.first);
             embedding_map[name] = kv.second;
-            tokenizer.add_special_token(name);
+            tokenizer->add_special_token(name);
         }
         bool force_clip_f32 = !embedding_map.empty();
         if (sd_version_is_sd1(version)) {
@@ -365,16 +372,15 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
         return load_embedding(name, iter->second, bpe_tokens);
     }
 
-    std::vector<int> convert_token_to_id(std::string text) {
+    bool convert_token_to_id(const std::string& text, std::vector<int>& tokens) {
         auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
             return append_embedding_tokens(str, bpe_tokens);
         };
-        std::vector<int> curr_tokens = tokenizer.encode(text, on_new_token_cb);
+        return tokenizer->encode(text, tokens, on_new_token_cb);
-        return curr_tokens;
     }
 
-    std::string decode(const std::vector<int>& tokens) {
+    bool decode(const std::vector<int>& tokens, std::string& text) {
-        return tokenizer.decode(tokens);
+        return tokenizer->decode(tokens, text);
     }
 
     std::pair<std::vector<int>, std::vector<float>> tokenize(std::string text,
@@ -412,18 +418,21 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
 
                 if (padding_size > 0) {
                     LOG_VERBOSE("BREAK token encountered, padding current chunk by %zu tokens.", padding_size);
-                    tokens.insert(tokens.end(), padding_size, tokenizer.EOS_TOKEN_ID);
+                    tokens.insert(tokens.end(), padding_size, tokenizer->EOS_TOKEN_ID);
                     weights.insert(weights.end(), padding_size, 1.0f);
                 }
                 continue;  // Skip to the next item after handling BREAK
             }
 
-            std::vector<int> curr_tokens = tokenizer.encode(curr_text, on_new_token_cb);
+            std::vector<int> curr_tokens;
+            if (!tokenizer->encode(curr_text, curr_tokens, on_new_token_cb)) {
+                return {};
+            }
             tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
             weights.insert(weights.end(), curr_tokens.size(), curr_weight);
         }
 
-        tokenizer.pad_tokens(tokens, &weights, nullptr, min_length, max_length, allow_overflow_expand);
+        tokenizer->pad_tokens(tokens, &weights, nullptr, min_length, max_length, allow_overflow_expand);
 
         // for (int i = 0; i < tokens.size(); i++) {
         //     std::cout << tokens[i] << ":" << weights[i] << ", ";
@@ -460,7 +469,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
             sd::Tensor<int32_t> input_ids2;
             size_t max_token_idx = 0;
             if (sd_version_is_sdxl(version)) {
-                auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), tokenizer.EOS_TOKEN_ID);
+                auto it = std::find(chunk_tokens.begin(), chunk_tokens.end(), tokenizer->EOS_TOKEN_ID);
                 if (it != chunk_tokens.end()) {
                     std::fill(std::next(it), chunk_tokens.end(), 0);
                 }
@@ -561,7 +570,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
 
     SDCondition get_learned_condition(int n_threads,
                                       const ConditionerParams& conditioner_params) override {
-        auto tokens_and_weights     = tokenize(conditioner_params.text, text_model->model.n_token, text_model->model.n_token, true);
+        auto tokens_and_weights = tokenize(conditioner_params.text, text_model->model.n_token, text_model->model.n_token, true);
+        if (tokens_and_weights.first.empty()) {
+            return {};
+        }
         std::vector<int>& tokens    = tokens_and_weights.first;
         std::vector<float>& weights = tokens_and_weights.second;
         return get_learned_condition_common(n_threads,
@@ -629,8 +641,8 @@ struct FrozenCLIPVisionEmbedder : public GGMLRunner {
 };
 
 struct SD3CLIPEmbedder : public Conditioner {
-    CLIPTokenizer clip_l_tokenizer;
+    std::shared_ptr<Tokenizer> clip_l_tokenizer;
-    CLIPTokenizer clip_g_tokenizer;
+    std::shared_ptr<Tokenizer> clip_g_tokenizer;
     T5UniGramTokenizer t5_tokenizer;
     std::shared_ptr<CLIPTextModelRunner> clip_l;
     std::shared_ptr<CLIPTextModelRunner> clip_g;
@@ -638,8 +650,8 @@ struct SD3CLIPEmbedder : public Conditioner {
 
     SD3CLIPEmbedder(ggml_backend_t backend,
                     const String2TensorStorage& tensor_storage_map      = {},
-                    std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+                    std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
-        : clip_g_tokenizer(0) {
+                    const TokenizerConfig& tokenizers                   = {}) {
         bool use_clip_l = false;
         bool use_clip_g = false;
         bool use_t5     = false;
@@ -657,9 +669,17 @@ struct SD3CLIPEmbedder : public Conditioner {
             return;
         }
         if (use_clip_l) {
+            clip_l_tokenizer = tokenizers.create(TokenizerConfig::CLIP_L, 49408, 49407, false, true);
+            if (!clip_l_tokenizer) {
+                clip_l_tokenizer = std::make_shared<CLIPTokenizer>();
+            }
             clip_l = std::make_shared<CLIPTextModelRunner>(backend, tensor_storage_map, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, false, false, weight_manager);
         }
         if (use_clip_g) {
+            clip_g_tokenizer = tokenizers.create(TokenizerConfig::CLIP_G, 49408, 0, false, true);
+            if (!clip_g_tokenizer) {
+                clip_g_tokenizer = std::make_shared<CLIPTokenizer>(0);
+            }
             clip_g = std::make_shared<CLIPTextModelRunner>(backend, tensor_storage_map, "text_encoders.clip_g.transformer.text_model", OPEN_CLIP_VIT_BIGG_14, false, false, weight_manager);
         }
         if (use_t5) {
@@ -811,27 +831,36 @@ struct SD3CLIPEmbedder : public Conditioner {
             const std::string& curr_text = item.first;
             float curr_weight            = item.second;
             if (clip_l) {
-                std::vector<int> curr_tokens = clip_l_tokenizer.encode(curr_text, on_new_token_cb);
+                std::vector<int> curr_tokens;
+                if (!clip_l_tokenizer->encode(curr_text, curr_tokens, on_new_token_cb)) {
+                    return {};
+                }
                 clip_l_tokens.insert(clip_l_tokens.end(), curr_tokens.begin(), curr_tokens.end());
                 clip_l_weights.insert(clip_l_weights.end(), curr_tokens.size(), curr_weight);
             }
             if (clip_g) {
-                std::vector<int> curr_tokens = clip_g_tokenizer.encode(curr_text, on_new_token_cb);
+                std::vector<int> curr_tokens;
+                if (!clip_g_tokenizer->encode(curr_text, curr_tokens, on_new_token_cb)) {
+                    return {};
+                }
                 clip_g_tokens.insert(clip_g_tokens.end(), curr_tokens.begin(), curr_tokens.end());
                 clip_g_weights.insert(clip_g_weights.end(), curr_tokens.size(), curr_weight);
             }
             if (t5) {
-                std::vector<int> curr_tokens = t5_tokenizer.encode(curr_text);
+                std::vector<int> curr_tokens;
+                if (!t5_tokenizer.encode(curr_text, curr_tokens)) {
+                    return {};
+                }
                 t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
                 t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
             }
         }
 
         if (clip_l) {
-            clip_l_tokenizer.pad_tokens(clip_l_tokens, &clip_l_weights, nullptr, min_length, max_length, allow_overflow_expand);
+            clip_l_tokenizer->pad_tokens(clip_l_tokens, &clip_l_weights, nullptr, min_length, max_length, allow_overflow_expand);
         }
         if (clip_g) {
-            clip_g_tokenizer.pad_tokens(clip_g_tokens, &clip_g_weights, nullptr, min_length, max_length, allow_overflow_expand);
+            clip_g_tokenizer->pad_tokens(clip_g_tokens, &clip_g_weights, nullptr, min_length, max_length, allow_overflow_expand);
         }
         if (t5) {
             t5_tokenizer.pad_tokens(t5_tokens, &t5_weights, nullptr, min_length, max_length, true);
@@ -902,7 +931,7 @@ struct SD3CLIPEmbedder : public Conditioner {
                 chunk_hidden_states_l = ::apply_token_weights(std::move(chunk_hidden_states_l), chunk_weights);
 
                 if (chunk_idx == 0) {
-                    auto it       = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_l_tokenizer.EOS_TOKEN_ID);
+                    auto it       = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_l_tokenizer->EOS_TOKEN_ID);
                     max_token_idx = std::min<size_t>(std::distance(chunk_tokens.begin(), it), chunk_tokens.size() - 1);
                     pooled_l      = clip_l->compute(n_threads,
                                                     input_ids,
@@ -945,7 +974,7 @@ struct SD3CLIPEmbedder : public Conditioner {
                 chunk_hidden_states_g = ::apply_token_weights(std::move(chunk_hidden_states_g), chunk_weights);
 
                 if (chunk_idx == 0) {
-                    auto it       = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_g_tokenizer.EOS_TOKEN_ID);
+                    auto it       = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_g_tokenizer->EOS_TOKEN_ID);
                     max_token_idx = std::min<size_t>(std::distance(chunk_tokens.begin(), it), chunk_tokens.size() - 1);
                     pooled_g      = clip_g->compute(n_threads,
                                                     input_ids,
@@ -1023,6 +1052,9 @@ struct SD3CLIPEmbedder : public Conditioner {
     SDCondition get_learned_condition(int n_threads,
                                       const ConditionerParams& conditioner_params) override {
         auto tokens_and_weights = tokenize(conditioner_params.text, 77, 77, true);
+        if (tokens_and_weights.empty()) {
+            return {};
+        }
         return get_learned_condition_common(n_threads,
                                             tokens_and_weights,
                                             conditioner_params.clip_skip,
@@ -1031,7 +1063,7 @@ struct SD3CLIPEmbedder : public Conditioner {
 };
 
 struct FluxCLIPEmbedder : public Conditioner {
-    CLIPTokenizer clip_l_tokenizer;
+    std::shared_ptr<Tokenizer> clip_l_tokenizer;
     T5UniGramTokenizer t5_tokenizer;
     std::shared_ptr<CLIPTextModelRunner> clip_l;
     std::shared_ptr<T5Runner> t5;
@@ -1039,7 +1071,8 @@ struct FluxCLIPEmbedder : public Conditioner {
 
     FluxCLIPEmbedder(ggml_backend_t backend,
                      const String2TensorStorage& tensor_storage_map      = {},
-                     std::shared_ptr<RunnerWeightManager> weight_manager = nullptr) {
+                     std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
+                     const TokenizerConfig& tokenizers                   = {}) {
         bool use_clip_l = false;
         bool use_t5     = false;
         for (auto pair : tensor_storage_map) {
@@ -1056,6 +1089,11 @@ struct FluxCLIPEmbedder : public Conditioner {
         }
 
         if (use_clip_l) {
+            auto slot        = tokenizers.has(TokenizerConfig::CLIP_L) ? TokenizerConfig::CLIP_L : TokenizerConfig::MAIN;
+            clip_l_tokenizer = tokenizers.create(slot, 49408, 49407, false, true);
+            if (!clip_l_tokenizer) {
+                clip_l_tokenizer = std::make_shared<CLIPTokenizer>();
+            }
             clip_l = std::make_shared<CLIPTextModelRunner>(backend, tensor_storage_map, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, true, false, weight_manager);
         } else {
             LOG_WARN("clip_l text encoder not found! Prompt adherence might be degraded.");
@@ -1181,19 +1219,25 @@ struct FluxCLIPEmbedder : public Conditioner {
             const std::string& curr_text = item.first;
             float curr_weight            = item.second;
             if (clip_l) {
-                std::vector<int> curr_tokens = clip_l_tokenizer.encode(curr_text, on_new_token_cb);
+                std::vector<int> curr_tokens;
+                if (!clip_l_tokenizer->encode(curr_text, curr_tokens, on_new_token_cb)) {
+                    return {};
+                }
                 clip_l_tokens.insert(clip_l_tokens.end(), curr_tokens.begin(), curr_tokens.end());
                 clip_l_weights.insert(clip_l_weights.end(), curr_tokens.size(), curr_weight);
             }
             if (t5) {
-                std::vector<int> curr_tokens = t5_tokenizer.encode(curr_text);
+                std::vector<int> curr_tokens;
+                if (!t5_tokenizer.encode(curr_text, curr_tokens)) {
+                    return {};
+                }
                 t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
                 t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
             }
         }
 
         if (clip_l) {
-            clip_l_tokenizer.pad_tokens(clip_l_tokens, &clip_l_weights, nullptr, 77, 77, true);
+            clip_l_tokenizer->pad_tokens(clip_l_tokens, &clip_l_weights, nullptr, 77, 77, true);
         }
         if (t5) {
             t5_tokenizer.pad_tokens(t5_tokens, &t5_weights, nullptr, min_length, max_length, true);
@@ -1243,7 +1287,7 @@ struct FluxCLIPEmbedder : public Conditioner {
                     sd::Tensor<int32_t> input_ids({static_cast<int64_t>(chunk_tokens.size())}, chunk_tokens);
                     size_t max_token_idx = 0;
 
-                    auto it       = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_l_tokenizer.EOS_TOKEN_ID);
+                    auto it       = std::find(chunk_tokens.begin(), chunk_tokens.end(), clip_l_tokenizer->EOS_TOKEN_ID);
                     max_token_idx = std::min<size_t>(std::distance(chunk_tokens.begin(), it), chunk_tokens.size() - 1);
 
                     pooled = clip_l->compute(n_threads,
@@ -1306,6 +1350,9 @@ struct FluxCLIPEmbedder : public Conditioner {
     SDCondition get_learned_condition(int n_threads,
                                       const ConditionerParams& conditioner_params) override {
         auto tokens_and_weights = tokenize(conditioner_params.text, chunk_len, chunk_len);
+        if (tokens_and_weights.empty()) {
+            return {};
+        }
         return get_learned_condition_common(n_threads,
                                             tokens_and_weights,
                                             conditioner_params.clip_skip,
@@ -1449,7 +1496,10 @@ struct T5CLIPEmbedder : public Conditioner {
                 const std::string& curr_text = item.first;
                 float curr_weight            = item.second;
 
-                std::vector<int> curr_tokens = t5_tokenizer.encode(curr_text);
+                std::vector<int> curr_tokens;
+                if (!t5_tokenizer.encode(curr_text, curr_tokens)) {
+                    return {};
+                }
                 t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
                 t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
             }
@@ -1547,6 +1597,9 @@ struct T5CLIPEmbedder : public Conditioner {
     SDCondition get_learned_condition(int n_threads,
                                       const ConditionerParams& conditioner_params) override {
         auto tokens_and_weights = tokenize(conditioner_params.text, chunk_len, chunk_len);
+        if (std::get<0>(tokens_and_weights).empty()) {
+            return {};
+        }
         return get_learned_condition_common(n_threads,
                                             tokens_and_weights,
                                             conditioner_params.clip_skip,
@@ -1645,7 +1698,10 @@ struct MiniT2IConditioner : public Conditioner {
             return result;
         }
 
-        std::vector<int> tokens = tokenizer.encode(conditioner_params.text);
+        std::vector<int> tokens;
+        if (!tokenizer.encode(conditioner_params.text, tokens)) {
+            return {};
+        }
         if (tokens.size() > prompt_length) {
             tokens.resize(prompt_length);
         }
@@ -1712,7 +1768,10 @@ struct SenseNovaU1Conditioner : public Conditioner {
     }
 
     SDCondition tokenize_condition(const std::string& text, bool is_negative) {
-        auto tokens = tokenizer.encode(build_query(text, is_negative));
+        std::vector<int> tokens;
+        if (!tokenizer.encode(build_query(text, is_negative), tokens)) {
+            return {};
+        }
         if (tokens.empty() || tokens.size() > kMaxPromptTokens) {
             LOG_ERROR("SenseNova U1.5 prompt token count %zu is outside [1, %zu]",
                       tokens.size(),
@@ -1737,20 +1796,24 @@ struct SenseNovaU1Conditioner : public Conditioner {
 };
 
 struct AnimaConditioner : public Conditioner {
-    std::shared_ptr<BPETokenizer> qwen_tokenizer;
+    std::shared_ptr<Tokenizer> qwen_tokenizer;
     T5UniGramTokenizer t5_tokenizer;
     std::shared_ptr<LLM::LLMRunner> llm;
 
     AnimaConditioner(ggml_backend_t backend,
                      const String2TensorStorage& tensor_storage_map      = {},
-                     std::shared_ptr<RunnerWeightManager> weight_manager = nullptr) {
+                     std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
-        qwen_tokenizer = std::make_shared<Qwen2Tokenizer>();
+                     const TokenizerConfig& tokenizers                   = {}) {
         llm            = std::make_shared<LLM::LLMRunner>(LLM::LLMArch::QWEN3,
                                                backend,
                                                tensor_storage_map,
                                                "text_encoders.llm",
                                                false,
                                                weight_manager);
+        qwen_tokenizer = tokenizers.create(TokenizerConfig::MAIN, llm->config.vocab_size, 151643);
+        if (!qwen_tokenizer) {
+            qwen_tokenizer = std::make_shared<Qwen2Tokenizer>();
+        }
     }
 
     void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
@@ -1817,7 +1880,10 @@ struct AnimaConditioner : public Conditioner {
 
         for (const auto& item : parsed_attention) {
             const std::string& curr_text = item.first;
-            std::vector<int> curr_tokens = qwen_tokenizer->tokenize(curr_text, nullptr);
+            std::vector<int> curr_tokens;
+            if (!qwen_tokenizer->tokenize(curr_text, curr_tokens, nullptr)) {
+                return {};
+            }
             qwen_tokens.insert(qwen_tokens.end(), curr_tokens.begin(), curr_tokens.end());
             // Anima uses uniform Qwen token weights.
             qwen_weights.insert(qwen_weights.end(), curr_tokens.size(), 1.f);
@@ -1830,7 +1896,10 @@ struct AnimaConditioner : public Conditioner {
         for (const auto& item : parsed_attention) {
             const std::string& curr_text = item.first;
             float curr_weight            = item.second;
-            std::vector<int> curr_tokens = t5_tokenizer.encode(curr_text);
+            std::vector<int> curr_tokens;
+            if (!t5_tokenizer.encode(curr_text, curr_tokens)) {
+                return {};
+            }
             t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
             t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
         }
@@ -1849,6 +1918,10 @@ struct AnimaConditioner : public Conditioner {
         auto& t5_tokens    = std::get<2>(tokenized);
         auto& t5_weights   = std::get<3>(tokenized);
 
+        if (qwen_tokens.empty()) {
+            return {};
+        }
+
         sd::Tensor<int32_t> input_ids({static_cast<int64_t>(qwen_tokens.size()), 1}, qwen_tokens);
         auto hidden_states = llm->compute(n_threads,
                                           input_ids,
@@ -1875,7 +1948,7 @@ struct AnimaConditioner : public Conditioner {
 
 struct LLMEmbedder : public Conditioner {
     SDVersion version;
-    std::shared_ptr<BPETokenizer> tokenizer;
+    std::shared_ptr<Tokenizer> tokenizer;
     std::shared_ptr<LLM::LLMRunner> llm;
     std::shared_ptr<T5Runner> byt5;
 
@@ -1884,7 +1957,8 @@ struct LLMEmbedder : public Conditioner {
                 SDVersion version                                   = VERSION_QWEN_IMAGE,
                 const std::string prefix                            = "",
                 bool enable_vision                                  = false,
-                std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+                std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
+                const TokenizerConfig& tokenizers                   = {})
         : version(version) {
         LLM::LLMArch arch = LLM::LLMArch::QWEN2_5_VL;
         if (version == VERSION_FLUX2) {
@@ -1906,21 +1980,32 @@ struct LLMEmbedder : public Conditioner {
         } else if (sd_version_is_z_image(version) || version == VERSION_OVIS_IMAGE || version == VERSION_FLUX2_KLEIN) {
             arch = LLM::LLMArch::QWEN3;
         }
-        if (arch == LLM::LLMArch::MISTRAL_SMALL_3_2 || arch == LLM::LLMArch::MINISTRAL_3_3B) {
+        llm        = std::make_shared<LLM::LLMRunner>(arch,
-            tokenizer = std::make_shared<MistralTokenizer>();
-        } else if (arch == LLM::LLMArch::GPT_OSS_20B) {
-            tokenizer = std::make_shared<GPTOSSTokenizer>();
-        } else if (arch == LLM::LLMArch::GEMMA2_2B) {
-            tokenizer = std::make_shared<Gemma2Tokenizer>();
-        } else {
-            tokenizer = std::make_shared<Qwen2Tokenizer>();
-        }
-        llm = std::make_shared<LLM::LLMRunner>(arch,
                                                backend,
                                                tensor_storage_map,
                                                "text_encoders.llm",
                                                enable_vision,
                                                weight_manager);
+        int pad_id = 151643;
+        if (arch == LLM::LLMArch::MISTRAL_SMALL_3_2 || arch == LLM::LLMArch::MINISTRAL_3_3B) {
+            pad_id = 11;
+        } else if (arch == LLM::LLMArch::GPT_OSS_20B) {
+            pad_id = 199999;
+        } else if (arch == LLM::LLMArch::GEMMA2_2B) {
+            pad_id = 0;
+        }
+        tokenizer = tokenizers.create(TokenizerConfig::MAIN, llm->config.vocab_size, pad_id);
+        if (!tokenizer) {
+            if (arch == LLM::LLMArch::MISTRAL_SMALL_3_2 || arch == LLM::LLMArch::MINISTRAL_3_3B) {
+                tokenizer = std::make_shared<MistralTokenizer>();
+            } else if (arch == LLM::LLMArch::GPT_OSS_20B) {
+                tokenizer = std::make_shared<GPTOSSTokenizer>();
+            } else if (arch == LLM::LLMArch::GEMMA2_2B) {
+                tokenizer = std::make_shared<Gemma2Tokenizer>();
+            } else {
+                tokenizer = std::make_shared<Qwen2Tokenizer>();
+            }
+        }
         if (sd_version_is_hunyuan_video(version)) {
             const std::string byt5_prefix = "text_encoders.t5xxl.transformer";
             for (const auto& [name, _] : tensor_storage_map) {
@@ -2059,7 +2144,10 @@ struct LLMEmbedder : public Conditioner {
         for (const auto& item : parsed_attention) {
             const std::string& curr_text = item.first;
             float curr_weight            = item.second;
-            std::vector<int> curr_tokens = tokenizer->encode(curr_text, nullptr);
+            std::vector<int> curr_tokens;
+            if (!tokenizer->encode(curr_text, curr_tokens, nullptr)) {
+                return {};
+            }
             tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
             weights.insert(weights.end(), curr_tokens.size(), curr_weight);
         }
@@ -2092,6 +2180,10 @@ struct LLMEmbedder : public Conditioner {
         auto& weights            = std::get<1>(tokens_weights_mask);
         auto& mask               = std::get<2>(tokens_weights_mask);
 
+        if (tokens.empty()) {
+            return {};
+        }
+
         sd::Tensor<int32_t> input_ids({static_cast<int64_t>(tokens.size())}, tokens);
         sd::Tensor<float> attention_mask;
         if (!mask.empty()) {
@@ -2251,7 +2343,11 @@ struct LLMEmbedder : public Conditioner {
                     GGML_ASSERT(image_outputs.size() == 4);
                     auto image_embed = std::move(image_outputs[0]);
                     prompt += "<|vision_start|>";
-                    int image_embed_idx = static_cast<int>(tokenizer->encode(prompt, nullptr).size());
+                    std::vector<int> prefix_tokens;
+                    if (!tokenizer->encode(prompt, prefix_tokens, nullptr)) {
+                        return false;
+                    }
+                    int image_embed_idx = static_cast<int>(prefix_tokens.size());
                     image_embeds.emplace_back(image_embed_idx, image_embed);
                     if (deepstack_image_embeds.empty()) {
                         deepstack_image_embeds.resize(image_outputs.size() - 1);
@@ -2267,6 +2363,7 @@ struct LLMEmbedder : public Conditioner {
                         prompt += placeholder;
                     }
                     prompt += "<|vision_end|>";
+                    return true;
                 };
 
                 const auto* references = conditioner_params.minimax_h3_references;
@@ -2283,11 +2380,13 @@ struct LLMEmbedder : public Conditioner {
                             GGML_ASSERT(item.frames.size() == 1);
                             auto resized = resize_for_vision(item.frames[0]);
                             prompt += "<Picture " + std::to_string(++picture_index) + ">: ";
-                            add_vision_outputs(llm->encode_image_outputs(n_threads,
+                            if (!add_vision_outputs(llm->encode_image_outputs(n_threads,
-                                                                         resized,
+                                                                              resized,
-                                                                         false),
+                                                                              false),
-                                               static_cast<int>(resized.shape()[1]) / patch_size,
+                                                    static_cast<int>(resized.shape()[1]) / patch_size,
-                                               static_cast<int>(resized.shape()[0]) / patch_size);
+                                                    static_cast<int>(resized.shape()[0]) / patch_size)) {
+                                return {};
+                            }
                             continue;
                         }
 
@@ -2311,22 +2410,26 @@ struct LLMEmbedder : public Conditioner {
                                                                                    second.shape()[3]});
                             }
                             auto pair = sd::ops::concat(first.unsqueeze(2), second.unsqueeze(2), 2);
-                            add_vision_outputs(llm->encode_video_block_outputs(n_threads,
+                            if (!add_vision_outputs(llm->encode_video_block_outputs(n_threads,
-                                                                               pair,
+                                                                                    pair,
-                                                                               false),
+                                                                                    false),
-                                               static_cast<int>(first.shape()[1]) / patch_size,
+                                                    static_cast<int>(first.shape()[1]) / patch_size,
-                                               static_cast<int>(first.shape()[0]) / patch_size);
+                                                    static_cast<int>(first.shape()[0]) / patch_size)) {
+                                return {};
+                            }
                         }
                     }
                 } else if (conditioner_params.ref_images != nullptr) {
                     for (size_t i = 0; i < conditioner_params.ref_images->size(); ++i) {
                         auto resized = resize_for_vision((*conditioner_params.ref_images)[i]);
                         prompt += "<Picture " + std::to_string(i + 1) + ">: ";
-                        add_vision_outputs(llm->encode_image_outputs(n_threads,
+                        if (!add_vision_outputs(llm->encode_image_outputs(n_threads,
-                                                                     resized,
+                                                                          resized,
-                                                                     false),
+                                                                          false),
-                                           static_cast<int>(resized.shape()[1]) / patch_size,
+                                                static_cast<int>(resized.shape()[1]) / patch_size,
-                                           static_cast<int>(resized.shape()[0]) / patch_size);
+                                                static_cast<int>(resized.shape()[0]) / patch_size)) {
+                            return {};
+                        }
                     }
                 }
             }
@@ -2362,7 +2465,10 @@ struct LLMEmbedder : public Conditioner {
                 "enhanced description for the prompt below and avoid including any additional "
                 "commentary or evaluations:<|im_end|>\n<|im_start|>user\n";
 
-            auto prefix_tokens               = tokenizer->encode(prompt_prefix, nullptr);
+            std::vector<int> prefix_tokens;
+            if (!tokenizer->encode(prompt_prefix, prefix_tokens, nullptr)) {
+                return {};
+            }
             prompt_template_encode_start_idx = 0;
             for (int token : prefix_tokens) {
                 if (token != pad_token) {
@@ -2415,7 +2521,11 @@ struct LLMEmbedder : public Conditioner {
                     GGML_ASSERT(!image_embed.empty());
 
                     std::string image_prefix = prompt + img_prompt + "<|vision_start|>";
-                    int image_embed_idx      = static_cast<int>(tokenizer->encode(image_prefix, nullptr).size());
+                    std::vector<int> prefix_tokens;
+                    if (!tokenizer->encode(image_prefix, prefix_tokens, nullptr)) {
+                        return {};
+                    }
+                    int image_embed_idx = static_cast<int>(prefix_tokens.size());
                     image_embeds.emplace_back(image_embed_idx, image_embed);
 
                     img_prompt += "<|vision_start|>";
@@ -2557,7 +2667,11 @@ struct LLMEmbedder : public Conditioner {
                     GGML_ASSERT(!image_embed.empty());
 
                     std::string image_prefix = prompt_prefix + img_prompt + "<|vision_start|>";
-                    int image_embed_idx      = static_cast<int>(tokenizer->encode(image_prefix, nullptr).size());
+                    std::vector<int> prefix_tokens;
+                    if (!tokenizer->encode(image_prefix, prefix_tokens, nullptr)) {
+                        return {};
+                    }
+                    int image_embed_idx = static_cast<int>(prefix_tokens.size());
                     image_embeds.emplace_back(image_embed_idx, image_embed);
 
                     img_prompt += "<|vision_start|>";
@@ -2625,7 +2739,11 @@ struct LLMEmbedder : public Conditioner {
                     GGML_ASSERT(!image_embed.empty());
 
                     std::string image_prefix = prompt + img_prompt + "Picture " + std::to_string(i + 1) + ": <|vision_start|>";
-                    int image_embed_idx      = static_cast<int>(tokenizer->encode(image_prefix, nullptr).size());
+                    std::vector<int> prefix_tokens;
+                    if (!tokenizer->encode(image_prefix, prefix_tokens, nullptr)) {
+                        return {};
+                    }
+                    int image_embed_idx = static_cast<int>(prefix_tokens.size());
                     image_embeds.emplace_back(image_embed_idx, image_embed);
 
                     img_prompt += "Picture " + std::to_string(i + 1) + ": <|vision_start|>";
@@ -2830,7 +2948,10 @@ struct LLMEmbedder : public Conditioner {
                 "- User Prompt: A busy city street -> Enhanced: A bustling city street scene at dusk, featuring glowing street lamps, a diverse crowd of people in colorful clothing, and a double-decker bus passing by towering glass skyscrapers.\n"
                 "Please generate only the enhanced description for the prompt below and avoid including any additional commentary or evaluations:\n"
                 "User Prompt: ";
-            auto chi_tokens       = std::get<0>(tokenize(chi_prompt, {0, 0}));
+            auto chi_tokens = std::get<0>(tokenize(chi_prompt, {0, 0}));
+            if (chi_tokens.empty()) {
+                return {};
+            }
             size_t num_chi_tokens = chi_tokens.size();
             max_length            = (int)num_chi_tokens + pixeldit_max_length - 2;
             min_length            = max_length;
@@ -2849,7 +2970,9 @@ struct LLMEmbedder : public Conditioner {
                                                0,
                                                false,
                                                max_length);
-            GGML_ASSERT(!hidden_states.empty());
+            if (hidden_states.empty()) {
+                return {};
+            }
 
             if (hidden_states.shape()[1] > pixeldit_max_length) {
                 auto bos      = sd::ops::slice(hidden_states, 1, 0, 1);
@@ -2882,6 +3005,9 @@ struct LLMEmbedder : public Conditioner {
                                            max_length,
                                            deepstack_image_embeds,
                                            image_grids);
+        if (hidden_states.empty()) {
+            return {};
+        }
         std::vector<sd::Tensor<float>> extra_hidden_states_vec;
         if (sd_version_is_hunyuan_video(version) && byt5) {
             std::vector<std::string> quoted_texts;
@@ -2932,6 +3058,9 @@ struct LLMEmbedder : public Conditioner {
                                                      prompt_template_encode_start_idx,
                                                      spell_quotes,
                                                      max_length);
+            if (extra_hidden_states.empty()) {
+                return {};
+            }
             extra_hidden_states_vec.push_back(std::move(extra_hidden_states));
         }
 
@@ -3030,7 +3159,7 @@ struct LTXAVEmbedder : public Conditioner {
     static constexpr int64_t kNumStates  = 49;
     static constexpr int64_t kMinLength  = 1024;
 
-    std::shared_ptr<GemmaTokenizer> tokenizer;
+    std::shared_ptr<Tokenizer> tokenizer;
     std::shared_ptr<LLM::LLMRunner> llm;
     std::shared_ptr<LTXAVTextProjectionRunner> projector;
     std::string projector_prefix;
@@ -3057,17 +3186,21 @@ struct LTXAVEmbedder : public Conditioner {
                   const String2TensorStorage& tensor_storage_map      = {},
                   const std::string& llm_prefix                       = "text_encoders.llm",
                   const std::string& projector_prefix                 = "text_embedding_projection",
-                  std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+                  std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
+                  const TokenizerConfig& tokenizers                   = {})
         : projector_prefix(projector_prefix) {
         LLM::LLMArch arch = detect_gemma_arch(tensor_storage_map, llm_prefix);
         LOG_INFO("ltxav text encoder: %s", arch == LLM::LLMArch::GEMMA4_12B ? "gemma 4" : "gemma 3");
-        tokenizer       = std::make_shared<GemmaTokenizer>();
+        llm       = std::make_shared<LLM::LLMRunner>(arch,
-        llm             = std::make_shared<LLM::LLMRunner>(arch,
                                                backend,
                                                tensor_storage_map,
                                                llm_prefix,
                                                false,
                                                weight_manager);
+        tokenizer = tokenizers.create(TokenizerConfig::MAIN, llm->config.vocab_size, 0, true);
+        if (!tokenizer) {
+            tokenizer = std::make_shared<GemmaTokenizer>();
+        }
         dual_projection = tensor_storage_map.find(projector_prefix + ".video_aggregate_embed.weight") != tensor_storage_map.end();
         projector       = std::make_shared<LTXAVTextProjectionRunner>(backend,
                                                                 tensor_storage_map,
@@ -3146,7 +3279,10 @@ struct LTXAVEmbedder : public Conditioner {
         std::vector<int> tokens;
         std::vector<float> weights;
         for (const auto& item : parsed_attention) {
-            auto curr_tokens = tokenizer->encode(item.first, nullptr);
+            std::vector<int> curr_tokens;
+            if (!tokenizer->encode(item.first, curr_tokens, nullptr)) {
+                return {};
+            }
             tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
             weights.insert(weights.end(), curr_tokens.size(), item.second);
         }
@@ -3164,6 +3300,10 @@ struct LTXAVEmbedder : public Conditioner {
         auto& weights            = std::get<1>(tokens_weights_mask);
         auto& mask               = std::get<2>(tokens_weights_mask);
 
+        if (tokens.empty()) {
+            return {};
+        }
+
         sd::Tensor<int32_t> input_ids({static_cast<int64_t>(tokens.size())}, std::vector<int32_t>(tokens.begin(), tokens.end()));
         sd::Tensor<float> attention_mask;
         if (!mask.empty()) {
@@ -3262,7 +3402,9 @@ struct LTXAVEmbedder : public Conditioner {
         prompt_attn_range.second = static_cast<int>(prompt.size());
 
         auto hidden_states = encode_prompt(n_threads, prompt, prompt_attn_range);
-        GGML_ASSERT(!hidden_states.empty());
+        if (hidden_states.empty()) {
+            return {};
+        }
 
         int64_t t1 = ggml_time_ms();
         LOG_VERBOSE("computing LTXAV condition graph completed, taking %" PRId64 " ms", t1 - t0);

src/core/regex.cpp

@@ -0,0 +1,171 @@
+#include "regex.h"
+
+#include <cstdint>
+#include <limits>
+#include <mutex>
+
+#define ONIG_ESCAPE_UCHAR_COLLISION
+#define ONIG_ESCAPE_REGEX_T_COLLISION
+#include <oniguruma.h>
+
+namespace sd {
+
+    struct Regex::Impl {
+        OnigRegex regex = nullptr;
+
+        ~Impl() {
+            if (regex) {
+                onig_free(regex);
+            }
+        }
+    };
+
+    struct RegexRegionDeleter {
+        void operator()(OnigRegion* region) const {
+            onig_region_free(region, 1);
+        }
+    };
+
+    static bool regex_error(std::string* error, const std::string& message) {
+        if (error) {
+            *error = message;
+        }
+        return false;
+    }
+
+    static bool regex_onig_error(std::string* error, int code, OnigErrorInfo* info = nullptr) {
+        OnigUChar buffer[ONIG_MAX_ERROR_MESSAGE_LEN];
+        onig_error_code_to_str(buffer, code, info);
+        return regex_error(error, reinterpret_cast<const char*>(buffer));
+    }
+
+    static int regex_initialize() {
+        static std::once_flag once;
+        static int result = ONIG_NORMAL;
+        std::call_once(once, [] {
+            OnigEncoding encodings[] = {ONIG_ENCODING_UTF8};
+            result                   = onig_initialize(encodings, 1);
+        });
+        // onig_end() would invalidate expressions held by other Regex instances.
+        return result;
+    }
+
+    // Rust str excludes overlong encodings, surrogates and extended UTF-8 accepted by Oniguruma.
+    static bool regex_valid_utf8(const std::string& text) {
+        size_t position = 0;
+        while (position < text.size()) {
+            const auto lead = static_cast<unsigned char>(text[position++]);
+            if (lead < 0x80) {
+                continue;
+            }
+            int count = 0;
+            if (lead >= 0xC2 && lead <= 0xDF) {
+                count = 1;
+            } else if (lead >= 0xE0 && lead <= 0xEF) {
+                count = 2;
+            } else if (lead >= 0xF0 && lead <= 0xF4) {
+                count = 3;
+            }
+            if (count == 0 || text.size() - position < static_cast<size_t>(count)) {
+                return false;
+            }
+            uint32_t codepoint = lead & (0x7F >> count);
+            for (int i = 0; i < count; ++i) {
+                const auto byte = static_cast<unsigned char>(text[position++]);
+                if ((byte & 0xC0) != 0x80) {
+                    return false;
+                }
+                codepoint = (codepoint << 6) | (byte & 0x3F);
+            }
+            constexpr uint32_t minimum[] = {0, 0x80, 0x800, 0x10000};
+            if (codepoint < minimum[count] || codepoint > 0x10FFFF || (codepoint >= 0xD800 && codepoint <= 0xDFFF)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    Regex::Regex()                            = default;
+    Regex::~Regex()                           = default;
+    Regex::Regex(Regex&&) noexcept            = default;
+    Regex& Regex::operator=(Regex&&) noexcept = default;
+
+    bool Regex::compile(const std::string& pattern, std::string* error) {
+        if (error) {
+            error->clear();
+        }
+        const int initialized = regex_initialize();
+        if (initialized != ONIG_NORMAL) {
+            return regex_onig_error(error, initialized);
+        }
+        if (pattern.size() > static_cast<size_t>(std::numeric_limits<int>::max())) {
+            return regex_error(error, "regex pattern exceeds Oniguruma's offset range");
+        }
+        const auto* begin = reinterpret_cast<const OnigUChar*>(pattern.data());
+        const auto* end   = begin + pattern.size();
+        if (!regex_valid_utf8(pattern)) {
+            return regex_error(error, "regex pattern is not valid UTF-8");
+        }
+
+        auto next = std::make_unique<Impl>();
+        OnigErrorInfo info{};
+        static std::mutex compile_mutex;
+        std::lock_guard<std::mutex> lock(compile_mutex);
+        const int result = onig_new(&next->regex, begin, end, ONIG_OPTION_NONE,
+                                    ONIG_ENCODING_UTF8, ONIG_SYNTAX_ONIGURUMA, &info);
+        if (result != ONIG_NORMAL) {
+            return regex_onig_error(error, result, &info);
+        }
+        impl_ = std::move(next);
+        return true;
+    }
+
+    bool Regex::find_matches(const std::string& text, std::vector<Match>& matches, std::string* error) const {
+        matches.clear();
+        if (error) {
+            error->clear();
+        }
+        if (!impl_) {
+            return regex_error(error, "regex has not been compiled");
+        }
+        if (text.size() > static_cast<size_t>(std::numeric_limits<int>::max())) {
+            return regex_error(error, "regex input exceeds Oniguruma's offset range");
+        }
+        const auto* begin = reinterpret_cast<const OnigUChar*>(text.data());
+        const auto* end   = begin + text.size();
+        if (!regex_valid_utf8(text)) {
+            return regex_error(error, "regex input is not valid UTF-8");
+        }
+        std::unique_ptr<OnigRegion, RegexRegionDeleter> region(onig_region_new());
+        if (!region) {
+            return regex_error(error, "failed to allocate regex match region");
+        }
+
+        size_t position = 0;
+        while (position <= text.size()) {
+            const int result = onig_search(impl_->regex, begin, end, begin + position, end,
+                                           region.get(), ONIG_OPTION_NONE);
+            if (result == ONIG_MISMATCH) {
+                break;
+            }
+            if (result < 0) {
+                matches.clear();
+                return regex_onig_error(error, result);
+            }
+            const size_t match_begin = static_cast<size_t>(region->beg[0]);
+            const size_t match_end   = static_cast<size_t>(region->end[0]);
+            // Match rust-onig's find_iter: suppress an empty match at the previous match's end.
+            if (match_begin == match_end && !matches.empty() && matches.back().second == match_end) {
+                if (position == text.size()) {
+                    break;
+                }
+                position += static_cast<size_t>(ONIGENC_MBC_ENC_LEN(ONIG_ENCODING_UTF8, begin + position));
+                continue;
+            }
+            matches.emplace_back(match_begin, match_end);
+            position = match_end;
+        }
+        return true;
+    }
+
+}  // namespace sd

src/core/regex.h

@@ -0,0 +1,32 @@
+#ifndef __SD_CORE_REGEX_H__
+#define __SD_CORE_REGEX_H__
+
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace sd {
+
+    class Regex {
+        struct Impl;
+        std::unique_ptr<Impl> impl_;
+
+    public:
+        using Match = std::pair<size_t, size_t>;
+
+        Regex();
+        ~Regex();
+        Regex(Regex&&) noexcept;
+        Regex& operator=(Regex&&) noexcept;
+
+        // Failed compilation leaves the previous expression intact.
+        bool compile(const std::string& pattern, std::string* error = nullptr);
+        // Matches are non-overlapping UTF-8 byte ranges; each call owns its search state.
+        bool find_matches(const std::string& text, std::vector<Match>& matches, std::string* error = nullptr) const;
+    };
+
+}  // namespace sd
+
+#endif  // __SD_CORE_REGEX_H__

src/extensions/photomaker_extension.cpp

@@ -18,12 +18,15 @@ tokenize_photomaker_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
     auto tokens_and_weights           = clip_conditioner.tokenize(text);
     std::vector<int> source_tokens    = std::move(tokens_and_weights.first);
     std::vector<float> source_weights = std::move(tokens_and_weights.second);
+    if (source_tokens.empty()) {
+        return {};
+    }
 
-    if (!source_tokens.empty() && source_tokens.front() == clip_conditioner.tokenizer.BOS_TOKEN_ID) {
+    if (!source_tokens.empty() && source_tokens.front() == clip_conditioner.tokenizer->BOS_TOKEN_ID) {
         source_tokens.erase(source_tokens.begin());
         source_weights.erase(source_weights.begin());
     }
-    if (!source_tokens.empty() && source_tokens.back() == clip_conditioner.tokenizer.EOS_TOKEN_ID) {
+    if (!source_tokens.empty() && source_tokens.back() == clip_conditioner.tokenizer->EOS_TOKEN_ID) {
         source_tokens.pop_back();
         source_weights.pop_back();
     }
@@ -49,12 +52,12 @@ tokenize_photomaker_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
         weights.push_back(source_weights[i]);
     }
 
-    clip_conditioner.tokenizer.pad_tokens(tokens,
+    clip_conditioner.tokenizer->pad_tokens(tokens,
-                                          &weights,
+                                           &weights,
-                                          nullptr,
+                                           nullptr,
-                                          clip_conditioner.text_model->model.n_token,
+                                           clip_conditioner.text_model->model.n_token,
-                                          clip_conditioner.text_model->model.n_token,
+                                           clip_conditioner.text_model->model.n_token,
-                                          true);
+                                           true);
     std::vector<bool> class_token_mask;
     for (int i = 0; i < tokens.size(); i++) {
         class_token_mask.push_back(class_idx >= 0 && class_idx + 1 <= i && i < class_idx + 1 + trigger_token_count);
@@ -69,8 +72,14 @@ get_photomaker_condition_with_trigger(FrozenCLIPEmbedderWithCustomWords& clip_co
                                       const ConditionerParams& conditioner_params,
                                       const std::string& trigger_word,
                                       int trigger_token_count) {
-    auto image_tokens = clip_conditioner.convert_token_to_id(trigger_word);
+    std::vector<int> image_tokens;
-    GGML_ASSERT(image_tokens.size() == 1);
+    if (!clip_conditioner.convert_token_to_id(trigger_word, image_tokens)) {
+        return {};
+    }
+    if (image_tokens.size() != 1) {
+        LOG_ERROR("PhotoMaker trigger word must encode to one token");
+        return {};
+    }
     auto tokens_and_weights         = tokenize_photomaker_trigger(clip_conditioner,
                                                                   conditioner_params.text,
                                                                   trigger_token_count,
@@ -78,27 +87,43 @@ get_photomaker_condition_with_trigger(FrozenCLIPEmbedderWithCustomWords& clip_co
     std::vector<int>& tokens        = std::get<0>(tokens_and_weights);
     std::vector<float>& weights     = std::get<1>(tokens_and_weights);
     std::vector<bool>& trigger_mask = std::get<2>(tokens_and_weights);
-    auto cond                       = clip_conditioner.get_learned_condition_common(n_threads,
+    if (tokens.empty()) {
-                                                                                    tokens,
+        return {};
-                                                                                    weights,
+    }
-                                                                                    conditioner_params.clip_skip,
+    auto cond = clip_conditioner.get_learned_condition_common(n_threads,
-                                                                                    conditioner_params.width,
+                                                              tokens,
-                                                                                    conditioner_params.height,
+                                                              weights,
-                                                                                    conditioner_params.zero_out_masked);
+                                                              conditioner_params.clip_skip,
+                                                              conditioner_params.width,
+                                                              conditioner_params.height,
+                                                              conditioner_params.zero_out_masked);
     return std::make_tuple(std::move(cond), trigger_mask);
 }
 
-static std::string remove_photomaker_trigger_from_prompt(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
+static bool remove_photomaker_trigger_from_prompt(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
-                                                         const std::string& prompt,
+                                                  const std::string& prompt,
-                                                         const std::string& trigger_word) {
+                                                  const std::string& trigger_word,
-    auto image_tokens = clip_conditioner.convert_token_to_id(trigger_word);
+                                                  std::string& result) {
-    GGML_ASSERT(image_tokens.size() == 1);
+    std::vector<int> image_tokens;
+    if (!clip_conditioner.convert_token_to_id(trigger_word, image_tokens)) {
+        return false;
+    }
+    if (image_tokens.size() != 1) {
+        LOG_ERROR("PhotoMaker trigger word must encode to one token");
+        return false;
+    }
     auto tokens_and_weights  = clip_conditioner.tokenize(prompt);
     std::vector<int>& tokens = tokens_and_weights.first;
-    auto it                  = std::find(tokens.begin(), tokens.end(), image_tokens[0]);
+    if (tokens.empty()) {
-    GGML_ASSERT(it != tokens.end());
+        return false;
+    }
+    auto it = std::find(tokens.begin(), tokens.end(), image_tokens[0]);
+    if (it == tokens.end()) {
+        LOG_ERROR("PhotoMaker trigger word was not found in tokenized prompt");
+        return false;
+    }
     tokens.erase(it);
-    return clip_conditioner.decode(tokens);
+    return clip_conditioner.decode(tokens, result);
 }
 
 struct PhotoMakerExtension : public GenerationExtension {
@@ -223,6 +248,10 @@ struct PhotoMakerExtension : public GenerationExtension {
                                                                                   trigger_token_count);
         SDCondition prepared_id_condition = std::get<0>(cond_tup);
         auto class_tokens_mask            = std::get<1>(cond_tup);
+        if (prepared_id_condition.empty()) {
+            LOG_ERROR("failed to encode PhotoMaker prompt");
+            return false;
+        }
         if (std::find(class_tokens_mask.begin(), class_tokens_mask.end(), true) == class_tokens_mask.end()) {
             LOG_WARN("PhotoMaker trigger word '%s' was not found in prompt", trigger_word.c_str());
             LOG_WARN("Turn off PhotoMaker for this request");
@@ -263,11 +292,16 @@ struct PhotoMakerExtension : public GenerationExtension {
 
         prepared_id_condition.c_crossattn = std::move(res);
         int64_t t1                        = ggml_time_ms();
-        id_condition                      = std::move(prepared_id_condition);
+        std::string prompt;
-        start_merge_step                  = int(ctx.pm_params.style_strength / 100.f * ctx.total_steps);
+        if (!remove_photomaker_trigger_from_prompt(*clip_conditioner,
-        ctx.condition_params.text         = remove_photomaker_trigger_from_prompt(*clip_conditioner,
+                                                   ctx.condition_params.text,
-                                                                                  ctx.condition_params.text,
+                                                   trigger_word,
-                                                                                  trigger_word);
+                                                   prompt)) {
+            return false;
+        }
+        id_condition              = std::move(prepared_id_condition);
+        start_merge_step          = int(ctx.pm_params.style_strength / 100.f * ctx.total_steps);
+        ctx.condition_params.text = std::move(prompt);
         LOG_INFO("Photomaker ID Stacking, taking %" PRId64 " ms", t1 - t0);
         LOG_INFO("PHOTOMAKER: start_merge_step: %d", start_merge_step);
 

src/model/diffusion/hidream_o1.hpp

@@ -484,13 +484,19 @@ namespace HiDreamO1 {
     };
 
     struct HiDreamO1Conditioner : public Conditioner {
-        Qwen2Tokenizer tokenizer;
+        std::shared_ptr<Tokenizer> tokenizer;
         std::shared_ptr<HiDreamO1VisionRunner> vision_runner;
 
         HiDreamO1Conditioner(ggml_backend_t backend,
                              const String2TensorStorage& tensor_storage_map      = {},
-                             std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+                             std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
-            : vision_runner(std::make_shared<HiDreamO1VisionRunner>(backend, tensor_storage_map, "model.visual", weight_manager)) {}
+                             const TokenizerConfig& tokenizers                   = {})
+            : vision_runner(std::make_shared<HiDreamO1VisionRunner>(backend, tensor_storage_map, "model.visual", weight_manager)) {
+            tokenizer = tokenizers.create(TokenizerConfig::MAIN, HiDreamO1Config::detect_from_weights(tensor_storage_map, "").llm.vocab_size, 151643);
+            if (!tokenizer) {
+                tokenizer = std::make_shared<Qwen2Tokenizer>();
+            }
+        }
 
         void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
             vision_runner->get_param_tensors(tensors);
@@ -538,7 +544,10 @@ namespace HiDreamO1 {
             if (ref_images.empty()) {
                 prompt += conditioner_params.text;
                 prompt += "<|im_end|>\n<|im_start|>assistant\n<|boi_token|><|tms_token|>";
-                auto input_ids = tokenizer.encode(prompt, nullptr);
+                std::vector<int> input_ids;
+                if (!tokenizer->encode(prompt, input_ids, nullptr)) {
+                    return {};
+                }
 
                 std::vector<int32_t> input_ids_pad = input_ids;
                 input_ids_pad.push_back(VISION_START_TOKEN_ID);
@@ -612,7 +621,11 @@ namespace HiDreamO1 {
 
                 auto patch_img = resized_ref * 2.0f - 1.0f;
                 result.c_ref_images.push_back(std::move(patch_img));
-                int64_t prompt_start = static_cast<int64_t>(tokenizer.encode(prompt + "<|vision_start|>", nullptr).size());
+                std::vector<int> prefix_tokens;
+                if (!tokenizer->encode(prompt + "<|vision_start|>", prefix_tokens, nullptr)) {
+                    return {};
+                }
+                int64_t prompt_start = static_cast<int64_t>(prefix_tokens.size());
                 prompt += "<|vision_start|>";
                 prompt += repeat_special_token("<|image_pad|>", image_tokens);
                 prompt += "<|vision_end|>";
@@ -623,7 +636,10 @@ namespace HiDreamO1 {
 
             prompt += conditioner_params.text;
             prompt += "<|im_end|>\n<|im_start|>assistant\n<|boi_token|><|tms_token|>";
-            auto input_ids = tokenizer.encode(prompt, nullptr);
+            std::vector<int> input_ids;
+            if (!tokenizer->encode(prompt, input_ids, nullptr)) {
+                return {};
+            }
 
             std::vector<int32_t> input_ids_pad = input_ids;
             input_ids_pad.push_back(VISION_START_TOKEN_ID);

src/model/te/llm.hpp

@@ -2402,7 +2402,10 @@ namespace LLM {
             for (const auto& item : parsed_attention) {
                 const std::string& curr_text = item.first;
                 float curr_weight            = item.second;
-                std::vector<int> curr_tokens = tokenizer->tokenize(curr_text, nullptr);
+                std::vector<int> curr_tokens;
+                if (!tokenizer->tokenize(curr_text, curr_tokens, nullptr)) {
+                    return {};
+                }
                 tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
                 weights.insert(weights.end(), curr_tokens.size(), curr_weight);
             }

src/model/te/t5.hpp

@@ -567,7 +567,10 @@ struct T5Embedder {
         for (const auto& item : parsed_attention) {
             const std::string& curr_text = item.first;
             float curr_weight            = item.second;
-            std::vector<int> curr_tokens = tokenizer.encode(curr_text);
+            std::vector<int> curr_tokens;
+            if (!tokenizer.encode(curr_text, curr_tokens)) {
+                return {};
+            }
             tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
             weights.insert(weights.end(), curr_tokens.size(), curr_weight);
         }

src/pipeline/diffusion_engine.h

@@ -128,6 +128,7 @@ public:
                                 &sd_ctx_params_t::clip_g_path, &sd_ctx_params_t::clip_vision_path,
                                 &sd_ctx_params_t::t5xxl_path, &sd_ctx_params_t::llm_path,
                                 &sd_ctx_params_t::llm_vision_path, &sd_ctx_params_t::diffusion_model_path,
+                                &sd_ctx_params_t::tokenizer,
                                 &sd_ctx_params_t::high_noise_diffusion_model_path, &sd_ctx_params_t::uncond_diffusion_model_path,
                                 &sd_ctx_params_t::embeddings_connectors_path, &sd_ctx_params_t::vae_path,
                                 &sd_ctx_params_t::audio_vae_path, &sd_ctx_params_t::taesd_path,

src/pipeline/model_builders.cpp

@@ -73,12 +73,13 @@ namespace sd::model_builders {
         }
     }
 
-    bool build_core_runners(const Context& ctx, CoreRunners& runners) {
+    bool build_core_runners(const Context& ctx, CoreRunners& runners) try {
         const auto* sd_ctx_params      = &ctx.params;
         const auto& tensor_storage_map = ctx.tensor_storage_map;
         const auto version             = ctx.version;
         const auto& weight_manager     = ctx.weight_manager;
         CoreRunners result;
+        TokenizerConfig tokenizers(sd_ctx_params->tokenizer);
         if (!ensure_backend_pair(ctx.backends, SDBackendModule::TE) ||
             !ensure_backend_pair(ctx.backends, SDBackendModule::DIFFUSION)) {
             return false;
@@ -87,7 +88,8 @@ namespace sd::model_builders {
         if (sd_version_is_sd3(version)) {
             result.conditioner = std::make_shared<SD3CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
                                                                    tensor_storage_map,
-                                                                   weight_manager);
+                                                                   weight_manager,
+                                                                   tokenizers);
             result.diffusion   = std::make_shared<MMDiTRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                              tensor_storage_map,
                                                              "model.diffusion_model",
@@ -98,7 +100,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                false,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<Pid::PiDRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                 tensor_storage_map,
                                                                 "model.diffusion_model.net",
@@ -109,7 +112,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                false,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<Ideogram4::Ideogram4Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                             tensor_storage_map,
                                                                             "model.diffusion_model",
@@ -120,7 +124,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                true,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<Krea2::Krea2Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                     tensor_storage_map,
                                                                     "model.diffusion_model",
@@ -147,11 +152,13 @@ namespace sd::model_builders {
                                                                    version,
                                                                    "",
                                                                    false,
-                                                                   weight_manager);
+                                                                   weight_manager,
+                                                                   tokenizers);
             } else {
                 result.conditioner = std::make_shared<FluxCLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
                                                                         tensor_storage_map,
-                                                                        weight_manager);
+                                                                        weight_manager,
+                                                                        tokenizers);
             }
             result.diffusion = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                   tensor_storage_map,
@@ -166,7 +173,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                false,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                   tensor_storage_map,
                                                                   "model.diffusion_model",
@@ -178,7 +186,8 @@ namespace sd::model_builders {
                                                                  tensor_storage_map,
                                                                  "text_encoders.llm",
                                                                  "text_embedding_projection",
-                                                                 weight_manager);
+                                                                 weight_manager,
+                                                                 tokenizers);
             result.diffusion   = std::make_shared<LTXV::LTXAVRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                    tensor_storage_map,
                                                                    "model.diffusion_model",
@@ -189,7 +198,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                true,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<MiniMaxH3::MiniMaxH3Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                             tensor_storage_map,
                                                                             "model.diffusion_model",
@@ -200,7 +210,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                false,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<Hunyuan::HunyuanVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                              tensor_storage_map,
                                                                              "model.diffusion_model",
@@ -258,7 +269,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                enable_vision,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<LingBotVideo::LingBotVideoRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                                   tensor_storage_map,
                                                                                   "model.diffusion_model",
@@ -271,7 +283,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                enable_vision,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<Qwen::QwenImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                        tensor_storage_map,
                                                                        "model.diffusion_model",
@@ -284,7 +297,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                true,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<MageFlow::MageFlowRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                           tensor_storage_map,
                                                                           "model.diffusion_model",
@@ -295,7 +309,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                true,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<Flux::FluxRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                   tensor_storage_map,
                                                                   "model.diffusion_model",
@@ -305,7 +320,8 @@ namespace sd::model_builders {
         } else if (version == VERSION_HIDREAM_O1) {
             result.conditioner = std::make_shared<HiDreamO1::HiDreamO1Conditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
                                                                                    tensor_storage_map,
-                                                                                   weight_manager);
+                                                                                   weight_manager,
+                                                                                   tokenizers);
             result.diffusion   = std::make_shared<HiDreamO1::HiDreamO1Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                             tensor_storage_map,
                                                                             "model",
@@ -327,7 +343,8 @@ namespace sd::model_builders {
         } else if (sd_version_is_anima(version)) {
             result.conditioner = std::make_shared<AnimaConditioner>(ctx.backends.runtime_backend(SDBackendModule::TE),
                                                                     tensor_storage_map,
-                                                                    weight_manager);
+                                                                    weight_manager,
+                                                                    tokenizers);
             result.diffusion   = std::make_shared<Anima::AnimaRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                     tensor_storage_map,
                                                                     "model.diffusion_model",
@@ -338,7 +355,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                false,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<ZImage::ZImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                       tensor_storage_map,
                                                                       "model.diffusion_model",
@@ -350,7 +368,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                true,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<Boogu::BooguImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                          tensor_storage_map,
                                                                          "model.diffusion_model",
@@ -362,7 +381,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                false,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<ErnieImage::ErnieImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                               tensor_storage_map,
                                                                               "model.diffusion_model",
@@ -373,7 +393,8 @@ namespace sd::model_builders {
                                                                version,
                                                                "",
                                                                false,
-                                                               weight_manager);
+                                                               weight_manager,
+                                                               tokenizers);
             result.diffusion   = std::make_shared<Lens::LensRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                   tensor_storage_map,
                                                                   "model.diffusion_model",
@@ -387,7 +408,8 @@ namespace sd::model_builders {
                                                                                      tensor_storage_map,
                                                                                      embbeding_map,
                                                                                      version,
-                                                                                     weight_manager);
+                                                                                     weight_manager,
+                                                                                     tokenizers);
             result.diffusion   = std::make_shared<UNetModelRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                  tensor_storage_map,
                                                                  "model.diffusion_model",
@@ -429,8 +451,12 @@ namespace sd::model_builders {
         if (result.ip_adapter) {
             result.ip_adapter->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
         }
+        tokenizers.validate_usage();
         runners = std::move(result);
         return true;
+    } catch (const std::exception& error) {
+        LOG_ERROR("failed to build model runners: %s", error.what());
+        return false;
     }
 
     bool build_vae_runners(const Context& ctx, const VAEOptions& options, VAERunners& runners) {

src/pipeline/video.cpp

@@ -1139,10 +1139,10 @@ namespace sd::pipeline {
         return latents;
     }
 
-    static ImageGenerationEmbeds prepare_video_generation_embeds(StableDiffusionGGML* sd,
+    static std::optional<ImageGenerationEmbeds> prepare_video_generation_embeds(StableDiffusionGGML* sd,
-                                                                 const sd_vid_gen_params_t* sd_vid_gen_params,
+                                                                                const sd_vid_gen_params_t* sd_vid_gen_params,
-                                                                 const GenerationRequest& request,
+                                                                                const GenerationRequest& request,
-                                                                 const ImageGenerationLatents& latents) {
+                                                                                const ImageGenerationLatents& latents) {
         ConditionerRunnerEndOnExit conditioner_runner_end{sd->cond_stage_model.get()};
 
         ImageGenerationEmbeds embeds;
@@ -1159,8 +1159,12 @@ namespace sd::pipeline {
         int64_t prepare_start_ms = ggml_time_ms();
         embeds.cond              = sd->cond_stage_model->get_learned_condition(sd->n_threads,
                                                                                condition_params);
-        embeds.cond.c_concat     = latents.concat_latent;
+        if (embeds.cond.empty()) {
-        embeds.cond.c_vector     = latents.clip_vision_output;
+            LOG_ERROR("failed to encode video prompt");
+            return std::nullopt;
+        }
+        embeds.cond.c_concat = latents.concat_latent;
+        embeds.cond.c_vector = latents.clip_vision_output;
         if (sd_version_is_minimax_h3(sd->version)) {
             embeds.cond.c_ref_images       = latents.ref_latents;
             embeds.cond.c_ref_audios       = latents.reference_audio_latents;
@@ -1178,9 +1182,13 @@ namespace sd::pipeline {
             }
         }
         if (request.use_uncond) {
-            condition_params.text  = request.negative_prompt;
+            condition_params.text = request.negative_prompt;
-            embeds.uncond          = sd->cond_stage_model->get_learned_condition(sd->n_threads,
+            embeds.uncond         = sd->cond_stage_model->get_learned_condition(sd->n_threads,
-                                                                                 condition_params);
+                                                                                condition_params);
+            if (embeds.uncond.empty()) {
+                LOG_ERROR("failed to encode negative video prompt");
+                return std::nullopt;
+            }
             embeds.uncond.c_concat = latents.concat_latent;
             embeds.uncond.c_vector = latents.clip_vision_output;
             if (sd_version_is_minimax_h3(sd->version)) {
@@ -1574,10 +1582,14 @@ namespace sd::pipeline {
         }
         ImageGenerationLatents latents = std::move(*latent_inputs_opt);
 
-        ImageGenerationEmbeds embeds = prepare_video_generation_embeds(sd,
+        auto embeds_opt = prepare_video_generation_embeds(sd,
-                                                                       sd_vid_gen_params,
+                                                          sd_vid_gen_params,
-                                                                       request,
+                                                          request,
-                                                                       latents);
+                                                          latents);
+        if (!embeds_opt) {
+            return false;
+        }
+        ImageGenerationEmbeds embeds = std::move(*embeds_opt);
         if (latent_upscale_enabled) {
             LOG_INFO("generate_video %dx%dx%d -> LTX latent spatial upscale",
                      request.width,

src/stable-diffusion.cpp

@@ -349,6 +349,7 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) {
              "t5xxl_path: %s\n"
              "llm_path: %s\n"
              "llm_vision_path: %s\n"
+             "tokenizer: %s\n"
              "diffusion_model_path: %s\n"
              "high_noise_diffusion_model_path: %s\n"
              "uncond_diffusion_model_path: %s\n"
@@ -387,6 +388,7 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) {
              SAFE_STR(sd_ctx_params->t5xxl_path),
              SAFE_STR(sd_ctx_params->llm_path),
              SAFE_STR(sd_ctx_params->llm_vision_path),
+             SAFE_STR(sd_ctx_params->tokenizer),
              SAFE_STR(sd_ctx_params->diffusion_model_path),
              SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path),
              SAFE_STR(sd_ctx_params->uncond_diffusion_model_path),

src/tokenizers/bpe_tokenizer.cpp

@@ -130,7 +130,11 @@ std::vector<std::u32string> BPETokenizer::bpe(const std::u32string& token) const
     return word;
 }
 
-std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t on_new_token_cb) {
+bool BPETokenizer::encode(const std::string& text, std::vector<int>& result, on_new_token_cb_t on_new_token_cb, std::string* error) {
+    result.clear();
+    if (error) {
+        error->clear();
+    }
     std::vector<int32_t> bpe_tokens;
     std::vector<std::string> token_strs;
 
@@ -206,7 +210,8 @@ std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t
     }
     ss << "]";
     LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), bpe_tokens.size(), ss.str().c_str());
-    return bpe_tokens;
+    result = std::move(bpe_tokens);
+    return true;
 }
 
 std::string BPETokenizer::decode_token(int token_id) const {

src/tokenizers/bpe_tokenizer.h

@@ -36,7 +36,7 @@ public:
     BPETokenizer()          = default;
     virtual ~BPETokenizer() = default;
 
-    std::vector<int> encode(const std::string& text, on_new_token_cb_t on_new_token_cb = nullptr) override;
+    bool encode(const std::string& text, std::vector<int>& tokens, on_new_token_cb_t on_new_token_cb = nullptr, std::string* error = nullptr) override;
 };
 
 #endif  // __SD_TOKENIZERS_BPE_TOKENIZER_H__

src/tokenizers/hf_tokenizer.cpp

@@ -0,0 +1,856 @@
+#include "hf_tokenizer.h"
+
+#include <algorithm>
+#include <array>
+#include <climits>
+#include <cstdlib>
+#include <fstream>
+#include <queue>
+#include <sstream>
+#include <stdexcept>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "core/regex.h"
+#include "core/util.h"
+#include "json.hpp"
+#include "utf8proc.h"
+
+using TokenizerJSON = nlohmann::json;
+
+static void tokenizer_require(bool condition, const std::string& message) {
+    if (!condition) {
+        throw std::runtime_error("tokenizer.json: " + message);
+    }
+}
+
+static std::string tokenizer_utf8(int32_t codepoint) {
+    utf8proc_uint8_t bytes[4];
+    auto count = utf8proc_encode_char(codepoint, bytes);
+    return std::string(reinterpret_cast<const char*>(bytes), count);
+}
+
+static bool tokenizer_error(std::string* error, const std::string& message) {
+    if (error) {
+        *error = "tokenizer.json: " + message;
+    } else {
+        LOG_ERROR("tokenizer.json: %s", message.c_str());
+    }
+    return false;
+}
+
+static bool tokenizer_next(const std::string& text, size_t& offset, int32_t& codepoint, std::string* error) {
+    auto size = utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(text.data() + offset), text.size() - offset, &codepoint);
+    if (size <= 0) {
+        return tokenizer_error(error, "invalid UTF-8 input");
+    }
+    offset += size;
+    return true;
+}
+
+static int tokenizer_id(const TokenizerJSON& value) {
+    tokenizer_require(value.is_number_integer(), "token ID must be an integer");
+    auto id = value.get<int64_t>();
+    tokenizer_require(id >= 0 && id <= INT_MAX, "token ID outside int32 range");
+    return static_cast<int>(id);
+}
+
+static uint64_t tokenizer_pair(int left, int right) {
+    return (static_cast<uint64_t>(left) << 32) | static_cast<uint32_t>(right);
+}
+
+struct HFTokenizer::Impl {
+    struct Pattern {
+        std::string literal;
+        std::shared_ptr<sd::Regex> regex;
+
+        explicit Pattern(const TokenizerJSON& config) {
+            tokenizer_require(config.is_object() && config.size() == 1, "invalid String/Regex pattern");
+            if (config.contains("String")) {
+                literal = config.at("String").get<std::string>();
+            } else {
+                tokenizer_require(config.contains("Regex"), "unsupported pattern");
+                regex = std::make_shared<sd::Regex>();
+                std::string error;
+                bool ok = regex->compile(config.at("Regex").get<std::string>(), &error);
+                tokenizer_require(ok, "invalid regex: " + error);
+            }
+        }
+
+        bool matches(const std::string& text, std::vector<sd::Regex::Match>& result, std::string* error) const {
+            result.clear();
+            if (regex) {
+                std::string regex_error;
+                if (!regex->find_matches(text, result, &regex_error)) {
+                    return tokenizer_error(error, "regex search failed: " + regex_error);
+                }
+            } else if (literal.empty()) {
+                size_t offset = 0;
+                for (;;) {
+                    result.emplace_back(offset, offset);
+                    if (offset == text.size()) {
+                        break;
+                    }
+                    int32_t cp;
+                    if (!tokenizer_next(text, offset, cp, error)) {
+                        result.clear();
+                        return false;
+                    }
+                }
+            } else {
+                size_t offset = 0;
+                while ((offset = text.find(literal, offset)) != std::string::npos) {
+                    result.emplace_back(offset, offset + literal.size());
+                    offset += literal.size();
+                }
+            }
+            return true;
+        }
+
+        bool replace(const std::string& text, const std::string& replacement, std::string& result, std::string* error) const {
+            result.clear();
+            size_t offset = 0;
+            std::vector<sd::Regex::Match> found;
+            if (!matches(text, found, error)) {
+                return false;
+            }
+            for (const auto& match : found) {
+                result.append(text, offset, match.first - offset);
+                result += replacement;
+                offset = match.second;
+            }
+            result.append(text, offset, std::string::npos);
+            return true;
+        }
+
+        bool split(const std::string& text, const std::string& behavior, bool invert, std::vector<std::string>& result, std::string* error) const {
+            result.clear();
+            struct Part {
+                size_t start, end;
+                bool matched;
+            };
+            std::vector<Part> parts;
+            size_t offset = 0;
+            std::vector<sd::Regex::Match> found;
+            if (!matches(text, found, error)) {
+                return false;
+            }
+            for (const auto& match : found) {
+                if (match.first > offset) {
+                    parts.push_back({offset, match.first, invert});
+                }
+                parts.push_back({match.first, match.second, !invert});
+                offset = match.second;
+            }
+            if (offset < text.size()) {
+                parts.push_back({offset, text.size(), invert});
+            }
+            if (behavior == "MergedWithNext") {
+                std::reverse(parts.begin(), parts.end());
+            }
+            std::vector<Part> merged;
+            bool previous = false;
+            for (const auto& part : parts) {
+                bool join = (behavior == "Contiguous" && part.matched == previous) ||
+                            ((behavior == "MergedWithPrevious" || behavior == "MergedWithNext") && part.matched && !previous);
+                if (join && !merged.empty()) {
+                    merged.back().start = std::min(merged.back().start, part.start);
+                    merged.back().end   = std::max(merged.back().end, part.end);
+                } else if (behavior != "Removed" || !part.matched) {
+                    merged.push_back(part);
+                }
+                previous = part.matched;
+            }
+            if (behavior == "MergedWithNext") {
+                std::reverse(merged.begin(), merged.end());
+            }
+            for (const auto& part : merged) {
+                if (part.start != part.end) {
+                    result.push_back(text.substr(part.start, part.end - part.start));
+                }
+            }
+            return true;
+        }
+    };
+
+    struct Step {
+        std::string type, content, behavior;
+        std::shared_ptr<Pattern> pattern;
+        bool invert = false, prefix_space = false;
+    };
+
+    struct Trie {
+        struct Node {
+            std::unordered_map<unsigned char, size_t> children;
+            int id = -1;
+        };
+        std::vector<Node> nodes{1};
+
+        void add(const std::string& text, int id) {
+            size_t index = 0;
+            for (unsigned char c : text) {
+                auto found = nodes[index].children.find(c);
+                if (found == nodes[index].children.end()) {
+                    size_t next = nodes.size();
+                    nodes[index].children.emplace(c, next);
+                    nodes.emplace_back();
+                    index = next;
+                } else {
+                    index = found->second;
+                }
+            }
+            nodes[index].id = id;
+        }
+
+        std::pair<size_t, int> match(const std::string& text, size_t start) const {
+            size_t index = 0;
+            std::pair<size_t, int> result{start, -1};
+            for (size_t end = start; end < text.size(); ++end) {
+                auto found = nodes[index].children.find(static_cast<unsigned char>(text[end]));
+                if (found == nodes[index].children.end()) {
+                    break;
+                }
+                index = found->second;
+                if (nodes[index].id >= 0) {
+                    result = {end + 1, nodes[index].id};
+                }
+            }
+            return result;
+        }
+    };
+
+    struct Merge {
+        size_t rank;
+        int id;
+    };
+    std::unordered_map<std::string, int> vocab;
+    std::unordered_map<std::string, int> added_vocab;
+    std::unordered_map<int, std::string> tokens;
+    std::unordered_map<uint64_t, Merge> merges;
+    std::unordered_set<int> special_ids;
+    std::vector<std::string> custom_tokens;
+    std::vector<Step> normalizers, pre_tokenizers, decoders;
+    Trie raw_added, normalized_added;
+    std::array<std::string, 256> byte_encoder;
+    std::unordered_map<int32_t, unsigned char> byte_decoder;
+    std::string suffix;
+    int unk       = -1;
+    bool fuse_unk = false, byte_fallback = false, ignore_merges = false, has_decoder = false;
+
+    Impl() {
+        int extra = 256;
+        for (int byte = 0; byte < 256; ++byte) {
+            int cp             = ((byte >= 33 && byte <= 126) || (byte >= 161 && byte <= 172) || byte >= 174) ? byte : extra++;
+            byte_encoder[byte] = tokenizer_utf8(cp);
+            byte_decoder[cp]   = static_cast<unsigned char>(byte);
+        }
+    }
+
+    static void parse_steps(const TokenizerJSON& config, const std::string& stage, std::vector<Step>& out, int depth = 0) {
+        tokenizer_require(depth < 32, stage + " nesting is too deep");
+        if (config.is_null()) {
+            return;
+        }
+        Step step;
+        step.type = config.at("type").get<std::string>();
+        if (step.type == "Sequence") {
+            const char* key = stage == "normalizer" ? "normalizers" : stage == "pre_tokenizer" ? "pretokenizers"
+                                                                                               : "decoders";
+            for (const auto& child : config.at(key)) {
+                parse_steps(child, stage, out, depth + 1);
+            }
+            return;
+        }
+        if ((stage == "normalizer" || stage == "decoder") && step.type == "Replace") {
+            step.pattern = std::make_shared<Pattern>(config.at("pattern"));
+            step.content = config.at("content").get<std::string>();
+        } else if (stage == "normalizer" && (step.type == "NFC" || step.type == "Lowercase")) {
+        } else if (stage == "pre_tokenizer" && step.type == "Split") {
+            step.pattern  = std::make_shared<Pattern>(config.at("pattern"));
+            step.behavior = config.at("behavior").get<std::string>();
+            tokenizer_require(step.behavior == "Removed" || step.behavior == "Isolated" || step.behavior == "Contiguous" || step.behavior == "MergedWithPrevious" || step.behavior == "MergedWithNext", "unsupported Split behavior: " + step.behavior);
+            step.invert = config.value("invert", false);
+        } else if ((stage == "pre_tokenizer" || stage == "decoder") && step.type == "ByteLevel") {
+            step.prefix_space = config.value("add_prefix_space", true);
+            if (stage == "pre_tokenizer" && config.value("use_regex", true)) {
+                step.pattern = std::make_shared<Pattern>(TokenizerJSON{{"Regex", R"('s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+)"}});
+            }
+        } else if (stage == "decoder" && (step.type == "ByteFallback" || step.type == "Fuse")) {
+        } else {
+            tokenizer_require(false, "unsupported " + stage + ": " + step.type);
+        }
+        out.push_back(std::move(step));
+    }
+
+    bool normalize(std::string text, std::string& result, std::string* error) const {
+        result.clear();
+        for (const auto& step : normalizers) {
+            if (step.type == "Replace") {
+                std::string replaced;
+                if (!step.pattern->replace(text, step.content, replaced, error)) {
+                    return false;
+                }
+                text = std::move(replaced);
+            } else if (step.type == "NFC") {
+                utf8proc_uint8_t* output = nullptr;
+                auto size                = utf8proc_map(reinterpret_cast<const utf8proc_uint8_t*>(text.data()), text.size(), &output, static_cast<utf8proc_option_t>(UTF8PROC_STABLE | UTF8PROC_COMPOSE));
+                std::unique_ptr<utf8proc_uint8_t, decltype(&std::free)> buffer(output, &std::free);
+                if (size < 0) {
+                    return tokenizer_error(error, std::string("NFC normalization failed: ") + utf8proc_errmsg(size));
+                }
+                text.assign(reinterpret_cast<const char*>(output), size);
+            } else {
+                std::string lower;
+                for (size_t i = 0; i < text.size();) {
+                    int32_t cp;
+                    if (!tokenizer_next(text, i, cp, error)) {
+                        return false;
+                    }
+                    // Rust char::to_lowercase uses full, context-free lowercase. U+0130 expands.
+                    lower += cp == 0x130 ? "i\xcc\x87" : tokenizer_utf8(utf8proc_tolower(cp));
+                }
+                text = std::move(lower);
+            }
+        }
+        result = std::move(text);
+        return true;
+    }
+
+    bool pre_tokenize(const std::string& text, std::vector<std::string>& result, std::string* error) const {
+        result.clear();
+        std::vector<std::string> pieces{text};
+        for (const auto& step : pre_tokenizers) {
+            std::vector<std::string> next;
+            for (auto piece : pieces) {
+                if (piece.empty()) {
+                    continue;
+                }
+                std::vector<std::string> split;
+                if (step.type == "Split") {
+                    if (!step.pattern->split(piece, step.behavior, step.invert, split, error)) {
+                        return false;
+                    }
+                    next.insert(next.end(), split.begin(), split.end());
+                } else {
+                    if (step.prefix_space && piece.front() != ' ') {
+                        piece.insert(piece.begin(), ' ');
+                    }
+                    if (step.pattern) {
+                        if (!step.pattern->split(piece, "Isolated", false, split, error)) {
+                            return false;
+                        }
+                    } else {
+                        split.push_back(piece);
+                    }
+                    for (const auto& part : split) {
+                        std::string encoded;
+                        for (unsigned char byte : part) {
+                            encoded += byte_encoder[byte];
+                        }
+                        next.push_back(std::move(encoded));
+                    }
+                }
+            }
+            pieces = std::move(next);
+        }
+        result = std::move(pieces);
+        return true;
+    }
+
+    bool bpe(const std::string& text, std::vector<int>& ids, std::string* error) const {
+        ids.clear();
+        if (ignore_merges) {
+            auto found = vocab.find(text);
+            if (found != vocab.end()) {
+                ids.push_back(found->second);
+                return true;
+            }
+        }
+        bool pending_unk = false;
+        for (size_t i = 0; i < text.size();) {
+            int32_t cp;
+            size_t end = i;
+            if (!tokenizer_next(text, end, cp, error)) {
+                ids.clear();
+                return false;
+            }
+            std::string symbol = text.substr(i, end - i);
+            if (end == text.size()) {
+                symbol += suffix;
+            }
+            i          = end;
+            auto found = vocab.find(symbol);
+            if (found != vocab.end()) {
+                if (pending_unk) {
+                    ids.push_back(unk);
+                    pending_unk = false;
+                }
+                ids.push_back(found->second);
+                continue;
+            }
+            if (byte_fallback) {
+                std::vector<int> bytes;
+                for (unsigned char byte : symbol) {
+                    const char* hex   = "0123456789ABCDEF";
+                    std::string token = "<0x00>";
+                    token[3]          = hex[byte >> 4];
+                    token[4]          = hex[byte & 15];
+                    auto fallback     = vocab.find(token);
+                    if (fallback == vocab.end()) {
+                        break;
+                    }
+                    bytes.push_back(fallback->second);
+                }
+                if (bytes.size() == symbol.size()) {
+                    ids.insert(ids.end(), bytes.begin(), bytes.end());
+                    continue;
+                }
+            }
+            if (unk >= 0) {
+                if (pending_unk && !fuse_unk) {
+                    ids.push_back(unk);
+                }
+                pending_unk = true;
+            }
+        }
+        if (pending_unk) {
+            ids.push_back(unk);
+        }
+        struct Symbol {
+            int id;
+            size_t prev, next, generation = 0;
+            bool alive = true;
+        };
+        struct Candidate {
+            size_t rank, left, right, left_generation, right_generation;
+            int id;
+            bool operator<(const Candidate& other) const {
+                return rank != other.rank ? rank > other.rank : left > other.left;
+            }
+        };
+        const size_t none = ids.size();
+        std::vector<Symbol> symbols;
+        for (size_t i = 0; i < ids.size(); ++i) {
+            symbols.push_back({ids[i], i == 0 ? none : i - 1, i + 1});
+        }
+        std::priority_queue<Candidate> queue;
+        auto push = [&](size_t left) {
+            if (left == none || symbols[left].next == none) {
+                return;
+            }
+            size_t right = symbols[left].next;
+            auto found   = merges.find(tokenizer_pair(symbols[left].id, symbols[right].id));
+            if (found != merges.end()) {
+                queue.push({found->second.rank, left, right, symbols[left].generation, symbols[right].generation, found->second.id});
+            }
+        };
+        for (size_t i = 0; i < symbols.size(); ++i) {
+            push(i);
+        }
+        while (!queue.empty()) {
+            Candidate item = queue.top();
+            queue.pop();
+            auto& left  = symbols[item.left];
+            auto& right = symbols[item.right];
+            if (!left.alive || !right.alive || left.next != item.right || left.generation != item.left_generation || right.generation != item.right_generation) {
+                continue;
+            }
+            left.id   = item.id;
+            left.next = right.next;
+            ++left.generation;
+            right.alive = false;
+            if (left.next != none) {
+                symbols[left.next].prev = item.left;
+            }
+            push(left.prev);
+            push(item.left);
+        }
+        ids.clear();
+        for (const auto& symbol : symbols) {
+            if (symbol.alive) {
+                ids.push_back(symbol.id);
+            }
+        }
+        return true;
+    }
+
+    int lookup(const std::string& token) const {
+        auto added = added_vocab.find(token);
+        if (added != added_vocab.end()) {
+            return added->second;
+        }
+        auto found = vocab.find(token);
+        return found == vocab.end() ? -1 : found->second;
+    }
+
+    void add_token(const std::string& token, int id, bool added = false) {
+        auto old = tokens.find(id);
+        tokenizer_require(old == tokens.end() || old->second == token, "conflicting token ID " + std::to_string(id));
+        int old_id = lookup(token);
+        tokenizer_require(old_id < 0 || old_id == id, "conflicting ID for token " + token);
+        tokens[id]                           = token;
+        (added ? added_vocab : vocab)[token] = id;
+    }
+};
+
+HFTokenizer::HFTokenizer(const std::string& path)
+    : impl_(new Impl) {
+    std::ifstream stream(path, std::ios::binary);
+    tokenizer_require(stream.good(), "cannot open " + path);
+    TokenizerJSON config;
+    stream >> config;
+    tokenizer_require(config.value("version", std::string("1.0")) == "1.0", "unsupported version");
+    tokenizer_require(config.value("padding", TokenizerJSON()).is_null(), "JSON padding is unsupported; padding is controlled by the text encoder");
+    tokenizer_require(config.value("truncation", TokenizerJSON()).is_null(), "JSON truncation is unsupported; truncation is controlled by the text encoder");
+    const auto& model = config.at("model");
+    tokenizer_require(model.at("type") == "BPE", "only BPE models are supported");
+    tokenizer_require(model.value("dropout", TokenizerJSON()).is_null() || model.at("dropout") == 0, "BPE dropout is unsupported");
+    const auto& prefix = model.value("continuing_subword_prefix", TokenizerJSON());
+    tokenizer_require(prefix.is_null() || prefix == "", "nonempty continuing_subword_prefix is unsupported");
+    const auto& suffix   = model.value("end_of_word_suffix", TokenizerJSON());
+    impl_->suffix        = suffix.is_null() ? "" : suffix.get<std::string>();
+    impl_->fuse_unk      = model.value("fuse_unk", false);
+    impl_->byte_fallback = model.value("byte_fallback", false);
+    impl_->ignore_merges = model.value("ignore_merges", false);
+    tokenizer_require(model.at("vocab").is_object(), "BPE vocab must be an object");
+    impl_->vocab.reserve(model.at("vocab").size());
+    impl_->tokens.reserve(model.at("vocab").size());
+    for (const auto& entry : model.at("vocab").items()) {
+        impl_->add_token(entry.key(), tokenizer_id(entry.value()));
+    }
+    if (!model.value("unk_token", TokenizerJSON()).is_null()) {
+        UNK_TOKEN  = model.at("unk_token").get<std::string>();
+        impl_->unk = impl_->lookup(UNK_TOKEN);
+        tokenizer_require(impl_->unk >= 0, "unk_token is absent from vocab");
+    }
+    UNK_TOKEN_ID = impl_->unk;
+    tokenizer_require(model.at("merges").is_array(), "BPE merges must be an array");
+    impl_->merges.reserve(model.at("merges").size());
+    size_t rank = 0;
+    for (const auto& merge : model.at("merges")) {
+        std::string left, right;
+        if (merge.is_string()) {
+            auto value = merge.get<std::string>();
+            auto space = value.find(' ');
+            tokenizer_require(space != std::string::npos && value.find(' ', space + 1) == std::string::npos, "invalid legacy BPE merge");
+            left  = value.substr(0, space);
+            right = value.substr(space + 1);
+        } else {
+            tokenizer_require(merge.is_array() && merge.size() == 2, "BPE merge must contain two tokens");
+            left  = merge.at(0).get<std::string>();
+            right = merge.at(1).get<std::string>();
+        }
+        int a = impl_->lookup(left), b = impl_->lookup(right), id = impl_->lookup(left + right);
+        tokenizer_require(a >= 0 && b >= 0 && id >= 0, "BPE merge references a missing vocab token");
+        impl_->merges[tokenizer_pair(a, b)] = {rank++, id};
+    }
+    Impl::parse_steps(config.value("normalizer", TokenizerJSON()), "normalizer", impl_->normalizers);
+    Impl::parse_steps(config.value("pre_tokenizer", TokenizerJSON()), "pre_tokenizer", impl_->pre_tokenizers);
+    impl_->has_decoder = !config.value("decoder", TokenizerJSON()).is_null();
+    Impl::parse_steps(config.value("decoder", TokenizerJSON()), "decoder", impl_->decoders);
+    size_t next_added_id = impl_->vocab.size();
+    for (const auto& token : config.value("added_tokens", TokenizerJSON::array())) {
+        for (const char* flag : {"single_word", "lstrip", "rstrip"}) {
+            tokenizer_require(!token.value(flag, false), std::string("added_tokens.") + flag + "=true is unsupported");
+        }
+        auto content = token.at("content").get<std::string>();
+        tokenizer_require(!content.empty(), "empty added token is unsupported");
+        int id = tokenizer_id(token.at("id"));
+        if (impl_->lookup(content) < 0) {
+            tokenizer_require(static_cast<size_t>(id) == next_added_id++, "nonconsecutive added token IDs would be reassigned by Hugging Face tokenizers");
+        }
+        impl_->add_token(content, id, true);
+        if (token.value("special", false)) {
+            special_tokens.push_back(content);
+            impl_->special_ids.insert(id);
+        }
+        bool normalized     = token.value("normalized", true);
+        std::string pattern = content;
+        if (normalized) {
+            std::string error;
+            bool ok = impl_->normalize(content, pattern, &error);
+            tokenizer_require(ok, error);
+        }
+        tokenizer_require(!pattern.empty(), "added token normalizes to an empty string");
+        (normalized ? impl_->normalized_added : impl_->raw_added).add(pattern, id);
+    }
+    const auto& processor = config.value("post_processor", TokenizerJSON());
+    BOS_TOKEN_ID = EOS_TOKEN_ID = -1;
+    if (!processor.is_null()) {
+        auto type    = processor.at("type").get<std::string>();
+        auto special = [&](const TokenizerJSON& pair) {
+            tokenizer_require(pair.is_array() && pair.size() == 2, "invalid postprocessor special token");
+            int id = tokenizer_id(pair.at(1));
+            tokenizer_require(impl_->lookup(pair.at(0).get<std::string>()) == id, "postprocessor token/ID does not match vocab");
+            return id;
+        };
+        if (type == "RobertaProcessing") {
+            BOS_TOKEN_ID = special(processor.at("cls"));
+            EOS_TOKEN_ID = special(processor.at("sep"));
+        } else if (type == "TemplateProcessing") {
+            bool seen_sequence = false;
+            for (const auto& item : processor.at("single")) {
+                if (item.contains("Sequence")) {
+                    tokenizer_require(!seen_sequence && item.at("Sequence").at("id") == "A", "single template must contain exactly one sequence A");
+                    seen_sequence = true;
+                } else {
+                    auto name         = item.at("SpecialToken").at("id").get<std::string>();
+                    const auto& token = processor.at("special_tokens").at(name);
+                    tokenizer_require(token.at("ids").size() == 1 && token.at("tokens").size() == 1, "multi-ID template special tokens are unsupported");
+                    int id      = special(TokenizerJSON::array({token.at("tokens").at(0), token.at("ids").at(0)}));
+                    int& target = seen_sequence ? EOS_TOKEN_ID : BOS_TOKEN_ID;
+                    tokenizer_require(target < 0, "single template supports at most one prefix and one suffix token");
+                    target = id;
+                }
+            }
+            tokenizer_require(seen_sequence, "single template has no sequence A");
+        } else {
+            tokenizer_require(type == "ByteLevel", "unsupported post_processor: " + type);
+        }
+    }
+    add_bos_token = BOS_TOKEN_ID >= 0;
+    add_eos_token = EOS_TOKEN_ID >= 0;
+    BOS_TOKEN     = decode_token(BOS_TOKEN_ID);
+    EOS_TOKEN     = decode_token(EOS_TOKEN_ID);
+    set_padding(0, false);
+}
+
+HFTokenizer::~HFTokenizer() = default;
+
+void HFTokenizer::set_padding(int token_id, bool left) {
+    PAD_TOKEN_ID = token_id;
+    PAD_TOKEN    = decode_token(token_id);
+    pad_left     = left;
+}
+
+void HFTokenizer::validate_vocab_size(int64_t embedding_rows) const {
+    tokenizer_require(embedding_rows > 0, "text encoder has no token embedding rows");
+    for (const auto& token : impl_->tokens) {
+        tokenizer_require(token.first < embedding_rows, "token ID " + std::to_string(token.first) + " exceeds text encoder vocabulary (" + std::to_string(embedding_rows) + ")");
+    }
+    tokenizer_require(PAD_TOKEN_ID >= 0 && PAD_TOKEN_ID < embedding_rows, "padding ID exceeds text encoder vocabulary");
+}
+
+int HFTokenizer::token_to_id(const std::string& token) const {
+    return impl_->lookup(token);
+}
+
+void HFTokenizer::add_special_token(const std::string& token) {
+    Tokenizer::add_special_token(token);
+    if (!token.empty()) {
+        impl_->custom_tokens.push_back(token);
+    }
+}
+
+bool HFTokenizer::encode(const std::string& text, std::vector<int>& tokens, on_new_token_cb_t callback, std::string* error) {
+    tokens.clear();
+    if (error) {
+        error->clear();
+    }
+    for (size_t i = 0; i < text.size();) {
+        int32_t cp;
+        if (!tokenizer_next(text, i, cp, error)) {
+            return false;
+        }
+    }
+    std::vector<int> result;
+    Impl::Trie raw_custom, normalized_custom;
+    if (callback) {
+        for (size_t index = 0; index < impl_->custom_tokens.size(); ++index) {
+            const auto& token = impl_->custom_tokens[index];
+            raw_custom.add(token, static_cast<int>(index));
+            std::string normalized;
+            if (!impl_->normalize(token, normalized, error)) {
+                return false;
+            }
+            if (!normalized.empty()) {
+                normalized_custom.add(normalized, static_cast<int>(index));
+            }
+        }
+    }
+    auto encode_plain = [&](const std::string& value) {
+        std::vector<std::string> pieces;
+        if (!impl_->pre_tokenize(value, pieces, error)) {
+            return false;
+        }
+        for (auto& piece : pieces) {
+            if (callback && callback(piece, result)) {
+                continue;
+            }
+            std::vector<int> ids;
+            if (!impl_->bpe(piece, ids, error)) {
+                return false;
+            }
+            result.insert(result.end(), ids.begin(), ids.end());
+        }
+        return true;
+    };
+    auto extract = [&](const std::string& value, const Impl::Trie& added, const Impl::Trie& custom_tokens, const auto& encode_gap) {
+        size_t start = 0, i = 0;
+        while (i < value.size()) {
+            auto match      = added.match(value, i);
+            auto custom     = custom_tokens.match(value, i);
+            bool use_custom = custom.second >= 0 && custom.first >= match.first;
+            if (match.second < 0 && !use_custom) {
+                ++i;
+                continue;
+            }
+            if (!encode_gap(value.substr(start, i - start))) {
+                return false;
+            }
+            if (use_custom) {
+                auto token = impl_->custom_tokens[custom.second];
+                if (!callback(token, result)) {
+                    if (match.second >= 0 && match.first == custom.first) {
+                        result.push_back(match.second);
+                    } else if (!encode_gap(value.substr(i, custom.first - i))) {
+                        return false;
+                    }
+                }
+            } else {
+                result.push_back(match.second);
+            }
+            start = i = use_custom ? custom.first : match.first;
+        }
+        return encode_gap(value.substr(start));
+    };
+    auto encode_normalized = [&](const std::string& value) {
+        std::string normalized;
+        if (!impl_->normalize(value, normalized, error)) {
+            return false;
+        }
+        return extract(normalized, impl_->normalized_added, normalized_custom, encode_plain);
+    };
+    if (!extract(text, impl_->raw_added, raw_custom, encode_normalized)) {
+        return false;
+    }
+    std::stringstream ss;
+    ss << "[";
+    for (int id : result) {
+        auto token = impl_->tokens.find(id);
+        if (token != impl_->tokens.end()) {
+            ss << "\"" << token->second << "\", ";
+        } else {
+            ss << "\"<id:" << id << ">\", ";
+        }
+    }
+    ss << "]";
+    LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), result.size(), ss.str().c_str());
+    tokens = std::move(result);
+    return true;
+}
+
+std::string HFTokenizer::decode_token(int id) const {
+    auto found = impl_->tokens.find(id);
+    return found == impl_->tokens.end() ? "" : found->second;
+}
+
+static std::string tokenizer_lossy_utf8(const std::string& bytes, bool fallback) {
+    std::string result;
+    for (size_t i = 0; i < bytes.size();) {
+        int32_t cp;
+        auto count = utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(bytes.data() + i), bytes.size() - i, &cp);
+        if (count > 0) {
+            result.append(bytes, i, count);
+            i += count;
+        } else if (fallback) {
+            result.clear();
+            for (size_t j = 0; j < bytes.size(); ++j) {
+                result += "\xef\xbf\xbd";
+            }
+            return result;
+        } else {
+            result += "\xef\xbf\xbd";
+            unsigned char lead = bytes[i++];
+            size_t expected    = lead >= 0xc2 && lead <= 0xdf ? 2 : lead >= 0xe0 && lead <= 0xef ? 3
+                                                                : lead >= 0xf0 && lead <= 0xf4   ? 4
+                                                                                                 : 1;
+            for (size_t j = 1; j < expected && i < bytes.size(); ++j) {
+                unsigned char c = bytes[i];
+                if (c < 0x80 || c > 0xbf || (j == 1 && ((lead == 0xe0 && c < 0xa0) || (lead == 0xed && c > 0x9f) || (lead == 0xf0 && c < 0x90) || (lead == 0xf4 && c > 0x8f)))) {
+                    break;
+                }
+                ++i;
+            }
+        }
+    }
+    return result;
+}
+
+bool HFTokenizer::decode(const std::vector<int>& ids, std::string& text, std::string* error) const {
+    text.clear();
+    if (error) {
+        error->clear();
+    }
+    std::vector<std::string> pieces;
+    for (int id : ids) {
+        if (!impl_->special_ids.count(id) && impl_->tokens.count(id)) {
+            pieces.push_back(decode_token(id));
+        }
+    }
+    for (const auto& step : impl_->decoders) {
+        if (step.type == "Replace") {
+            for (auto& piece : pieces) {
+                std::string replaced;
+                if (!step.pattern->replace(piece, step.content, replaced, error)) {
+                    return false;
+                }
+                piece = std::move(replaced);
+            }
+        } else if (step.type == "ByteLevel" || step.type == "Fuse") {
+            std::string joined;
+            for (const auto& piece : pieces) {
+                std::string bytes;
+                if (step.type == "ByteLevel") {
+                    for (size_t i = 0; i < piece.size();) {
+                        int32_t cp;
+                        if (!tokenizer_next(piece, i, cp, error)) {
+                            return false;
+                        }
+                        auto found = impl_->byte_decoder.find(cp);
+                        if (found == impl_->byte_decoder.end()) {
+                            bytes = piece;
+                            break;
+                        }
+                        bytes += static_cast<char>(found->second);
+                    }
+                } else {
+                    bytes = piece;
+                }
+                joined += bytes;
+            }
+            pieces = {step.type == "ByteLevel" ? tokenizer_lossy_utf8(joined, false) : joined};
+        } else {
+            std::vector<std::string> decoded;
+            std::string bytes;
+            auto flush = [&] {
+                if (!bytes.empty()) {
+                    decoded.push_back(tokenizer_lossy_utf8(bytes, true));
+                    bytes.clear();
+                }
+            };
+            for (const auto& piece : pieces) {
+                auto hex = [](char c) { return c >= '0' && c <= '9' ? c - '0' : c >= 'A' && c <= 'F' ? c - 'A' + 10
+                                                                            : c >= 'a' && c <= 'f'   ? c - 'a' + 10
+                                                                                                     : -1; };
+                if (piece.size() == 6 && piece.compare(0, 3, "<0x") == 0 && piece[5] == '>' && hex(piece[3]) >= 0 && hex(piece[4]) >= 0) {
+                    bytes += static_cast<char>((hex(piece[3]) << 4) | hex(piece[4]));
+                } else {
+                    flush();
+                    decoded.push_back(piece);
+                }
+            }
+            flush();
+            pieces = std::move(decoded);
+        }
+    }
+    std::string result;
+    for (size_t i = 0; i < pieces.size(); ++i) {
+        if (i && !impl_->has_decoder) {
+            result += ' ';
+        }
+        result += pieces[i];
+    }
+    text = std::move(result);
+    return true;
+}

src/tokenizers/hf_tokenizer.h

@@ -0,0 +1,26 @@
+#ifndef __SD_TOKENIZERS_HF_TOKENIZER_H__
+#define __SD_TOKENIZERS_HF_TOKENIZER_H__
+
+#include <memory>
+
+#include "tokenizer.h"
+
+class HFTokenizer : public Tokenizer {
+    struct Impl;
+    std::unique_ptr<Impl> impl_;
+    std::string decode_token(int token_id) const override;
+
+public:
+    explicit HFTokenizer(const std::string& path);
+    ~HFTokenizer() override;
+
+    // Padding belongs to the encoder; tokenizer.json supplies the single-sequence template.
+    void set_padding(int token_id, bool left);
+    void validate_vocab_size(int64_t embedding_rows) const;
+    int token_to_id(const std::string& token) const;
+    void add_special_token(const std::string& token) override;
+    bool encode(const std::string& text, std::vector<int>& tokens, on_new_token_cb_t on_new_token_cb = nullptr, std::string* error = nullptr) override;
+    bool decode(const std::vector<int>& tokens, std::string& text, std::string* error = nullptr) const override;
+};
+
+#endif  // __SD_TOKENIZERS_HF_TOKENIZER_H__

src/tokenizers/t5_unigram_tokenizer.cpp

@@ -287,7 +287,11 @@ std::string T5UniGramTokenizer::normalize(const std::string& input) const {
     return normalized;
 }
 
-std::vector<int> T5UniGramTokenizer::encode(const std::string& input, on_new_token_cb_t on_new_token_cb) {
+bool T5UniGramTokenizer::encode(const std::string& input, std::vector<int>& result, on_new_token_cb_t on_new_token_cb, std::string* error) {
+    result.clear();
+    if (error) {
+        error->clear();
+    }
     std::vector<int32_t> tokens;
     std::vector<std::string> token_strs;
     std::string normalized = normalize(input);
@@ -335,5 +339,6 @@ std::vector<int> T5UniGramTokenizer::encode(const std::string& input, on_new_tok
     ss << "]";
     LOG_VERBOSE("split prompt \"%s\" to tokens %s", input.c_str(), ss.str().c_str());
 
-    return tokens;
+    result = std::move(tokens);
+    return true;
 }

src/tokenizers/t5_unigram_tokenizer.h

@@ -64,7 +64,7 @@ public:
     explicit T5UniGramTokenizer(bool is_umt5 = false);
     ~T5UniGramTokenizer();
 
-    std::vector<int> encode(const std::string& input, on_new_token_cb_t on_new_token_cb = nullptr) override;
+    bool encode(const std::string& input, std::vector<int>& tokens, on_new_token_cb_t on_new_token_cb = nullptr, std::string* error = nullptr) override;
 };
 
 #endif  // __SD_TOKENIZERS_T5_UNIGRAM_TOKENIZER_H__

src/tokenizers/tokenizer.cpp

@@ -23,17 +23,22 @@ std::string Tokenizer::normalize(const std::string& text) const {
     return text;
 }
 
-std::vector<int> Tokenizer::tokenize(const std::string& text,
+bool Tokenizer::tokenize(const std::string& text,
-                                     on_new_token_cb_t on_new_token_cb,
+                         std::vector<int>& tokens,
-                                     bool padding,
+                         on_new_token_cb_t on_new_token_cb,
-                                     size_t min_length,
+                         bool padding,
-                                     size_t max_length,
+                         size_t min_length,
-                                     bool allow_overflow_expand) {
+                         size_t max_length,
-    std::vector<int> tokens = encode(text, on_new_token_cb);
+                         bool allow_overflow_expand,
+                         std::string* error) {
+    if (!encode(text, tokens, on_new_token_cb, error)) {
+        tokens.clear();
+        return false;
+    }
     if (padding) {
         pad_tokens(tokens, nullptr, nullptr, min_length, max_length, allow_overflow_expand);
     }
-    return tokens;
+    return true;
 }
 
 void Tokenizer::pad_tokens(std::vector<int>& tokens,
@@ -200,8 +205,11 @@ static std::string clean_up_tokenization(std::string& text) {
     return std::regex_replace(text, pattern, ",");
 }
 
-std::string Tokenizer::decode(const std::vector<int>& tokens) const {
+bool Tokenizer::decode(const std::vector<int>& tokens, std::string& text, std::string* error) const {
-    std::string text;
+    text.clear();
+    if (error) {
+        error->clear();
+    }
 
     for (int token_id : tokens) {
         if (token_id == BOS_TOKEN_ID || token_id == EOS_TOKEN_ID || token_id == PAD_TOKEN_ID) {
@@ -218,5 +226,6 @@ std::string Tokenizer::decode(const std::vector<int>& tokens) const {
     }
 
     text = clean_up_tokenization(text);
-    return trim(text);
+    text = trim(text);
+    return true;
 }

src/tokenizers/tokenizer.h

@@ -33,22 +33,25 @@ public:
 
     virtual ~Tokenizer() = default;
 
-    void add_special_token(const std::string& token);
+    virtual void add_special_token(const std::string& token);
     bool is_special_token(const std::string& token) const;
-    virtual std::vector<int> encode(const std::string& text, on_new_token_cb_t on_new_token_cb = nullptr) = 0;
+    // An empty output may be valid; failures return false and clear the output.
-    std::vector<int> tokenize(const std::string& text,
+    virtual bool encode(const std::string& text, std::vector<int>& tokens, on_new_token_cb_t on_new_token_cb = nullptr, std::string* error = nullptr) = 0;
-                              on_new_token_cb_t on_new_token_cb = nullptr,
+    bool tokenize(const std::string& text,
-                              bool padding                      = false,
+                  std::vector<int>& tokens,
-                              size_t min_length                 = 0,
+                  on_new_token_cb_t on_new_token_cb = nullptr,
-                              size_t max_length                 = 100000000,
+                  bool padding                      = false,
-                              bool allow_overflow_expand        = false);
+                  size_t min_length                 = 0,
+                  size_t max_length                 = 100000000,
+                  bool allow_overflow_expand        = false,
+                  std::string* error                = nullptr);
     void pad_tokens(std::vector<int>& tokens,
                     std::vector<float>* weights,
                     std::vector<float>* mask,
                     size_t min_length          = 0,
                     size_t max_length          = 100000000,
                     bool allow_overflow_expand = false);
-    std::string decode(const std::vector<int>& tokens) const;
+    virtual bool decode(const std::vector<int>& tokens, std::string& text, std::string* error = nullptr) const;
 };
 
 #endif  // __SD_TOKENIZERS_TOKENIZER_H__

src/tokenizers/tokenizer_config.cpp

@@ -0,0 +1,77 @@
+#include "tokenizer_config.h"
+
+#include <stdexcept>
+
+#include "core/util.h"
+#include "hf_tokenizer.h"
+
+TokenizerConfig::TokenizerConfig(const char* config) {
+    if (!config || !*config) {
+        return;
+    }
+    const std::string value = config;
+    const bool assignments  = value.find('=') != std::string::npos;
+    size_t begin            = 0;
+    for (;;) {
+        const auto end   = assignments ? value.find(',', begin) : std::string::npos;
+        const auto entry = value.substr(begin, end == std::string::npos ? end : end - begin);
+        const auto equal = entry.find('=');
+        if (assignments && equal == std::string::npos) {
+            throw std::runtime_error("invalid tokenizer entry '" + entry + "'; expected main=FILE,clip-l=FILE,clip-g=FILE");
+        }
+        const auto key  = assignments ? entry.substr(0, equal) : "main";
+        const auto path = assignments ? entry.substr(equal + 1) : entry;
+        Slot slot       = MAIN;
+        if (key == "clip-l") {
+            slot = CLIP_L;
+        } else if (key == "clip-g") {
+            slot = CLIP_G;
+        } else if (key != "main") {
+            throw std::runtime_error("unknown tokenizer slot '" + key + "'; expected main, clip-l or clip-g");
+        }
+        if (path.empty()) {
+            throw std::runtime_error("tokenizer slot '" + key + "' requires a nonempty path");
+        }
+        if (!paths_[slot].empty()) {
+            throw std::runtime_error("tokenizer slot '" + key + "' is specified more than once");
+        }
+        paths_[slot] = path;
+        if (end == std::string::npos) {
+            break;
+        }
+        begin = end + 1;
+    }
+}
+
+bool TokenizerConfig::has(Slot slot) const {
+    return !paths_[slot].empty();
+}
+
+std::shared_ptr<Tokenizer> TokenizerConfig::create(Slot slot, int64_t embedding_rows, int padding_id, bool pad_left, bool clip) const {
+    if (!has(slot)) {
+        return nullptr;
+    }
+    try {
+        auto tokenizer = std::make_shared<HFTokenizer>(paths_[slot]);
+        tokenizer->set_padding(padding_id, pad_left);
+        tokenizer->validate_vocab_size(embedding_rows);
+        if (clip && (tokenizer->BOS_TOKEN_ID < 0 || tokenizer->EOS_TOKEN_ID < 0)) {
+            throw std::runtime_error("CLIP requires a single BOS + A + EOS template");
+        }
+        used_[slot]         = true;
+        const char* names[] = {"main", "clip-l", "clip-g"};
+        LOG_INFO("using external tokenizer (%s): %s", names[slot], paths_[slot].c_str());
+        return tokenizer;
+    } catch (const std::exception& error) {
+        throw std::runtime_error("failed to load tokenizer '" + paths_[slot] + "': " + error.what());
+    }
+}
+
+void TokenizerConfig::validate_usage() const {
+    const char* names[] = {"main", "clip-l", "clip-g"};
+    for (size_t i = 0; i < paths_.size(); ++i) {
+        if (!paths_[i].empty() && !used_[i]) {
+            throw std::runtime_error(std::string("tokenizer slot '") + names[i] + "' does not target an active, supported text encoder; SD3 uses the clip-l and clip-g slots");
+        }
+    }
+}

src/tokenizers/tokenizer_config.h

@@ -0,0 +1,28 @@
+#ifndef __SD_TOKENIZERS_TOKENIZER_CONFIG_H__
+#define __SD_TOKENIZERS_TOKENIZER_CONFIG_H__
+
+#include <array>
+#include <memory>
+#include <string>
+
+#include "tokenizer.h"
+
+class TokenizerConfig {
+public:
+    enum Slot { MAIN,
+                CLIP_L,
+                CLIP_G };
+
+private:
+    std::array<std::string, 3> paths_;
+    mutable std::array<bool, 3> used_{};
+
+public:
+    TokenizerConfig() = default;
+    explicit TokenizerConfig(const char* config);
+    bool has(Slot slot) const;
+    std::shared_ptr<Tokenizer> create(Slot slot, int64_t embedding_rows, int padding_id, bool pad_left = false, bool clip = false) const;
+    void validate_usage() const;
+};
+
+#endif  // __SD_TOKENIZERS_TOKENIZER_CONFIG_H__

thirdparty/CMakeLists.txt

@@ -2,6 +2,70 @@ set(Z_TARGET zip)
 add_library(${Z_TARGET} OBJECT zip.c zip.h miniz.h)
 target_include_directories(${Z_TARGET} PUBLIC .)
 
+function(sd_add_oniguruma)
+    include(CheckIncludeFiles)
+    include(CheckSymbolExists)
+    include(CheckTypeSize)
+    include(GNUInstallDirs)
+
+    foreach(header IN ITEMS alloca.h stdint.h sys/times.h sys/time.h sys/types.h unistd.h inttypes.h)
+        string(TOUPPER "${header}" name)
+        string(REGEX REPLACE "[./]" "_" name "${name}")
+        check_include_files("${header}" SD_ONIG_HAVE_${name})
+        set(HAVE_${name} "${SD_ONIG_HAVE_${name}}")
+    endforeach()
+    check_type_size(int SD_ONIG_SIZEOF_INT)
+    check_type_size(long SD_ONIG_SIZEOF_LONG)
+    check_type_size("long long" SD_ONIG_SIZEOF_LONG_LONG)
+    check_type_size("void*" SD_ONIG_SIZEOF_VOIDP)
+    foreach(name IN ITEMS SIZEOF_INT SIZEOF_LONG SIZEOF_LONG_LONG SIZEOF_VOIDP)
+        set(${name} "${SD_ONIG_${name}}")
+    endforeach()
+    if(HAVE_ALLOCA_H)
+        check_symbol_exists(alloca "alloca.h" SD_ONIG_HAVE_ALLOCA)
+    else()
+        check_symbol_exists(alloca "stdlib.h;malloc.h" SD_ONIG_HAVE_ALLOCA)
+    endif()
+    set(HAVE_ALLOCA "${SD_ONIG_HAVE_ALLOCA}")
+    set(PACKAGE onig)
+    set(PACKAGE_VERSION 6.9.10)
+    set(VERSION "${PACKAGE_VERSION}")
+    set(USE_CRNL_AS_LINE_TERMINATOR 0)
+    configure_file(oniguruma/config.h.cmake.in oniguruma/config.h)
+
+    # unicode.c includes its four data tables; they are not separate translation units.
+    add_library(onig OBJECT
+        oniguruma/ascii.c
+        oniguruma/regcomp.c
+        oniguruma/regenc.c
+        oniguruma/regerror.c
+        oniguruma/regexec.c
+        oniguruma/regparse.c
+        oniguruma/st.c
+        oniguruma/unicode.c
+        oniguruma/unicode_fold1_key.c
+        oniguruma/unicode_fold2_key.c
+        oniguruma/unicode_fold3_key.c
+        oniguruma/unicode_unfold_key.c
+        oniguruma/utf8.c)
+    target_include_directories(onig PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/oniguruma"
+                                   PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/oniguruma")
+    target_compile_definitions(onig PUBLIC ONIG_STATIC)
+    set_target_properties(onig PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
+    install(FILES oniguruma/COPYING
+            DESTINATION "${CMAKE_INSTALL_DATADIR}/licenses/stable-diffusion/oniguruma")
+endfunction()
+
+sd_add_oniguruma()
+
+add_library(sd-utf8proc OBJECT utf8proc/utf8proc.c)
+target_include_directories(sd-utf8proc PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/utf8proc")
+target_compile_definitions(sd-utf8proc PUBLIC UTF8PROC_STATIC)
+set_target_properties(sd-utf8proc PROPERTIES POSITION_INDEPENDENT_CODE ON)
+include(GNUInstallDirs)
+install(FILES utf8proc/LICENSE.md DESTINATION "${CMAKE_INSTALL_DATADIR}/licenses/stable-diffusion/utf8proc")
+
 if(SD_WEBP AND NOT SD_USE_SYSTEM_WEBP)
     set(WEBP_BUILD_ANIM_UTILS OFF)
     set(WEBP_BUILD_CWEBP OFF)

thirdparty/README.md

@@ -7,4 +7,10 @@
 - httplib.h from: https://github.com/yhirose/cpp-httplib/blob/master/httplib.h
     - LICENSE: https://github.com/yhirose/cpp-httplib/blob/master/LICENSE
 - stb_image.h/stb_image_resize.h/stb_image_write.h from: https://github.com/nothings/stb
-    - LICENSE: https://github.com/nothings/stb/blob/master/LICENSE
+    - LICENSE: https://github.com/nothings/stb/blob/master/LICENSE
+- Oniguruma from: https://github.com/kkos/oniguruma
+    - Version and source details: [README](oniguruma/README.md)
+    - LICENSE: [BSD-2-Clause](oniguruma/COPYING)
+- utf8proc from: https://github.com/JuliaStrings/utf8proc
+    - Version and source details: [README](utf8proc/README.md)
+    - LICENSE: [MIT and Unicode data licenses](utf8proc/LICENSE.md)

thirdparty/oniguruma/COPYING

@@ -0,0 +1,26 @@
+Oniguruma LICENSE
+-----------------
+
+Copyright (c) 2002-2021  K.Kosako  <kkosako0@gmail.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.

thirdparty/oniguruma/README.md

@@ -0,0 +1,88 @@
+# Oniguruma source subset
+
+This directory contains the Oniguruma sources needed by sd.cpp's
+[UTF-8 regex wrapper](../../src/core/regex.cpp).
+
+## Upstream source
+
+- Repository: [kkos/oniguruma](https://github.com/kkos/oniguruma)
+- Release: `v6.9.10`
+- Commit: [`4ef89209a239c1aea328cf13c05a2807e5c146d1`](https://github.com/kkos/oniguruma/tree/4ef89209a239c1aea328cf13c05a2807e5c146d1)
+- License: [BSD-2-Clause](COPYING)
+
+The 24 upstream files below were copied without local modifications. `COPYING`
+comes from the upstream repository root; all other files come from upstream
+`src/` and retain their original filenames, flattened into this directory.
+This README is maintained by sd.cpp and is not part of the upstream copy.
+
+## File selection
+
+The following 13 C files are compiled separately:
+
+```text
+ascii.c
+regcomp.c
+regenc.c
+regerror.c
+regexec.c
+regparse.c
+st.c
+unicode.c
+unicode_fold1_key.c
+unicode_fold2_key.c
+unicode_fold3_key.c
+unicode_unfold_key.c
+utf8.c
+```
+
+Four Unicode data tables are included by `unicode.c` and must not be compiled
+as separate translation units. They were copied from upstream as supplied;
+sd.cpp does not regenerate them:
+
+```text
+unicode_fold_data.c
+unicode_property_data.c
+unicode_egcb_data.c
+unicode_wb_data.c
+```
+
+The remaining files are five headers, the platform configuration template,
+and the license:
+
+```text
+oniguruma.h
+regint.h
+regenc.h
+regparse.h
+st.h
+config.h.cmake.in
+COPYING
+```
+
+The wrapper uses `ONIG_ENCODING_UTF8` and `ONIG_SYNTAX_ONIGURUMA`. ASCII support
+is also retained because the engine requires it for initialization and error
+handling. Unicode property, case-folding, grapheme-cluster and word-boundary
+support remain enabled as in upstream.
+
+Other encodings, GNU/POSIX compatibility APIs, unused API implementations,
+upstream tests, examples, build scripts and packaging files are omitted.
+This subset supports sd.cpp's internal wrapper; it does not provide the full
+Oniguruma API declared in `oniguruma.h`.
+
+## Build integration
+
+[The parent CMake file](../CMakeLists.txt) detects platform headers and type
+sizes, then generates `config.h` from `config.h.cmake.in` in the build directory.
+It builds the 13 C files as the `onig` OBJECT target with `ONIG_STATIC` and
+position-independent code enabled. The resulting objects are included directly
+in the sd.cpp static or shared library, with no separate Oniguruma library
+required by consumers.
+
+Keep all 24 upstream files under version control, including the Unicode tables,
+configuration template and license. Generated `config.h` and build artifacts
+belong in the build directory.
+
+When refreshing this subset, copy the listed files from the selected upstream
+revision, retain `COPYING`, and update the revision recorded here.
+Recheck source dependencies and the CMake configuration, then validate the
+regex wrapper and tokenizer output on supported platforms.

thirdparty/oniguruma/ascii.c

@@ -0,0 +1,121 @@
+/**********************************************************************
+  ascii.c -  Oniguruma (regular expression library)
+**********************************************************************/
+/*-
+ * Copyright (c) 2002-2024  K.Kosako
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "regint.h"   /* for USE_CALLOUT */
+
+static int
+init(void)
+{
+#ifdef USE_CALLOUT
+
+    int id;
+    OnigEncoding enc;
+    char* name;
+    unsigned int args[4];
+    OnigValue    opts[4];
+
+    enc = ONIG_ENCODING_ASCII;
+
+    name = "FAIL";        BC0_P(name, fail);
+    name = "MISMATCH";    BC0_P(name, mismatch);
+#ifdef USE_SKIP_SEARCH
+    name = "SKIP";        BC0_P(name, skip);
+#endif
+
+    name = "MAX";
+    args[0] = ONIG_TYPE_TAG | ONIG_TYPE_LONG;
+    args[1] = ONIG_TYPE_CHAR;
+    opts[0].c = 'X';
+    BC_B_O(name, max, 2, args, 1, opts);
+
+    name = "ERROR";
+    args[0] = ONIG_TYPE_LONG; opts[0].l = ONIG_ABORT;
+    BC_P_O(name, error, 1, args, 1, opts);
+
+    name = "COUNT";
+    args[0] = ONIG_TYPE_CHAR; opts[0].c = '>';
+    BC_B_O(name, count, 1, args, 1, opts);
+
+    name = "TOTAL_COUNT";
+    args[0] = ONIG_TYPE_CHAR; opts[0].c = '>';
+    BC_B_O(name, total_count, 1, args, 1, opts);
+
+    name = "CMP";
+    args[0] = ONIG_TYPE_TAG | ONIG_TYPE_LONG;
+    args[1] = ONIG_TYPE_STRING;
+    args[2] = ONIG_TYPE_TAG | ONIG_TYPE_LONG;
+    BC_P(name, cmp, 3, args);
+
+#endif /* USE_CALLOUT */
+
+  return ONIG_NORMAL;
+}
+
+#if 0
+static int
+is_initialized(void)
+{
+  /* Don't use this function */
+  /* can't answer, because builtin callout entries removed in onig_end() */
+  return 0;
+}
+#endif
+
+static int
+ascii_is_code_ctype(OnigCodePoint code, unsigned int ctype)
+{
+  if (code < 128)
+    return ONIGENC_IS_ASCII_CODE_CTYPE(code, ctype);
+  else
+    return FALSE;
+}
+
+OnigEncodingType OnigEncodingASCII = {
+  onigenc_single_byte_mbc_enc_len,
+  "US-ASCII",  /* name */
+  1,           /* max enc length */
+  1,           /* min enc length */
+  onigenc_is_mbc_newline_0x0a,
+  onigenc_single_byte_mbc_to_code,
+  onigenc_single_byte_code_to_mbclen,
+  onigenc_single_byte_code_to_mbc,
+  onigenc_ascii_mbc_case_fold,
+  onigenc_ascii_apply_all_case_fold,
+  onigenc_ascii_get_case_fold_codes_by_str,
+  onigenc_minimum_property_name_to_ctype,
+  ascii_is_code_ctype,
+  onigenc_not_support_get_ctype_code_range,
+  onigenc_single_byte_left_adjust_char_head,
+  onigenc_always_true_is_allowed_reverse_match,
+  init,
+  0, /* is_initialized */
+  onigenc_always_true_is_valid_mbc_string,
+  ENC_FLAG_ASCII_COMPATIBLE|ENC_FLAG_SKIP_OFFSET_1,
+  0, 0
+};

thirdparty/oniguruma/config.h.cmake.in

@@ -0,0 +1,56 @@
+/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP
+   systems. This function is required for `alloca.c' support on those systems.
+   */
+#cmakedefine CRAY_STACKSEG_END
+
+/* Define to 1 if using `alloca.c'. */
+#cmakedefine C_ALLOCA
+
+/* Define to 1 if you have `alloca', as a function or macro. */
+#cmakedefine HAVE_ALLOCA  ${HAVE_ALLOCA}
+
+/* Define to 1 if you have <alloca.h> and it should be used (not on Ultrix).
+   */
+#cmakedefine HAVE_ALLOCA_H  ${HAVE_ALLOCA_H}
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#cmakedefine HAVE_STDINT_H  ${HAVE_STDINT_H}
+
+/* Define to 1 if you have the <sys/times.h> header file. */
+#cmakedefine HAVE_SYS_TIMES_H  ${HAVE_SYS_TIMES_H}
+
+/* Define to 1 if you have the <sys/time.h> header file. */
+#cmakedefine HAVE_SYS_TIME_H  ${HAVE_SYS_TIME_H}
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#cmakedefine HAVE_SYS_TYPES_H  ${HAVE_SYS_TYPES_H}
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#cmakedefine HAVE_UNISTD_H  ${HAVE_UNISTD_H}
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#cmakedefine HAVE_INTTYPES_H  ${HAVE_INTTYPES_H}
+
+/* Name of package */
+#cmakedefine PACKAGE  ${PACKAGE}
+
+/* Define to the version of this package. */
+#cmakedefine PACKAGE_VERSION  ${PACKAGE_VERSION}
+
+/* The size of `int', as computed by sizeof. */
+#cmakedefine SIZEOF_INT  ${SIZEOF_INT}
+
+/* The size of `long', as computed by sizeof. */
+#cmakedefine SIZEOF_LONG  ${SIZEOF_LONG}
+
+/* The size of `long long', as computed by sizeof. */
+#cmakedefine SIZEOF_LONG_LONG  ${SIZEOF_LONG_LONG}
+
+/* The size of `void*', as computed by sizeof. */
+#cmakedefine SIZEOF_VOIDP  ${SIZEOF_VOIDP}
+
+/* Define if enable CR+NL as line terminator */
+#cmakedefine USE_CRNL_AS_LINE_TERMINATOR  ${USE_CRNL_AS_LINE_TERMINATOR}
+
+/* Version number of package */
+#cmakedefine VERSION  ${VERSION}

thirdparty/oniguruma/regenc.c

@@ -0,0 +1,994 @@
+/**********************************************************************
+  regenc.c -  Oniguruma (regular expression library)
+**********************************************************************/
+/*-
+ * Copyright (c) 2002-2020  K.Kosako
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "regint.h"
+
+#define LARGE_S   0x53
+#define SMALL_S   0x73
+
+OnigEncoding OnigEncDefaultCharEncoding = ONIG_ENCODING_INIT_DEFAULT;
+
+#define INITED_LIST_SIZE  20
+
+static int InitedListNum;
+
+static struct {
+  OnigEncoding enc;
+  int          inited;
+} InitedList[INITED_LIST_SIZE];
+
+static int
+enc_inited_entry(OnigEncoding enc)
+{
+  int i;
+
+  for (i = 0; i < InitedListNum; i++) {
+    if (InitedList[i].enc == enc) {
+      InitedList[i].inited = 1;
+      return i;
+    }
+  }
+
+  i = InitedListNum;
+  if (i < INITED_LIST_SIZE - 1) {
+    InitedList[i].enc    = enc;
+    InitedList[i].inited = 1;
+    InitedListNum++;
+    return i;
+  }
+
+  return -1;
+}
+
+static int
+enc_is_inited(OnigEncoding enc)
+{
+  int i;
+
+  for (i = 0; i < InitedListNum; i++) {
+    if (InitedList[i].enc == enc) {
+      return InitedList[i].inited;
+    }
+  }
+
+  return 0;
+}
+
+static int OnigEncInited;
+
+extern int
+onigenc_init(void)
+{
+  if (OnigEncInited != 0) return 0;
+
+  OnigEncInited = 1;
+  return 0;
+}
+
+extern int
+onigenc_end(void)
+{
+  int i;
+
+  for (i = 0; i < InitedListNum; i++) {
+    InitedList[i].enc    = 0;
+    InitedList[i].inited = 0;
+  }
+  InitedListNum = 0;
+
+  OnigEncInited = 0;
+  return ONIG_NORMAL;
+}
+
+extern int
+onig_initialize_encoding(OnigEncoding enc)
+{
+  int r;
+
+  if (enc != ONIG_ENCODING_ASCII &&
+      ONIGENC_IS_ASCII_COMPATIBLE_ENCODING(enc)) {
+    OnigEncoding ascii = ONIG_ENCODING_ASCII;
+    if (ascii->init != 0 && enc_is_inited(ascii) == 0) {
+      r = ascii->init();
+      if (r != ONIG_NORMAL) return r;
+      enc_inited_entry(ascii);
+    }
+  }
+
+  if (enc->init != 0 &&
+      enc_is_inited(enc) == 0) {
+    r = (enc->init)();
+    if (r == ONIG_NORMAL)
+      enc_inited_entry(enc);
+    return r;
+  }
+
+  return 0;
+}
+
+extern OnigEncoding
+onigenc_get_default_encoding(void)
+{
+  return OnigEncDefaultCharEncoding;
+}
+
+extern int
+onigenc_set_default_encoding(OnigEncoding enc)
+{
+  OnigEncDefaultCharEncoding = enc;
+  return 0;
+}
+
+extern UChar*
+onigenc_strdup(OnigEncoding enc, const UChar* s, const UChar* end)
+{
+  int slen, term_len, i;
+  UChar *r;
+
+  slen = (int )(end - s);
+  term_len = ONIGENC_MBC_MINLEN(enc);
+
+  r = (UChar* )xmalloc(slen + term_len);
+  CHECK_NULL_RETURN(r);
+  xmemcpy(r, s, slen);
+
+  for (i = 0; i < term_len; i++)
+    r[slen + i] = (UChar )0;
+
+  return r;
+}
+
+extern UChar*
+onigenc_get_right_adjust_char_head(OnigEncoding enc, const UChar* start, const UChar* s)
+{
+  UChar* p = ONIGENC_LEFT_ADJUST_CHAR_HEAD(enc, start, s);
+  if (p < s) {
+    p += enclen(enc, p);
+  }
+  return p;
+}
+
+extern UChar*
+onigenc_get_right_adjust_char_head_with_prev(OnigEncoding enc,
+            const UChar* start, const UChar* s, const UChar** prev)
+{
+  UChar* p = ONIGENC_LEFT_ADJUST_CHAR_HEAD(enc, start, s);
+
+  if (p < s) {
+    if (prev) *prev = (const UChar* )p;
+    p += enclen(enc, p);
+  }
+  else {
+    if (prev)
+      *prev = onigenc_get_prev_char_head(enc, start, p);
+  }
+  return p;
+}
+
+extern UChar*
+onigenc_get_prev_char_head(OnigEncoding enc, const UChar* start, const UChar* s)
+{
+  if (s <= start)
+    return (UChar* )NULL;
+
+  return ONIGENC_LEFT_ADJUST_CHAR_HEAD(enc, start, s - 1);
+}
+
+extern UChar*
+onigenc_step_back(OnigEncoding enc, const UChar* start, const UChar* s, int n)
+{
+  while (ONIG_IS_NOT_NULL(s) && n-- > 0) {
+    if (s <= start)
+      return (UChar* )NULL;
+
+    s = ONIGENC_LEFT_ADJUST_CHAR_HEAD(enc, start, s - 1);
+  }
+  return (UChar* )s;
+}
+
+extern UChar*
+onigenc_step(OnigEncoding enc, const UChar* p, const UChar* end, int n)
+{
+  UChar* q = (UChar* )p;
+  while (n-- > 0) {
+    q += ONIGENC_MBC_ENC_LEN(enc, q);
+  }
+  return (q <= end ? q : NULL);
+}
+
+extern int
+onigenc_strlen(OnigEncoding enc, const UChar* p, const UChar* end)
+{
+  int n = 0;
+  UChar* q = (UChar* )p;
+
+  while (q < end) {
+    q += ONIGENC_MBC_ENC_LEN(enc, q);
+    n++;
+  }
+  return n;
+}
+
+extern int
+onigenc_strlen_null(OnigEncoding enc, const UChar* s)
+{
+  int n = 0;
+  UChar* p = (UChar* )s;
+
+  while (1) {
+    if (*p == '\0') {
+      UChar* q;
+      int len = ONIGENC_MBC_MINLEN(enc);
+
+      if (len == 1) return n;
+      q = p + 1;
+      while (len > 1) {
+        if (*q != '\0') break;
+        q++;
+        len--;
+      }
+      if (len == 1) return n;
+    }
+    p += ONIGENC_MBC_ENC_LEN(enc, p);
+    n++;
+  }
+}
+
+extern int
+onigenc_str_bytelen_null(OnigEncoding enc, const UChar* s)
+{
+  const UChar* start = s;
+  const UChar* p = s;
+
+  while (1) {
+    if (*p == '\0') {
+      const UChar* q;
+      int len = ONIGENC_MBC_MINLEN(enc);
+
+      if (len == 1) return (int )(p - start);
+      q = p + 1;
+      while (len > 1) {
+        if (*q != '\0') break;
+        q++;
+        len--;
+      }
+      if (len == 1) return (int )(p - start);
+    }
+    p += ONIGENC_MBC_ENC_LEN(enc, p);
+  }
+}
+
+const UChar OnigEncAsciiToLowerCaseTable[] = {
+  '\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+  '\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+  '\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+  '\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+  '\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+  '\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+  '\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+  '\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+  '\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+  '\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+  '\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+  '\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+  '\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+  '\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+  '\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+  '\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+  '\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+  '\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+  '\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+  '\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+  '\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+  '\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+  '\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+  '\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+  '\300', '\301', '\302', '\303', '\304', '\305', '\306', '\307',
+  '\310', '\311', '\312', '\313', '\314', '\315', '\316', '\317',
+  '\320', '\321', '\322', '\323', '\324', '\325', '\326', '\327',
+  '\330', '\331', '\332', '\333', '\334', '\335', '\336', '\337',
+  '\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+  '\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+  '\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+  '\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+#ifdef USE_UPPER_CASE_TABLE
+const UChar OnigEncAsciiToUpperCaseTable[256] = {
+  '\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+  '\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+  '\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+  '\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+  '\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+  '\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+  '\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+  '\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+  '\100', '\101', '\102', '\103', '\104', '\105', '\106', '\107',
+  '\110', '\111', '\112', '\113', '\114', '\115', '\116', '\117',
+  '\120', '\121', '\122', '\123', '\124', '\125', '\126', '\127',
+  '\130', '\131', '\132', '\133', '\134', '\135', '\136', '\137',
+  '\140', '\101', '\102', '\103', '\104', '\105', '\106', '\107',
+  '\110', '\111', '\112', '\113', '\114', '\115', '\116', '\117',
+  '\120', '\121', '\122', '\123', '\124', '\125', '\126', '\127',
+  '\130', '\131', '\132', '\173', '\174', '\175', '\176', '\177',
+  '\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+  '\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+  '\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+  '\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+  '\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+  '\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+  '\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+  '\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+  '\300', '\301', '\302', '\303', '\304', '\305', '\306', '\307',
+  '\310', '\311', '\312', '\313', '\314', '\315', '\316', '\317',
+  '\320', '\321', '\322', '\323', '\324', '\325', '\326', '\327',
+  '\330', '\331', '\332', '\333', '\334', '\335', '\336', '\337',
+  '\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+  '\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+  '\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+  '\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+#endif
+
+const unsigned short OnigEncAsciiCtypeTable[256] = {
+  0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
+  0x4008, 0x420c, 0x4209, 0x4208, 0x4208, 0x4208, 0x4008, 0x4008,
+  0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
+  0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
+  0x4284, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
+  0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
+  0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0,
+  0x78b0, 0x78b0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
+  0x41a0, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x74a2,
+  0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2,
+  0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2,
+  0x74a2, 0x74a2, 0x74a2, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x51a0,
+  0x41a0, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x70e2,
+  0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2,
+  0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2,
+  0x70e2, 0x70e2, 0x70e2, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x4008,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+  0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000
+};
+
+const UChar OnigEncISO_8859_1_ToLowerCaseTable[256] = {
+  '\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+  '\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+  '\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+  '\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+  '\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+  '\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+  '\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+  '\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+  '\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+  '\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+  '\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+  '\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+  '\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+  '\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+  '\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+  '\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+  '\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+  '\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+  '\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+  '\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+  '\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+  '\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+  '\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+  '\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+  '\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+  '\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+  '\360', '\361', '\362', '\363', '\364', '\365', '\366', '\327',
+  '\370', '\371', '\372', '\373', '\374', '\375', '\376', '\337',
+  '\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+  '\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+  '\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+  '\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377'
+};
+
+#ifdef USE_UPPER_CASE_TABLE
+const UChar OnigEncISO_8859_1_ToUpperCaseTable[256] = {
+  '\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+  '\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+  '\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+  '\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+  '\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+  '\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+  '\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+  '\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+  '\100', '\101', '\102', '\103', '\104', '\105', '\106', '\107',
+  '\110', '\111', '\112', '\113', '\114', '\115', '\116', '\117',
+  '\120', '\121', '\122', '\123', '\124', '\125', '\126', '\127',
+  '\130', '\131', '\132', '\133', '\134', '\135', '\136', '\137',
+  '\140', '\101', '\102', '\103', '\104', '\105', '\106', '\107',
+  '\110', '\111', '\112', '\113', '\114', '\115', '\116', '\117',
+  '\120', '\121', '\122', '\123', '\124', '\125', '\126', '\127',
+  '\130', '\131', '\132', '\173', '\174', '\175', '\176', '\177',
+  '\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+  '\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+  '\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+  '\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+  '\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+  '\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+  '\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+  '\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+  '\300', '\301', '\302', '\303', '\304', '\305', '\306', '\307',
+  '\310', '\311', '\312', '\313', '\314', '\315', '\316', '\317',
+  '\320', '\321', '\322', '\323', '\324', '\325', '\326', '\327',
+  '\330', '\331', '\332', '\333', '\334', '\335', '\336', '\337',
+  '\300', '\301', '\302', '\303', '\304', '\305', '\306', '\307',
+  '\310', '\311', '\312', '\313', '\314', '\315', '\316', '\317',
+  '\320', '\321', '\322', '\323', '\324', '\325', '\326', '\367',
+  '\330', '\331', '\332', '\333', '\334', '\335', '\336', '\377',
+};
+#endif
+
+extern void
+onigenc_set_default_caseconv_table(const UChar* table ARG_UNUSED)
+{
+  /* nothing */
+  /* obsoleted. */
+}
+
+extern UChar*
+onigenc_get_left_adjust_char_head(OnigEncoding enc, const UChar* start, const UChar* s)
+{
+  return ONIGENC_LEFT_ADJUST_CHAR_HEAD(enc, start, s);
+}
+
+const OnigPairCaseFoldCodes OnigAsciiLowerMap[] = {
+  { 0x41, 0x61 },
+  { 0x42, 0x62 },
+  { 0x43, 0x63 },
+  { 0x44, 0x64 },
+  { 0x45, 0x65 },
+  { 0x46, 0x66 },
+  { 0x47, 0x67 },
+  { 0x48, 0x68 },
+  { 0x49, 0x69 },
+  { 0x4a, 0x6a },
+  { 0x4b, 0x6b },
+  { 0x4c, 0x6c },
+  { 0x4d, 0x6d },
+  { 0x4e, 0x6e },
+  { 0x4f, 0x6f },
+  { 0x50, 0x70 },
+  { 0x51, 0x71 },
+  { 0x52, 0x72 },
+  { 0x53, 0x73 },
+  { 0x54, 0x74 },
+  { 0x55, 0x75 },
+  { 0x56, 0x76 },
+  { 0x57, 0x77 },
+  { 0x58, 0x78 },
+  { 0x59, 0x79 },
+  { 0x5a, 0x7a }
+};
+
+extern int
+onigenc_ascii_apply_all_case_fold(OnigCaseFoldType flag ARG_UNUSED,
+                                  OnigApplyAllCaseFoldFunc f, void* arg)
+{
+  OnigCodePoint code;
+  int i, r;
+
+  for (i = 0;
+       i < (int )(sizeof(OnigAsciiLowerMap)/sizeof(OnigPairCaseFoldCodes));
+       i++) {
+    code = OnigAsciiLowerMap[i].to;
+    r = (*f)(OnigAsciiLowerMap[i].from, &code, 1, arg);
+    if (r != 0) return r;
+
+    code = OnigAsciiLowerMap[i].from;
+    r = (*f)(OnigAsciiLowerMap[i].to, &code, 1, arg);
+    if (r != 0) return r;
+  }
+
+  return 0;
+}
+
+extern int
+onigenc_ascii_get_case_fold_codes_by_str(OnigCaseFoldType flag ARG_UNUSED,
+    const OnigUChar* p, const OnigUChar* end ARG_UNUSED,
+    OnigCaseFoldCodeItem items[])
+{
+  if (0x41 <= *p && *p <= 0x5a) {
+    items[0].byte_len = 1;
+    items[0].code_len = 1;
+    items[0].code[0] = (OnigCodePoint )(*p + 0x20);
+    return 1;
+  }
+  else if (0x61 <= *p && *p <= 0x7a) {
+    items[0].byte_len = 1;
+    items[0].code_len = 1;
+    items[0].code[0] = (OnigCodePoint )(*p - 0x20);
+    return 1;
+  }
+  else
+    return 0;
+}
+
+static int
+ss_apply_all_case_fold(OnigCaseFoldType flag ARG_UNUSED,
+                       OnigApplyAllCaseFoldFunc f, void* arg)
+{
+  static OnigCodePoint ss[] = { SMALL_S, SMALL_S };
+
+  return (*f)((OnigCodePoint )0xdf, ss, 2, arg);
+}
+
+extern int
+onigenc_apply_all_case_fold_with_map(int map_size,
+    const OnigPairCaseFoldCodes map[],
+    int ess_tsett_flag, OnigCaseFoldType flag,
+    OnigApplyAllCaseFoldFunc f, void* arg)
+{
+  OnigCodePoint code;
+  int i, r;
+
+  r = onigenc_ascii_apply_all_case_fold(flag, f, arg);
+  if (r != 0) return r;
+
+  if (CASE_FOLD_IS_ASCII_ONLY(flag))
+    return 0;
+
+  for (i = 0; i < map_size; i++) {
+    code = map[i].to;
+    r = (*f)(map[i].from, &code, 1, arg);
+    if (r != 0) return r;
+
+    code = map[i].from;
+    r = (*f)(map[i].to, &code, 1, arg);
+    if (r != 0) return r;
+  }
+
+  if (ess_tsett_flag != 0)
+    return ss_apply_all_case_fold(flag, f, arg);
+
+  return 0;
+}
+
+extern int
+onigenc_get_case_fold_codes_by_str_with_map(int map_size,
+    const OnigPairCaseFoldCodes map[],
+    int ess_tsett_flag, OnigCaseFoldType flag,
+    const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[])
+{
+  int i, j, n;
+  static OnigUChar sa[] = { LARGE_S, SMALL_S };
+
+  if (0x41 <= *p && *p <= 0x5a) { /* A - Z */
+    if (*p == LARGE_S && ess_tsett_flag != 0 && end > p + 1
+        && (*(p+1) == LARGE_S || *(p+1) == SMALL_S) /* SS */
+        && CASE_FOLD_IS_NOT_ASCII_ONLY(flag)) {
+    ss_combination:
+      items[0].byte_len = 2;
+      items[0].code_len = 1;
+      items[0].code[0] = (OnigCodePoint )0xdf;
+
+      n = 1;
+      for (i = 0; i < 2; i++) {
+        for (j = 0; j < 2; j++) {
+          if (sa[i] == *p && sa[j] == *(p+1))
+            continue;
+
+          items[n].byte_len = 2;
+          items[n].code_len = 2;
+          items[n].code[0] = (OnigCodePoint )sa[i];
+          items[n].code[1] = (OnigCodePoint )sa[j];
+          n++;
+        }
+      }
+      return 4;
+    }
+
+    items[0].byte_len = 1;
+    items[0].code_len = 1;
+    items[0].code[0] = (OnigCodePoint )(*p + 0x20);
+    return 1;
+  }
+  else if (0x61 <= *p && *p <= 0x7a) { /* a - z */
+    if (*p == SMALL_S && ess_tsett_flag != 0 && end > p + 1
+        && (*(p+1) == SMALL_S || *(p+1) == LARGE_S)
+        && CASE_FOLD_IS_NOT_ASCII_ONLY(flag)) {
+      goto ss_combination;
+    }
+
+    items[0].byte_len = 1;
+    items[0].code_len = 1;
+    items[0].code[0] = (OnigCodePoint )(*p - 0x20);
+    return 1;
+  }
+  else if (*p == 0xdf && ess_tsett_flag != 0
+           && CASE_FOLD_IS_NOT_ASCII_ONLY(flag)) {
+    items[0].byte_len = 1;
+    items[0].code_len = 2;
+    items[0].code[0] = (OnigCodePoint )'s';
+    items[0].code[1] = (OnigCodePoint )'s';
+
+    items[1].byte_len = 1;
+    items[1].code_len = 2;
+    items[1].code[0] = (OnigCodePoint )'S';
+    items[1].code[1] = (OnigCodePoint )'S';
+
+    items[2].byte_len = 1;
+    items[2].code_len = 2;
+    items[2].code[0] = (OnigCodePoint )'s';
+    items[2].code[1] = (OnigCodePoint )'S';
+
+    items[3].byte_len = 1;
+    items[3].code_len = 2;
+    items[3].code[0] = (OnigCodePoint )'S';
+    items[3].code[1] = (OnigCodePoint )'s';
+
+    return 4;
+  }
+  else {
+    int i;
+
+    if (CASE_FOLD_IS_ASCII_ONLY(flag))
+      return 0;
+
+    for (i = 0; i < map_size; i++) {
+      if (*p == map[i].from) {
+        items[0].byte_len = 1;
+        items[0].code_len = 1;
+        items[0].code[0] = map[i].to;
+        return 1;
+      }
+      else if (*p == map[i].to) {
+        items[0].byte_len = 1;
+        items[0].code_len = 1;
+        items[0].code[0] = map[i].from;
+        return 1;
+      }
+    }
+  }
+
+  return 0;
+}
+
+
+extern int
+onigenc_not_support_get_ctype_code_range(OnigCtype ctype ARG_UNUSED,
+                                         OnigCodePoint* sb_out ARG_UNUSED,
+                                         const OnigCodePoint* ranges[] ARG_UNUSED)
+{
+  return ONIG_NO_SUPPORT_CONFIG;
+}
+
+extern int
+onigenc_is_mbc_newline_0x0a(const UChar* p, const UChar* end)
+{
+  if (p < end) {
+    if (*p == NEWLINE_CODE) return 1;
+  }
+  return 0;
+}
+
+/* for single byte encodings */
+extern int
+onigenc_ascii_mbc_case_fold(OnigCaseFoldType flag ARG_UNUSED, const UChar** p,
+                            const UChar*end ARG_UNUSED, UChar* lower)
+{
+  *lower = ONIGENC_ASCII_CODE_TO_LOWER_CASE(**p);
+
+  (*p)++;
+  return 1; /* return byte length of converted char to lower */
+}
+
+extern int
+onigenc_single_byte_mbc_enc_len(const UChar* p ARG_UNUSED)
+{
+  return 1;
+}
+
+extern OnigCodePoint
+onigenc_single_byte_mbc_to_code(const UChar* p, const UChar* end ARG_UNUSED)
+{
+  return (OnigCodePoint )(*p);
+}
+
+extern int
+onigenc_single_byte_code_to_mbclen(OnigCodePoint code ARG_UNUSED)
+{
+  return (code < 0x100 ? 1 : ONIGERR_INVALID_CODE_POINT_VALUE);
+}
+
+extern int
+onigenc_single_byte_code_to_mbc(OnigCodePoint code, UChar *buf)
+{
+  *buf = (UChar )(code & 0xff);
+  return 1;
+}
+
+extern UChar*
+onigenc_single_byte_left_adjust_char_head(const UChar* start ARG_UNUSED,
+                                          const UChar* s)
+{
+  return (UChar* )s;
+}
+
+extern int
+onigenc_always_true_is_allowed_reverse_match(const UChar* s   ARG_UNUSED,
+                                             const UChar* end ARG_UNUSED)
+{
+  return TRUE;
+}
+
+extern int
+onigenc_always_false_is_allowed_reverse_match(const UChar* s   ARG_UNUSED,
+                                              const UChar* end ARG_UNUSED)
+{
+  return FALSE;
+}
+
+extern int
+onigenc_always_true_is_valid_mbc_string(const UChar* s   ARG_UNUSED,
+                                        const UChar* end ARG_UNUSED)
+{
+  return TRUE;
+}
+
+extern int
+onigenc_length_check_is_valid_mbc_string(OnigEncoding enc,
+                                         const UChar* p, const UChar* end)
+{
+  while (p < end) {
+    p += enclen(enc, p);
+  }
+
+  if (p != end)
+    return FALSE;
+  else
+    return TRUE;
+}
+
+extern int
+onigenc_is_valid_mbc_string(OnigEncoding enc, const UChar* s, const UChar* end)
+{
+  return ONIGENC_IS_VALID_MBC_STRING(enc, s, end);
+}
+
+extern OnigCodePoint
+onigenc_mbn_mbc_to_code(OnigEncoding enc, const UChar* p, const UChar* end)
+{
+  int c, i, len;
+  OnigCodePoint n;
+
+  len = enclen(enc, p);
+  n = (OnigCodePoint )(*p++);
+  if (len == 1) return n;
+
+  for (i = 1; i < len; i++) {
+    if (p >= end) break;
+    c = *p++;
+    n <<= 8;  n += c;
+  }
+  return n;
+}
+
+extern int
+onigenc_mbn_mbc_case_fold(OnigEncoding enc, OnigCaseFoldType flag ARG_UNUSED,
+                          const UChar** pp, const UChar* end ARG_UNUSED,
+                          UChar* lower)
+{
+  int len;
+  const UChar *p = *pp;
+
+  if (ONIGENC_IS_MBC_ASCII(p)) {
+    *lower = ONIGENC_ASCII_CODE_TO_LOWER_CASE(*p);
+    (*pp)++;
+    return 1;
+  }
+  else {
+    int i;
+
+    len = enclen(enc, p);
+    for (i = 0; i < len; i++) {
+      *lower++ = *p++;
+    }
+    (*pp) += len;
+    return len; /* return byte length of converted to lower char */
+  }
+}
+
+extern int
+onigenc_mb2_code_to_mbc(OnigEncoding enc, OnigCodePoint code, UChar *buf)
+{
+  UChar *p = buf;
+
+  if ((code & 0xff00) != 0) {
+    *p++ = (UChar )((code >>  8) & 0xff);
+  }
+  *p++ = (UChar )(code & 0xff);
+
+#if 1
+  if (enclen(enc, buf) != (p - buf))
+    return ONIGERR_INVALID_CODE_POINT_VALUE;
+#endif
+  return (int )(p - buf);
+}
+
+extern int
+onigenc_mb4_code_to_mbc(OnigEncoding enc, OnigCodePoint code, UChar *buf)
+{
+  UChar *p = buf;
+
+  if ((code & 0xff000000) != 0) {
+    *p++ = (UChar )((code >> 24) & 0xff);
+  }
+  if ((code & 0xff0000) != 0 || p != buf) {
+    *p++ = (UChar )((code >> 16) & 0xff);
+  }
+  if ((code & 0xff00) != 0 || p != buf) {
+    *p++ = (UChar )((code >> 8) & 0xff);
+  }
+  *p++ = (UChar )(code & 0xff);
+
+#if 1
+  if (enclen(enc, buf) != (p - buf))
+    return ONIGERR_INVALID_CODE_POINT_VALUE;
+#endif
+  return (int )(p - buf);
+}
+
+extern int
+onigenc_minimum_property_name_to_ctype(OnigEncoding enc, UChar* p, UChar* end)
+{
+  static PosixBracketEntryType PBS[] = {
+    { (UChar* )"Alnum",  ONIGENC_CTYPE_ALNUM,  5 },
+    { (UChar* )"Alpha",  ONIGENC_CTYPE_ALPHA,  5 },
+    { (UChar* )"Blank",  ONIGENC_CTYPE_BLANK,  5 },
+    { (UChar* )"Cntrl",  ONIGENC_CTYPE_CNTRL,  5 },
+    { (UChar* )"Digit",  ONIGENC_CTYPE_DIGIT,  5 },
+    { (UChar* )"Graph",  ONIGENC_CTYPE_GRAPH,  5 },
+    { (UChar* )"Lower",  ONIGENC_CTYPE_LOWER,  5 },
+    { (UChar* )"Print",  ONIGENC_CTYPE_PRINT,  5 },
+    { (UChar* )"Punct",  ONIGENC_CTYPE_PUNCT,  5 },
+    { (UChar* )"Space",  ONIGENC_CTYPE_SPACE,  5 },
+    { (UChar* )"Upper",  ONIGENC_CTYPE_UPPER,  5 },
+    { (UChar* )"XDigit", ONIGENC_CTYPE_XDIGIT, 6 },
+    { (UChar* )"ASCII",  ONIGENC_CTYPE_ASCII,  5 },
+    { (UChar* )"Word",   ONIGENC_CTYPE_WORD,   4 },
+    { (UChar* )NULL, -1, 0 }
+  };
+
+  PosixBracketEntryType *pb;
+  int len;
+
+  len = onigenc_strlen(enc, p, end);
+  for (pb = PBS; IS_NOT_NULL(pb->name); pb++) {
+    if (len == pb->len &&
+        onigenc_with_ascii_strncmp(enc, p, end, pb->name, pb->len) == 0)
+      return pb->ctype;
+  }
+
+  return ONIGERR_INVALID_CHAR_PROPERTY_NAME;
+}
+
+extern int
+onigenc_is_mbc_word_ascii(OnigEncoding enc, UChar* s, const UChar* end)
+{
+  OnigCodePoint code = ONIGENC_MBC_TO_CODE(enc, s, end);
+
+  if (code > ASCII_LIMIT) return 0;
+
+  return ONIGENC_IS_ASCII_CODE_WORD(code);
+}
+
+extern int
+onigenc_mb2_is_code_ctype(OnigEncoding enc, OnigCodePoint code,
+                          unsigned int ctype)
+{
+  if (code < 128)
+    return ONIGENC_IS_ASCII_CODE_CTYPE(code, ctype);
+  else {
+    if (CTYPE_IS_WORD_GRAPH_PRINT(ctype)) {
+      return (ONIGENC_CODE_TO_MBCLEN(enc, code) > 1 ? TRUE : FALSE);
+    }
+  }
+
+  return FALSE;
+}
+
+extern int
+onigenc_mb4_is_code_ctype(OnigEncoding enc, OnigCodePoint code,
+                          unsigned int ctype)
+{
+  if (code < 128)
+    return ONIGENC_IS_ASCII_CODE_CTYPE(code, ctype);
+  else {
+    if (CTYPE_IS_WORD_GRAPH_PRINT(ctype)) {
+      return (ONIGENC_CODE_TO_MBCLEN(enc, code) > 1 ? TRUE : FALSE);
+    }
+  }
+
+  return FALSE;
+}
+
+extern int
+onigenc_with_ascii_strncmp(OnigEncoding enc, const UChar* p, const UChar* end,
+                           const UChar* sascii /* ascii */, int n)
+{
+  int x, c;
+
+  while (n-- > 0) {
+    if (p >= end) return (int )(*sascii);
+
+    c = (int )ONIGENC_MBC_TO_CODE(enc, p, end);
+    x = *sascii - c;
+    if (x) return x;
+
+    sascii++;
+    p += enclen(enc, p);
+  }
+  return 0;
+}
+
+extern int
+onig_codes_cmp(OnigCodePoint a[], OnigCodePoint b[], int n)
+{
+  int i;
+
+  for (i = 0; i < n; i++) {
+    if (a[i] != b[i])
+      return -1;
+  }
+
+  return 0;
+}
+
+extern int
+onig_codes_byte_at(OnigCodePoint codes[], int at)
+{
+  int index;
+  int b;
+  OnigCodePoint code;
+
+  index = at / 3;
+  b     = at % 3;
+  code = codes[index];
+
+  return ((code >> ((2 - b) * 8)) & 0xff);
+}

thirdparty/oniguruma/regenc.h

@@ -0,0 +1,286 @@
+#ifndef REGENC_H
+#define REGENC_H
+/**********************************************************************
+  regenc.h -  Oniguruma (regular expression library)
+**********************************************************************/
+/*-
+ * Copyright (c) 2002-2020  K.Kosako
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef ONIGURUMA_EXPORT
+#define ONIGURUMA_EXPORT
+#endif
+
+#include "config.h"
+
+#ifndef ONIG_NO_STANDARD_C_HEADERS
+#include <stddef.h>
+#endif
+
+#ifdef ONIG_ESCAPE_UCHAR_COLLISION
+#undef ONIG_ESCAPE_UCHAR_COLLISION
+#endif
+
+#include "oniguruma.h"
+
+typedef struct {
+  OnigCodePoint from;
+  OnigCodePoint to;
+} OnigPairCaseFoldCodes;
+
+
+#ifndef NULL
+#define NULL   ((void* )0)
+#endif
+
+#ifndef TRUE
+#define TRUE    1
+#endif
+
+#ifndef FALSE
+#define FALSE   0
+#endif
+
+#ifndef ARG_UNUSED
+#if defined(__GNUC__)
+#  define ARG_UNUSED  __attribute__ ((unused))
+#else
+#  define ARG_UNUSED
+#endif
+#endif
+
+#define ONIG_IS_NULL(p)                    (((void*)(p)) == (void*)0)
+#define ONIG_IS_NOT_NULL(p)                (((void*)(p)) != (void*)0)
+#define ONIG_CHECK_NULL_RETURN(p)          if (ONIG_IS_NULL(p)) return NULL
+#define ONIG_CHECK_NULL_RETURN_VAL(p,val)  if (ONIG_IS_NULL(p)) return (val)
+
+#define MAX_CODE_POINT         (~((OnigCodePoint )0))
+#define ASCII_LIMIT            127
+#define NEWLINE_CODE           0x0a
+
+#define enclen(enc,p)          ONIGENC_MBC_ENC_LEN(enc,p)
+
+/* character types bit flag */
+#define BIT_CTYPE_NEWLINE  (1<< ONIGENC_CTYPE_NEWLINE)
+#define BIT_CTYPE_ALPHA    (1<< ONIGENC_CTYPE_ALPHA)
+#define BIT_CTYPE_BLANK    (1<< ONIGENC_CTYPE_BLANK)
+#define BIT_CTYPE_CNTRL    (1<< ONIGENC_CTYPE_CNTRL)
+#define BIT_CTYPE_DIGIT    (1<< ONIGENC_CTYPE_DIGIT)
+#define BIT_CTYPE_GRAPH    (1<< ONIGENC_CTYPE_GRAPH)
+#define BIT_CTYPE_LOWER    (1<< ONIGENC_CTYPE_LOWER)
+#define BIT_CTYPE_PRINT    (1<< ONIGENC_CTYPE_PRINT)
+#define BIT_CTYPE_PUNCT    (1<< ONIGENC_CTYPE_PUNCT)
+#define BIT_CTYPE_SPACE    (1<< ONIGENC_CTYPE_SPACE)
+#define BIT_CTYPE_UPPER    (1<< ONIGENC_CTYPE_UPPER)
+#define BIT_CTYPE_XDIGIT   (1<< ONIGENC_CTYPE_XDIGIT)
+#define BIT_CTYPE_WORD     (1<< ONIGENC_CTYPE_WORD)
+#define BIT_CTYPE_ALNUM    (1<< ONIGENC_CTYPE_ALNUM)
+#define BIT_CTYPE_ASCII    (1<< ONIGENC_CTYPE_ASCII)
+
+#define CTYPE_TO_BIT(ctype)  (1<<(ctype))
+#define CTYPE_IS_WORD_GRAPH_PRINT(ctype) \
+  ((ctype) == ONIGENC_CTYPE_WORD || (ctype) == ONIGENC_CTYPE_GRAPH ||\
+   (ctype) == ONIGENC_CTYPE_PRINT)
+
+
+typedef struct {
+  UChar    *name;
+  int       ctype;
+  short int len;
+} PosixBracketEntryType;
+
+struct PropertyNameCtype {
+  char *name;
+  int ctype;
+};
+
+/* #define USE_CRNL_AS_LINE_TERMINATOR */
+#define USE_UNICODE_PROPERTIES
+#define USE_UNICODE_EXTENDED_GRAPHEME_CLUSTER
+#define USE_UNICODE_WORD_BREAK
+/* #define USE_UNICODE_CASE_FOLD_TURKISH_AZERI */
+/* #define USE_UNICODE_ALL_LINE_TERMINATORS */  /* see Unicode.org UTS #18 */
+
+
+#define ONIG_ENCODING_INIT_DEFAULT           ONIG_ENCODING_ASCII
+
+
+#define ENC_SKIP_OFFSET_1_OR_0             7
+
+#define ENC_FLAG_ASCII_COMPATIBLE      (1<<0)
+#define ENC_FLAG_UNICODE               (1<<1)
+#define ENC_FLAG_SKIP_OFFSET_MASK      (7<<2)
+#define ENC_FLAG_SKIP_OFFSET_0             0
+#define ENC_FLAG_SKIP_OFFSET_1         (1<<2)
+#define ENC_FLAG_SKIP_OFFSET_2         (2<<2)
+#define ENC_FLAG_SKIP_OFFSET_3         (3<<2)
+#define ENC_FLAG_SKIP_OFFSET_4         (4<<2)
+#define ENC_FLAG_SKIP_OFFSET_1_OR_0    (ENC_SKIP_OFFSET_1_OR_0<<2)
+
+#define ENC_GET_SKIP_OFFSET(enc) \
+  (((enc)->flag & ENC_FLAG_SKIP_OFFSET_MASK)>>2)
+
+#define CASE_FOLD_IS_ASCII_ONLY(flag) \
+  (((flag) & ONIGENC_CASE_FOLD_ASCII_ONLY) != 0)
+#define CASE_FOLD_IS_NOT_ASCII_ONLY(flag) \
+  (((flag) & ONIGENC_CASE_FOLD_ASCII_ONLY) == 0)
+
+/* for encoding system implementation (internal) */
+extern int onigenc_end(void);
+extern int onigenc_ascii_apply_all_case_fold P_((OnigCaseFoldType flag, OnigApplyAllCaseFoldFunc f, void* arg));
+extern int onigenc_ascii_get_case_fold_codes_by_str P_((OnigCaseFoldType flag, const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[]));
+extern int onigenc_apply_all_case_fold_with_map P_((int map_size, const OnigPairCaseFoldCodes map[], int ess_tsett_flag, OnigCaseFoldType flag, OnigApplyAllCaseFoldFunc f, void* arg));
+extern int onigenc_get_case_fold_codes_by_str_with_map P_((int map_size, const OnigPairCaseFoldCodes map[], int ess_tsett_flag, OnigCaseFoldType flag, const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[]));
+extern int onigenc_not_support_get_ctype_code_range P_((OnigCtype ctype, OnigCodePoint* sb_out, const OnigCodePoint* ranges[]));
+extern int onigenc_is_mbc_newline_0x0a P_((const UChar* p, const UChar* end));
+
+
+/* methods for single byte encoding */
+extern int onigenc_ascii_mbc_case_fold P_((OnigCaseFoldType flag, const UChar** p, const UChar* end, UChar* lower));
+extern int onigenc_single_byte_mbc_enc_len P_((const UChar* p));
+extern OnigCodePoint onigenc_single_byte_mbc_to_code P_((const UChar* p, const UChar* end));
+extern int onigenc_single_byte_code_to_mbclen P_((OnigCodePoint code));
+extern int onigenc_single_byte_code_to_mbc P_((OnigCodePoint code, UChar *buf));
+extern UChar* onigenc_single_byte_left_adjust_char_head P_((const UChar* start, const UChar* s));
+extern int onigenc_always_true_is_allowed_reverse_match P_((const UChar* s, const UChar* end));
+extern int onigenc_always_false_is_allowed_reverse_match P_((const UChar* s, const UChar* end));
+extern int onigenc_always_true_is_valid_mbc_string P_((const UChar* s, const UChar* end));
+extern int onigenc_length_check_is_valid_mbc_string P_((OnigEncoding enc, const UChar* s, const UChar* end));
+
+/* methods for multi byte encoding */
+extern OnigCodePoint onigenc_mbn_mbc_to_code P_((OnigEncoding enc, const UChar* p, const UChar* end));
+extern int onigenc_mbn_mbc_case_fold P_((OnigEncoding enc, OnigCaseFoldType flag, const UChar** p, const UChar* end, UChar* lower));
+extern int onigenc_mb2_code_to_mbc P_((OnigEncoding enc, OnigCodePoint code, UChar *buf));
+extern int onigenc_minimum_property_name_to_ctype P_((OnigEncoding enc, UChar* p, UChar* end));
+extern int onigenc_unicode_property_name_to_ctype P_((OnigEncoding enc, UChar* p, UChar* end));
+extern int onigenc_is_mbc_word_ascii P_((OnigEncoding enc, UChar* s, const UChar* end));
+extern int onigenc_mb2_is_code_ctype P_((OnigEncoding enc, OnigCodePoint code, unsigned int ctype));
+extern int onigenc_mb4_code_to_mbc P_((OnigEncoding enc, OnigCodePoint code, UChar *buf));
+extern int onigenc_mb4_is_code_ctype P_((OnigEncoding enc, OnigCodePoint code, unsigned int ctype));
+extern struct PropertyNameCtype* onigenc_euc_jp_lookup_property_name P_((register const char *str, register size_t len));
+extern struct PropertyNameCtype* onigenc_sjis_lookup_property_name P_((register const char *str, register size_t len));
+
+/* in unicode.c */
+extern int onigenc_unicode_is_code_ctype P_((OnigCodePoint code, unsigned int ctype));
+extern int onigenc_utf16_32_get_ctype_code_range P_((OnigCtype ctype, OnigCodePoint *sb_out, const OnigCodePoint* ranges[]));
+extern int onigenc_unicode_ctype_code_range P_((OnigCtype ctype, const OnigCodePoint* ranges[]));
+extern int onigenc_unicode_get_case_fold_codes_by_str P_((OnigEncoding enc, OnigCaseFoldType flag, const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[]));
+extern int onigenc_unicode_mbc_case_fold P_((OnigEncoding enc, OnigCaseFoldType flag, const UChar** pp, const UChar* end, UChar* fold));
+extern int onigenc_unicode_apply_all_case_fold P_((OnigCaseFoldType flag, OnigApplyAllCaseFoldFunc f, void* arg));
+
+extern int onigenc_egcb_is_break_position P_((OnigEncoding enc, UChar* p, UChar* prev, const UChar* start, const UChar* end));
+
+#ifdef USE_UNICODE_WORD_BREAK
+extern int onigenc_wb_is_break_position P_((OnigEncoding enc, UChar* p, UChar* prev, const UChar* start, const UChar* end));
+#endif
+
+#define UTF16_IS_SURROGATE_FIRST(c)    (((c) & 0xfc) == 0xd8)
+#define UTF16_IS_SURROGATE_SECOND(c)   (((c) & 0xfc) == 0xdc)
+
+/* from unicode generated codes */
+#define FOLDS1_FOLD(i)         (OnigUnicodeFolds1 + (i))
+#define FOLDS2_FOLD(i)         (OnigUnicodeFolds2 + (i))
+#define FOLDS3_FOLD(i)         (OnigUnicodeFolds3 + (i))
+#define FOLDS1_UNFOLDS_NUM(i)  (OnigUnicodeFolds1[(i)+1])
+#define FOLDS2_UNFOLDS_NUM(i)  (OnigUnicodeFolds2[(i)+2])
+#define FOLDS3_UNFOLDS_NUM(i)  (OnigUnicodeFolds3[(i)+3])
+#define FOLDS1_UNFOLDS(i)      (FOLDS1_FOLD(i) + 2)
+#define FOLDS2_UNFOLDS(i)      (FOLDS2_FOLD(i) + 3)
+#define FOLDS3_UNFOLDS(i)      (FOLDS3_FOLD(i) + 4)
+#define FOLDS1_NEXT_INDEX(i)   ((i) + 2 + FOLDS1_UNFOLDS_NUM(i))
+#define FOLDS2_NEXT_INDEX(i)   ((i) + 3 + FOLDS2_UNFOLDS_NUM(i))
+#define FOLDS3_NEXT_INDEX(i)   ((i) + 4 + FOLDS3_UNFOLDS_NUM(i))
+
+#define FOLDS_FOLD_ADDR_BUK(buk, addr) do {\
+  if ((buk)->fold_len == 1)\
+    addr = OnigUnicodeFolds1 + (buk)->index;\
+  else if ((buk)->fold_len == 2)\
+    addr = OnigUnicodeFolds2 + (buk)->index;\
+  else if ((buk)->fold_len == 3)\
+    addr = OnigUnicodeFolds3 + (buk)->index;\
+  else\
+    return ONIGERR_INVALID_CODE_POINT_VALUE;\
+} while (0)
+
+extern OnigCodePoint OnigUnicodeFolds1[];
+extern OnigCodePoint OnigUnicodeFolds2[];
+extern OnigCodePoint OnigUnicodeFolds3[];
+
+struct ByUnfoldKey {
+  OnigCodePoint code;
+  short int     index;
+  short int     fold_len;
+};
+
+extern const struct ByUnfoldKey* onigenc_unicode_unfold_key(OnigCodePoint code);
+extern int onigenc_unicode_fold1_key(OnigCodePoint code[]);
+extern int onigenc_unicode_fold2_key(OnigCodePoint code[]);
+extern int onigenc_unicode_fold3_key(OnigCodePoint code[]);
+
+extern int onig_codes_cmp(OnigCodePoint a[], OnigCodePoint b[], int n);
+extern int onig_codes_byte_at(OnigCodePoint code[], int at);
+
+
+
+#define ONIGENC_ISO_8859_1_TO_LOWER_CASE(c) \
+  OnigEncISO_8859_1_ToLowerCaseTable[c]
+#define ONIGENC_ISO_8859_1_TO_UPPER_CASE(c) \
+  OnigEncISO_8859_1_ToUpperCaseTable[c]
+
+extern const UChar OnigEncISO_8859_1_ToLowerCaseTable[];
+extern const UChar OnigEncISO_8859_1_ToUpperCaseTable[];
+
+extern int
+onigenc_with_ascii_strncmp P_((OnigEncoding enc, const UChar* p, const UChar* end, const UChar* sascii /* ascii */, int n));
+extern UChar*
+onigenc_step P_((OnigEncoding enc, const UChar* p, const UChar* end, int n));
+
+/* defined in regexec.c, but used in enc/xxx.c */
+extern int  onig_is_in_code_range P_((const UChar* p, OnigCodePoint code));
+
+extern OnigEncoding  OnigEncDefaultCharEncoding;
+extern const UChar  OnigEncAsciiToLowerCaseTable[];
+extern const UChar  OnigEncAsciiToUpperCaseTable[];
+extern const unsigned short OnigEncAsciiCtypeTable[];
+
+
+#define ONIGENC_IS_ASCII_CODE(code)  ((code) < 0x80)
+#define ONIGENC_ASCII_CODE_TO_LOWER_CASE(c) OnigEncAsciiToLowerCaseTable[c]
+#define ONIGENC_ASCII_CODE_TO_UPPER_CASE(c) OnigEncAsciiToUpperCaseTable[c]
+#define ONIGENC_IS_ASCII_CODE_CTYPE(code,ctype) \
+  ((OnigEncAsciiCtypeTable[code] & CTYPE_TO_BIT(ctype)) != 0)
+#define ONIGENC_IS_ASCII_CODE_WORD(code) \
+  ((OnigEncAsciiCtypeTable[code] & CTYPE_TO_BIT(ONIGENC_CTYPE_WORD)) != 0)
+#define ONIGENC_IS_ASCII_CODE_CASE_AMBIG(code) \
+ (ONIGENC_IS_ASCII_CODE_CTYPE(code, ONIGENC_CTYPE_UPPER) ||\
+  ONIGENC_IS_ASCII_CODE_CTYPE(code, ONIGENC_CTYPE_LOWER))
+
+#define ONIGENC_IS_UNICODE_ENCODING(enc) \
+  (((enc)->flag & ENC_FLAG_UNICODE) != 0)
+
+#define ONIGENC_IS_ASCII_COMPATIBLE_ENCODING(enc)  \
+  (((enc)->flag & ENC_FLAG_ASCII_COMPATIBLE) != 0)
+
+#endif /* REGENC_H */

thirdparty/oniguruma/regerror.c

@@ -0,0 +1,414 @@
+/**********************************************************************
+  regerror.c -  Oniguruma (regular expression library)
+**********************************************************************/
+/*-
+ * Copyright (c) 2002-2022  K.Kosako
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef NEED_TO_INCLUDE_STDIO
+/* for vsnprintf() */
+#define NEED_TO_INCLUDE_STDIO
+#endif
+
+#include "regint.h"
+
+extern UChar*
+onig_error_code_to_format(int code)
+{
+  char *p;
+
+  switch (code) {
+  case ONIG_MISMATCH:
+    p = "mismatch"; break;
+  case ONIG_NO_SUPPORT_CONFIG:
+    p = "no support in this configuration"; break;
+  case ONIG_ABORT:
+    p = "abort"; break;
+  case ONIGERR_MEMORY:
+    p = "fail to memory allocation"; break;
+  case ONIGERR_MATCH_STACK_LIMIT_OVER:
+    p = "match-stack limit over"; break;
+  case ONIGERR_PARSE_DEPTH_LIMIT_OVER:
+    p = "parse depth limit over"; break;
+  case ONIGERR_RETRY_LIMIT_IN_MATCH_OVER:
+    p = "retry-limit-in-match over"; break;
+  case ONIGERR_RETRY_LIMIT_IN_SEARCH_OVER:
+    p = "retry-limit-in-search over"; break;
+  case ONIGERR_SUBEXP_CALL_LIMIT_IN_SEARCH_OVER:
+    p = "subexp-call-limit-in-search over"; break;
+  case ONIGERR_TYPE_BUG:
+    p = "undefined type (bug)"; break;
+  case ONIGERR_PARSER_BUG:
+    p = "internal parser error (bug)"; break;
+  case ONIGERR_STACK_BUG:
+    p = "stack error (bug)"; break;
+  case ONIGERR_UNDEFINED_BYTECODE:
+    p = "undefined bytecode (bug)"; break;
+  case ONIGERR_UNEXPECTED_BYTECODE:
+    p = "unexpected bytecode (bug)"; break;
+  case ONIGERR_DEFAULT_ENCODING_IS_NOT_SET:
+    p = "default multibyte-encoding is not set"; break;
+  case ONIGERR_SPECIFIED_ENCODING_CANT_CONVERT_TO_WIDE_CHAR:
+    p = "can't convert to wide-char on specified multibyte-encoding"; break;
+  case ONIGERR_FAIL_TO_INITIALIZE:
+    p = "fail to initialize"; break;
+  case ONIGERR_INVALID_ARGUMENT:
+    p = "invalid argument"; break;
+  case ONIGERR_END_PATTERN_AT_LEFT_BRACE:
+    p = "end pattern at left brace"; break;
+  case ONIGERR_END_PATTERN_AT_LEFT_BRACKET:
+    p = "end pattern at left bracket"; break;
+  case ONIGERR_EMPTY_CHAR_CLASS:
+    p = "empty char-class"; break;
+  case ONIGERR_PREMATURE_END_OF_CHAR_CLASS:
+    p = "premature end of char-class"; break;
+  case ONIGERR_END_PATTERN_AT_ESCAPE:
+    p = "end pattern at escape"; break;
+  case ONIGERR_END_PATTERN_AT_META:
+    p = "end pattern at meta"; break;
+  case ONIGERR_END_PATTERN_AT_CONTROL:
+    p = "end pattern at control"; break;
+  case ONIGERR_META_CODE_SYNTAX:
+    p = "invalid meta-code syntax"; break;
+  case ONIGERR_CONTROL_CODE_SYNTAX:
+    p = "invalid control-code syntax"; break;
+  case ONIGERR_CHAR_CLASS_VALUE_AT_END_OF_RANGE:
+    p = "char-class value at end of range"; break;
+  case ONIGERR_CHAR_CLASS_VALUE_AT_START_OF_RANGE:
+    p = "char-class value at start of range"; break;
+  case ONIGERR_UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS:
+    p = "unmatched range specifier in char-class"; break;
+  case ONIGERR_TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED:
+    p = "target of repeat operator is not specified"; break;
+  case ONIGERR_TARGET_OF_REPEAT_OPERATOR_INVALID:
+    p = "target of repeat operator is invalid"; break;
+  case ONIGERR_NESTED_REPEAT_OPERATOR:
+    p = "nested repeat operator"; break;
+  case ONIGERR_UNMATCHED_CLOSE_PARENTHESIS:
+    p = "unmatched close parenthesis"; break;
+  case ONIGERR_END_PATTERN_WITH_UNMATCHED_PARENTHESIS:
+    p = "end pattern with unmatched parenthesis"; break;
+  case ONIGERR_END_PATTERN_IN_GROUP:
+    p = "end pattern in group"; break;
+  case ONIGERR_UNDEFINED_GROUP_OPTION:
+    p = "undefined group option"; break;
+  case ONIGERR_INVALID_GROUP_OPTION:
+    p = "invalid group option"; break;
+  case ONIGERR_INVALID_POSIX_BRACKET_TYPE:
+    p = "invalid POSIX bracket type"; break;
+  case ONIGERR_INVALID_LOOK_BEHIND_PATTERN:
+    p = "invalid pattern in look-behind"; break;
+  case ONIGERR_INVALID_REPEAT_RANGE_PATTERN:
+    p = "invalid repeat range {lower,upper}"; break;
+  case ONIGERR_TOO_BIG_NUMBER:
+    p = "too big number"; break;
+  case ONIGERR_TOO_BIG_NUMBER_FOR_REPEAT_RANGE:
+    p = "too big number for repeat range"; break;
+  case ONIGERR_UPPER_SMALLER_THAN_LOWER_IN_REPEAT_RANGE:
+    p = "upper is smaller than lower in repeat range"; break;
+  case ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS:
+    p = "empty range in char class"; break;
+  case ONIGERR_MISMATCH_CODE_LENGTH_IN_CLASS_RANGE:
+    p = "mismatch multibyte code length in char-class range"; break;
+  case ONIGERR_TOO_MANY_MULTI_BYTE_RANGES:
+    p = "too many multibyte code ranges are specified"; break;
+  case ONIGERR_TOO_SHORT_MULTI_BYTE_STRING:
+    p = "too short multibyte code string"; break;
+  case ONIGERR_TOO_BIG_BACKREF_NUMBER:
+    p = "too big backref number"; break;
+  case ONIGERR_INVALID_BACKREF:
+    p = "invalid backref number/name"; break;
+  case ONIGERR_NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED:
+    p = "numbered backref/call is not allowed. (use name)"; break;
+  case ONIGERR_TOO_MANY_CAPTURES:
+    p = "too many captures"; break;
+  case ONIGERR_TOO_BIG_WIDE_CHAR_VALUE:
+    p = "too big wide-char value"; break;
+  case ONIGERR_TOO_LONG_WIDE_CHAR_VALUE:
+    p = "too long wide-char value"; break;
+  case ONIGERR_UNDEFINED_OPERATOR:
+    p = "undefined operator"; break;
+  case ONIGERR_INVALID_CODE_POINT_VALUE:
+    p = "invalid code point value"; break;
+  case ONIGERR_EMPTY_GROUP_NAME:
+    p = "group name is empty"; break;
+  case ONIGERR_INVALID_GROUP_NAME:
+    p = "invalid group name <%n>"; break;
+  case ONIGERR_INVALID_CHAR_IN_GROUP_NAME:
+    p = "invalid char in group name <%n>"; break;
+  case ONIGERR_UNDEFINED_NAME_REFERENCE:
+    p = "undefined name <%n> reference"; break;
+  case ONIGERR_UNDEFINED_GROUP_REFERENCE:
+    p = "undefined group <%n> reference"; break;
+  case ONIGERR_MULTIPLEX_DEFINED_NAME:
+    p = "multiplex defined name <%n>"; break;
+  case ONIGERR_MULTIPLEX_DEFINITION_NAME_CALL:
+    p = "multiplex definition name <%n> call"; break;
+  case ONIGERR_NEVER_ENDING_RECURSION:
+    p = "never ending recursion"; break;
+  case ONIGERR_GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY:
+    p = "group number is too big for capture history"; break;
+  case ONIGERR_INVALID_CHAR_PROPERTY_NAME:
+    p = "invalid character property name {%n}"; break;
+  case ONIGERR_INVALID_IF_ELSE_SYNTAX:
+    p = "invalid if-else syntax"; break;
+  case ONIGERR_INVALID_ABSENT_GROUP_PATTERN:
+    p = "invalid absent group pattern"; break;
+  case ONIGERR_INVALID_ABSENT_GROUP_GENERATOR_PATTERN:
+    p = "invalid absent group generator pattern"; break;
+  case ONIGERR_INVALID_CALLOUT_PATTERN:
+    p = "invalid callout pattern"; break;
+  case ONIGERR_INVALID_CALLOUT_NAME:
+    p = "invalid callout name"; break;
+  case ONIGERR_UNDEFINED_CALLOUT_NAME:
+    p = "undefined callout name"; break;
+  case ONIGERR_INVALID_CALLOUT_BODY:
+    p = "invalid callout body"; break;
+  case ONIGERR_INVALID_CALLOUT_TAG_NAME:
+    p = "invalid callout tag name"; break;
+  case ONIGERR_INVALID_CALLOUT_ARG:
+    p = "invalid callout arg"; break;
+  case ONIGERR_NOT_SUPPORTED_ENCODING_COMBINATION:
+    p = "not supported encoding combination"; break;
+  case ONIGERR_INVALID_COMBINATION_OF_OPTIONS:
+    p = "invalid combination of options"; break;
+  case ONIGERR_VERY_INEFFICIENT_PATTERN:
+    p = "very inefficient pattern"; break;
+  case ONIGERR_LIBRARY_IS_NOT_INITIALIZED:
+    p = "library is not initialized"; break;
+
+  default:
+    p = "undefined error code"; break;
+  }
+
+  return (UChar* )p;
+}
+
+static void sprint_byte(char* s, unsigned int v)
+{
+  xsnprintf(s, 3, "%02x", (v & 0377));
+}
+
+static void sprint_byte_with_x(char* s, unsigned int v)
+{
+  xsnprintf(s, 5, "\\x%02x", (v & 0377));
+}
+
+static int to_ascii(OnigEncoding enc, UChar *s, UChar *end,
+                    UChar buf[], int buf_size, int *is_over)
+{
+  int len;
+  UChar *p;
+  OnigCodePoint code;
+
+  if (!s) {
+    len = 0;
+    *is_over = 0;
+  }
+  else if (ONIGENC_MBC_MINLEN(enc) > 1) {
+    p = s;
+    len = 0;
+    while (p < end) {
+      code = ONIGENC_MBC_TO_CODE(enc, p, end);
+      if (code >= 0x80) {
+        if (code > 0xffff && len + 10 <= buf_size) {
+          sprint_byte_with_x((char*)(&(buf[len])), (unsigned int)(code >> 24));
+          sprint_byte((char*)(&(buf[len+4])),      (unsigned int)(code >> 16));
+          sprint_byte((char*)(&(buf[len+6])),      (unsigned int)(code >>  8));
+          sprint_byte((char*)(&(buf[len+8])),      (unsigned int)code);
+          len += 10;
+        }
+        else if (len + 6 <= buf_size) {
+          sprint_byte_with_x((char*)(&(buf[len])), (unsigned int)(code >> 8));
+          sprint_byte((char*)(&(buf[len+4])),      (unsigned int)code);
+          len += 6;
+        }
+        else {
+          break;
+        }
+      }
+      else {
+        buf[len++] = (UChar )code;
+      }
+
+      p += enclen(enc, p);
+      if (len >= buf_size) break;
+    }
+
+    *is_over = p < end;
+  }
+  else {
+    len = MIN((int )(end - s), buf_size);
+    xmemcpy(buf, s, (size_t )len);
+    *is_over = ((buf_size < (end - s)) ? 1 : 0);
+  }
+
+  return len;
+}
+
+
+extern int
+onig_is_error_code_needs_param(int code)
+{
+  switch (code) {
+  case ONIGERR_UNDEFINED_NAME_REFERENCE:
+  case ONIGERR_UNDEFINED_GROUP_REFERENCE:
+  case ONIGERR_MULTIPLEX_DEFINED_NAME:
+  case ONIGERR_MULTIPLEX_DEFINITION_NAME_CALL:
+  case ONIGERR_INVALID_GROUP_NAME:
+  case ONIGERR_INVALID_CHAR_IN_GROUP_NAME:
+  case ONIGERR_INVALID_CHAR_PROPERTY_NAME:
+    return 1;
+  default:
+    return 0;
+  }
+}
+
+/* for ONIG_MAX_ERROR_MESSAGE_LEN */
+#define MAX_ERROR_PAR_LEN   30
+
+extern int ONIG_VARIADIC_FUNC_ATTR
+onig_error_code_to_str(UChar* s, int code, ...)
+{
+  UChar *p, *q;
+  OnigErrorInfo* einfo;
+  int len, is_over;
+  UChar parbuf[MAX_ERROR_PAR_LEN];
+  va_list vargs;
+
+  va_start(vargs, code);
+
+  switch (code) {
+  case ONIGERR_UNDEFINED_NAME_REFERENCE:
+  case ONIGERR_UNDEFINED_GROUP_REFERENCE:
+  case ONIGERR_MULTIPLEX_DEFINED_NAME:
+  case ONIGERR_MULTIPLEX_DEFINITION_NAME_CALL:
+  case ONIGERR_INVALID_GROUP_NAME:
+  case ONIGERR_INVALID_CHAR_IN_GROUP_NAME:
+  case ONIGERR_INVALID_CHAR_PROPERTY_NAME:
+    einfo = va_arg(vargs, OnigErrorInfo*);
+    len = to_ascii(einfo->enc, einfo->par, einfo->par_end,
+                   parbuf, MAX_ERROR_PAR_LEN - 3, &is_over);
+    q = onig_error_code_to_format(code);
+    p = s;
+    while (*q != '\0') {
+      if (*q == '%') {
+        q++;
+        if (*q == 'n') { /* '%n': name */
+          xmemcpy(p, parbuf, len);
+          p += len;
+          if (is_over != 0) {
+            xmemcpy(p, "...", 3);
+            p += 3;
+          }
+          q++;
+        }
+        else
+          goto normal_char;
+      }
+      else {
+      normal_char:
+        *p++ = *q++;
+      }
+    }
+    *p = '\0';
+    len = (int )(p - s);
+    break;
+
+  default:
+    q = onig_error_code_to_format(code);
+    len = onigenc_str_bytelen_null(ONIG_ENCODING_ASCII, q);
+    xmemcpy(s, q, len);
+    s[len] = '\0';
+    break;
+  }
+
+  va_end(vargs);
+  return len;
+}
+
+
+void ONIG_VARIADIC_FUNC_ATTR
+onig_snprintf_with_pattern(UChar buf[], int bufsize, OnigEncoding enc,
+                           UChar* pat, UChar* pat_end, const char *fmt, ...)
+{
+  int n, need, len;
+  UChar *p, *s, *bp;
+  UChar bs[6];
+  va_list args;
+
+  va_start(args, fmt);
+  n = xvsnprintf((char* )buf, bufsize, fmt, args);
+  va_end(args);
+
+  need = (int )(pat_end - pat) * 4 + 4;
+
+  if (n + need < bufsize) {
+    xstrcat((char* )buf, ": /", bufsize);
+    s = buf + onigenc_str_bytelen_null(ONIG_ENCODING_ASCII, buf);
+
+    p = pat;
+    while (p < pat_end) {
+      if (ONIGENC_IS_MBC_HEAD(enc, p)) {
+        len = enclen(enc, p);
+        if (ONIGENC_MBC_MINLEN(enc) == 1) {
+          while (len-- > 0) *s++ = *p++;
+        }
+        else { /* for UTF16/32 */
+          int blen;
+
+          while (len-- > 0) {
+            sprint_byte_with_x((char* )bs, (unsigned int )(*p++));
+            blen = onigenc_str_bytelen_null(ONIG_ENCODING_ASCII, bs);
+            bp = bs;
+            while (blen-- > 0) *s++ = *bp++;
+          }
+        }
+      }
+      else if (*p == '\\') {
+        *s++ = *p++;
+        len = enclen(enc, p);
+        while (len-- > 0) *s++ = *p++;
+      }
+      else if (*p == '/') {
+        *s++ = (unsigned char )'\\';
+        *s++ = *p++;
+      }
+      else if (!ONIGENC_IS_CODE_PRINT(enc, *p) &&
+               !ONIGENC_IS_CODE_SPACE(enc, *p)) {
+        sprint_byte_with_x((char* )bs, (unsigned int )(*p++));
+        len = onigenc_str_bytelen_null(ONIG_ENCODING_ASCII, bs);
+        bp = bs;
+        while (len-- > 0) *s++ = *bp++;
+      }
+      else {
+        *s++ = *p++;
+      }
+    }
+
+    *s++ = '/';
+    *s   = '\0';
+  }
+}

thirdparty/oniguruma/regparse.h

@@ -0,0 +1,494 @@
+#ifndef REGPARSE_H
+#define REGPARSE_H
+/**********************************************************************
+  regparse.h -  Oniguruma (regular expression library)
+**********************************************************************/
+/*-
+ * Copyright (c) 2002-2022  K.Kosako
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "regint.h"
+
+#define ND_STRING_MARGIN     16
+#define ND_STRING_BUF_SIZE   24  /* sizeof(CClassNode) - sizeof(int)*4 */
+#define ND_BACKREFS_SIZE      6
+
+/* node type */
+typedef enum {
+  ND_STRING  =  0,
+  ND_CCLASS  =  1,
+  ND_CTYPE   =  2,
+  ND_BACKREF =  3,
+  ND_QUANT   =  4,
+  ND_BAG     =  5,
+  ND_ANCHOR  =  6,
+  ND_LIST    =  7,
+  ND_ALT     =  8,
+  ND_CALL    =  9,
+  ND_GIMMICK = 10
+} NodeType;
+
+enum BagType {
+  BAG_MEMORY         = 0,
+  BAG_OPTION         = 1,
+  BAG_STOP_BACKTRACK = 2,
+  BAG_IF_ELSE        = 3,
+};
+
+enum GimmickType {
+  GIMMICK_FAIL       = 0,
+  GIMMICK_SAVE       = 1,
+  GIMMICK_UPDATE_VAR = 2,
+#ifdef USE_CALLOUT
+  GIMMICK_CALLOUT    = 3,
+#endif
+};
+
+enum BodyEmptyType {
+  BODY_IS_NOT_EMPTY     = 0,
+  BODY_MAY_BE_EMPTY     = 1,
+  BODY_MAY_BE_EMPTY_MEM = 2,
+  BODY_MAY_BE_EMPTY_REC = 3
+};
+
+/* bytes buffer */
+typedef struct _BBuf {
+  UChar* p;
+  unsigned int used;
+  unsigned int alloc;
+} BBuf;
+
+
+struct _Node;
+
+typedef struct {
+  NodeType node_type;
+  int status;
+  struct _Node* parent;
+
+  UChar* s;
+  UChar* end;
+  unsigned int flag;
+  UChar  buf[ND_STRING_BUF_SIZE];
+  int    capacity;  /* (allocated size - 1) or 0: use buf[] */
+} StrNode;
+
+typedef struct {
+  NodeType node_type;
+  int status;
+  struct _Node* parent;
+
+  unsigned int flags;
+  BitSet bs;
+  BBuf*  mbuf;   /* multi-byte info or NULL */
+} CClassNode;
+
+typedef struct {
+  NodeType node_type;
+  int status;
+  struct _Node* parent;
+  struct _Node* body;
+
+  int lower;
+  int upper;
+  int greedy;
+  enum BodyEmptyType emptiness;
+  struct _Node* head_exact;
+  struct _Node* next_head_exact;
+  int include_referred;  /* include called node. don't eliminate even if {0} */
+  MemStatusType empty_status_mem;
+} QuantNode;
+
+typedef struct {
+  NodeType node_type;
+  int status;
+  struct _Node* parent;
+  struct _Node* body;
+
+  enum BagType type;
+  union {
+    struct {
+      int regnum;
+      AbsAddrType called_addr;
+      int entry_count;
+      int called_state;
+    } m;
+    struct {
+      OnigOptionType options;
+    } o;
+    struct {
+      /* body is condition */
+      struct _Node* Then;
+      struct _Node* Else;
+    } te;
+  };
+  /* for multiple call reference */
+  OnigLen min_len;   /* min length (byte) */
+  OnigLen max_len;   /* max length (byte) */
+  OnigLen min_char_len;
+  OnigLen max_char_len;
+  int opt_count;     /* referenced count in optimize_nodes() */
+} BagNode;
+
+#ifdef USE_CALL
+
+typedef struct {
+  int           offset;
+  struct _Node* target;
+} UnsetAddr;
+
+typedef struct {
+  int        num;
+  int        alloc;
+  UnsetAddr* us;
+} UnsetAddrList;
+
+typedef struct {
+  NodeType node_type;
+  int status;
+  struct _Node* parent;
+  struct _Node* body; /* to BagNode : BAG_MEMORY */
+
+  int     by_number;
+  int     called_gnum;
+  UChar*  name;
+  UChar*  name_end;
+  int     entry_count;
+} CallNode;
+
+#endif
+
+typedef struct {
+  NodeType node_type;
+  int status;
+  struct _Node* parent;
+
+  int  back_num;
+  int  back_static[ND_BACKREFS_SIZE];
+  int* back_dynamic;
+  int  nest_level;
+} BackRefNode;
+
+typedef struct {
+  NodeType node_type;
+  int status;
+  struct _Node* parent;
+  struct _Node* body;
+
+  int type;
+  OnigLen char_min_len;
+  OnigLen char_max_len;
+  int ascii_mode;
+  struct _Node* lead_node;
+} AnchorNode;
+
+typedef struct {
+  NodeType node_type;
+  int status;
+  struct _Node* parent;
+
+  struct _Node* car;
+  struct _Node* cdr;
+} ConsAltNode;
+
+typedef struct {
+  NodeType node_type;
+  int status;
+  struct _Node* parent;
+
+  int ctype;
+  int not;
+  int ascii_mode;
+} CtypeNode;
+
+typedef struct {
+  NodeType node_type;
+  int status;
+  struct _Node* parent;
+
+  enum GimmickType type;
+  int  detail_type;
+  int  num;
+  int  id;
+} GimmickNode;
+
+typedef struct _Node {
+  union {
+    struct {
+      NodeType node_type;
+      int status;
+      struct _Node* parent;
+      struct _Node* body;
+    } base;
+
+    StrNode       str;
+    CClassNode    cclass;
+    QuantNode     quant;
+    BagNode       bag;
+    BackRefNode   backref;
+    AnchorNode    anchor;
+    ConsAltNode   cons;
+    CtypeNode     ctype;
+#ifdef USE_CALL
+    CallNode      call;
+#endif
+    GimmickNode   gimmick;
+  } u;
+} Node;
+
+typedef struct {
+  int new_val;
+} GroupNumMap;
+
+
+#define NULL_NODE  ((Node* )0)
+
+
+/* node type bit */
+#define ND_TYPE2BIT(type)      (1<<(type))
+
+#define ND_BIT_STRING     ND_TYPE2BIT(ND_STRING)
+#define ND_BIT_CCLASS     ND_TYPE2BIT(ND_CCLASS)
+#define ND_BIT_CTYPE      ND_TYPE2BIT(ND_CTYPE)
+#define ND_BIT_BACKREF    ND_TYPE2BIT(ND_BACKREF)
+#define ND_BIT_QUANT      ND_TYPE2BIT(ND_QUANT)
+#define ND_BIT_BAG        ND_TYPE2BIT(ND_BAG)
+#define ND_BIT_ANCHOR     ND_TYPE2BIT(ND_ANCHOR)
+#define ND_BIT_LIST       ND_TYPE2BIT(ND_LIST)
+#define ND_BIT_ALT        ND_TYPE2BIT(ND_ALT)
+#define ND_BIT_CALL       ND_TYPE2BIT(ND_CALL)
+#define ND_BIT_GIMMICK    ND_TYPE2BIT(ND_GIMMICK)
+
+#define ND_TYPE(node)             ((node)->u.base.node_type)
+#define ND_SET_TYPE(node, ntype)   (node)->u.base.node_type = (ntype)
+
+#define STR_(node)         (&((node)->u.str))
+#define CCLASS_(node)      (&((node)->u.cclass))
+#define CTYPE_(node)       (&((node)->u.ctype))
+#define BACKREF_(node)     (&((node)->u.backref))
+#define QUANT_(node)       (&((node)->u.quant))
+#define BAG_(node)         (&((node)->u.bag))
+#define ANCHOR_(node)      (&((node)->u.anchor))
+#define CONS_(node)        (&((node)->u.cons))
+#define CALL_(node)        (&((node)->u.call))
+#define GIMMICK_(node)     (&((node)->u.gimmick))
+
+#define ND_CAR(node)     (CONS_(node)->car)
+#define ND_CDR(node)     (CONS_(node)->cdr)
+
+#define CTYPE_ANYCHAR      -1
+#define ND_IS_ANYCHAR(node) \
+  (ND_TYPE(node) == ND_CTYPE && CTYPE_(node)->ctype == CTYPE_ANYCHAR)
+
+
+#define ANCR_ANYCHAR_INF_MASK  (ANCR_ANYCHAR_INF | ANCR_ANYCHAR_INF_ML)
+#define ANCR_END_BUF_MASK      (ANCR_END_BUF | ANCR_SEMI_END_BUF)
+
+#define ND_STRING_CRUDE           (1<<0)
+#define ND_STRING_CASE_EXPANDED   (1<<1)
+
+#define ND_STRING_LEN(node)            (int )((node)->u.str.end - (node)->u.str.s)
+#define ND_STRING_SET_CRUDE(node)         (node)->u.str.flag |= ND_STRING_CRUDE
+#define ND_STRING_CLEAR_CRUDE(node)       (node)->u.str.flag &= ~ND_STRING_CRUDE
+#define ND_STRING_SET_CASE_EXPANDED(node) (node)->u.str.flag |= ND_STRING_CASE_EXPANDED
+#define ND_STRING_IS_CRUDE(node) \
+  (((node)->u.str.flag & ND_STRING_CRUDE) != 0)
+#define ND_STRING_IS_CASE_EXPANDED(node) \
+  (((node)->u.str.flag & ND_STRING_CASE_EXPANDED) != 0)
+
+#define BACKREFS_P(br) \
+  (IS_NOT_NULL((br)->back_dynamic) ? (br)->back_dynamic : (br)->back_static)
+
+/* node status bits */
+#define ND_ST_FIXED_MIN           (1<<0)
+#define ND_ST_FIXED_MAX           (1<<1)
+#define ND_ST_FIXED_CLEN          (1<<2)
+#define ND_ST_MARK1               (1<<3)
+#define ND_ST_MARK2               (1<<4)
+#define ND_ST_STRICT_REAL_REPEAT  (1<<5)
+#define ND_ST_RECURSION           (1<<6)
+#define ND_ST_CALLED              (1<<7)
+#define ND_ST_FIXED_ADDR          (1<<8)
+#define ND_ST_NAMED_GROUP         (1<<9)
+#define ND_ST_IN_REAL_REPEAT      (1<<10) /* STK_REPEAT is nested in stack. */
+#define ND_ST_IN_ZERO_REPEAT      (1<<11) /* (....){0} */
+#define ND_ST_IN_MULTI_ENTRY      (1<<12)
+#define ND_ST_NEST_LEVEL          (1<<13)
+#define ND_ST_BY_NUMBER           (1<<14) /* {n,m} */
+#define ND_ST_BY_NAME             (1<<15) /* backref by name */
+#define ND_ST_BACKREF             (1<<16)
+#define ND_ST_CHECKER             (1<<17)
+#define ND_ST_PROHIBIT_RECURSION  (1<<18)
+#define ND_ST_SUPER               (1<<19)
+#define ND_ST_EMPTY_STATUS_CHECK  (1<<20)
+#define ND_ST_IGNORECASE          (1<<21)
+#define ND_ST_MULTILINE           (1<<22)
+#define ND_ST_TEXT_SEGMENT_WORD   (1<<23)
+#define ND_ST_ABSENT_WITH_SIDE_EFFECTS (1<<24)  /* stopper or clear */
+#define ND_ST_FIXED_CLEN_MIN_SURE (1<<25)
+#define ND_ST_REFERENCED          (1<<26)
+#define ND_ST_INPEEK              (1<<27)
+#define ND_ST_WHOLE_OPTIONS       (1<<28)
+
+
+#define ND_STATUS(node)           (((Node* )node)->u.base.status)
+#define ND_STATUS_ADD(node,f)     (ND_STATUS(node) |= (ND_ST_ ## f))
+#define ND_STATUS_REMOVE(node,f)  (ND_STATUS(node) &= ~(ND_ST_ ## f))
+
+#define ND_IS_BY_NUMBER(node)       ((ND_STATUS(node) & ND_ST_BY_NUMBER)      != 0)
+#define ND_IS_IN_REAL_REPEAT(node)  ((ND_STATUS(node) & ND_ST_IN_REAL_REPEAT) != 0)
+#define ND_IS_CALLED(node)          ((ND_STATUS(node) & ND_ST_CALLED)         != 0)
+#define ND_IS_IN_MULTI_ENTRY(node)  ((ND_STATUS(node) & ND_ST_IN_MULTI_ENTRY) != 0)
+#define ND_IS_RECURSION(node)       ((ND_STATUS(node) & ND_ST_RECURSION)      != 0)
+#define ND_IS_IN_ZERO_REPEAT(node)  ((ND_STATUS(node) & ND_ST_IN_ZERO_REPEAT) != 0)
+#define ND_IS_NAMED_GROUP(node)     ((ND_STATUS(node) & ND_ST_NAMED_GROUP)  != 0)
+#define ND_IS_FIXED_ADDR(node)      ((ND_STATUS(node) & ND_ST_FIXED_ADDR)   != 0)
+#define ND_IS_FIXED_CLEN(node)      ((ND_STATUS(node) & ND_ST_FIXED_CLEN)   != 0)
+#define ND_IS_FIXED_MIN(node)       ((ND_STATUS(node) & ND_ST_FIXED_MIN)    != 0)
+#define ND_IS_FIXED_MAX(node)       ((ND_STATUS(node) & ND_ST_FIXED_MAX)    != 0)
+#define ND_IS_MARK1(node)           ((ND_STATUS(node) & ND_ST_MARK1)        != 0)
+#define ND_IS_MARK2(node)           ((ND_STATUS(node) & ND_ST_MARK2)        != 0)
+#define ND_IS_NEST_LEVEL(node)      ((ND_STATUS(node) & ND_ST_NEST_LEVEL)   != 0)
+#define ND_IS_BY_NAME(node)         ((ND_STATUS(node) & ND_ST_BY_NAME)      != 0)
+#define ND_IS_BACKREF(node)         ((ND_STATUS(node) & ND_ST_BACKREF)      != 0)
+#define ND_IS_CHECKER(node)         ((ND_STATUS(node) & ND_ST_CHECKER)      != 0)
+#define ND_IS_SUPER(node)           ((ND_STATUS(node) & ND_ST_SUPER)        != 0)
+#define ND_IS_PROHIBIT_RECURSION(node) \
+    ((ND_STATUS(node) & ND_ST_PROHIBIT_RECURSION) != 0)
+#define ND_IS_STRICT_REAL_REPEAT(node) \
+    ((ND_STATUS(node) & ND_ST_STRICT_REAL_REPEAT) != 0)
+#define ND_IS_EMPTY_STATUS_CHECK(node) \
+    ((ND_STATUS(node) & ND_ST_EMPTY_STATUS_CHECK) != 0)
+#define ND_IS_IGNORECASE(node)      ((ND_STATUS(node) & ND_ST_IGNORECASE) != 0)
+#define ND_IS_MULTILINE(node)       ((ND_STATUS(node) & ND_ST_MULTILINE) != 0)
+#define ND_IS_TEXT_SEGMENT_WORD(node)  ((ND_STATUS(node) & ND_ST_TEXT_SEGMENT_WORD) != 0)
+#define ND_IS_ABSENT_WITH_SIDE_EFFECTS(node)  ((ND_STATUS(node) & ND_ST_ABSENT_WITH_SIDE_EFFECTS) != 0)
+#define ND_IS_FIXED_CLEN_MIN_SURE(node)  ((ND_STATUS(node) & ND_ST_FIXED_CLEN_MIN_SURE) != 0)
+#define ND_IS_REFERENCED(node)      ((ND_STATUS(node) & ND_ST_REFERENCED) != 0)
+#define ND_IS_INPEEK(node)          ((ND_STATUS(node) & ND_ST_INPEEK) != 0)
+#define ND_IS_WHOLE_OPTIONS(node)   ((ND_STATUS(node) & ND_ST_WHOLE_OPTIONS) != 0)
+
+#define ND_PARENT(node)         ((node)->u.base.parent)
+#define ND_BODY(node)           ((node)->u.base.body)
+#define ND_QUANT_BODY(node)     ((node)->body)
+#define ND_BAG_BODY(node)       ((node)->body)
+#define ND_CALL_BODY(node)      ((node)->body)
+#define ND_ANCHOR_BODY(node)    ((node)->body)
+
+#define PARSEENV_MEMENV_SIZE  8
+#define PARSEENV_MEMENV(senv) \
+ (IS_NOT_NULL((senv)->mem_env_dynamic) ? \
+    (senv)->mem_env_dynamic : (senv)->mem_env_static)
+
+#define IS_SYNTAX_OP(syn, opm)    (((syn)->op  & (opm)) != 0)
+#define IS_SYNTAX_OP2(syn, opm)   (((syn)->op2 & (opm)) != 0)
+#define IS_SYNTAX_BV(syn, bvm)    (((syn)->behavior & (bvm)) != 0)
+
+#define ID_ENTRY(env, id) do {\
+  id = (env)->id_num++;\
+} while(0)
+
+
+typedef struct {
+  Node* mem_node;
+  Node* empty_repeat_node;
+} MemEnv;
+
+typedef struct {
+  enum SaveType type;
+} SaveItem;
+
+typedef struct {
+  OnigOptionType   options;
+  OnigCaseFoldType case_fold_flag;
+  OnigEncoding     enc;
+  OnigSyntaxType*  syntax;
+  MemStatusType    cap_history;
+  MemStatusType    backtrack_mem; /* backtrack/recursion */
+  MemStatusType    backrefed_mem;
+  UChar*           pattern;
+  UChar*           pattern_end;
+  UChar*           error;
+  UChar*           error_end;
+  regex_t*         reg;       /* for reg->names only */
+  int              num_call;
+  int              num_mem;
+  int              num_named;
+  int              mem_alloc;
+  MemEnv           mem_env_static[PARSEENV_MEMENV_SIZE];
+  MemEnv*          mem_env_dynamic;
+  int              backref_num;
+  int              keep_num;
+  int              id_num;
+  int              save_alloc_num;
+  SaveItem*        saves;
+#ifdef USE_CALL
+  UnsetAddrList*   unset_addr_list;
+#endif
+  unsigned int     parse_depth;
+#ifdef ONIG_DEBUG_PARSE
+  unsigned int     max_parse_depth;
+#endif
+  unsigned int     flags;
+} ParseEnv;
+
+#define PE_FLAG_HAS_CALL_ZERO      (1<<0)
+#define PE_FLAG_HAS_WHOLE_OPTIONS  (1<<1)
+#define PE_FLAG_HAS_ABSENT_STOPPER (1<<2)
+
+
+extern int    onig_renumber_name_table P_((regex_t* reg, GroupNumMap* map));
+
+extern int    onig_strncmp P_((const UChar* s1, const UChar* s2, int n));
+extern void   onig_strcpy P_((UChar* dest, const UChar* src, const UChar* end));
+extern void   onig_scan_env_set_error_string P_((ParseEnv* env, int ecode, UChar* arg, UChar* arg_end));
+extern int    onig_reduce_nested_quantifier P_((Node* pnode));
+extern int    onig_node_copy(Node** rcopy, Node* from);
+extern int    onig_node_str_cat P_((Node* node, const UChar* s, const UChar* end));
+extern int    onig_node_str_set P_((Node* node, const UChar* s, const UChar* end, int need_free));
+extern void   onig_node_str_clear P_((Node* node, int need_free));
+extern void   onig_node_free P_((Node* node));
+extern int    onig_node_reset_empty P_((Node* node));
+extern int    onig_node_reset_fail P_((Node* node));
+extern Node*  onig_node_new_bag P_((enum BagType type));
+extern Node*  onig_node_new_str P_((const UChar* s, const UChar* end));
+extern Node*  onig_node_new_list P_((Node* left, Node* right));
+extern Node*  onig_node_new_alt P_((Node* left, Node* right));
+extern int    onig_names_free P_((regex_t* reg));
+extern int    onig_parse_tree P_((Node** root, const UChar* pattern, const UChar* end, regex_t* reg, ParseEnv* env));
+extern int    onig_free_shared_cclass_table P_((void));
+extern int    onig_is_code_in_cc P_((OnigEncoding enc, OnigCodePoint code, CClassNode* cc));
+extern int    onig_new_cclass_with_code_list(Node** rnode, OnigEncoding enc, int n, OnigCodePoint codes[]);
+extern OnigLen onig_get_tiny_min_len(Node* node, unsigned int inhibit_node_types, int* invalid_node);
+
+#ifdef USE_CALLOUT
+extern int onig_global_callout_names_free(void);
+#endif
+
+#ifdef ONIG_DEBUG
+extern int onig_print_names(FILE*, regex_t*);
+#endif
+
+#endif /* REGPARSE_H */

thirdparty/oniguruma/st.c

@@ -0,0 +1,562 @@
+/* This is a public domain general purpose hash table package written by Peter Moore @ UCB. */
+
+/* static	char	sccsid[] = "@(#) st.c 5.1 89/12/14 Crucible"; */
+
+#ifndef NEED_TO_INCLUDE_STDIO
+#define NEED_TO_INCLUDE_STDIO
+#endif
+
+#include "regint.h"
+#include "st.h"
+
+
+typedef struct st_table_entry st_table_entry;
+
+struct st_table_entry {
+    unsigned int hash;
+    st_data_t key;
+    st_data_t record;
+    st_table_entry *next;
+};
+
+#define ST_DEFAULT_MAX_DENSITY 5
+#define ST_DEFAULT_INIT_TABLE_SIZE 11
+
+    /*
+     * DEFAULT_MAX_DENSITY is the default for the largest we allow the
+     * average number of items per bin before increasing the number of
+     * bins
+     *
+     * DEFAULT_INIT_TABLE_SIZE is the default for the number of bins
+     * allocated initially
+     *
+     */
+
+static int numcmp(st_data_t, st_data_t);
+static int numhash(st_data_t);
+static struct st_hash_type type_numhash = {
+    numcmp,
+    numhash,
+};
+
+static int str_cmp(st_data_t, st_data_t);
+static int str_hash(st_data_t);
+static struct st_hash_type type_strhash = {
+    str_cmp,
+    str_hash,
+};
+
+static void rehash(st_table *);
+
+#define alloc(type) (type*)xmalloc((unsigned)sizeof(type))
+#define Calloc(n,s) (char*)xcalloc((n),(s))
+
+#define EQUAL(table,x,y) ((x)==(y) || (*table->type->compare)((x),(y)) == 0)
+
+#define do_hash(key,table) (unsigned int)(*(table)->type->hash)((key))
+#define do_hash_bin(key,table) (do_hash(key, table)%(table)->num_bins)
+
+/*
+ * MINSIZE is the minimum size of a dictionary.
+ */
+
+#define MINSIZE 8
+
+/*
+Table of prime numbers 2^n+a, 2<=n<=30.
+*/
+static const long primes[] = {
+	8 + 3,
+	16 + 3,
+	32 + 5,
+	64 + 3,
+	128 + 3,
+	256 + 27,
+	512 + 9,
+	1024 + 9,
+	2048 + 5,
+	4096 + 3,
+	8192 + 27,
+	16384 + 43,
+	32768 + 3,
+	65536 + 45,
+	131072 + 29,
+	262144 + 3,
+	524288 + 21,
+	1048576 + 7,
+	2097152 + 17,
+	4194304 + 15,
+	8388608 + 9,
+	16777216 + 43,
+	33554432 + 35,
+	67108864 + 15,
+	134217728 + 29,
+	268435456 + 3,
+	536870912 + 11,
+	1073741824 + 85,
+	0
+};
+
+static int new_size(int size)
+{
+    int i;
+
+#if 0
+    for (i=3; i<31; i++) {
+      if ((1<<i) > size) return 1<<i;
+    }
+    return -1;
+#else
+    int newsize;
+
+    for (i = 0, newsize = MINSIZE;
+         i < (int )(sizeof(primes)/sizeof(primes[0]));
+         i++, newsize <<= 1) {
+      if (newsize > size) return primes[i];
+    }
+    /* Ran out of polynomials */
+    return -1;			/* should raise exception */
+#endif
+}
+
+#ifdef HASH_LOG
+static int collision = 0;
+static int init_st = 0;
+
+static void
+stat_col(void)
+{
+  FILE *f = fopen("/tmp/col", "w");
+  if (f == 0) return ;
+
+  (void) fprintf(f, "collision: %d\n", collision);
+  (void) fclose(f);
+}
+#endif
+
+extern st_table*
+st_init_table_with_size(struct st_hash_type* type, int size)
+{
+  st_table *tbl;
+
+#ifdef HASH_LOG
+  if (init_st == 0) {
+    init_st = 1;
+    atexit(stat_col);
+  }
+#endif
+
+  size = new_size(size);	/* round up to prime number */
+  if (size <= 0) return 0;
+
+  tbl = alloc(st_table);
+  if (tbl == 0) return 0;
+
+  tbl->type = type;
+  tbl->num_entries = 0;
+  tbl->num_bins = size;
+  tbl->bins = (st_table_entry **)Calloc(size, sizeof(st_table_entry*));
+  if (tbl->bins == 0) {
+    free(tbl);
+    return 0;
+  }
+
+  return tbl;
+}
+
+extern st_table*
+st_init_table(struct st_hash_type* type)
+{
+  return st_init_table_with_size(type, 0);
+}
+
+extern st_table*
+st_init_numtable(void)
+{
+  return st_init_table(&type_numhash);
+}
+
+extern st_table*
+st_init_numtable_with_size(int size)
+{
+  return st_init_table_with_size(&type_numhash, size);
+}
+
+extern st_table*
+st_init_strtable(void)
+{
+  return st_init_table(&type_strhash);
+}
+
+extern st_table*
+st_init_strtable_with_size(int size)
+{
+  return st_init_table_with_size(&type_strhash, size);
+}
+
+extern void
+st_free_table(st_table* table)
+{
+  register st_table_entry *ptr, *next;
+  int i;
+
+  for(i = 0; i < table->num_bins; i++) {
+    ptr = table->bins[i];
+    while (ptr != 0) {
+	    next = ptr->next;
+	    free(ptr);
+	    ptr = next;
+    }
+  }
+  free(table->bins);
+  free(table);
+}
+
+#define PTR_NOT_EQUAL(table, ptr, hash_val, key) \
+((ptr) != 0 && (ptr->hash != (hash_val) || !EQUAL((table), (key), (ptr)->key)))
+
+#ifdef HASH_LOG
+#define COLLISION collision++
+#else
+#define COLLISION
+#endif
+
+#define FIND_ENTRY(table, ptr, hash_val, bin_pos) do {\
+    bin_pos = hash_val%(table)->num_bins;\
+    ptr = (table)->bins[bin_pos];\
+    if (PTR_NOT_EQUAL(table, ptr, hash_val, key)) {\
+      COLLISION;\
+      while (PTR_NOT_EQUAL(table, ptr->next, hash_val, key)) {\
+        ptr = ptr->next;\
+      }\
+      ptr = ptr->next;\
+    }\
+} while (0)
+
+extern int
+st_lookup(st_table* table, register st_data_t key, st_data_t* value)
+{
+  unsigned int hash_val, bin_pos;
+  register st_table_entry *ptr;
+
+  hash_val = do_hash(key, table);
+  FIND_ENTRY(table, ptr, hash_val, bin_pos);
+
+  if (ptr == 0) {
+    return 0;
+  }
+  else {
+    if (value != 0)  *value = ptr->record;
+    return 1;
+  }
+}
+
+#define ADD_DIRECT(table, key, value, hash_val, bin_pos, ret) \
+do {\
+  st_table_entry *entry;\
+  if (table->num_entries/(table->num_bins) > ST_DEFAULT_MAX_DENSITY) {\
+    rehash(table);\
+    bin_pos = hash_val % table->num_bins;\
+  }\
+  entry = alloc(st_table_entry);\
+  if (IS_NULL(entry)) return ret;\
+  entry->hash = hash_val;\
+  entry->key = key;\
+  entry->record = value;\
+  entry->next = table->bins[bin_pos];\
+  table->bins[bin_pos] = entry;\
+  table->num_entries++;\
+} while (0)
+
+extern int
+st_insert(register st_table* table, register st_data_t key, st_data_t value)
+{
+  unsigned int hash_val, bin_pos;
+  register st_table_entry *ptr;
+
+  hash_val = do_hash(key, table);
+  FIND_ENTRY(table, ptr, hash_val, bin_pos);
+
+  if (ptr == 0) {
+    ADD_DIRECT(table, key, value, hash_val, bin_pos, ONIGERR_MEMORY);
+    return 0;
+  }
+  else {
+    ptr->record = value;
+    return 1;
+  }
+}
+
+extern void
+st_add_direct(st_table* table, st_data_t key, st_data_t value)
+{
+  unsigned int hash_val, bin_pos;
+
+  hash_val = do_hash(key, table);
+  bin_pos = hash_val % table->num_bins;
+  ADD_DIRECT(table, key, value, hash_val, bin_pos,);
+}
+
+static void
+rehash(register st_table* table)
+{
+  register st_table_entry *ptr, *next, **new_bins;
+  int i, new_num_bins, old_num_bins;
+  unsigned int hash_val;
+
+  old_num_bins = table->num_bins;
+  new_num_bins = new_size(old_num_bins + 1);
+  if (new_num_bins <= 0) return ;
+
+  new_bins = (st_table_entry**)Calloc(new_num_bins, sizeof(st_table_entry*));
+  if (new_bins == 0) {
+    return ;
+  }
+
+  for(i = 0; i < old_num_bins; i++) {
+    ptr = table->bins[i];
+    while (ptr != 0) {
+	    next = ptr->next;
+	    hash_val = ptr->hash % new_num_bins;
+	    ptr->next = new_bins[hash_val];
+	    new_bins[hash_val] = ptr;
+	    ptr = next;
+    }
+  }
+  free(table->bins);
+  table->num_bins = new_num_bins;
+  table->bins = new_bins;
+}
+
+extern st_table*
+st_copy(st_table* old_table)
+{
+  st_table *new_table;
+  st_table_entry *ptr, *entry;
+  int i, num_bins = old_table->num_bins;
+
+  new_table = alloc(st_table);
+  if (new_table == 0) {
+    return 0;
+  }
+
+  *new_table = *old_table;
+  new_table->bins = (st_table_entry**)
+    Calloc((unsigned)num_bins, sizeof(st_table_entry*));
+
+  if (new_table->bins == 0) {
+    free(new_table);
+    return 0;
+  }
+
+  for(i = 0; i < num_bins; i++) {
+    new_table->bins[i] = 0;
+    ptr = old_table->bins[i];
+    while (ptr != 0) {
+	    entry = alloc(st_table_entry);
+	    if (entry == 0) {
+        free(new_table->bins);
+        free(new_table);
+        return 0;
+	    }
+	    *entry = *ptr;
+	    entry->next = new_table->bins[i];
+	    new_table->bins[i] = entry;
+	    ptr = ptr->next;
+    }
+  }
+  return new_table;
+}
+
+extern int
+st_delete(register st_table* table, register st_data_t* key, st_data_t* value)
+{
+  unsigned int hash_val;
+  st_table_entry *tmp;
+  register st_table_entry *ptr;
+
+  hash_val = do_hash_bin(*key, table);
+  ptr = table->bins[hash_val];
+
+  if (ptr == 0) {
+    if (value != 0) *value = 0;
+    return 0;
+  }
+
+  if (EQUAL(table, *key, ptr->key)) {
+    table->bins[hash_val] = ptr->next;
+    table->num_entries--;
+    if (value != 0) *value = ptr->record;
+    *key = ptr->key;
+    free(ptr);
+    return 1;
+  }
+
+  for(; ptr->next != 0; ptr = ptr->next) {
+    if (EQUAL(table, ptr->next->key, *key)) {
+	    tmp = ptr->next;
+	    ptr->next = ptr->next->next;
+	    table->num_entries--;
+	    if (value != 0) *value = tmp->record;
+	    *key = tmp->key;
+	    free(tmp);
+	    return 1;
+    }
+  }
+
+  return 0;
+}
+
+extern int
+st_delete_safe(register st_table* table, register st_data_t* key, st_data_t* value, st_data_t never)
+{
+  unsigned int hash_val;
+  register st_table_entry *ptr;
+
+  hash_val = do_hash_bin(*key, table);
+  ptr = table->bins[hash_val];
+
+  if (ptr == 0) {
+    if (value != 0) *value = 0;
+    return 0;
+  }
+
+  for(; ptr != 0; ptr = ptr->next) {
+    if ((ptr->key != never) && EQUAL(table, ptr->key, *key)) {
+	    table->num_entries--;
+	    *key = ptr->key;
+	    if (value != 0) *value = ptr->record;
+	    ptr->key = ptr->record = never;
+	    return 1;
+    }
+  }
+
+  return 0;
+}
+
+static int
+#if defined(__GNUC__)
+delete_never(st_data_t key __attribute__ ((unused)), st_data_t value,
+	     st_data_t never)
+#else
+delete_never(st_data_t key, st_data_t value, st_data_t never)
+#endif
+{
+    if (value == never) return ST_DELETE;
+    return ST_CONTINUE;
+}
+
+extern void
+st_cleanup_safe(st_table* table, st_data_t never)
+{
+  int num_entries = table->num_entries;
+
+  st_foreach(table, delete_never, never);
+  table->num_entries = num_entries;
+}
+
+extern int
+st_foreach(st_table* table, int (*func)(st_data_t, st_data_t, st_data_t), st_data_t arg)
+{
+  st_table_entry *ptr, *last, *tmp;
+  enum st_retval retval;
+  int i;
+
+  for(i = 0; i < table->num_bins; i++) {
+    last = 0;
+    for(ptr = table->bins[i]; ptr != 0;) {
+	    retval = (*func)(ptr->key, ptr->record, arg);
+	    switch (retval) {
+	    case ST_CHECK:	/* check if hash is modified during iteration */
+        tmp = 0;
+        if (i < table->num_bins) {
+          for (tmp = table->bins[i]; tmp; tmp=tmp->next) {
+            if (tmp == ptr) break;
+          }
+        }
+        if (!tmp) {
+          /* call func with error notice */
+          return 1;
+        }
+        /* fall through */
+	    case ST_CONTINUE:
+        last = ptr;
+        ptr = ptr->next;
+        break;
+	    case ST_STOP:
+        return 0;
+	    case ST_DELETE:
+        tmp = ptr;
+        if (last == 0) {
+          table->bins[i] = ptr->next;
+        }
+        else {
+          last->next = ptr->next;
+        }
+        ptr = ptr->next;
+        free(tmp);
+        table->num_entries--;
+	    }
+    }
+  }
+  return 0;
+}
+
+static int
+str_cmp(st_data_t a1, st_data_t a2)
+{
+  const char* s1 = (const char* )a1;
+  const char* s2 = (const char* )a2;
+  return strcmp(s1, s2);
+}
+
+static int
+str_hash(st_data_t astring)
+{
+  const char* string = (const char* )astring;
+  register int c;
+
+#ifdef HASH_ELFHASH
+  register unsigned int h = 0, g;
+
+  while ((c = *string++) != '\0') {
+    h = ( h << 4 ) + c;
+    if ( g = h & 0xF0000000 )
+	    h ^= g >> 24;
+    h &= ~g;
+  }
+  return h;
+#elif HASH_PERL
+  register int val = 0;
+
+  while ((c = *string++) != '\0') {
+    val += c;
+    val += (val << 10);
+    val ^= (val >> 6);
+  }
+  val += (val << 3);
+  val ^= (val >> 11);
+
+  return val + (val << 15);
+#else
+  register int val = 0;
+
+  while ((c = *string++) != '\0') {
+    val = val*997 + c;
+  }
+
+  return val + (val>>5);
+#endif
+}
+
+static int
+numcmp(st_data_t x, st_data_t y)
+{
+  return x != y;
+}
+
+static int
+numhash(st_data_t n)
+{
+  return n;
+}

thirdparty/oniguruma/st.h

@@ -0,0 +1,60 @@
+/* This is a public domain general purpose hash table package written by Peter Moore @ UCB. */
+
+/* @(#) st.h 5.1 89/12/14 */
+
+#ifndef ST_INCLUDED
+#define ST_INCLUDED
+
+#if SIZEOF_VOIDP == SIZEOF_LONG
+typedef unsigned long st_data_t;
+#elif SIZEOF_VOIDP == SIZEOF_LONG_LONG
+typedef unsigned long long st_data_t;
+#endif
+
+#define ST_DATA_T_DEFINED
+
+typedef struct st_table st_table;
+
+struct st_hash_type {
+    int (*compare)(st_data_t, st_data_t);
+    int (*hash)(st_data_t);
+};
+
+struct st_table {
+    struct st_hash_type *type;
+    int num_bins;
+    int num_entries;
+    struct st_table_entry **bins;
+};
+
+#define st_is_member(table,key) st_lookup(table,key,(st_data_t *)0)
+
+enum st_retval {ST_CONTINUE, ST_STOP, ST_DELETE, ST_CHECK};
+
+#ifndef _
+# define _(args) args
+#endif
+
+st_table *st_init_table _((struct st_hash_type *));
+st_table *st_init_table_with_size _((struct st_hash_type *, int));
+st_table *st_init_numtable _((void));
+st_table *st_init_numtable_with_size _((int));
+st_table *st_init_strtable _((void));
+st_table *st_init_strtable_with_size _((int));
+int st_delete _((st_table *, st_data_t *, st_data_t *));
+int st_delete_safe _((st_table *, st_data_t *, st_data_t *, st_data_t));
+int st_insert _((st_table *, st_data_t, st_data_t));
+int st_lookup _((st_table *, st_data_t, st_data_t *));
+int st_foreach _((st_table *, int (*)(st_data_t, st_data_t, st_data_t), st_data_t));
+void st_add_direct _((st_table *, st_data_t, st_data_t));
+void st_free_table _((st_table *));
+void st_cleanup_safe _((st_table *, st_data_t));
+st_table *st_copy _((st_table *));
+
+#define ST_NUMCMP	((int (*)()) 0)
+#define ST_NUMHASH	((int (*)()) -2)
+
+#define st_numcmp	ST_NUMCMP
+#define st_numhash	ST_NUMHASH
+
+#endif /* ST_INCLUDED */

thirdparty/oniguruma/unicode_fold2_key.c

@@ -0,0 +1,226 @@
+/* This file was converted by gperf_fold_key_conv.py
+      from gperf output file. */
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf -n -C -T -c -t -j1 -L ANSI-C -F,-1 -N onigenc_unicode_fold2_key unicode_fold2_key.gperf  */
+/* Computed positions: -k'3,6' */
+
+
+
+/* This gperf source file was generated by make_unicode_fold_data.py */
+
+/*-
+ * Copyright (c) 2017-2024  K.Kosako
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "regint.h"
+
+#define TOTAL_KEYWORDS 59
+#define MIN_WORD_LENGTH 6
+#define MAX_WORD_LENGTH 6
+#define MIN_HASH_VALUE 0
+#define MAX_HASH_VALUE 58
+/* maximum key range = 59, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+/*ARGSUSED*/
+static unsigned int
+hash(OnigCodePoint codes[])
+{
+  static const unsigned char asso_values[] =
+    {
+      58, 57, 56, 55, 54, 53, 52, 16, 50, 59,
+      15, 59, 25, 59, 59, 59, 59, 59, 59,  3,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 49, 48, 47, 46, 45, 44, 43, 42,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 21,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 59, 59, 59, 59,  2, 59, 59, 59,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 59, 59, 59, 59, 40, 20, 39, 38,
+      37, 14,  5, 36, 20,  7, 25, 34, 29, 32,
+      16, 59, 31, 59, 59,  2,  1, 59, 25, 15,
+      59, 14, 59, 59, 28, 59,  2, 59, 59, 59,
+      11, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 24, 59, 22, 59, 59, 11, 59, 59,
+      59, 59, 59,  7, 59,  0, 59, 59, 16, 59,
+       1, 59, 59, 16, 59, 59, 59, 15, 59, 59,
+      59,  6, 59, 59, 59, 59,  0, 59, 59, 59,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+      59, 59, 59, 59, 59, 59
+    };
+  return asso_values[(unsigned char)onig_codes_byte_at(codes, 5)] + asso_values[(unsigned char)onig_codes_byte_at(codes, 2)];
+}
+
+int
+onigenc_unicode_fold2_key(OnigCodePoint codes[])
+{
+  static const short int wordlist[] =
+    {
+
+      101,
+
+      253,
+
+      76,
+
+      29,
+
+      24,
+
+      239,
+
+      96,
+
+      71,
+
+      92,
+
+      67,
+
+      4,
+
+      62,
+
+      8,
+
+      58,
+
+      234,
+
+      109,
+
+      164,
+
+      88,
+
+      84,
+
+      80,
+
+      214,
+
+      0,
+
+      54,
+
+      261,
+
+      50,
+
+      105,
+
+      121,
+
+      125,
+
+      257,
+
+      42,
+
+      38,
+
+      249,
+
+      46,
+
+      117,
+
+      12,
+
+      113,
+
+      244,
+
+      229,
+
+      224,
+
+      219,
+
+      209,
+
+      16,
+
+      204,
+
+      199,
+
+      194,
+
+      189,
+
+      184,
+
+      179,
+
+      174,
+
+      169,
+
+      20,
+
+      34,
+
+      159,
+
+      154,
+
+      149,
+
+      144,
+
+      139,
+
+      134,
+
+      129
+    };
+
+
+    {
+      int key = hash(codes);
+
+      if (key <= MAX_HASH_VALUE)
+        {
+          int index = wordlist[key];
+
+          if (index >= 0 && onig_codes_cmp(codes, OnigUnicodeFolds2 + index, 2) == 0)
+            return index;
+        }
+    }
+  return -1;
+}

thirdparty/oniguruma/unicode_fold3_key.c

@@ -0,0 +1,136 @@
+/* This file was converted by gperf_fold_key_conv.py
+      from gperf output file. */
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf -n -C -T -c -t -j1 -L ANSI-C -F,-1 -N onigenc_unicode_fold3_key unicode_fold3_key.gperf  */
+/* Computed positions: -k'3,6,9' */
+
+
+
+/* This gperf source file was generated by make_unicode_fold_data.py */
+
+/*-
+ * Copyright (c) 2017-2024  K.Kosako
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "regint.h"
+
+#define TOTAL_KEYWORDS 14
+#define MIN_WORD_LENGTH 9
+#define MAX_WORD_LENGTH 9
+#define MIN_HASH_VALUE 0
+#define MAX_HASH_VALUE 13
+/* maximum key range = 14, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+/*ARGSUSED*/
+static unsigned int
+hash(OnigCodePoint codes[])
+{
+  static const unsigned char asso_values[] =
+    {
+       6,  3, 14, 14, 14, 14, 14, 14,  1, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14,  0,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14,  0, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14,  4, 14, 14,  5, 14, 14,  4, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 10, 14, 14,
+      14, 14, 14,  9, 14,  1, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14,  0, 14, 14,
+      14,  8, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+      14, 14, 14, 14, 14, 14
+    };
+  return asso_values[(unsigned char)onig_codes_byte_at(codes, 8)] + asso_values[(unsigned char)onig_codes_byte_at(codes, 5)] + asso_values[(unsigned char)onig_codes_byte_at(codes, 2)];
+}
+
+int
+onigenc_unicode_fold3_key(OnigCodePoint codes[])
+{
+  static const short int wordlist[] =
+    {
+
+      62,
+
+      47,
+
+      31,
+
+      57,
+
+      41,
+
+      25,
+
+      52,
+
+      36,
+
+      20,
+
+      67,
+
+      15,
+
+      10,
+
+      5,
+
+      0
+    };
+
+
+    {
+      int key = hash(codes);
+
+      if (key <= MAX_HASH_VALUE)
+        {
+          int index = wordlist[key];
+
+          if (index >= 0 && onig_codes_cmp(codes, OnigUnicodeFolds3 + index, 3) == 0)
+            return index;
+        }
+    }
+  return -1;
+}

thirdparty/oniguruma/utf8.c

@@ -0,0 +1,290 @@
+/**********************************************************************
+  utf8.c -  Oniguruma (regular expression library)
+**********************************************************************/
+/*-
+ * Copyright (c) 2002-2019  K.Kosako
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "regenc.h"
+
+/* U+0000 - U+10FFFF */
+#define USE_RFC3629_RANGE
+
+/* #define USE_INVALID_CODE_SCHEME */
+
+#ifdef USE_INVALID_CODE_SCHEME
+/* virtual codepoint values for invalid encoding byte 0xfe and 0xff */
+#define INVALID_CODE_FE   0xfffffffe
+#define INVALID_CODE_FF   0xffffffff
+#define VALID_CODE_LIMIT  0x7fffffff
+#endif
+
+#define utf8_islead(c)     ((UChar )((c) & 0xc0) != 0x80)
+#define utf8_istail(c)     ((UChar )((c) & 0xc0) == 0x80)
+
+static const int EncLen_UTF8[] = {
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+#ifdef USE_RFC3629_RANGE
+  4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
+#else
+  4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1
+#endif
+};
+
+static int
+mbc_enc_len(const UChar* p)
+{
+  return EncLen_UTF8[*p];
+}
+
+static int
+is_valid_mbc_string(const UChar* p, const UChar* end)
+{
+  int i, len;
+
+  while (p < end) {
+    if (! utf8_islead(*p))
+      return FALSE;
+
+    len = mbc_enc_len(p++);
+    if (len > 1) {
+      for (i = 1; i < len; i++) {
+        if (p == end)
+          return FALSE;
+
+        if (! utf8_istail(*p++))
+          return FALSE;
+      }
+    }
+  }
+
+  return TRUE;
+}
+
+static OnigCodePoint
+mbc_to_code(const UChar* p, const UChar* end)
+{
+  int c, len;
+  OnigCodePoint n;
+
+  len = mbc_enc_len(p);
+  if (len > (int )(end - p)) len = (int )(end - p);
+
+  c = *p++;
+  if (len > 1) {
+    len--;
+    n = c & ((1 << (6 - len)) - 1);
+    while (len--) {
+      c = *p++;
+      n = (n << 6) | (c & ((1 << 6) - 1));
+    }
+    return n;
+  }
+  else {
+#ifdef USE_INVALID_CODE_SCHEME
+    if (c > 0xfd) {
+      return ((c == 0xfe) ? INVALID_CODE_FE : INVALID_CODE_FF);
+    }
+#endif
+    return (OnigCodePoint )c;
+  }
+}
+
+static int
+code_to_mbclen(OnigCodePoint code)
+{
+  if      ((code & 0xffffff80) == 0) return 1;
+  else if ((code & 0xfffff800) == 0) return 2;
+  else if ((code & 0xffff0000) == 0) return 3;
+  else if ((code & 0xffe00000) == 0) return 4;
+#ifndef USE_RFC3629_RANGE
+  else if ((code & 0xfc000000) == 0) return 5;
+  else if ((code & 0x80000000) == 0) return 6;
+#endif
+#ifdef USE_INVALID_CODE_SCHEME
+  else if (code == INVALID_CODE_FE) return 1;
+  else if (code == INVALID_CODE_FF) return 1;
+#endif
+  else
+    return ONIGERR_INVALID_CODE_POINT_VALUE;
+}
+
+static int
+code_to_mbc(OnigCodePoint code, UChar *buf)
+{
+#define UTF8_TRAILS(code, shift) (UChar )((((code) >> (shift)) & 0x3f) | 0x80)
+#define UTF8_TRAIL0(code)        (UChar )(((code) & 0x3f) | 0x80)
+
+  if ((code & 0xffffff80) == 0) {
+    *buf = (UChar )code;
+    return 1;
+  }
+  else {
+    UChar *p = buf;
+
+    if ((code & 0xfffff800) == 0) {
+      *p++ = (UChar )(((code>>6)& 0x1f) | 0xc0);
+    }
+    else if ((code & 0xffff0000) == 0) {
+      *p++ = (UChar )(((code>>12) & 0x0f) | 0xe0);
+      *p++ = UTF8_TRAILS(code, 6);
+    }
+    else if ((code & 0xffe00000) == 0) {
+      *p++ = (UChar )(((code>>18) & 0x07) | 0xf0);
+      *p++ = UTF8_TRAILS(code, 12);
+      *p++ = UTF8_TRAILS(code,  6);
+    }
+#ifndef USE_RFC3629_RANGE
+    else if ((code & 0xfc000000) == 0) {
+      *p++ = (UChar )(((code>>24) & 0x03) | 0xf8);
+      *p++ = UTF8_TRAILS(code, 18);
+      *p++ = UTF8_TRAILS(code, 12);
+      *p++ = UTF8_TRAILS(code,  6);
+    }
+    else if ((code & 0x80000000) == 0) {
+      *p++ = (UChar )(((code>>30) & 0x01) | 0xfc);
+      *p++ = UTF8_TRAILS(code, 24);
+      *p++ = UTF8_TRAILS(code, 18);
+      *p++ = UTF8_TRAILS(code, 12);
+      *p++ = UTF8_TRAILS(code,  6);
+    }
+#endif
+#ifdef USE_INVALID_CODE_SCHEME
+    else if (code == INVALID_CODE_FE) {
+      *p = 0xfe;
+      return 1;
+    }
+    else if (code == INVALID_CODE_FF) {
+      *p = 0xff;
+      return 1;
+    }
+#endif
+    else {
+      return ONIGERR_TOO_BIG_WIDE_CHAR_VALUE;
+    }
+
+    *p++ = UTF8_TRAIL0(code);
+    return (int )(p - buf);
+  }
+}
+
+static int
+mbc_case_fold(OnigCaseFoldType flag, const UChar** pp,
+              const UChar* end, UChar* fold)
+{
+  const UChar* p = *pp;
+
+  if (ONIGENC_IS_MBC_ASCII(p)) {
+#ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
+    if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) != 0) {
+      if (*p == 0x49) {
+        *fold++ = 0xc4;
+        *fold   = 0xb1;
+        (*pp)++;
+        return 2;
+      }
+    }
+#endif
+
+    *fold = ONIGENC_ASCII_CODE_TO_LOWER_CASE(*p);
+    (*pp)++;
+    return 1; /* return byte length of converted char to lower */
+  }
+  else {
+    return onigenc_unicode_mbc_case_fold(ONIG_ENCODING_UTF8, flag,
+                                         pp, end, fold);
+  }
+}
+
+static int
+get_ctype_code_range(OnigCtype ctype, OnigCodePoint *sb_out,
+                     const OnigCodePoint* ranges[])
+{
+  *sb_out = 0x80;
+  return onigenc_unicode_ctype_code_range(ctype, ranges);
+}
+
+
+static UChar*
+left_adjust_char_head(const UChar* start, const UChar* s)
+{
+  const UChar *p;
+
+  if (s <= start) return (UChar* )s;
+  p = s;
+
+  while (!utf8_islead(*p) && p > start) p--;
+  return (UChar* )p;
+}
+
+static int
+get_case_fold_codes_by_str(OnigCaseFoldType flag,
+    const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[])
+{
+  return onigenc_unicode_get_case_fold_codes_by_str(ONIG_ENCODING_UTF8,
+                                                    flag, p, end, items);
+}
+
+OnigEncodingType OnigEncodingUTF8 = {
+  mbc_enc_len,
+  "UTF-8",     /* name */
+#ifdef USE_RFC3629_RANGE
+  4,           /* max enc length */
+#else
+  6,
+#endif
+  1,           /* min enc length */
+  onigenc_is_mbc_newline_0x0a,
+  mbc_to_code,
+  code_to_mbclen,
+  code_to_mbc,
+  mbc_case_fold,
+  onigenc_unicode_apply_all_case_fold,
+  get_case_fold_codes_by_str,
+  onigenc_unicode_property_name_to_ctype,
+  onigenc_unicode_is_code_ctype,
+  get_ctype_code_range,
+  left_adjust_char_head,
+  onigenc_always_true_is_allowed_reverse_match,
+  NULL, /* init */
+  NULL, /* is_initialized */
+  is_valid_mbc_string,
+  ENC_FLAG_ASCII_COMPATIBLE|ENC_FLAG_UNICODE|ENC_FLAG_SKIP_OFFSET_1_OR_0,
+  0, 0
+};

thirdparty/utf8proc/LICENSE.md

@@ -0,0 +1,93 @@
+## utf8proc license ##
+
+**utf8proc** is a software package originally developed
+by Jan Behrens and the rest of the Public Software Group, who
+deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers.  Like the original utf8proc,
+whose copyright and license statements are reproduced below, all new
+work on the utf8proc library is licensed under the [MIT "expat"
+license](http://opensource.org/licenses/MIT):
+
+*Copyright © 2014-2021 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+## Original utf8proc license ##
+
+*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+## Unicode data license ##
+
+This software contains data (`utf8proc_data.c`) derived from processing
+the Unicode data files. The following license applies to that data:
+
+**COPYRIGHT AND PERMISSION NOTICE**
+
+*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
+under the Terms of Use in http://www.unicode.org/copyright.html.*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of the Unicode data files and any associated documentation (the "Data
+Files") or Unicode software and any associated documentation (the
+"Software") to deal in the Data Files or Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, and/or sell copies of the Data Files or Software, and
+to permit persons to whom the Data Files or Software are furnished to do
+so, provided that (a) the above copyright notice(s) and this permission
+notice appear with all copies of the Data Files or Software, (b) both the
+above copyright notice(s) and this permission notice appear in associated
+documentation, and (c) there is clear notice in each modified Data File or
+in the Software as well as in the documentation associated with the Data
+File(s) or Software that the data or software has been modified.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
+INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
+CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
+USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.
+
+Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
+registered in some jurisdictions. All other trademarks and registered
+trademarks mentioned herein are the property of their respective owners.

thirdparty/utf8proc/README.md

@@ -0,0 +1,45 @@
+# utf8proc source subset
+
+This directory contains the utf8proc sources used by sd.cpp's
+[JSON tokenizer](../../src/tokenizers/json_tokenizer.cpp) for NFC normalization,
+lowercase mapping, and UTF-8 decoding and encoding.
+
+## Upstream source
+
+- Repository: [JuliaStrings/utf8proc](https://github.com/JuliaStrings/utf8proc)
+- Release: `v2.10.0`
+- Commit: [`a1b99daa2a3393884220264c927a48ba1251a9c6`](https://github.com/JuliaStrings/utf8proc/tree/a1b99daa2a3393884220264c927a48ba1251a9c6)
+- Unicode version: `16.0.0`
+- License: [MIT and Unicode data licenses](LICENSE.md)
+
+The following four files were copied from the upstream repository root without
+local modifications. Their original filenames are preserved. This README is
+maintained by sd.cpp and replaces the upstream README.
+
+| File | Purpose |
+| --- | --- |
+| `utf8proc.c` | Library implementation; the only separately compiled C file. |
+| `utf8proc.h` | Public declarations, types and version definitions. |
+| `utf8proc_data.c` | Unicode data tables included by `utf8proc.c`. |
+| `LICENSE.md` | Library and Unicode data license notices. |
+
+The library implementation and Unicode tables are retained in full.
+`utf8proc_data.c` is copied as supplied by upstream; sd.cpp does not regenerate
+it or compile it as a separate translation unit. Upstream tests, benchmarks,
+documentation, data-generation tools and build/packaging files are omitted.
+
+## Build integration
+
+[The parent CMake file](../CMakeLists.txt) compiles `utf8proc.c` as the
+`sd-utf8proc` OBJECT target with `UTF8PROC_STATIC` and position-independent code
+enabled. The object is included directly in the sd.cpp static or shared library,
+with no separate utf8proc library required by consumers. The license is installed
+alongside sd.cpp.
+
+All four upstream files belong under version control. Build artifacts belong
+in the build directory; this subset requires no generated configuration header.
+
+When refreshing this subset, copy all four files from the same upstream revision
+and retain `LICENSE.md`. Update the release, commit and Unicode version recorded
+here. Recheck source dependencies and validate tokenizer normalization, lowercase
+conversion and token IDs on supported platforms.

thirdparty/utf8proc/utf8proc.c

@@ -0,0 +1,823 @@
+/* -*- mode: c; c-basic-offset: 2; tab-width: 2; indent-tabs-mode: nil -*- */
+/*
+ *  Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
+ *  Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *  DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ *  This library contains derived data from a modified version of the
+ *  Unicode data files.
+ *
+ *  The original data files are available at
+ *  https://www.unicode.org/Public/UNIDATA/
+ *
+ *  Please notice the copyright statement in the file "utf8proc_data.c".
+ */
+
+
+/*
+ *  File name:    utf8proc.c
+ *
+ *  Description:
+ *  Implementation of libutf8proc.
+ */
+
+
+#include "utf8proc.h"
+
+#ifndef SSIZE_MAX
+#define SSIZE_MAX ((size_t)SIZE_MAX/2)
+#endif
+#ifndef UINT16_MAX
+#  define UINT16_MAX 65535U
+#endif
+
+#include "utf8proc_data.c"
+
+
+UTF8PROC_DLLEXPORT const utf8proc_int8_t utf8proc_utf8class[256] = {
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+  4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0 };
+
+#define UTF8PROC_HANGUL_SBASE 0xAC00
+#define UTF8PROC_HANGUL_LBASE 0x1100
+#define UTF8PROC_HANGUL_VBASE 0x1161
+#define UTF8PROC_HANGUL_TBASE 0x11A7
+#define UTF8PROC_HANGUL_LCOUNT 19
+#define UTF8PROC_HANGUL_VCOUNT 21
+#define UTF8PROC_HANGUL_TCOUNT 28
+#define UTF8PROC_HANGUL_NCOUNT 588
+#define UTF8PROC_HANGUL_SCOUNT 11172
+/* END is exclusive */
+#define UTF8PROC_HANGUL_L_START  0x1100
+#define UTF8PROC_HANGUL_L_END    0x115A
+#define UTF8PROC_HANGUL_L_FILLER 0x115F
+#define UTF8PROC_HANGUL_V_START  0x1160
+#define UTF8PROC_HANGUL_V_END    0x11A3
+#define UTF8PROC_HANGUL_T_START  0x11A8
+#define UTF8PROC_HANGUL_T_END    0x11FA
+#define UTF8PROC_HANGUL_S_START  0xAC00
+#define UTF8PROC_HANGUL_S_END    0xD7A4
+
+/* Should follow semantic-versioning rules (semver.org) based on API
+   compatibility.  (Note that the shared-library version number will
+   be different, being based on ABI compatibility.): */
+#define STRINGIZEx(x) #x
+#define STRINGIZE(x) STRINGIZEx(x)
+UTF8PROC_DLLEXPORT const char *utf8proc_version(void) {
+  return STRINGIZE(UTF8PROC_VERSION_MAJOR) "." STRINGIZE(UTF8PROC_VERSION_MINOR) "." STRINGIZE(UTF8PROC_VERSION_PATCH) "";
+}
+
+UTF8PROC_DLLEXPORT const char *utf8proc_unicode_version(void) {
+  return "16.0.0";
+}
+
+UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode) {
+  switch (errcode) {
+    case UTF8PROC_ERROR_NOMEM:
+    return "Memory for processing UTF-8 data could not be allocated.";
+    case UTF8PROC_ERROR_OVERFLOW:
+    return "UTF-8 string is too long to be processed.";
+    case UTF8PROC_ERROR_INVALIDUTF8:
+    return "Invalid UTF-8 string";
+    case UTF8PROC_ERROR_NOTASSIGNED:
+    return "Unassigned Unicode code point found in UTF-8 string.";
+    case UTF8PROC_ERROR_INVALIDOPTS:
+    return "Invalid options for UTF-8 processing chosen.";
+    default:
+    return "An unknown error occurred while processing UTF-8 data.";
+  }
+}
+
+#define utf_cont(ch)  (((ch) & 0xc0) == 0x80)
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
+) {
+  utf8proc_int32_t uc;
+  const utf8proc_uint8_t *end;
+
+  *dst = -1;
+  if (!strlen) return 0;
+  end = str + ((strlen < 0) ? 4 : strlen);
+  uc = *str++;
+  if (uc < 0x80) {
+    *dst = uc;
+    return 1;
+  }
+  // Must be between 0xc2 and 0xf4 inclusive to be valid
+  if ((utf8proc_uint32_t)(uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8;
+  if (uc < 0xe0) {         // 2-byte sequence
+     // Must have valid continuation character
+     if (str >= end || !utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8;
+     *dst = ((uc & 0x1f)<<6) | (*str & 0x3f);
+     return 2;
+  }
+  if (uc < 0xf0) {        // 3-byte sequence
+     if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1]))
+        return UTF8PROC_ERROR_INVALIDUTF8;
+     // Check for surrogate chars
+     if (uc == 0xed && *str > 0x9f)
+         return UTF8PROC_ERROR_INVALIDUTF8;
+     uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f);
+     if (uc < 0x800)
+         return UTF8PROC_ERROR_INVALIDUTF8;
+     *dst = uc;
+     return 3;
+  }
+  // 4-byte sequence
+  // Must have 3 valid continuation characters
+  if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2]))
+     return UTF8PROC_ERROR_INVALIDUTF8;
+  // Make sure in correct range (0x10000 - 0x10ffff)
+  if (uc == 0xf0) {
+    if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8;
+  } else if (uc == 0xf4) {
+    if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8;
+  }
+  *dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f);
+  return 4;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
+    return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
+  if (uc < 0x00) {
+    return 0;
+  } else if (uc < 0x80) {
+    dst[0] = (utf8proc_uint8_t) uc;
+    return 1;
+  } else if (uc < 0x800) {
+    dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
+    dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+    return 2;
+  // Note: we allow encoding 0xd800-0xdfff here, so as not to change
+  // the API, however, these are actually invalid in UTF-8
+  } else if (uc < 0x10000) {
+    dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
+    dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+    dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+    return 3;
+  } else if (uc < 0x110000) {
+    dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
+    dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
+    dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+    dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+    return 4;
+  } else return 0;
+}
+
+/* internal version used for inserting 0xff bytes between graphemes */
+static utf8proc_ssize_t charbound_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
+   if (uc < 0x00) {
+      if (uc == -1) { /* internal value used for grapheme breaks */
+        dst[0] = (utf8proc_uint8_t)0xFF;
+        return 1;
+      }
+      return 0;
+   } else if (uc < 0x80) {
+      dst[0] = (utf8proc_uint8_t)uc;
+      return 1;
+   } else if (uc < 0x800) {
+      dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
+      dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+      return 2;
+   } else if (uc < 0x10000) {
+      dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
+      dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+      dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+      return 3;
+   } else if (uc < 0x110000) {
+      dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
+      dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
+      dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+      dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+      return 4;
+   } else return 0;
+}
+
+/* internal "unsafe" version that does not check whether uc is in range */
+static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) {
+  /* ASSERT: uc >= 0 && uc < 0x110000 */
+  return utf8proc_properties + (
+    utf8proc_stage2table[
+      utf8proc_stage1table[uc >> 8] + (uc & 0xFF)
+    ]
+  );
+}
+
+UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) {
+  return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc);
+}
+
+/* return whether there is a grapheme break between boundclasses lbc and tbc
+   (according to the definition of extended grapheme clusters)
+
+  Rule numbering refers to TR29 Version 29 (Unicode 9.0.0):
+  http://www.unicode.org/reports/tr29/tr29-29.html
+
+  CAVEATS:
+   Please note that evaluation of GB10 (grapheme breaks between emoji zwj sequences)
+   and GB 12/13 (regional indicator code points) require knowledge of previous characters
+   and are thus not handled by this function. This may result in an incorrect break before
+   an E_Modifier class codepoint and an incorrectly missing break between two
+   REGIONAL_INDICATOR class code points if such support does not exist in the caller.
+
+   See the special support in grapheme_break_extended, for required bookkeeping by the caller.
+*/
+static utf8proc_bool grapheme_break_simple(int lbc, int tbc) {
+  return
+    (lbc == UTF8PROC_BOUNDCLASS_START) ? true :       // GB1
+    (lbc == UTF8PROC_BOUNDCLASS_CR &&                 // GB3
+     tbc == UTF8PROC_BOUNDCLASS_LF) ? false :         // ---
+    (lbc >= UTF8PROC_BOUNDCLASS_CR && lbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true :  // GB4
+    (tbc >= UTF8PROC_BOUNDCLASS_CR && tbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true :  // GB5
+    (lbc == UTF8PROC_BOUNDCLASS_L &&                  // GB6
+     (tbc == UTF8PROC_BOUNDCLASS_L ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_V ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_LV ||                // ---
+      tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false :      // ---
+    ((lbc == UTF8PROC_BOUNDCLASS_LV ||                // GB7
+      lbc == UTF8PROC_BOUNDCLASS_V) &&                // ---
+     (tbc == UTF8PROC_BOUNDCLASS_V ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_T)) ? false :        // ---
+    ((lbc == UTF8PROC_BOUNDCLASS_LVT ||               // GB8
+      lbc == UTF8PROC_BOUNDCLASS_T) &&                // ---
+     tbc == UTF8PROC_BOUNDCLASS_T) ? false :          // ---
+    (tbc == UTF8PROC_BOUNDCLASS_EXTEND ||             // GB9
+     tbc == UTF8PROC_BOUNDCLASS_ZWJ ||                // ---
+     tbc == UTF8PROC_BOUNDCLASS_SPACINGMARK ||        // GB9a
+     lbc == UTF8PROC_BOUNDCLASS_PREPEND) ? false :    // GB9b
+    (lbc == UTF8PROC_BOUNDCLASS_E_ZWG &&              // GB11 (requires additional handling below)
+     tbc == UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC) ? false : // ----
+    (lbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR &&          // GB12/13 (requires additional handling below)
+     tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR) ? false :  // ----
+    true; // GB999
+}
+
+static utf8proc_bool grapheme_break_extended(int lbc, int tbc, int licb, int ticb, utf8proc_int32_t *state)
+{
+  if (state) {
+    int state_bc, state_icb; /* boundclass and indic_conjunct_break state */
+    if (*state == 0) { /* state initialization */
+      state_bc = lbc;
+      state_icb = licb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT ? licb : UTF8PROC_INDIC_CONJUNCT_BREAK_NONE;
+    }
+    else { /* lbc and licb are already encoded in *state */
+      state_bc = *state & 0xff;  // 1st byte of state is bound class
+      state_icb = *state >> 8;   // 2nd byte of state is indic conjunct break
+    }
+
+    utf8proc_bool break_permitted = grapheme_break_simple(state_bc, tbc) &&
+       !(state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER
+        && ticb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT); // GB9c
+
+    // Special support for GB9c.  Don't break between two consonants
+    // separated 1+ linker characters and 0+ extend characters in any order.
+    // After a consonant, we enter LINKER state after at least one linker.
+    if (ticb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT
+        || state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT
+        || state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_EXTEND)
+      state_icb = ticb;
+    else if (state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER)
+      state_icb = ticb == UTF8PROC_INDIC_CONJUNCT_BREAK_EXTEND ?
+                  UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER : ticb;
+
+    // Special support for GB 12/13 made possible by GB999. After two RI
+    // class codepoints we want to force a break. Do this by resetting the
+    // second RI's bound class to UTF8PROC_BOUNDCLASS_OTHER, to force a break
+    // after that character according to GB999 (unless of course such a break is
+    // forbidden by a different rule such as GB9).
+    if (state_bc == tbc && tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR)
+      state_bc = UTF8PROC_BOUNDCLASS_OTHER;
+    // Special support for GB11 (emoji extend* zwj / emoji)
+    else if (state_bc == UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC) {
+      if (tbc == UTF8PROC_BOUNDCLASS_EXTEND) // fold EXTEND codepoints into emoji
+        state_bc = UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC;
+      else if (tbc == UTF8PROC_BOUNDCLASS_ZWJ)
+        state_bc = UTF8PROC_BOUNDCLASS_E_ZWG; // state to record emoji+zwg combo
+      else
+        state_bc = tbc;
+    }
+    else
+      state_bc = tbc;
+
+    *state = state_bc + (state_icb << 8);
+    return break_permitted;
+  }
+  else
+    return grapheme_break_simple(lbc, tbc);
+}
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
+    utf8proc_int32_t c1, utf8proc_int32_t c2, utf8proc_int32_t *state) {
+
+  const utf8proc_property_t *p1 = utf8proc_get_property(c1);
+  const utf8proc_property_t *p2 = utf8proc_get_property(c2);
+  return grapheme_break_extended(p1->boundclass,
+                                 p2->boundclass,
+                                 p1->indic_conjunct_break,
+                                 p2->indic_conjunct_break,
+                                 state);
+}
+
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
+    utf8proc_int32_t c1, utf8proc_int32_t c2) {
+  return utf8proc_grapheme_break_stateful(c1, c2, NULL);
+}
+
+static utf8proc_int32_t seqindex_decode_entry(const utf8proc_uint16_t **entry)
+{
+  utf8proc_int32_t entry_cp = **entry;
+  if ((entry_cp & 0xF800) == 0xD800) {
+    *entry = *entry + 1;
+    entry_cp = ((entry_cp & 0x03FF) << 10) | (**entry & 0x03FF);
+    entry_cp += 0x10000;
+  }
+  return entry_cp;
+}
+
+static utf8proc_int32_t seqindex_decode_index(const utf8proc_uint32_t seqindex)
+{
+  const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex];
+  return seqindex_decode_entry(&entry);
+}
+
+static utf8proc_ssize_t seqindex_write_char_decomposed(utf8proc_uint16_t seqindex, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
+  utf8proc_ssize_t written = 0;
+  const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex & 0x3FFF];
+  int len = seqindex >> 14;
+  if (len >= 3) {
+    len = *entry;
+    entry++;
+  }
+  for (; len >= 0; entry++, len--) {
+    utf8proc_int32_t entry_cp = seqindex_decode_entry(&entry);
+
+    written += utf8proc_decompose_char(entry_cp, dst+written,
+      (bufsize > written) ? (bufsize - written) : 0, options,
+    last_boundclass);
+    if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
+  }
+  return written;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cl = utf8proc_get_property(c)->lowercase_seqindex;
+  return cl != UINT16_MAX ? seqindex_decode_index((utf8proc_uint32_t)cl) : c;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cu = utf8proc_get_property(c)->uppercase_seqindex;
+  return cu != UINT16_MAX ? seqindex_decode_index((utf8proc_uint32_t)cu) : c;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cu = utf8proc_get_property(c)->titlecase_seqindex;
+  return cu != UINT16_MAX ? seqindex_decode_index((utf8proc_uint32_t)cu) : c;
+}
+
+UTF8PROC_DLLEXPORT int utf8proc_islower(utf8proc_int32_t c)
+{
+  const utf8proc_property_t *p = utf8proc_get_property(c);
+  return p->lowercase_seqindex != p->uppercase_seqindex && p->lowercase_seqindex == UINT16_MAX;
+}
+
+UTF8PROC_DLLEXPORT int utf8proc_isupper(utf8proc_int32_t c)
+{
+  const utf8proc_property_t *p = utf8proc_get_property(c);
+  return p->lowercase_seqindex != p->uppercase_seqindex && p->uppercase_seqindex == UINT16_MAX && p->category != UTF8PROC_CATEGORY_LT;
+}
+
+/* return a character width analogous to wcwidth (except portable and
+   hopefully less buggy than most system wcwidth functions). */
+UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t c) {
+  return utf8proc_get_property(c)->charwidth;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_charwidth_ambiguous(utf8proc_int32_t c) {
+  return utf8proc_get_property(c)->ambiguous_width;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t c) {
+  return (utf8proc_category_t) utf8proc_get_property(c)->category;
+}
+
+UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t c) {
+  static const char s[][3] = {"Cn","Lu","Ll","Lt","Lm","Lo","Mn","Mc","Me","Nd","Nl","No","Pc","Pd","Ps","Pe","Pi","Pf","Po","Sm","Sc","Sk","So","Zs","Zl","Zp","Cc","Cf","Cs","Co"};
+  return s[utf8proc_category(c)];
+}
+
+#define utf8proc_decompose_lump(replacement_uc) \
+  return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
+  options & ~(unsigned int)UTF8PROC_LUMP, last_boundclass)
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
+  const utf8proc_property_t *property;
+  utf8proc_propval_t category;
+  utf8proc_int32_t hangul_sindex;
+  if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED;
+  property = unsafe_get_property(uc);
+  category = property->category;
+  hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
+  if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
+    if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT) {
+      utf8proc_int32_t hangul_tindex;
+      if (bufsize >= 1) {
+        dst[0] = UTF8PROC_HANGUL_LBASE +
+          hangul_sindex / UTF8PROC_HANGUL_NCOUNT;
+        if (bufsize >= 2) dst[1] = UTF8PROC_HANGUL_VBASE +
+          (hangul_sindex % UTF8PROC_HANGUL_NCOUNT) / UTF8PROC_HANGUL_TCOUNT;
+      }
+      hangul_tindex = hangul_sindex % UTF8PROC_HANGUL_TCOUNT;
+      if (!hangul_tindex) return 2;
+      if (bufsize >= 3) dst[2] = UTF8PROC_HANGUL_TBASE + hangul_tindex;
+      return 3;
+    }
+  }
+  if (options & UTF8PROC_REJECTNA) {
+    if (!category) return UTF8PROC_ERROR_NOTASSIGNED;
+  }
+  if (options & UTF8PROC_IGNORE) {
+    if (property->ignorable) return 0;
+  }
+  if (options & UTF8PROC_STRIPNA) {
+    if (!category) return 0;
+  }
+  if (options & UTF8PROC_LUMP) {
+    if (category == UTF8PROC_CATEGORY_ZS) utf8proc_decompose_lump(0x0020);
+    if (uc == 0x2018 || uc == 0x2019 || uc == 0x02BC || uc == 0x02C8)
+      utf8proc_decompose_lump(0x0027);
+    if (category == UTF8PROC_CATEGORY_PD || uc == 0x2212)
+      utf8proc_decompose_lump(0x002D);
+    if (uc == 0x2044 || uc == 0x2215) utf8proc_decompose_lump(0x002F);
+    if (uc == 0x2236) utf8proc_decompose_lump(0x003A);
+    if (uc == 0x2039 || uc == 0x2329 || uc == 0x3008)
+      utf8proc_decompose_lump(0x003C);
+    if (uc == 0x203A || uc == 0x232A || uc == 0x3009)
+      utf8proc_decompose_lump(0x003E);
+    if (uc == 0x2216) utf8proc_decompose_lump(0x005C);
+    if (uc == 0x02C4 || uc == 0x02C6 || uc == 0x2038 || uc == 0x2303)
+      utf8proc_decompose_lump(0x005E);
+    if (category == UTF8PROC_CATEGORY_PC || uc == 0x02CD)
+      utf8proc_decompose_lump(0x005F);
+    if (uc == 0x02CB) utf8proc_decompose_lump(0x0060);
+    if (uc == 0x2223) utf8proc_decompose_lump(0x007C);
+    if (uc == 0x223C) utf8proc_decompose_lump(0x007E);
+    if ((options & UTF8PROC_NLF2LS) && (options & UTF8PROC_NLF2PS)) {
+      if (category == UTF8PROC_CATEGORY_ZL ||
+          category == UTF8PROC_CATEGORY_ZP)
+        utf8proc_decompose_lump(0x000A);
+    }
+  }
+  if (options & UTF8PROC_STRIPMARK) {
+    if (category == UTF8PROC_CATEGORY_MN ||
+      category == UTF8PROC_CATEGORY_MC ||
+      category == UTF8PROC_CATEGORY_ME) return 0;
+  }
+  if (options & UTF8PROC_CASEFOLD) {
+    if (property->casefold_seqindex != UINT16_MAX) {
+      return seqindex_write_char_decomposed(property->casefold_seqindex, dst, bufsize, options, last_boundclass);
+    }
+  }
+  if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
+    if (property->decomp_seqindex != UINT16_MAX &&
+        (!property->decomp_type || (options & UTF8PROC_COMPAT))) {
+      return seqindex_write_char_decomposed(property->decomp_seqindex, dst, bufsize, options, last_boundclass);
+    }
+  }
+  if (options & UTF8PROC_CHARBOUND) {
+    utf8proc_bool boundary;
+    boundary = grapheme_break_extended(0, property->boundclass, 0, property->indic_conjunct_break,
+                                       last_boundclass);
+    if (boundary) {
+      if (bufsize >= 1) dst[0] = -1; /* sentinel value for grapheme break */
+      if (bufsize >= 2) dst[1] = uc;
+      return 2;
+    }
+  }
+  if (bufsize >= 1) *dst = uc;
+  return 1;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
+) {
+    return utf8proc_decompose_custom(str, strlen, buffer, bufsize, options, NULL, NULL);
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+) {
+  /* strlen will be ignored, if UTF8PROC_NULLTERM is set in options */
+  utf8proc_ssize_t wpos = 0;
+  if ((options & UTF8PROC_COMPOSE) && (options & UTF8PROC_DECOMPOSE))
+    return UTF8PROC_ERROR_INVALIDOPTS;
+  if ((options & UTF8PROC_STRIPMARK) &&
+      !(options & UTF8PROC_COMPOSE) && !(options & UTF8PROC_DECOMPOSE))
+    return UTF8PROC_ERROR_INVALIDOPTS;
+  {
+    utf8proc_int32_t uc;
+    utf8proc_ssize_t rpos = 0;
+    utf8proc_ssize_t decomp_result;
+    int boundclass = UTF8PROC_BOUNDCLASS_START;
+    while (1) {
+      if (options & UTF8PROC_NULLTERM) {
+        rpos += utf8proc_iterate(str + rpos, -1, &uc);
+        /* checking of return value is not necessary,
+           as 'uc' is < 0 in case of error */
+        if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
+        if (rpos < 0) return UTF8PROC_ERROR_OVERFLOW;
+        if (uc == 0) break;
+      } else {
+        if (rpos >= strlen) break;
+        rpos += utf8proc_iterate(str + rpos, strlen - rpos, &uc);
+        if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
+      }
+      if (custom_func != NULL) {
+        uc = custom_func(uc, custom_data);   /* user-specified custom mapping */
+      }
+      decomp_result = utf8proc_decompose_char(
+        uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
+        &boundclass
+      );
+      if (decomp_result < 0) return decomp_result;
+      wpos += decomp_result;
+      /* prohibiting integer overflows due to too long strings: */
+      if (wpos < 0 ||
+          wpos > (utf8proc_ssize_t)(SSIZE_MAX/sizeof(utf8proc_int32_t)/2))
+        return UTF8PROC_ERROR_OVERFLOW;
+    }
+  }
+  if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
+    utf8proc_ssize_t pos = 0;
+    while (pos < wpos-1) {
+      utf8proc_int32_t uc1, uc2;
+      const utf8proc_property_t *property1, *property2;
+      uc1 = buffer[pos];
+      uc2 = buffer[pos+1];
+      property1 = unsafe_get_property(uc1);
+      property2 = unsafe_get_property(uc2);
+      if (property1->combining_class > property2->combining_class &&
+          property2->combining_class > 0) {
+        buffer[pos] = uc2;
+        buffer[pos+1] = uc1;
+        if (pos > 0) pos--; else pos++;
+      } else {
+        pos++;
+      }
+    }
+  }
+  return wpos;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
+  /* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored */
+  if (options & (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS | UTF8PROC_STRIPCC)) {
+    utf8proc_ssize_t rpos;
+    utf8proc_ssize_t wpos = 0;
+    utf8proc_int32_t uc;
+    for (rpos = 0; rpos < length; rpos++) {
+      uc = buffer[rpos];
+      if (uc == 0x000D && rpos < length-1 && buffer[rpos+1] == 0x000A) rpos++;
+      if (uc == 0x000A || uc == 0x000D || uc == 0x0085 ||
+          ((options & UTF8PROC_STRIPCC) && (uc == 0x000B || uc == 0x000C))) {
+        if (options & UTF8PROC_NLF2LS) {
+          if (options & UTF8PROC_NLF2PS) {
+            buffer[wpos++] = 0x000A;
+          } else {
+            buffer[wpos++] = 0x2028;
+          }
+        } else {
+          if (options & UTF8PROC_NLF2PS) {
+            buffer[wpos++] = 0x2029;
+          } else {
+            buffer[wpos++] = 0x0020;
+          }
+        }
+      } else if ((options & UTF8PROC_STRIPCC) &&
+          (uc < 0x0020 || (uc >= 0x007F && uc < 0x00A0))) {
+        if (uc == 0x0009) buffer[wpos++] = 0x0020;
+      } else {
+        buffer[wpos++] = uc;
+      }
+    }
+    length = wpos;
+  }
+  if (options & UTF8PROC_COMPOSE) {
+    utf8proc_int32_t *starter = NULL;
+    const utf8proc_property_t *starter_property = NULL;
+    utf8proc_propval_t max_combining_class = -1;
+    utf8proc_ssize_t rpos;
+    utf8proc_ssize_t wpos = 0;
+    for (rpos = 0; rpos < length; rpos++) {
+      utf8proc_int32_t current_char = buffer[rpos];
+      const utf8proc_property_t *current_property = unsafe_get_property(current_char);
+      if (starter && current_property->combining_class > max_combining_class) {
+        /* combination perhaps possible */
+        utf8proc_int32_t hangul_lindex;
+        utf8proc_int32_t hangul_sindex;
+        hangul_lindex = *starter - UTF8PROC_HANGUL_LBASE;
+        if (hangul_lindex >= 0 && hangul_lindex < UTF8PROC_HANGUL_LCOUNT) {
+          utf8proc_int32_t hangul_vindex;
+          hangul_vindex = current_char - UTF8PROC_HANGUL_VBASE;
+          if (hangul_vindex >= 0 && hangul_vindex < UTF8PROC_HANGUL_VCOUNT) {
+            *starter = UTF8PROC_HANGUL_SBASE +
+              (hangul_lindex * UTF8PROC_HANGUL_VCOUNT + hangul_vindex) *
+              UTF8PROC_HANGUL_TCOUNT;
+            starter_property = NULL;
+            continue;
+          }
+        }
+        hangul_sindex = *starter - UTF8PROC_HANGUL_SBASE;
+        if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT &&
+            (hangul_sindex % UTF8PROC_HANGUL_TCOUNT) == 0) {
+          utf8proc_int32_t hangul_tindex;
+          hangul_tindex = current_char - UTF8PROC_HANGUL_TBASE;
+          if (hangul_tindex >= 0 && hangul_tindex < UTF8PROC_HANGUL_TCOUNT) {
+            *starter += hangul_tindex;
+            starter_property = NULL;
+            continue;
+          }
+        }
+        if (!starter_property) {
+          starter_property = unsafe_get_property(*starter);
+        }
+        int idx = starter_property->comb_index;
+        if (idx < 0x3FF && current_property->comb_issecond) {
+          int len = starter_property->comb_length;
+          utf8proc_int32_t max_second = utf8proc_combinations_second[idx + len - 1];
+          if (current_char <= max_second) {
+            // TODO: binary search? arithmetic search?
+            for (int off = 0; off < len; ++off) {
+              utf8proc_int32_t second = utf8proc_combinations_second[idx + off];
+              if (current_char < second) {
+                /* not found */
+                break;
+              }
+              if (current_char == second) {
+                /* found */
+                utf8proc_int32_t composition = utf8proc_combinations_combined[idx + off];
+                *starter = composition;
+                starter_property = NULL;
+                break;
+              }
+            }
+            if (starter_property == NULL) {
+              /* found */
+              continue;
+            }
+          }
+        }
+      }
+      buffer[wpos] = current_char;
+      if (current_property->combining_class) {
+        if (current_property->combining_class > max_combining_class) {
+          max_combining_class = current_property->combining_class;
+        }
+      } else {
+        starter = buffer + wpos;
+        starter_property = NULL;
+        max_combining_class = -1;
+      }
+      wpos++;
+    }
+    length = wpos;
+  }
+  return length;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
+  /* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored
+     ASSERT: 'buffer' has one spare byte of free space at the end! */
+  length = utf8proc_normalize_utf32(buffer, length, options);
+  if (length < 0) return length;
+  {
+    utf8proc_ssize_t rpos, wpos = 0;
+    utf8proc_int32_t uc;
+    if (options & UTF8PROC_CHARBOUND) {
+        for (rpos = 0; rpos < length; rpos++) {
+            uc = buffer[rpos];
+            wpos += charbound_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
+        }
+    } else {
+        for (rpos = 0; rpos < length; rpos++) {
+            uc = buffer[rpos];
+            wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
+        }
+    }
+    ((utf8proc_uint8_t *)buffer)[wpos] = 0;
+    return wpos;
+  }
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
+) {
+    return utf8proc_map_custom(str, strlen, dstptr, options, NULL, NULL);
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+) {
+  utf8proc_int32_t *buffer;
+  utf8proc_ssize_t result;
+  *dstptr = NULL;
+  result = utf8proc_decompose_custom(str, strlen, NULL, 0, options, custom_func, custom_data);
+  if (result < 0) return result;
+  buffer = (utf8proc_int32_t *) malloc(((utf8proc_size_t)result) * sizeof(utf8proc_int32_t) + 1);
+  if (!buffer) return UTF8PROC_ERROR_NOMEM;
+  result = utf8proc_decompose_custom(str, strlen, buffer, result, options, custom_func, custom_data);
+  if (result < 0) {
+    free(buffer);
+    return result;
+  }
+  result = utf8proc_reencode(buffer, result, options);
+  if (result < 0) {
+    free(buffer);
+    return result;
+  }
+  {
+    utf8proc_int32_t *newptr;
+    newptr = (utf8proc_int32_t *) realloc(buffer, (size_t)result+1);
+    if (newptr) buffer = newptr;
+  }
+  *dstptr = (utf8proc_uint8_t *)buffer;
+  return result;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_DECOMPOSE);
+  return retval;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_COMPOSE);
+  return retval;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
+  return retval;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
+  return retval;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC_Casefold(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD | UTF8PROC_IGNORE);
+  return retval;
+}

thirdparty/utf8proc/utf8proc.h

@@ -0,0 +1,789 @@
+/*
+ * Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
+ * Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+/**
+ * @mainpage
+ *
+ * utf8proc is a free/open-source (MIT/expat licensed) C library
+ * providing Unicode normalization, case-folding, and other operations
+ * for strings in the UTF-8 encoding, supporting up-to-date Unicode versions.
+ * See the utf8proc home page (http://julialang.org/utf8proc/)
+ * for downloads and other information, or the source code on github
+ * (https://github.com/JuliaLang/utf8proc).
+ *
+ * For the utf8proc API documentation, see: @ref utf8proc.h
+ *
+ * The features of utf8proc include:
+ *
+ * - Transformation of strings (utf8proc_map()) to:
+ *    - decompose (@ref UTF8PROC_DECOMPOSE) or compose (@ref UTF8PROC_COMPOSE) Unicode combining characters (http://en.wikipedia.org/wiki/Combining_character)
+ *    - canonicalize Unicode compatibility characters (@ref UTF8PROC_COMPAT)
+ *    - strip "ignorable" (@ref UTF8PROC_IGNORE) characters, control characters (@ref UTF8PROC_STRIPCC), or combining characters such as accents (@ref UTF8PROC_STRIPMARK)
+ *    - case-folding (@ref UTF8PROC_CASEFOLD)
+ * - Unicode normalization: utf8proc_NFD(), utf8proc_NFC(), utf8proc_NFKD(), utf8proc_NFKC()
+ * - Detecting grapheme boundaries (utf8proc_grapheme_break() and @ref UTF8PROC_CHARBOUND)
+ * - Character-width computation: utf8proc_charwidth()
+ * - Classification of characters by Unicode category: utf8proc_category() and utf8proc_category_string()
+ * - Encode (utf8proc_encode_char()) and decode (utf8proc_iterate()) Unicode codepoints to/from UTF-8.
+ */
+
+/** @file */
+
+#ifndef UTF8PROC_H
+#define UTF8PROC_H
+
+/** @name API version
+ *
+ * The utf8proc API version MAJOR.MINOR.PATCH, following
+ * semantic-versioning rules (http://semver.org) based on API
+ * compatibility.
+ *
+ * This is also returned at runtime by utf8proc_version(); however, the
+ * runtime version may append a string like "-dev" to the version number
+ * for prerelease versions.
+ *
+ * @note The shared-library version number in the Makefile
+ *       (and CMakeLists.txt, and MANIFEST) may be different,
+ *       being based on ABI compatibility rather than API compatibility.
+ */
+/** @{ */
+/** The MAJOR version number (increased when backwards API compatibility is broken). */
+#define UTF8PROC_VERSION_MAJOR 2
+/** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */
+#define UTF8PROC_VERSION_MINOR 10
+/** The PATCH version (increased for fixes that do not change the API). */
+#define UTF8PROC_VERSION_PATCH 0
+/** @} */
+
+#include <stdlib.h>
+
+#if defined(_MSC_VER) && _MSC_VER < 1800
+// MSVC prior to 2013 lacked stdbool.h and stdint.h
+typedef signed char utf8proc_int8_t;
+typedef unsigned char utf8proc_uint8_t;
+typedef short utf8proc_int16_t;
+typedef unsigned short utf8proc_uint16_t;
+typedef int utf8proc_int32_t;
+typedef unsigned int utf8proc_uint32_t;
+#  ifdef _WIN64
+typedef __int64 utf8proc_ssize_t;
+typedef unsigned __int64 utf8proc_size_t;
+#  else
+typedef int utf8proc_ssize_t;
+typedef unsigned int utf8proc_size_t;
+#  endif
+#  ifndef __cplusplus
+// emulate C99 bool
+typedef unsigned char utf8proc_bool;
+#    ifndef __bool_true_false_are_defined
+#      define false 0
+#      define true 1
+#      define __bool_true_false_are_defined 1
+#    endif
+#  else
+typedef bool utf8proc_bool;
+#  endif
+#else
+#  include <stddef.h>
+#  include <stdbool.h>
+#  include <stdint.h>
+typedef int8_t utf8proc_int8_t;
+typedef uint8_t utf8proc_uint8_t;
+typedef int16_t utf8proc_int16_t;
+typedef uint16_t utf8proc_uint16_t;
+typedef int32_t utf8proc_int32_t;
+typedef uint32_t utf8proc_uint32_t;
+typedef size_t utf8proc_size_t;
+typedef ptrdiff_t utf8proc_ssize_t;
+typedef bool utf8proc_bool;
+#endif
+#include <limits.h>
+
+#ifdef UTF8PROC_STATIC
+#  define UTF8PROC_DLLEXPORT
+#else
+#  ifdef _WIN32
+#    ifdef UTF8PROC_EXPORTS
+#      define UTF8PROC_DLLEXPORT __declspec(dllexport)
+#    else
+#      define UTF8PROC_DLLEXPORT __declspec(dllimport)
+#    endif
+#  elif __GNUC__ >= 4
+#    define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default")))
+#  else
+#    define UTF8PROC_DLLEXPORT
+#  endif
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Option flags used by several functions in the library.
+ */
+typedef enum {
+  /** The given UTF-8 input is NULL terminated. */
+  UTF8PROC_NULLTERM  = (1<<0),
+  /** Unicode Versioning Stability has to be respected. */
+  UTF8PROC_STABLE    = (1<<1),
+  /** Compatibility decomposition (i.e. formatting information is lost). */
+  UTF8PROC_COMPAT    = (1<<2),
+  /** Return a result with decomposed characters. */
+  UTF8PROC_COMPOSE   = (1<<3),
+  /** Return a result with decomposed characters. */
+  UTF8PROC_DECOMPOSE = (1<<4),
+  /** Strip "default ignorable characters" such as SOFT-HYPHEN or ZERO-WIDTH-SPACE. */
+  UTF8PROC_IGNORE    = (1<<5),
+  /** Return an error, if the input contains unassigned codepoints. */
+  UTF8PROC_REJECTNA  = (1<<6),
+  /**
+   * Indicating that NLF-sequences (LF, CRLF, CR, NEL) are representing a
+   * line break, and should be converted to the codepoint for line
+   * separation (LS).
+   */
+  UTF8PROC_NLF2LS    = (1<<7),
+  /**
+   * Indicating that NLF-sequences are representing a paragraph break, and
+   * should be converted to the codepoint for paragraph separation
+   * (PS).
+   */
+  UTF8PROC_NLF2PS    = (1<<8),
+  /** Indicating that the meaning of NLF-sequences is unknown. */
+  UTF8PROC_NLF2LF    = (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS),
+  /** Strips and/or convers control characters.
+   *
+   * NLF-sequences are transformed into space, except if one of the
+   * NLF2LS/PS/LF options is given. HorizontalTab (HT) and FormFeed (FF)
+   * are treated as a NLF-sequence in this case.  All other control
+   * characters are simply removed.
+   */
+  UTF8PROC_STRIPCC   = (1<<9),
+  /**
+   * Performs unicode case folding, to be able to do a case-insensitive
+   * string comparison.
+   */
+  UTF8PROC_CASEFOLD  = (1<<10),
+  /**
+   * Inserts 0xFF bytes at the beginning of each sequence which is
+   * representing a single grapheme cluster (see UAX#29).
+   */
+  UTF8PROC_CHARBOUND = (1<<11),
+  /** Lumps certain characters together.
+   *
+   * E.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-". See lump.md for details.
+   *
+   * If NLF2LF is set, this includes a transformation of paragraph and
+   * line separators to ASCII line-feed (LF).
+   */
+  UTF8PROC_LUMP      = (1<<12),
+  /** Strips all character markings.
+   *
+   * This includes non-spacing, spacing and enclosing (i.e. accents).
+   * @note This option works only with @ref UTF8PROC_COMPOSE or
+   *       @ref UTF8PROC_DECOMPOSE
+   */
+  UTF8PROC_STRIPMARK = (1<<13),
+  /**
+   * Strip unassigned codepoints.
+   */
+  UTF8PROC_STRIPNA    = (1<<14),
+} utf8proc_option_t;
+
+/** @name Error codes
+ * Error codes being returned by almost all functions.
+ */
+/** @{ */
+/** Memory could not be allocated. */
+#define UTF8PROC_ERROR_NOMEM -1
+/** The given string is too long to be processed. */
+#define UTF8PROC_ERROR_OVERFLOW -2
+/** The given string is not a legal UTF-8 string. */
+#define UTF8PROC_ERROR_INVALIDUTF8 -3
+/** The @ref UTF8PROC_REJECTNA flag was set and an unassigned codepoint was found. */
+#define UTF8PROC_ERROR_NOTASSIGNED -4
+/** Invalid options have been used. */
+#define UTF8PROC_ERROR_INVALIDOPTS -5
+/** @} */
+
+/* @name Types */
+
+/** Holds the value of a property. */
+typedef utf8proc_int16_t utf8proc_propval_t;
+
+/** Struct containing information about a codepoint. */
+typedef struct utf8proc_property_struct {
+  /**
+   * Unicode category.
+   * @see utf8proc_category_t.
+   */
+  utf8proc_propval_t category;
+  utf8proc_propval_t combining_class;
+  /**
+   * Bidirectional class.
+   * @see utf8proc_bidi_class_t.
+   */
+  utf8proc_propval_t bidi_class;
+  /**
+   * @anchor Decomposition type.
+   * @see utf8proc_decomp_type_t.
+   */
+  utf8proc_propval_t decomp_type;
+  utf8proc_uint16_t decomp_seqindex;
+  utf8proc_uint16_t casefold_seqindex;
+  utf8proc_uint16_t uppercase_seqindex;
+  utf8proc_uint16_t lowercase_seqindex;
+  utf8proc_uint16_t titlecase_seqindex;
+  /**
+   * Character combining table.
+   *
+   * The character combining table is formally indexed by two
+   * characters, the first and second character that might form a
+   * combining pair. The table entry then contains the combined
+   * character. Most character pairs cannot be combined. There are
+   * about 1,000 characters that can be the first character in a
+   * combining pair, and for most, there are only a handful for
+   * possible second characters.
+   *
+   * The combining table is stored as sparse matrix in the CSR
+   * (compressed sparse row) format. That is, it is stored as two
+   * arrays, `utf8proc_uint32_t utf8proc_combinations_second[]` and
+   * `utf8proc_uint32_t utf8proc_combinations_combined[]`. These
+   * contain the second combining characters and the combined
+   * character of every combining pair.
+   *
+   * - `comb_index`: Index into the combining table if this character
+   *   is the first character in a combining pair, else 0x3ff
+   *
+   * - `comb_length`: Number of table entries for this first character
+   *
+   * - `comb_is_second`: As optimization we also record whether this
+   *   character is the second combining character in any pair. If
+   *   not, we can skip the table lookup.
+   *
+   * A table lookup starts from a given character pair. It first
+   * checks whether the first character is stored in the table
+   * (checking whether the index is 0x3ff) and whether the second
+   * index is stored in the table (looking at `comb_is_second`). If
+   * so, the `comb_length` table entries will be checked sequentially
+   * for a match.
+   */
+  utf8proc_uint16_t comb_index:10;
+  utf8proc_uint16_t comb_length:5;
+  utf8proc_uint16_t comb_issecond:1;
+  unsigned bidi_mirrored:1;
+  unsigned comp_exclusion:1;
+  /**
+   * Can this codepoint be ignored?
+   *
+   * Used by utf8proc_decompose_char() when @ref UTF8PROC_IGNORE is
+   * passed as an option.
+   */
+  unsigned ignorable:1;
+  unsigned control_boundary:1;
+  /** The width of the codepoint. */
+  unsigned charwidth:2;
+  /** East Asian width class A */
+  unsigned ambiguous_width:1;
+  unsigned pad:1;
+  /**
+   * Boundclass.
+   * @see utf8proc_boundclass_t.
+   */
+  unsigned boundclass:6;
+  unsigned indic_conjunct_break:2;
+} utf8proc_property_t;
+
+/** Unicode categories. */
+typedef enum {
+  UTF8PROC_CATEGORY_CN  = 0, /**< Other, not assigned */
+  UTF8PROC_CATEGORY_LU  = 1, /**< Letter, uppercase */
+  UTF8PROC_CATEGORY_LL  = 2, /**< Letter, lowercase */
+  UTF8PROC_CATEGORY_LT  = 3, /**< Letter, titlecase */
+  UTF8PROC_CATEGORY_LM  = 4, /**< Letter, modifier */
+  UTF8PROC_CATEGORY_LO  = 5, /**< Letter, other */
+  UTF8PROC_CATEGORY_MN  = 6, /**< Mark, nonspacing */
+  UTF8PROC_CATEGORY_MC  = 7, /**< Mark, spacing combining */
+  UTF8PROC_CATEGORY_ME  = 8, /**< Mark, enclosing */
+  UTF8PROC_CATEGORY_ND  = 9, /**< Number, decimal digit */
+  UTF8PROC_CATEGORY_NL = 10, /**< Number, letter */
+  UTF8PROC_CATEGORY_NO = 11, /**< Number, other */
+  UTF8PROC_CATEGORY_PC = 12, /**< Punctuation, connector */
+  UTF8PROC_CATEGORY_PD = 13, /**< Punctuation, dash */
+  UTF8PROC_CATEGORY_PS = 14, /**< Punctuation, open */
+  UTF8PROC_CATEGORY_PE = 15, /**< Punctuation, close */
+  UTF8PROC_CATEGORY_PI = 16, /**< Punctuation, initial quote */
+  UTF8PROC_CATEGORY_PF = 17, /**< Punctuation, final quote */
+  UTF8PROC_CATEGORY_PO = 18, /**< Punctuation, other */
+  UTF8PROC_CATEGORY_SM = 19, /**< Symbol, math */
+  UTF8PROC_CATEGORY_SC = 20, /**< Symbol, currency */
+  UTF8PROC_CATEGORY_SK = 21, /**< Symbol, modifier */
+  UTF8PROC_CATEGORY_SO = 22, /**< Symbol, other */
+  UTF8PROC_CATEGORY_ZS = 23, /**< Separator, space */
+  UTF8PROC_CATEGORY_ZL = 24, /**< Separator, line */
+  UTF8PROC_CATEGORY_ZP = 25, /**< Separator, paragraph */
+  UTF8PROC_CATEGORY_CC = 26, /**< Other, control */
+  UTF8PROC_CATEGORY_CF = 27, /**< Other, format */
+  UTF8PROC_CATEGORY_CS = 28, /**< Other, surrogate */
+  UTF8PROC_CATEGORY_CO = 29, /**< Other, private use */
+} utf8proc_category_t;
+
+/** Bidirectional character classes. */
+typedef enum {
+  UTF8PROC_BIDI_CLASS_L     = 1, /**< Left-to-Right */
+  UTF8PROC_BIDI_CLASS_LRE   = 2, /**< Left-to-Right Embedding */
+  UTF8PROC_BIDI_CLASS_LRO   = 3, /**< Left-to-Right Override */
+  UTF8PROC_BIDI_CLASS_R     = 4, /**< Right-to-Left */
+  UTF8PROC_BIDI_CLASS_AL    = 5, /**< Right-to-Left Arabic */
+  UTF8PROC_BIDI_CLASS_RLE   = 6, /**< Right-to-Left Embedding */
+  UTF8PROC_BIDI_CLASS_RLO   = 7, /**< Right-to-Left Override */
+  UTF8PROC_BIDI_CLASS_PDF   = 8, /**< Pop Directional Format */
+  UTF8PROC_BIDI_CLASS_EN    = 9, /**< European Number */
+  UTF8PROC_BIDI_CLASS_ES   = 10, /**< European Separator */
+  UTF8PROC_BIDI_CLASS_ET   = 11, /**< European Number Terminator */
+  UTF8PROC_BIDI_CLASS_AN   = 12, /**< Arabic Number */
+  UTF8PROC_BIDI_CLASS_CS   = 13, /**< Common Number Separator */
+  UTF8PROC_BIDI_CLASS_NSM  = 14, /**< Nonspacing Mark */
+  UTF8PROC_BIDI_CLASS_BN   = 15, /**< Boundary Neutral */
+  UTF8PROC_BIDI_CLASS_B    = 16, /**< Paragraph Separator */
+  UTF8PROC_BIDI_CLASS_S    = 17, /**< Segment Separator */
+  UTF8PROC_BIDI_CLASS_WS   = 18, /**< Whitespace */
+  UTF8PROC_BIDI_CLASS_ON   = 19, /**< Other Neutrals */
+  UTF8PROC_BIDI_CLASS_LRI  = 20, /**< Left-to-Right Isolate */
+  UTF8PROC_BIDI_CLASS_RLI  = 21, /**< Right-to-Left Isolate */
+  UTF8PROC_BIDI_CLASS_FSI  = 22, /**< First Strong Isolate */
+  UTF8PROC_BIDI_CLASS_PDI  = 23, /**< Pop Directional Isolate */
+} utf8proc_bidi_class_t;
+
+/** Decomposition type. */
+typedef enum {
+  UTF8PROC_DECOMP_TYPE_FONT      = 1, /**< Font */
+  UTF8PROC_DECOMP_TYPE_NOBREAK   = 2, /**< Nobreak */
+  UTF8PROC_DECOMP_TYPE_INITIAL   = 3, /**< Initial */
+  UTF8PROC_DECOMP_TYPE_MEDIAL    = 4, /**< Medial */
+  UTF8PROC_DECOMP_TYPE_FINAL     = 5, /**< Final */
+  UTF8PROC_DECOMP_TYPE_ISOLATED  = 6, /**< Isolated */
+  UTF8PROC_DECOMP_TYPE_CIRCLE    = 7, /**< Circle */
+  UTF8PROC_DECOMP_TYPE_SUPER     = 8, /**< Super */
+  UTF8PROC_DECOMP_TYPE_SUB       = 9, /**< Sub */
+  UTF8PROC_DECOMP_TYPE_VERTICAL = 10, /**< Vertical */
+  UTF8PROC_DECOMP_TYPE_WIDE     = 11, /**< Wide */
+  UTF8PROC_DECOMP_TYPE_NARROW   = 12, /**< Narrow */
+  UTF8PROC_DECOMP_TYPE_SMALL    = 13, /**< Small */
+  UTF8PROC_DECOMP_TYPE_SQUARE   = 14, /**< Square */
+  UTF8PROC_DECOMP_TYPE_FRACTION = 15, /**< Fraction */
+  UTF8PROC_DECOMP_TYPE_COMPAT   = 16, /**< Compat */
+} utf8proc_decomp_type_t;
+
+/** Boundclass property. (TR29) */
+typedef enum {
+  UTF8PROC_BOUNDCLASS_START              =  0, /**< Start */
+  UTF8PROC_BOUNDCLASS_OTHER              =  1, /**< Other */
+  UTF8PROC_BOUNDCLASS_CR                 =  2, /**< Cr */
+  UTF8PROC_BOUNDCLASS_LF                 =  3, /**< Lf */
+  UTF8PROC_BOUNDCLASS_CONTROL            =  4, /**< Control */
+  UTF8PROC_BOUNDCLASS_EXTEND             =  5, /**< Extend */
+  UTF8PROC_BOUNDCLASS_L                  =  6, /**< L */
+  UTF8PROC_BOUNDCLASS_V                  =  7, /**< V */
+  UTF8PROC_BOUNDCLASS_T                  =  8, /**< T */
+  UTF8PROC_BOUNDCLASS_LV                 =  9, /**< Lv */
+  UTF8PROC_BOUNDCLASS_LVT                = 10, /**< Lvt */
+  UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR = 11, /**< Regional indicator */
+  UTF8PROC_BOUNDCLASS_SPACINGMARK        = 12, /**< Spacingmark */
+  UTF8PROC_BOUNDCLASS_PREPEND            = 13, /**< Prepend */
+  UTF8PROC_BOUNDCLASS_ZWJ                = 14, /**< Zero Width Joiner */
+
+  /* the following are no longer used in Unicode 11, but we keep
+     the constants here for backward compatibility */
+  UTF8PROC_BOUNDCLASS_E_BASE             = 15, /**< Emoji Base */
+  UTF8PROC_BOUNDCLASS_E_MODIFIER         = 16, /**< Emoji Modifier */
+  UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ     = 17, /**< Glue_After_ZWJ */
+  UTF8PROC_BOUNDCLASS_E_BASE_GAZ         = 18, /**< E_BASE + GLUE_AFTER_ZJW */
+
+  /* the Extended_Pictographic property is used in the Unicode 11
+     grapheme-boundary rules, so we store it in the boundclass field */
+  UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC = 19,
+  UTF8PROC_BOUNDCLASS_E_ZWG = 20, /* UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC + ZWJ */
+} utf8proc_boundclass_t;
+
+/** Indic_Conjunct_Break property. (TR44) */
+typedef enum {
+  UTF8PROC_INDIC_CONJUNCT_BREAK_NONE = 0,
+  UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER = 1,
+  UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT = 2,
+  UTF8PROC_INDIC_CONJUNCT_BREAK_EXTEND = 3,
+} utf8proc_indic_conjunct_break_t;
+
+/**
+ * Function pointer type passed to utf8proc_map_custom() and
+ * utf8proc_decompose_custom(), which is used to specify a user-defined
+ * mapping of codepoints to be applied in conjunction with other mappings.
+ */
+typedef utf8proc_int32_t (*utf8proc_custom_func)(utf8proc_int32_t codepoint, void *data);
+
+/**
+ * Array containing the byte lengths of a UTF-8 encoded codepoint based
+ * on the first byte.
+ */
+UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256];
+
+/**
+ * Returns the utf8proc API version as a string MAJOR.MINOR.PATCH
+ * (http://semver.org format), possibly with a "-dev" suffix for
+ * development versions.
+ */
+UTF8PROC_DLLEXPORT const char *utf8proc_version(void);
+
+/**
+ * Returns the utf8proc supported Unicode version as a string MAJOR.MINOR.PATCH.
+ */
+UTF8PROC_DLLEXPORT const char *utf8proc_unicode_version(void);
+
+/**
+ * Returns an informative error string for the given utf8proc error code
+ * (e.g. the error codes returned by utf8proc_map()).
+ */
+UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode);
+
+/**
+ * Reads a single codepoint from the UTF-8 sequence being pointed to by `str`.
+ * The maximum number of bytes read is `strlen`, unless `strlen` is
+ * negative (in which case up to 4 bytes are read).
+ *
+ * If a valid codepoint could be read, it is stored in the variable
+ * pointed to by `codepoint_ref`, otherwise that variable will be set to -1.
+ * In case of success, the number of bytes read is returned; otherwise, a
+ * negative error code is returned.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);
+
+/**
+ * Check if a codepoint is valid (regardless of whether it has been
+ * assigned a value by the current Unicode standard).
+ *
+ * @return 1 if the given `codepoint` is valid and otherwise return 0.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t codepoint);
+
+/**
+ * Encodes the codepoint as an UTF-8 string in the byte array pointed
+ * to by `dst`. This array must be at least 4 bytes long.
+ *
+ * In case of success the number of bytes written is returned, and
+ * otherwise 0 is returned.
+ *
+ * This function does not check whether `codepoint` is valid Unicode.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t codepoint, utf8proc_uint8_t *dst);
+
+/**
+ * Look up the properties for a given codepoint.
+ *
+ * @param codepoint The Unicode codepoint.
+ *
+ * @returns
+ * A pointer to a (constant) struct containing information about
+ * the codepoint.
+ * @par
+ * If the codepoint is unassigned or invalid, a pointer to a special struct is
+ * returned in which `category` is 0 (@ref UTF8PROC_CATEGORY_CN).
+ */
+UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t codepoint);
+
+/** Decompose a codepoint into an array of codepoints.
+ *
+ * @param codepoint the codepoint.
+ * @param dst the destination buffer.
+ * @param bufsize the size of the destination buffer.
+ * @param options one or more of the following flags:
+ * - @ref UTF8PROC_REJECTNA  - return an error `codepoint` is unassigned
+ * - @ref UTF8PROC_IGNORE    - strip "default ignorable" codepoints
+ * - @ref UTF8PROC_CASEFOLD  - apply Unicode casefolding
+ * - @ref UTF8PROC_COMPAT    - replace certain codepoints with their
+ *                             compatibility decomposition
+ * - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
+ * - @ref UTF8PROC_LUMP      - lump certain different codepoints together
+ * - @ref UTF8PROC_STRIPMARK - remove all character marks
+ * - @ref UTF8PROC_STRIPNA   - remove unassigned codepoints
+ * @param last_boundclass
+ * Pointer to an integer variable containing
+ * the previous codepoint's (boundclass + indic_conjunct_break << 1) if the @ref UTF8PROC_CHARBOUND
+ * option is used.  If the string is being processed in order, this can be initialized to 0 for
+ * the beginning of the string, and is thereafter updated automatically.  Otherwise, this parameter is ignored.
+ *
+ * @return
+ * In case of success, the number of codepoints written is returned; in case
+ * of an error, a negative error code is returned (utf8proc_errmsg()).
+ * @par
+ * If the number of written codepoints would be bigger than `bufsize`, the
+ * required buffer size is returned, while the buffer will be overwritten with
+ * undefined data.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(
+  utf8proc_int32_t codepoint, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize,
+  utf8proc_option_t options, int *last_boundclass
+);
+
+/**
+ * The same as utf8proc_decompose_char(), but acts on a whole UTF-8
+ * string and orders the decomposed sequences correctly.
+ *
+ * If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing
+ * will be stopped, when a NULL byte is encountered, otherwise `strlen`
+ * bytes are processed.  The result (in the form of 32-bit unicode
+ * codepoints) is written into the buffer being pointed to by
+ * `buffer` (which must contain at least `bufsize` entries).  In case of
+ * success, the number of codepoints written is returned; in case of an
+ * error, a negative error code is returned (utf8proc_errmsg()).
+ * See utf8proc_decompose_custom() to supply additional transformations.
+ *
+ * If the number of written codepoints would be bigger than `bufsize`, the
+ * required buffer size is returned, while the buffer will be overwritten with
+ * undefined data.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
+);
+
+/**
+ * The same as utf8proc_decompose(), but also takes a `custom_func` mapping function
+ * that is called on each codepoint in `str` before any other transformations
+ * (along with a `custom_data` pointer that is passed through to `custom_func`).
+ * The `custom_func` argument is ignored if it is `NULL`.  See also utf8proc_map_custom().
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+);
+
+/**
+ * Normalizes the sequence of `length` codepoints pointed to by `buffer`
+ * in-place (i.e., the result is also stored in `buffer`).
+ *
+ * @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
+ * @param length the length (in codepoints) of the buffer.
+ * @param options a bitwise or (`|`) of one or more of the following flags:
+ * - @ref UTF8PROC_NLF2LS  - convert LF, CRLF, CR and NEL into LS
+ * - @ref UTF8PROC_NLF2PS  - convert LF, CRLF, CR and NEL into PS
+ * - @ref UTF8PROC_NLF2LF  - convert LF, CRLF, CR and NEL into LF
+ * - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
+ * - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
+ *                           codepoints
+ * - @ref UTF8PROC_STABLE  - prohibit combining characters that would violate
+ *                           the unicode versioning stability
+ *
+ * @return
+ * In case of success, the length (in codepoints) of the normalized UTF-32 string is
+ * returned; otherwise, a negative error code is returned (utf8proc_errmsg()).
+ *
+ * @warning The entries of the array pointed to by `str` have to be in the
+ *          range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
+
+/**
+ * Reencodes the sequence of `length` codepoints pointed to by `buffer`
+ * UTF-8 data in-place (i.e., the result is also stored in `buffer`).
+ * Can optionally normalize the UTF-32 sequence prior to UTF-8 conversion.
+ *
+ * @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
+ * @param length the length (in codepoints) of the buffer.
+ * @param options a bitwise or (`|`) of one or more of the following flags:
+ * - @ref UTF8PROC_NLF2LS  - convert LF, CRLF, CR and NEL into LS
+ * - @ref UTF8PROC_NLF2PS  - convert LF, CRLF, CR and NEL into PS
+ * - @ref UTF8PROC_NLF2LF  - convert LF, CRLF, CR and NEL into LF
+ * - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
+ * - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
+ *                           codepoints
+ * - @ref UTF8PROC_STABLE  - prohibit combining characters that would violate
+ *                           the unicode versioning stability
+ * - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
+ *
+ * @return
+ * In case of success, the length (in bytes) of the resulting nul-terminated
+ * UTF-8 string is returned; otherwise, a negative error code is returned
+ * (utf8proc_errmsg()).
+ *
+ * @warning The amount of free space pointed to by `buffer` must
+ *          exceed the amount of the input data by one byte, and the
+ *          entries of the array pointed to by `str` have to be in the
+ *          range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
+
+/**
+ * Given a pair of consecutive codepoints, return whether a grapheme break is
+ * permitted between them (as defined by the extended grapheme clusters in UAX#29).
+ *
+ * @param codepoint1 The first codepoint.
+ * @param codepoint2 The second codepoint, occurring consecutively after `codepoint1`.
+ * @param state Beginning with Version 29 (Unicode 9.0.0), this algorithm requires
+ *              state to break graphemes. This state can be passed in as a pointer
+ *              in the `state` argument and should initially be set to 0. If the
+ *              state is not passed in (i.e. a null pointer is passed), UAX#29 rules
+ *              GB10/12/13 which require this state will not be applied, essentially
+ *              matching the rules in Unicode 8.0.0.
+ *
+ * @warning If the state parameter is used, `utf8proc_grapheme_break_stateful` must
+ *          be called IN ORDER on ALL potential breaks in a string.  However, it
+ *          is safe to reset the state to zero after a grapheme break.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
+    utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state);
+
+/**
+ * Same as utf8proc_grapheme_break_stateful(), except without support for the
+ * Unicode 9 additions to the algorithm. Supported for legacy reasons.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
+    utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2);
+
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * lower-case character, if any; otherwise (if there is no lower-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * upper-case character, if any; otherwise (if there is no upper-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * title-case character, if any; otherwise (if there is no title-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint `c`, return `1` if the codepoint corresponds to a lower-case character
+ * and `0` otherwise.
+ */
+UTF8PROC_DLLEXPORT int utf8proc_islower(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint `c`, return `1` if the codepoint corresponds to an upper-case character
+ * and `0` otherwise.
+ */
+UTF8PROC_DLLEXPORT int utf8proc_isupper(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint, return a character width analogous to `wcwidth(codepoint)`,
+ * except that a width of 0 is returned for non-printable codepoints
+ * instead of -1 as in `wcwidth`.
+ *
+ * @note
+ * If you want to check for particular types of non-printable characters,
+ * (analogous to `isprint` or `iscntrl`), use utf8proc_category(). */
+UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t codepoint);
+
+/**
+ * Given a codepoint, return whether it has East Asian width class A (Ambiguous)
+ *
+ * Codepoints with this property are considered to have charwidth 1 (if they are printable)
+ * but some East Asian fonts render them as double width.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_charwidth_ambiguous(utf8proc_int32_t codepoint);
+
+/**
+ * Return the Unicode category for the codepoint (one of the
+ * @ref utf8proc_category_t constants.)
+ */
+UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t codepoint);
+
+/**
+ * Return the two-letter (nul-terminated) Unicode category string for
+ * the codepoint (e.g. `"Lu"` or `"Co"`).
+ */
+UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t codepoint);
+
+/**
+ * Maps the given UTF-8 string pointed to by `str` to a new UTF-8
+ * string, allocated dynamically by `malloc` and returned via `dstptr`.
+ *
+ * If the @ref UTF8PROC_NULLTERM flag in the `options` field is set,
+ * the length is determined by a NULL terminator, otherwise the
+ * parameter `strlen` is evaluated to determine the string length, but
+ * in any case the result will be NULL terminated (though it might
+ * contain NULL characters with the string if `str` contained NULL
+ * characters). Other flags in the `options` field are passed to the
+ * functions defined above, and regarded as described.  See also
+ * utf8proc_map_custom() to supply a custom codepoint transformation.
+ *
+ * In case of success the length of the new string is returned,
+ * otherwise a negative error code is returned.
+ *
+ * @note The memory of the new UTF-8 string will have been allocated
+ * with `malloc`, and should therefore be deallocated with `free`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
+);
+
+/**
+ * Like utf8proc_map(), but also takes a `custom_func` mapping function
+ * that is called on each codepoint in `str` before any other transformations
+ * (along with a `custom_data` pointer that is passed through to `custom_func`).
+ * The `custom_func` argument is ignored if it is `NULL`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+);
+
+/** @name Unicode normalization
+ *
+ * Returns a pointer to newly allocated memory of a NFD, NFC, NFKD, NFKC or
+ * NFKC_Casefold normalized version of the null-terminated string `str`.  These
+ * are shortcuts to calling utf8proc_map() with @ref UTF8PROC_NULLTERM
+ * combined with @ref UTF8PROC_STABLE and flags indicating the normalization.
+ */
+/** @{ */
+/** NFD normalization (@ref UTF8PROC_DECOMPOSE). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str);
+/** NFC normalization (@ref UTF8PROC_COMPOSE). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str);
+/** NFKD normalization (@ref UTF8PROC_DECOMPOSE and @ref UTF8PROC_COMPAT). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str);
+/** NFKC normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str);
+/**
+ * NFKC_Casefold normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT
+ * and @ref UTF8PROC_CASEFOLD and @ref UTF8PROC_IGNORE).
+ **/
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC_Casefold(const utf8proc_uint8_t *str);
+/** @} */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif