DarthAffe/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2025-12-12 13:28:37 +00:00
Code Issues Packages Projects Releases Wiki Activity
stable-diffusion.cpp/tokenize_util.cpp
leejet 66e27de9bd add flux2 support
2025-11-29 02:20:12 +08:00

988 lines
27 KiB
C++
Raw Permalink Blame History

#include <algorithm>
#include <iostream>
#include <string>
#include <vector>
#include "tokenize_util.h"
bool is_number(char32_t ch) {
return (ch >= U'0' && ch <= U'9');
}
bool is_letter(char32_t ch) {
static const struct { char32_t start, end; } ranges[] = {
{0x41, 0x5A},
{0x61, 0x7A},
{0xAA, 0xAA},
{0xB5, 0xB5},
{0xBA, 0xBA},
{0xC0, 0xD6},
{0xD8, 0xF6},
{0xF8, 0x2C1},
{0x2C6, 0x2D1},
{0x2E0, 0x2E4},
{0x2EC, 0x2EC},
{0x2EE, 0x2EE},
{0x370, 0x374},
{0x376, 0x377},
{0x37A, 0x37D},
{0x37F, 0x37F},
{0x386, 0x386},
{0x388, 0x38A},
{0x38C, 0x38C},
{0x38E, 0x3A1},
{0x3A3, 0x3F5},
{0x3F7, 0x481},
{0x48A, 0x52F},
{0x531, 0x556},
{0x559, 0x559},
{0x560, 0x588},
{0x5D0, 0x5EA},
{0x5EF, 0x5F2},
{0x620, 0x64A},
{0x66E, 0x66F},
{0x671, 0x6D3},
{0x6D5, 0x6D5},
{0x6E5, 0x6E6},
{0x6EE, 0x6EF},
{0x6FA, 0x6FC},
{0x6FF, 0x6FF},
{0x710, 0x710},
{0x712, 0x72F},
{0x74D, 0x7A5},
{0x7B1, 0x7B1},
{0x7CA, 0x7EA},
{0x7F4, 0x7F5},
{0x7FA, 0x7FA},
{0x800, 0x815},
{0x81A, 0x81A},
{0x824, 0x824},
{0x828, 0x828},
{0x840, 0x858},
{0x860, 0x86A},
{0x870, 0x887},
{0x889, 0x88F},
{0x8A0, 0x8C9},
{0x904, 0x939},
{0x93D, 0x93D},
{0x950, 0x950},
{0x958, 0x961},
{0x971, 0x980},
{0x985, 0x98C},
{0x98F, 0x990},
{0x993, 0x9A8},
{0x9AA, 0x9B0},
{0x9B2, 0x9B2},
{0x9B6, 0x9B9},
{0x9BD, 0x9BD},
{0x9CE, 0x9CE},
{0x9DC, 0x9DD},
{0x9DF, 0x9E1},
{0x9F0, 0x9F1},
{0x9FC, 0x9FC},
{0xA05, 0xA0A},
{0xA0F, 0xA10},
{0xA13, 0xA28},
{0xA2A, 0xA30},
{0xA32, 0xA33},
{0xA35, 0xA36},
{0xA38, 0xA39},
{0xA59, 0xA5C},
{0xA5E, 0xA5E},
{0xA72, 0xA74},
{0xA85, 0xA8D},
{0xA8F, 0xA91},
{0xA93, 0xAA8},
{0xAAA, 0xAB0},
{0xAB2, 0xAB3},
{0xAB5, 0xAB9},
{0xABD, 0xABD},
{0xAD0, 0xAD0},
{0xAE0, 0xAE1},
{0xAF9, 0xAF9},
{0xB05, 0xB0C},
{0xB0F, 0xB10},
{0xB13, 0xB28},
{0xB2A, 0xB30},
{0xB32, 0xB33},
{0xB35, 0xB39},
{0xB3D, 0xB3D},
{0xB5C, 0xB5D},
{0xB5F, 0xB61},
{0xB71, 0xB71},
{0xB83, 0xB83},
{0xB85, 0xB8A},
{0xB8E, 0xB90},
{0xB92, 0xB95},
{0xB99, 0xB9A},
{0xB9C, 0xB9C},
{0xB9E, 0xB9F},
{0xBA3, 0xBA4},
{0xBA8, 0xBAA},
{0xBAE, 0xBB9},
{0xBD0, 0xBD0},
{0xC05, 0xC0C},
{0xC0E, 0xC10},
{0xC12, 0xC28},
{0xC2A, 0xC39},
{0xC3D, 0xC3D},
{0xC58, 0xC5A},
{0xC5C, 0xC5D},
{0xC60, 0xC61},
{0xC80, 0xC80},
{0xC85, 0xC8C},
{0xC8E, 0xC90},
{0xC92, 0xCA8},
{0xCAA, 0xCB3},
{0xCB5, 0xCB9},
{0xCBD, 0xCBD},
{0xCDC, 0xCDE},
{0xCE0, 0xCE1},
{0xCF1, 0xCF2},
{0xD04, 0xD0C},
{0xD0E, 0xD10},
{0xD12, 0xD3A},
{0xD3D, 0xD3D},
{0xD4E, 0xD4E},
{0xD54, 0xD56},
{0xD5F, 0xD61},
{0xD7A, 0xD7F},
{0xD85, 0xD96},
{0xD9A, 0xDB1},
{0xDB3, 0xDBB},
{0xDBD, 0xDBD},
{0xDC0, 0xDC6},
{0xE01, 0xE30},
{0xE32, 0xE33},
{0xE40, 0xE46},
{0xE81, 0xE82},
{0xE84, 0xE84},
{0xE86, 0xE8A},
{0xE8C, 0xEA3},
{0xEA5, 0xEA5},
{0xEA7, 0xEB0},
{0xEB2, 0xEB3},
{0xEBD, 0xEBD},
{0xEC0, 0xEC4},
{0xEC6, 0xEC6},
{0xEDC, 0xEDF},
{0xF00, 0xF00},
{0xF40, 0xF47},
{0xF49, 0xF6C},
{0xF88, 0xF8C},
{0x1000, 0x102A},
{0x103F, 0x103F},
{0x1050, 0x1055},
{0x105A, 0x105D},
{0x1061, 0x1061},
{0x1065, 0x1066},
{0x106E, 0x1070},
{0x1075, 0x1081},
{0x108E, 0x108E},
{0x10A0, 0x10C5},
{0x10C7, 0x10C7},
{0x10CD, 0x10CD},
{0x10D0, 0x10FA},
{0x10FC, 0x1248},
{0x124A, 0x124D},
{0x1250, 0x1256},
{0x1258, 0x1258},
{0x125A, 0x125D},
{0x1260, 0x1288},
{0x128A, 0x128D},
{0x1290, 0x12B0},
{0x12B2, 0x12B5},
{0x12B8, 0x12BE},
{0x12C0, 0x12C0},
{0x12C2, 0x12C5},
{0x12C8, 0x12D6},
{0x12D8, 0x1310},
{0x1312, 0x1315},
{0x1318, 0x135A},
{0x1380, 0x138F},
{0x13A0, 0x13F5},
{0x13F8, 0x13FD},
{0x1401, 0x166C},
{0x166F, 0x167F},
{0x1681, 0x169A},
{0x16A0, 0x16EA},
{0x16F1, 0x16F8},
{0x1700, 0x1711},
{0x171F, 0x1731},
{0x1740, 0x1751},
{0x1760, 0x176C},
{0x176E, 0x1770},
{0x1780, 0x17B3},
{0x17D7, 0x17D7},
{0x17DC, 0x17DC},
{0x1820, 0x1878},
{0x1880, 0x1884},
{0x1887, 0x18A8},
{0x18AA, 0x18AA},
{0x18B0, 0x18F5},
{0x1900, 0x191E},
{0x1950, 0x196D},
{0x1970, 0x1974},
{0x1980, 0x19AB},
{0x19B0, 0x19C9},
{0x1A00, 0x1A16},
{0x1A20, 0x1A54},
{0x1AA7, 0x1AA7},
{0x1B05, 0x1B33},
{0x1B45, 0x1B4C},
{0x1B83, 0x1BA0},
{0x1BAE, 0x1BAF},
{0x1BBA, 0x1BE5},
{0x1C00, 0x1C23},
{0x1C4D, 0x1C4F},
{0x1C5A, 0x1C7D},
{0x1C80, 0x1C8A},
{0x1C90, 0x1CBA},
{0x1CBD, 0x1CBF},
{0x1CE9, 0x1CEC},
{0x1CEE, 0x1CF3},
{0x1CF5, 0x1CF6},
{0x1CFA, 0x1CFA},
{0x1D00, 0x1DBF},
{0x1E00, 0x1F15},
{0x1F18, 0x1F1D},
{0x1F20, 0x1F45},
{0x1F48, 0x1F4D},
{0x1F50, 0x1F57},
{0x1F59, 0x1F59},
{0x1F5B, 0x1F5B},
{0x1F5D, 0x1F5D},
{0x1F5F, 0x1F7D},
{0x1F80, 0x1FB4},
{0x1FB6, 0x1FBC},
{0x1FBE, 0x1FBE},
{0x1FC2, 0x1FC4},
{0x1FC6, 0x1FCC},
{0x1FD0, 0x1FD3},
{0x1FD6, 0x1FDB},
{0x1FE0, 0x1FEC},
{0x1FF2, 0x1FF4},
{0x1FF6, 0x1FFC},
{0x2071, 0x2071},
{0x207F, 0x207F},
{0x2090, 0x209C},
{0x2102, 0x2102},
{0x2107, 0x2107},
{0x210A, 0x2113},
{0x2115, 0x2115},
{0x2119, 0x211D},
{0x2124, 0x2124},
{0x2126, 0x2126},
{0x2128, 0x2128},
{0x212A, 0x212D},
{0x212F, 0x2139},
{0x213C, 0x213F},
{0x2145, 0x2149},
{0x214E, 0x214E},
{0x2183, 0x2184},
{0x2C00, 0x2CE4},
{0x2CEB, 0x2CEE},
{0x2CF2, 0x2CF3},
{0x2D00, 0x2D25},
{0x2D27, 0x2D27},
{0x2D2D, 0x2D2D},
{0x2D30, 0x2D67},
{0x2D6F, 0x2D6F},
{0x2D80, 0x2D96},
{0x2DA0, 0x2DA6},
{0x2DA8, 0x2DAE},
{0x2DB0, 0x2DB6},
{0x2DB8, 0x2DBE},
{0x2DC0, 0x2DC6},
{0x2DC8, 0x2DCE},
{0x2DD0, 0x2DD6},
{0x2DD8, 0x2DDE},
{0x2E2F, 0x2E2F},
{0x3005, 0x3006},
{0x3031, 0x3035},
{0x303B, 0x303C},
{0x3041, 0x3096},
{0x309D, 0x309F},
{0x30A1, 0x30FA},
{0x30FC, 0x30FF},
{0x3105, 0x312F},
{0x3131, 0x318E},
{0x31A0, 0x31BF},
{0x31F0, 0x31FF},
{0x3400, 0x4DBF},
{0x4E00, 0xA48C},
{0xA4D0, 0xA4FD},
{0xA500, 0xA60C},
{0xA610, 0xA61F},
{0xA62A, 0xA62B},
{0xA640, 0xA66E},
{0xA67F, 0xA69D},
{0xA6A0, 0xA6E5},
{0xA717, 0xA71F},
{0xA722, 0xA788},
{0xA78B, 0xA7DC},
{0xA7F1, 0xA801},
{0xA803, 0xA805},
{0xA807, 0xA80A},
{0xA80C, 0xA822},
{0xA840, 0xA873},
{0xA882, 0xA8B3},
{0xA8F2, 0xA8F7},
{0xA8FB, 0xA8FB},
{0xA8FD, 0xA8FE},
{0xA90A, 0xA925},
{0xA930, 0xA946},
{0xA960, 0xA97C},
{0xA984, 0xA9B2},
{0xA9CF, 0xA9CF},
{0xA9E0, 0xA9E4},
{0xA9E6, 0xA9EF},
{0xA9FA, 0xA9FE},
{0xAA00, 0xAA28},
{0xAA40, 0xAA42},
{0xAA44, 0xAA4B},
{0xAA60, 0xAA76},
{0xAA7A, 0xAA7A},
{0xAA7E, 0xAAAF},
{0xAAB1, 0xAAB1},
{0xAAB5, 0xAAB6},
{0xAAB9, 0xAABD},
{0xAAC0, 0xAAC0},
{0xAAC2, 0xAAC2},
{0xAADB, 0xAADD},
{0xAAE0, 0xAAEA},
{0xAAF2, 0xAAF4},
{0xAB01, 0xAB06},
{0xAB09, 0xAB0E},
{0xAB11, 0xAB16},
{0xAB20, 0xAB26},
{0xAB28, 0xAB2E},
{0xAB30, 0xAB5A},
{0xAB5C, 0xAB69},
{0xAB70, 0xABE2},
{0xAC00, 0xD7A3},
{0xD7B0, 0xD7C6},
{0xD7CB, 0xD7FB},
{0xF900, 0xFA6D},
{0xFA70, 0xFAD9},
{0xFB00, 0xFB06},
{0xFB13, 0xFB17},
{0xFB1D, 0xFB1D},
{0xFB1F, 0xFB28},
{0xFB2A, 0xFB36},
{0xFB38, 0xFB3C},
{0xFB3E, 0xFB3E},
{0xFB40, 0xFB41},
{0xFB43, 0xFB44},
{0xFB46, 0xFBB1},
{0xFBD3, 0xFD3D},
{0xFD50, 0xFD8F},
{0xFD92, 0xFDC7},
{0xFDF0, 0xFDFB},
{0xFE70, 0xFE74},
{0xFE76, 0xFEFC},
{0xFF21, 0xFF3A},
{0xFF41, 0xFF5A},
{0xFF66, 0xFFBE},
{0xFFC2, 0xFFC7},
{0xFFCA, 0xFFCF},
{0xFFD2, 0xFFD7},
{0xFFDA, 0xFFDC},
{0x10000, 0x1000B},
{0x1000D, 0x10026},
{0x10028, 0x1003A},
{0x1003C, 0x1003D},
{0x1003F, 0x1004D},
{0x10050, 0x1005D},
{0x10080, 0x100FA},
{0x10280, 0x1029C},
{0x102A0, 0x102D0},
{0x10300, 0x1031F},
{0x1032D, 0x10340},
{0x10342, 0x10349},
{0x10350, 0x10375},
{0x10380, 0x1039D},
{0x103A0, 0x103C3},
{0x103C8, 0x103CF},
{0x10400, 0x1049D},
{0x104B0, 0x104D3},
{0x104D8, 0x104FB},
{0x10500, 0x10527},
{0x10530, 0x10563},
{0x10570, 0x1057A},
{0x1057C, 0x1058A},
{0x1058C, 0x10592},
{0x10594, 0x10595},
{0x10597, 0x105A1},
{0x105A3, 0x105B1},
{0x105B3, 0x105B9},
{0x105BB, 0x105BC},
{0x105C0, 0x105F3},
{0x10600, 0x10736},
{0x10740, 0x10755},
{0x10760, 0x10767},
{0x10780, 0x10785},
{0x10787, 0x107B0},
{0x107B2, 0x107BA},
{0x10800, 0x10805},
{0x10808, 0x10808},
{0x1080A, 0x10835},
{0x10837, 0x10838},
{0x1083C, 0x1083C},
{0x1083F, 0x10855},
{0x10860, 0x10876},
{0x10880, 0x1089E},
{0x108E0, 0x108F2},
{0x108F4, 0x108F5},
{0x10900, 0x10915},
{0x10920, 0x10939},
{0x10940, 0x10959},
{0x10980, 0x109B7},
{0x109BE, 0x109BF},
{0x10A00, 0x10A00},
{0x10A10, 0x10A13},
{0x10A15, 0x10A17},
{0x10A19, 0x10A35},
{0x10A60, 0x10A7C},
{0x10A80, 0x10A9C},
{0x10AC0, 0x10AC7},
{0x10AC9, 0x10AE4},
{0x10B00, 0x10B35},
{0x10B40, 0x10B55},
{0x10B60, 0x10B72},
{0x10B80, 0x10B91},
{0x10C00, 0x10C48},
{0x10C80, 0x10CB2},
{0x10CC0, 0x10CF2},
{0x10D00, 0x10D23},
{0x10D4A, 0x10D65},
{0x10D6F, 0x10D85},
{0x10E80, 0x10EA9},
{0x10EB0, 0x10EB1},
{0x10EC2, 0x10EC7},
{0x10F00, 0x10F1C},
{0x10F27, 0x10F27},
{0x10F30, 0x10F45},
{0x10F70, 0x10F81},
{0x10FB0, 0x10FC4},
{0x10FE0, 0x10FF6},
{0x11003, 0x11037},
{0x11071, 0x11072},
{0x11075, 0x11075},
{0x11083, 0x110AF},
{0x110D0, 0x110E8},
{0x11103, 0x11126},
{0x11144, 0x11144},
{0x11147, 0x11147},
{0x11150, 0x11172},
{0x11176, 0x11176},
{0x11183, 0x111B2},
{0x111C1, 0x111C4},
{0x111DA, 0x111DA},
{0x111DC, 0x111DC},
{0x11200, 0x11211},
{0x11213, 0x1122B},
{0x1123F, 0x11240},
{0x11280, 0x11286},
{0x11288, 0x11288},
{0x1128A, 0x1128D},
{0x1128F, 0x1129D},
{0x1129F, 0x112A8},
{0x112B0, 0x112DE},
{0x11305, 0x1130C},
{0x1130F, 0x11310},
{0x11313, 0x11328},
{0x1132A, 0x11330},
{0x11332, 0x11333},
{0x11335, 0x11339},
{0x1133D, 0x1133D},
{0x11350, 0x11350},
{0x1135D, 0x11361},
{0x11380, 0x11389},
{0x1138B, 0x1138B},
{0x1138E, 0x1138E},
{0x11390, 0x113B5},
{0x113B7, 0x113B7},
{0x113D1, 0x113D1},
{0x113D3, 0x113D3},
{0x11400, 0x11434},
{0x11447, 0x1144A},
{0x1145F, 0x11461},
{0x11480, 0x114AF},
{0x114C4, 0x114C5},
{0x114C7, 0x114C7},
{0x11580, 0x115AE},
{0x115D8, 0x115DB},
{0x11600, 0x1162F},
{0x11644, 0x11644},
{0x11680, 0x116AA},
{0x116B8, 0x116B8},
{0x11700, 0x1171A},
{0x11740, 0x11746},
{0x11800, 0x1182B},
{0x118A0, 0x118DF},
{0x118FF, 0x11906},
{0x11909, 0x11909},
{0x1190C, 0x11913},
{0x11915, 0x11916},
{0x11918, 0x1192F},
{0x1193F, 0x1193F},
{0x11941, 0x11941},
{0x119A0, 0x119A7},
{0x119AA, 0x119D0},
{0x119E1, 0x119E1},
{0x119E3, 0x119E3},
{0x11A00, 0x11A00},
{0x11A0B, 0x11A32},
{0x11A3A, 0x11A3A},
{0x11A50, 0x11A50},
{0x11A5C, 0x11A89},
{0x11A9D, 0x11A9D},
{0x11AB0, 0x11AF8},
{0x11BC0, 0x11BE0},
{0x11C00, 0x11C08},
{0x11C0A, 0x11C2E},
{0x11C40, 0x11C40},
{0x11C72, 0x11C8F},
{0x11D00, 0x11D06},
{0x11D08, 0x11D09},
{0x11D0B, 0x11D30},
{0x11D46, 0x11D46},
{0x11D60, 0x11D65},
{0x11D67, 0x11D68},
{0x11D6A, 0x11D89},
{0x11D98, 0x11D98},
{0x11DB0, 0x11DDB},
{0x11EE0, 0x11EF2},
{0x11F02, 0x11F02},
{0x11F04, 0x11F10},
{0x11F12, 0x11F33},
{0x11FB0, 0x11FB0},
{0x12000, 0x12399},
{0x12480, 0x12543},
{0x12F90, 0x12FF0},
{0x13000, 0x1342F},
{0x13441, 0x13446},
{0x13460, 0x143FA},
{0x14400, 0x14646},
{0x16100, 0x1611D},
{0x16800, 0x16A38},
{0x16A40, 0x16A5E},
{0x16A70, 0x16ABE},
{0x16AD0, 0x16AED},
{0x16B00, 0x16B2F},
{0x16B40, 0x16B43},
{0x16B63, 0x16B77},
{0x16B7D, 0x16B8F},
{0x16D40, 0x16D6C},
{0x16E40, 0x16E7F},
{0x16EA0, 0x16EB8},
{0x16EBB, 0x16ED3},
{0x16F00, 0x16F4A},
{0x16F50, 0x16F50},
{0x16F93, 0x16F9F},
{0x16FE0, 0x16FE1},
{0x16FE3, 0x16FE3},
{0x16FF2, 0x16FF3},
{0x17000, 0x18CD5},
{0x18CFF, 0x18D1E},
{0x18D80, 0x18DF2},
{0x1AFF0, 0x1AFF3},
{0x1AFF5, 0x1AFFB},
{0x1AFFD, 0x1AFFE},
{0x1B000, 0x1B122},
{0x1B132, 0x1B132},
{0x1B150, 0x1B152},
{0x1B155, 0x1B155},
{0x1B164, 0x1B167},
{0x1B170, 0x1B2FB},
{0x1BC00, 0x1BC6A},
{0x1BC70, 0x1BC7C},
{0x1BC80, 0x1BC88},
{0x1BC90, 0x1BC99},
{0x1D400, 0x1D454},
{0x1D456, 0x1D49C},
{0x1D49E, 0x1D49F},
{0x1D4A2, 0x1D4A2},
{0x1D4A5, 0x1D4A6},
{0x1D4A9, 0x1D4AC},
{0x1D4AE, 0x1D4B9},
{0x1D4BB, 0x1D4BB},
{0x1D4BD, 0x1D4C3},
{0x1D4C5, 0x1D505},
{0x1D507, 0x1D50A},
{0x1D50D, 0x1D514},
{0x1D516, 0x1D51C},
{0x1D51E, 0x1D539},
{0x1D53B, 0x1D53E},
{0x1D540, 0x1D544},
{0x1D546, 0x1D546},
{0x1D54A, 0x1D550},
{0x1D552, 0x1D6A5},
{0x1D6A8, 0x1D6C0},
{0x1D6C2, 0x1D6DA},
{0x1D6DC, 0x1D6FA},
{0x1D6FC, 0x1D714},
{0x1D716, 0x1D734},
{0x1D736, 0x1D74E},
{0x1D750, 0x1D76E},
{0x1D770, 0x1D788},
{0x1D78A, 0x1D7A8},
{0x1D7AA, 0x1D7C2},
{0x1D7C4, 0x1D7CB},
{0x1DF00, 0x1DF1E},
{0x1DF25, 0x1DF2A},
{0x1E030, 0x1E06D},
{0x1E100, 0x1E12C},
{0x1E137, 0x1E13D},
{0x1E14E, 0x1E14E},
{0x1E290, 0x1E2AD},
{0x1E2C0, 0x1E2EB},
{0x1E4D0, 0x1E4EB},
{0x1E5D0, 0x1E5ED},
{0x1E5F0, 0x1E5F0},
{0x1E6C0, 0x1E6DE},
{0x1E6E0, 0x1E6E2},
{0x1E6E4, 0x1E6E5},
{0x1E6E7, 0x1E6ED},
{0x1E6F0, 0x1E6F4},
{0x1E6FE, 0x1E6FF},
{0x1E7E0, 0x1E7E6},
{0x1E7E8, 0x1E7EB},
{0x1E7ED, 0x1E7EE},
{0x1E7F0, 0x1E7FE},
{0x1E800, 0x1E8C4},
{0x1E900, 0x1E943},
{0x1E94B, 0x1E94B},
{0x1EE00, 0x1EE03},
{0x1EE05, 0x1EE1F},
{0x1EE21, 0x1EE22},
{0x1EE24, 0x1EE24},
{0x1EE27, 0x1EE27},
{0x1EE29, 0x1EE32},
{0x1EE34, 0x1EE37},
{0x1EE39, 0x1EE39},
{0x1EE3B, 0x1EE3B},
{0x1EE42, 0x1EE42},
{0x1EE47, 0x1EE47},
{0x1EE49, 0x1EE49},
{0x1EE4B, 0x1EE4B},
{0x1EE4D, 0x1EE4F},
{0x1EE51, 0x1EE52},
{0x1EE54, 0x1EE54},
{0x1EE57, 0x1EE57},
{0x1EE59, 0x1EE59},
{0x1EE5B, 0x1EE5B},
{0x1EE5D, 0x1EE5D},
{0x1EE5F, 0x1EE5F},
{0x1EE61, 0x1EE62},
{0x1EE64, 0x1EE64},
{0x1EE67, 0x1EE6A},
{0x1EE6C, 0x1EE72},
{0x1EE74, 0x1EE77},
{0x1EE79, 0x1EE7C},
{0x1EE7E, 0x1EE7E},
{0x1EE80, 0x1EE89},
{0x1EE8B, 0x1EE9B},
{0x1EEA1, 0x1EEA3},
{0x1EEA5, 0x1EEA9},
{0x1EEAB, 0x1EEBB},
{0x20000, 0x2A6DF},
{0x2A700, 0x2B81D},
{0x2B820, 0x2CEAD},
{0x2CEB0, 0x2EBE0},
{0x2EBF0, 0x2EE5D},
{0x2F800, 0x2FA1D},
{0x30000, 0x3134A},
{0x31350, 0x33479},
};
for (const auto& r : ranges) {
if (ch >= r.start && ch <= r.end)
return true;
}
return false;
}
bool is_space(char32_t cp) {
switch (cp) {
case 0x0009: // TAB \t
case 0x000A: // LF \n
case 0x000B: // VT
case 0x000C: // FF
case 0x000D: // CR \r
case 0x0020: // Space
case 0x00A0: // No-Break Space
case 0x1680: // Ogham Space Mark
case 0x2000: // En Quad
case 0x2001: // Em Quad
case 0x2002: // En Space
case 0x2003: // Em Space
case 0x2004: // Three-Per-Em Space
case 0x2005: // Four-Per-Em Space
case 0x2006: // Six-Per-Em Space
case 0x2007: // Figure Space
case 0x2008: // Punctuation Space
case 0x2009: // Thin Space
case 0x200A: // Hair Space
case 0x202F: // Narrow No-Break Space
case 0x205F: // Medium Mathematical Space
case 0x3000: // Ideographic Space
return true;
default:
return false;
}
}
std::string str_to_lower(const std::string& input) {
std::string result = input;
std::transform(result.begin(), result.end(), result.begin(),
[](unsigned char c) { return std::tolower(c); });
return result;
}
// UTF-8 -> Unicode code points
std::vector<char32_t> utf8_to_codepoints(const std::string& str) {
std::vector<char32_t> codepoints;
size_t i = 0;
while (i < str.size()) {
unsigned char c = str[i];
char32_t cp = 0;
size_t extra_bytes = 0;
if ((c & 0x80) == 0)
cp = c;
else if ((c & 0xE0) == 0xC0) {
cp = c & 0x1F;
extra_bytes = 1;
} else if ((c & 0xF0) == 0xE0) {
cp = c & 0x0F;
extra_bytes = 2;
} else if ((c & 0xF8) == 0xF0) {
cp = c & 0x07;
extra_bytes = 3;
} else {
++i;
continue;
} // Invalid UTF-8
if (i + extra_bytes >= str.size())
break;
for (size_t j = 1; j <= extra_bytes; ++j)
cp = (cp << 6) | (str[i + j] & 0x3F);
codepoints.push_back(cp);
i += 1 + extra_bytes;
}
return codepoints;
}
// Unicode code point -> UTF-8
std::string codepoint_to_utf8(char32_t cp) {
std::string out;
if (cp <= 0x7F)
out.push_back(static_cast<char>(cp));
else if (cp <= 0x7FF) {
out.push_back(static_cast<char>(0xC0 | (cp >> 6)));
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
} else if (cp <= 0xFFFF) {
out.push_back(static_cast<char>(0xE0 | (cp >> 12)));
out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
} else {
out.push_back(static_cast<char>(0xF0 | (cp >> 18)));
out.push_back(static_cast<char>(0x80 | ((cp >> 12) & 0x3F)));
out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
}
return out;
}
bool starts_with(const std::vector<char32_t>& text,
const std::vector<char32_t>& prefix,
std::size_t index) {
if (index > text.size()) {
return false;
}
if (prefix.size() > text.size() - index) {
return false;
}
return std::equal(prefix.begin(), prefix.end(), text.begin() + index);
}
// mistral: [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+
// qwen2: (?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+
std::vector<std::string> token_split(const std::string& text) {
std::vector<std::string> tokens;
auto cps = utf8_to_codepoints(text);
size_t i = 0;
while (i < cps.size()) {
char32_t cp = cps[i];
// `(?i:'s|'t|'re|'ve|'m|'ll|'d)`
if (cp == U'\'' && i + 1 < cps.size()) {
std::string next = str_to_lower(codepoint_to_utf8(cps[i + 1]));
if (next == "s" || next == "t" || next == "m") {
tokens.push_back("'" + next);
i += 2;
continue;
}
if (i + 2 < cps.size()) {
next += str_to_lower(codepoint_to_utf8(cps[i + 2]));
if (next == "re" || next == "ve" || next == "ll" || next == "d") {
tokens.push_back("'" + next);
i += 3;
continue;
}
}
}
// `\p{N}`
if (is_number(cp)) {
tokens.push_back(codepoint_to_utf8(cp));
++i;
continue;
}
// `[^\r\n\p{L}\p{N}]?\p{L}+`
{
// `[^\r\n\p{L}\p{N}]\p{L}+`
if (!is_letter(cp) && cp != U'\r' && cp != U'\n' && i + 1 < cps.size() && is_letter(cps[i + 1])) {
std::string token = codepoint_to_utf8(cp);
++i;
while (i < cps.size() && is_letter(cps[i])) {
token += codepoint_to_utf8(cps[i]);
++i;
}
tokens.push_back(token);
continue;
}
// `\p{L}+`
if (is_letter(cp)) {
std::string token = codepoint_to_utf8(cp);
++i;
while (i < cps.size() && is_letter(cps[i])) {
token += codepoint_to_utf8(cps[i]);
++i;
}
tokens.push_back(token);
continue;
}
}
// ` ?[^\s\p{L}\p{N}]+[\r\n]*`
{
// ` [^\s\p{L}\p{N}]+[\r\n]*`
if (cp == U' ' && i + 1 < cps.size() && !isspace(cps[i + 1]) && !is_letter(cps[i + 1]) && !is_number(cps[i + 1])) {
std::string token = codepoint_to_utf8(cp);
token += codepoint_to_utf8(cps[i + 1]);
i += 2;
while (i < cps.size() && !is_letter(cps[i]) && !is_number(cps[i]) && !isspace(cps[i])) {
token += codepoint_to_utf8(cps[i]);
++i;
}
while (i < cps.size() && (cps[i] == U'\r' || cps[i] == U'\n')) {
token += codepoint_to_utf8(cps[i]);
++i;
}
tokens.push_back(token);
continue;
}
// `[^\s\p{L}\p{N}]+[\r\n]*`
std::string token;
if (!is_letter(cps[i]) && !is_number(cps[i]) && !isspace(cps[i])) {
std::string token = codepoint_to_utf8(cp);
++i;
while (i < cps.size() && !is_letter(cps[i]) && !is_number(cps[i]) && !isspace(cps[i])) {
token += codepoint_to_utf8(cps[i]);
++i;
}
while (i < cps.size() && (cps[i] == U'\r' || cps[i] == U'\n')) {
token += codepoint_to_utf8(cps[i]);
++i;
}
tokens.push_back(token);
continue;
}
}
// `\s*[\r\n]+|\s+(?!\S)|\s+`
if (is_space(cp)) {
std::string token = codepoint_to_utf8(cp);
++i;
while (i < cps.size() && is_space(cps[i])) {
token += codepoint_to_utf8(cps[i]);
++i;
if (cps[i] == U'\r' || cps[i] == U'\n') {
break;
}
}
tokens.push_back(token);
continue;
}
// skip
++i;
}
return tokens;
}
std::vector<std::string> split_with_special_tokens(
const std::string& text,
const std::vector<std::string>& special_tokens) {
std::vector<std::string> result;
size_t pos = 0;
size_t text_len = text.size();
while (pos < text_len) {
size_t next_pos = text_len;
std::string matched_token;
for (const auto& token : special_tokens) {
size_t token_pos = text.find(token, pos);
if (token_pos != std::string::npos && token_pos < next_pos) {
next_pos = token_pos;
matched_token = token;
}
}
if (next_pos > pos) {
result.push_back(text.substr(pos, next_pos - pos));
}
if (!matched_token.empty()) {
result.push_back(matched_token);
pos = next_pos + matched_token.size();
} else {
break;
}
}
return result;
}
// int main() {
// std::string text = "I'm testing C++ token_split function. 你好，世界! 123";
// auto tokens = token_split(text);
// for (const auto& t : tokens) {
// std::cout << "[" << t << "] ";
// }
// std::cout << "\n";
// return 0;
// }
Reference in New Issue View Git Blame Copy Permalink