Method pre_tokenize

server/src/server/tokenizer.cpp:150–309

//   \s+

std::vector<std::string> Tokenizer::pre_tokenize(const std::string & text) const {
    std::vector<std::string> pieces;
    const char * s = text.c_str();
    const size_t len = text.size();
    size_t pos = 0;

    auto peek_cp = [&](size_t p, int * cplen) -> uint32_t {
        if (p >= len) { *cplen = 0; return 0; }
        return utf8_decode(s + p, len - p, cplen);
    };

    while (pos < len) {
        size_t start = pos;
        int cplen = 0;
        uint32_t cp = peek_cp(pos, &cplen);

        // Pattern 1: English contractions 's 't 're 've 'm 'll 'd
        if (cp == '\'') {
            size_t save = pos;
            pos++;
            bool matched = false;
            if (pos < len) {
                char c = s[pos] | 0x20;  // lowercase
                if (c == 's' || c == 't' || c == 'm' || c == 'd') {
                    pos++;
                    matched = true;
                } else if (c == 'r' && pos + 1 < len && (s[pos+1] | 0x20) == 'e') {
                    pos += 2;
                    matched = true;
                } else if (c == 'v' && pos + 1 < len && (s[pos+1] | 0x20) == 'e') {
                    pos += 2;
                    matched = true;
                } else if (c == 'l' && pos + 1 < len && (s[pos+1] | 0x20) == 'l') {
                    pos += 2;
                    matched = true;
                }
            }
            if (matched) {
                pieces.push_back(text.substr(start, pos - start));
                continue;
            }
            pos = save;  // reset, try other patterns
        }

        // Pattern 2: [^\r\n\p{L}\p{N}]?[\p{L}\p{M}]+
        {
            size_t p = pos;
            int cl = 0;
            uint32_t c = peek_cp(p, &cl);
            // Optional leading non-letter, non-digit, non-newline char
            if (cl > 0 && !is_newline(c) && !is_letter(c) && !is_digit(c)) {
                p += cl;
                c = peek_cp(p, &cl);
            }
            // One or more letter/mark chars
            if (cl > 0 && (is_letter(c) || is_mark(c))) {
                while (cl > 0 && (is_letter(c) || is_mark(c))) {
                    p += cl;