Qwen2 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
| 472 | |
| 473 | // Qwen2 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| 474 | static std::vector<size_t> unicode_regex_split_custom_qwen2(const std::string & text, const std::vector<size_t> & offsets) { |
| 475 | std::vector<size_t> bpe_offsets; // store the offset of each word |
| 476 | bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size |
| 477 | |
| 478 | const auto cpts = unicode_cpts_from_utf8(text); |
| 479 | |
| 480 | size_t start = 0; |
| 481 | for (auto offset : offsets) { |
| 482 | const size_t offset_ini = start; |
| 483 | const size_t offset_end = start + offset; |
| 484 | assert(offset_end <= cpts.size()); |
| 485 | start = offset_end; |
| 486 | |
| 487 | static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; |
| 488 | auto _get_cpt = [&] (const size_t pos) -> uint32_t { |
| 489 | return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; |
| 490 | }; |
| 491 | |
| 492 | auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { |
| 493 | return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; |
| 494 | }; |
| 495 | |
| 496 | size_t _prev_end = offset_ini; |
| 497 | auto _add_token = [&] (const size_t end) -> size_t { |
| 498 | assert(_prev_end <= end && end <= offset_end); |
| 499 | size_t len = end - _prev_end; |
| 500 | if (len > 0) { |
| 501 | bpe_offsets.push_back(len); |
| 502 | } |
| 503 | _prev_end = end; |
| 504 | //if (len > 0) { |
| 505 | // std::string s = ""; |
| 506 | // for(size_t p = end-len; p < end; p++) |
| 507 | // s += unicode_cpt_to_utf8(cpts[p]); |
| 508 | // printf(">>> '%s'\n", s.c_str()); |
| 509 | //} |
| 510 | return len; |
| 511 | }; |
| 512 | |
| 513 | for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { |
| 514 | const uint32_t cpt = _get_cpt(pos); |
| 515 | const auto flags = _get_flags(pos); |
| 516 | |
| 517 | // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive |
| 518 | if (cpt == '\'' && pos+1 < offset_end) { |
| 519 | uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1)); |
| 520 | if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { |
| 521 | pos += _add_token(pos+2); |
| 522 | continue; |
| 523 | } |
| 524 | if (pos+2 < offset_end) { |
| 525 | uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2)); |
| 526 | if ((cpt_next == 'r' && cpt_next_next == 'e') || |
| 527 | (cpt_next == 'v' && cpt_next_next == 'e') || |
| 528 | (cpt_next == 'l' && cpt_next_next == 'l')) { |
| 529 | pos += _add_token(pos+3); |
| 530 | continue; |
| 531 | } |
no test coverage detected