| 668 | } |
| 669 | |
| 670 | std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs) { |
| 671 | // unicode categories |
| 672 | static const std::map<std::string, int> k_ucat_enum = { |
| 673 | { "\\p{N}", unicode_cpt_flags::NUMBER }, |
| 674 | { "\\p{L}", unicode_cpt_flags::LETTER }, |
| 675 | { "\\p{P}", unicode_cpt_flags::PUNCTUATION }, |
| 676 | { "\\p{M}", unicode_cpt_flags::ACCENT_MARK }, |
| 677 | { "\\p{S}", unicode_cpt_flags::SYMBOL }, |
| 678 | }; |
| 679 | |
| 680 | static const std::map<int, int> k_ucat_cpt = { |
| 681 | { unicode_cpt_flags::NUMBER, 0xD1 }, |
| 682 | { unicode_cpt_flags::LETTER, 0xD2 }, |
| 683 | { unicode_cpt_flags::PUNCTUATION, 0xD3 }, |
| 684 | { unicode_cpt_flags::ACCENT_MARK, 0xD4 }, |
| 685 | { unicode_cpt_flags::SYMBOL, 0xD5 }, |
| 686 | }; |
| 687 | |
| 688 | static const std::map<int, std::string> k_ucat_map = { |
| 689 | { unicode_cpt_flags::NUMBER, "\x30-\x39" }, // 0-9 |
| 690 | { unicode_cpt_flags::LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z |
| 691 | { unicode_cpt_flags::PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\} |
| 692 | { unicode_cpt_flags::ACCENT_MARK, "" }, // no sub-128 codepoints |
| 693 | { unicode_cpt_flags::SYMBOL, "\\\x24\\\x2B\x3C-\x3E\x5E\x60\\\x7C" }, // $+<=>^`| |
| 694 | }; |
| 695 | |
| 696 | // compute collapsed codepoints only if needed by at least one regex |
| 697 | bool need_collapse = false; |
| 698 | for (const auto & regex_expr : regex_exprs) { |
| 699 | // search for unicode categories |
| 700 | for (const auto & ucat : k_ucat_enum) { |
| 701 | if (std::string::npos != regex_expr.find(ucat.first)) { |
| 702 | need_collapse = true; |
| 703 | break; |
| 704 | } |
| 705 | } |
| 706 | } |
| 707 | |
| 708 | const auto cpts = unicode_cpts_from_utf8(text); |
| 709 | |
| 710 | // generate a "collapsed" representation of the text, where all codepoints are replaced by a single byte |
| 711 | // ref: https://github.com/ggml-org/llama.cpp/pull/6920#issuecomment-2081479935 |
| 712 | std::string text_collapsed; |
| 713 | if (need_collapse) { |
| 714 | // collapse all unicode categories |
| 715 | text_collapsed.resize(cpts.size()); |
| 716 | |
| 717 | for (size_t i = 0; i < cpts.size(); ++i) { |
| 718 | // keep single-byte codepoints as is |
| 719 | if (cpts[i] < 128) { |
| 720 | text_collapsed[i] = cpts[i]; |
| 721 | continue; |
| 722 | } |
| 723 | |
| 724 | const auto flags = unicode_cpt_flags_from_cpt(cpts[i]); |
| 725 | |
| 726 | if (flags.is_whitespace) { |
| 727 | //NOTE: C++ std::regex \s does not mach 0x85, Rust and Python regex does. |
no test coverage detected