| 81 | } |
| 82 | |
| 83 | std::vector<std::string> bpe_initial_pieces( |
| 84 | std::string_view normalized_text, |
| 85 | const std::unordered_map<std::string, int32_t> & vocab) { |
| 86 | std::vector<std::string> pieces; |
| 87 | for (size_t offset = 0; offset < normalized_text.size();) { |
| 88 | const size_t start = offset; |
| 89 | (void) next_utf8_codepoint(normalized_text, offset); |
| 90 | std::string piece(normalized_text.substr(start, offset - start)); |
| 91 | if (vocab.find(piece) != vocab.end()) { |
| 92 | pieces.push_back(std::move(piece)); |
| 93 | continue; |
| 94 | } |
| 95 | for (size_t i = start; i < offset; ++i) { |
| 96 | pieces.push_back(byte_fallback_token(static_cast<unsigned char>(normalized_text[i]))); |
| 97 | } |
| 98 | } |
| 99 | return pieces; |
| 100 | } |
| 101 | |
| 102 | bool is_cjk_codepoint(uint32_t codepoint) { |
| 103 | return (codepoint >= 0x4E00 && codepoint <= 0x9FFF) || |
no test coverage detected