| 327 | } |
| 328 | |
| 329 | std::vector<std::string> BPETokenizer::byte_level_split(const std::string& text) const { |
| 330 | std::string unicode_text = bytes_to_unicode(text); |
| 331 | |
| 332 | std::vector<std::string> chars; |
| 333 | size_t i = 0; |
| 334 | while (i < unicode_text.length()) { |
| 335 | size_t char_len = 1; |
| 336 | |
| 337 | if ((unicode_text[i] & 0x80) == 0) { |
| 338 | char_len = 1; |
| 339 | } else if ((unicode_text[i] & 0xE0) == 0xC0) { |
| 340 | char_len = 2; |
| 341 | } else if ((unicode_text[i] & 0xF0) == 0xE0) { |
| 342 | char_len = 3; |
| 343 | } else if ((unicode_text[i] & 0xF8) == 0xF0) { |
| 344 | char_len = 4; |
| 345 | } |
| 346 | |
| 347 | if (i + char_len <= unicode_text.length()) { |
| 348 | chars.push_back(unicode_text.substr(i, char_len)); |
| 349 | } |
| 350 | i += char_len; |
| 351 | } |
| 352 | |
| 353 | return chars; |
| 354 | } |
| 355 | |
| 356 | std::vector<std::string> BPETokenizer::utf8_split(const std::string& text) const { |
| 357 | std::vector<std::string> chars; |
nothing calls this directly
no outgoing calls
no test coverage detected