| 432 | } |
| 433 | |
| 434 | static std::unordered_map<std::string, uint8_t> unicode_to_bytes_map_bpe() { |
| 435 | std::unordered_map<std::string, uint8_t> map; |
| 436 | for (int ch = u'!'; ch <= u'~'; ++ch) { |
| 437 | assert(0 <= ch && ch < 256); |
| 438 | map[codepoint_to_utf8(ch)] = ch; |
| 439 | } |
| 440 | for (int ch = u'¡'; ch <= u'¬'; ++ch) { |
| 441 | assert(0 <= ch && ch < 256); |
| 442 | map[codepoint_to_utf8(ch)] = ch; |
| 443 | } |
| 444 | for (int ch = u'®'; ch <= u'ÿ'; ++ch) { |
| 445 | assert(0 <= ch && ch < 256); |
| 446 | map[codepoint_to_utf8(ch)] = ch; |
| 447 | } |
| 448 | auto n = 0; |
| 449 | for (int ch = 0; ch < 256; ++ch) { |
| 450 | if (map.find(codepoint_to_utf8(ch)) == map.end()) { |
| 451 | map[codepoint_to_utf8(256 + n)] = ch; |
| 452 | ++n; |
| 453 | } |
| 454 | } |
| 455 | return map; |
| 456 | } |
| 457 | |
| 458 | static uint8_t unicode_to_bytes_bpe(const std::string & utf8) { |
| 459 | static std::unordered_map<std::string, uint8_t> map = unicode_to_bytes_map_bpe(); |
no test coverage detected