| 403 | } |
| 404 | |
| 405 | static std::unordered_map<uint8_t, std::string> bytes_to_unicode_map_bpe() { |
| 406 | std::unordered_map<uint8_t, std::string> map; |
| 407 | for (int ch = u'!'; ch <= u'~'; ++ch) { |
| 408 | assert(0 <= ch && ch < 256); |
| 409 | map[ch] = codepoint_to_utf8(ch); |
| 410 | } |
| 411 | for (int ch = u'¡'; ch <= u'¬'; ++ch) { |
| 412 | assert(0 <= ch && ch < 256); |
| 413 | map[ch] = codepoint_to_utf8(ch); |
| 414 | } |
| 415 | for (int ch = u'®'; ch <= u'ÿ'; ++ch) { |
| 416 | assert(0 <= ch && ch < 256); |
| 417 | map[ch] = codepoint_to_utf8(ch); |
| 418 | } |
| 419 | auto n = 0; |
| 420 | for (int ch = 0; ch < 256; ++ch) { |
| 421 | if (map.find(ch) == map.end()) { |
| 422 | map[ch] = codepoint_to_utf8(256 + n); |
| 423 | ++n; |
| 424 | } |
| 425 | } |
| 426 | return map; |
| 427 | } |
| 428 | |
| 429 | static std::string bytes_to_unicode_bpe(uint8_t byte) { |
| 430 | static std::unordered_map<uint8_t, std::string> map = bytes_to_unicode_map_bpe(); |
no test coverage detected