| 692 | // must reverse-map each codepoint back to its original byte. |
| 693 | |
| 694 | static uint8_t gpt2_unicode_to_byte(uint32_t cp) { |
| 695 | // Direct-mapped ranges: the codepoint IS the byte. |
| 696 | if ((cp >= 33 && cp <= 126) || |
| 697 | (cp >= 161 && cp <= 172) || |
| 698 | (cp >= 174 && cp <= 255)) { |
| 699 | return (uint8_t)cp; |
| 700 | } |
| 701 | // Offset-mapped range: U+0100..U+0143 → non-printable bytes. |
| 702 | // Build the reverse table once (thread-safe via C++11 static init). |
| 703 | static const auto table = []() { |
| 704 | std::array<uint8_t, 68> t{}; |
| 705 | int n = 0; |
| 706 | for (int b = 0; b < 256; b++) { |
| 707 | if ((b >= 33 && b <= 126) || |
| 708 | (b >= 161 && b <= 172) || |
| 709 | (b >= 174 && b <= 255)) continue; |
| 710 | t[n] = (uint8_t)b; |
| 711 | n++; |
| 712 | } |
| 713 | return t; |
| 714 | }(); |
| 715 | if (cp >= 256 && cp < 256 + 68) { |
| 716 | return table[cp - 256]; |
| 717 | } |
| 718 | // Shouldn't happen for valid BPE tokens — return replacement. |
| 719 | return '?'; |
| 720 | } |
| 721 | |
| 722 | static std::string decode_gpt2_bpe(const std::string & tok) { |
| 723 | std::string out; |