Function byte_to_gpt2_unicode

server/src/server/tokenizer.cpp:317–351

Forward GPT-2 byte encoding: raw byte → Unicode codepoint (UTF-8 string). This is the inverse of gpt2_unicode_to_byte (defined later, near decode). Bytes in {33-126, 161-172, 174-255} map to themselves as a codepoint; all others (0-32, 127-160, 173) map to U+0100..U+0143.

// Bytes in {33-126, 161-172, 174-255} map to themselves as a codepoint;
// all others (0-32, 127-160, 173) map to U+0100..U+0143.
static std::string byte_to_gpt2_unicode(uint8_t b) {
    // Build forward table once (thread-safe via C++11 static init).
    static const auto fwd = []() {
        std::array<uint32_t, 256> t{};
        int n = 0;
        for (int i = 0; i < 256; i++) {
            if ((i >= 33  && i <= 126) ||
                (i >= 161 && i <= 172) ||
                (i >= 174 && i <= 255)) {
                t[i] = (uint32_t)i;
            } else {
                t[i] = 256 + n;
                n++;
            }
        }
        return t;
    }();
    uint32_t cp = fwd[b];
    // Encode codepoint as UTF-8.
    char buf[4];
    int len;
    if (cp < 0x80) {
        buf[0] = (char)cp; len = 1;
    } else if (cp < 0x800) {
        buf[0] = (char)(0xC0 | (cp >> 6));
        buf[1] = (char)(0x80 | (cp & 0x3F));
        len = 2;
    } else {
        buf[0] = (char)(0xE0 | (cp >> 12));
        buf[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
        buf[2] = (char)(0x80 | (cp & 0x3F));
        len = 3;
    }
    return std::string(buf, len);
}

// Convert a raw UTF-8 text piece to GPT-2 byte-encoded form for BPE lookup.
static std::string encode_gpt2_bpe(const std::string & text) {