Function encode_gpt2_bpe

server/src/server/tokenizer.cpp:354–361

Convert a raw UTF-8 text piece to GPT-2 byte-encoded form for BPE lookup.

// Convert a raw UTF-8 text piece to GPT-2 byte-encoded form for BPE lookup.
static std::string encode_gpt2_bpe(const std::string & text) {
    std::string out;
    out.reserve(text.size() * 2);  // GPT-2 encoding may expand
    for (uint8_t b : text) {
        out += byte_to_gpt2_unicode(b);
    }
    return out;
}

// Encode a single pre-tokenized piece using BPE merges.
std::vector<int32_t> Tokenizer::bpe_encode_piece(const std::string & piece) const {