Method bpe_encode_piece

server/src/server/tokenizer.cpp:364–518

Encode a single pre-tokenized piece using BPE merges.

// Encode a single pre-tokenized piece using BPE merges.
std::vector<int32_t> Tokenizer::bpe_encode_piece(const std::string & piece) const {
    if (piece.empty()) return {};

    std::vector<std::string> symbols;

    if (is_sentencepiece_) {
        // SentencePiece: replace leading space with ▁, tokens are raw UTF-8.
        std::string sp_piece;
        sp_piece.reserve(piece.size() + 2);
        size_t start = 0;
        if (!piece.empty() && piece[0] == ' ') {
            sp_piece += "\xe2\x96\x81";  // ▁ (U+2581)
            start = 1;
        }
        sp_piece += piece.substr(start);
        // Replace any remaining spaces with ▁
        std::string encoded;
        encoded.reserve(sp_piece.size());
        for (char c : sp_piece) {
            if (c == ' ') {
                encoded += "\xe2\x96\x81";
            } else {
                encoded += c;
            }
        }

        // Try whole piece as single token.
        auto it = token_to_id_.find(encoded);
        if (it != token_to_id_.end()) {
            return { it->second };
        }

        // Split into individual UTF-8 characters as initial BPE symbols.
        const char * p = encoded.c_str();
        const char * end = p + encoded.size();
        while (p < end) {
            int cplen;
            utf8_decode(p, (size_t)(end - p), &cplen);
            if (cplen <= 0) cplen = 1;
            std::string sym(p, cplen);
            auto sit = token_to_id_.find(sym);
            if (sit != token_to_id_.end()) {
                symbols.push_back(sym);
            } else {
                // Byte-fallback: <0xNN>
                char buf[8];
                std::snprintf(buf, sizeof(buf), "<0x%02X>", (unsigned)(uint8_t)*p);
                symbols.push_back(buf);
            }
            p += cplen;
        }
    } else {
        // GPT-2 BPE: convert raw text to GPT-2 byte encoding for vocab lookup.
        std::string encoded = encode_gpt2_bpe(piece);

        // Try to find the encoded piece as a single token first.
        auto it = token_to_id_.find(encoded);
        if (it != token_to_id_.end()) {