Method token_text

server/src/server/tokenizer.cpp:736–778

}

std::string Tokenizer::token_text(int32_t id) const {
    if (id < 0 || id >= (int32_t)id_to_token_.size()) return "";
    const std::string & tok = id_to_token_[id];

    // Handle byte-fallback tokens like <0xNN>.
    if (tok.size() == 6 && tok[0] == '<' && tok[1] == '0' &&
        tok[2] == 'x' && tok[5] == '>') {
        unsigned val = 0;
        if (std::sscanf(tok.c_str(), "<0x%02X>", &val) == 1) {
            return std::string(1, (char)(uint8_t)val);
        }
    }

    // Special tokens (e.g. <|im_start|>, <turn|>) — return as-is.
    if (!tok.empty() && tok[0] == '<' && tok.back() == '>') {
        return tok;
    }

    if (is_sentencepiece_) {
        // SentencePiece: tokens are raw UTF-8 with ▁ (U+2581) for space.
        std::string out;
        out.reserve(tok.size());
        const char * p = tok.c_str();
        const char * end = p + tok.size();
        while (p < end) {
            // ▁ is 3 bytes: 0xE2 0x96 0x81
            if (end - p >= 3 &&
                (uint8_t)p[0] == 0xE2 &&
                (uint8_t)p[1] == 0x96 &&
                (uint8_t)p[2] == 0x81) {
                out.push_back(' ');
                p += 3;
            } else {
                out.push_back(*p);
                p++;
            }
        }
        return out;
    }

    // Decode GPT-2 byte-level BPE encoding → raw bytes.
    return decode_gpt2_bpe(tok);
}

std::string Tokenizer::decode(const std::vector<int32_t> & ids) const {
    std::string result;