Function decode_gpt2_bpe

server/src/server/tokenizer.cpp:722–734

}

static std::string decode_gpt2_bpe(const std::string & tok) {
    std::string out;
    out.reserve(tok.size());
    const char * p = tok.c_str();
    const char * end = p + tok.size();
    while (p < end) {
        int cplen;
        uint32_t cp = utf8_decode(p, (size_t)(end - p), &cplen);
        out.push_back((char)gpt2_unicode_to_byte(cp));
        p += cplen;
    }
    return out;
}

std::string Tokenizer::token_text(int32_t id) const {
    if (id < 0 || id >= (int32_t)id_to_token_.size()) return "";