Function decode_sentencepiece

src/framework/tokenizers/sentencepiece.cpp:127–139

}

std::string decode_sentencepiece(
    const std::vector<SentencePiecePiece> & pieces,
    const std::vector<int32_t> & token_ids) {
    const auto processor = lookup_processor_for_pieces(pieces);
    std::vector<int> ids;
    ids.reserve(token_ids.size());
    for (const int32_t id : token_ids) {
        ids.push_back(static_cast<int>(id));
    }
    std::string output;
    require_status(processor->Decode(ids, &output), "failed to decode with SentencePiece");
    return output;
}

SentencePieceTokenizer::SentencePieceTokenizer(std::vector<SentencePiecePiece> pieces)
    : pieces_(std::move(pieces)) {}