Method text_to_token

external/llama_tokenizer/bpe-core.cpp:530–533

}  // namespace

int32_t BpeVocabulary::text_to_token(const std::string & text) const {
    const auto it = token_to_id.find(text);
    return it == token_to_id.end() ? -1 : it->second;
}

int BpeVocabulary::find_bpe_rank(const std::string & left, const std::string & right) const {
    const auto it = bpe_ranks.find(pair_key(left, right));