Method GetToken

library/cpp/text_processing/dictionary/bpe_dictionary.cpp:156–163

}

TString TBpeDictionary::GetToken(TTokenId tokenId) const {
    TTokenId minId = GetMinTokenIdForUnits();
    if (tokenId < minId) {
        return Alphabet->GetToken(tokenId);
    }
    // TODO(nikitxskv): Add tokenId checks like in TDictionary.
    return StringTokens[tokenId - minId];
}

ui64 TBpeDictionary::GetCount(TTokenId tokenId) const {
    TTokenId minId = GetMinTokenIdForUnits();