Method Size

library/cpp/text_processing/dictionary/bpe_dictionary.cpp:152–154

}

ui32 TBpeDictionary::Size() const {
    return Alphabet->Size() + BpeUnits.size();
}

TString TBpeDictionary::GetToken(TTokenId tokenId) const {
    TTokenId minId = GetMinTokenIdForUnits();