Method Add

library/cpp/text_processing/dictionary/bpe_builder.cpp:74–76

}

void TBpeDictionaryBuilder::Add(TConstArrayRef<TStringBuf> tokens, ui64 weight) {
    AddImpl(tokens, weight, SkipUnknown, *Alphabet, &TokenIdsLists, &PairStats, &Counts);
}

void TBpeDictionaryBuilder::Add(TConstArrayRef<TString> tokens, ui64 weight) {
    AddImpl(tokens, weight, SkipUnknown, *Alphabet, &TokenIdsLists, &PairStats, &Counts);