Method AddImpl

library/cpp/text_processing/dictionary/dictionary_builder.cpp:110–126

    template <typename TTokenType>
    void TUnigramDictionaryBuilderImpl::AddImpl(TConstArrayRef<TTokenType> tokens, ui64 weight) {
        if (DictionaryOptions.TokenLevelType == ETokenLevelType::Word) {
            for (const auto& token : tokens) {
                TokenToCount[token] += weight;
            }
        } else {
            auto updateTokenToCountFunc = [&] (TStringBuf token) {
                TokenToCount[token] += weight;
            };
            ApplyFuncToLetterNGrams(
                tokens,
                DictionaryOptions.GramOrder,
                DictionaryOptions.EndOfWordTokenPolicy == EEndOfWordTokenPolicy::Insert,
                updateTokenToCountFunc
            );
        }
    }

    TIntrusivePtr<TDictionary> TUnigramDictionaryBuilderImpl::FinishBuilding() {
        Y_ENSURE(!IsBuildingFinish, "FinishBuilding method should be called only once.");