Function AddToken

library/cpp/text_processing/dictionary/bpe_builder.cpp:7–22

template <typename T>
static void AddToken(
    const T& token,
    bool skipUnknown,
    const TDictionary& alphabet,
    TEraseList<TTokenId>* tokenIdsList
) {
    auto tokenId = alphabet.Apply(token);
    if (tokenId != alphabet.GetUnknownTokenId()) {
        tokenIdsList->PushToken(tokenId);
    } else {
        if (skipUnknown) {
            return;
        }
        tokenIdsList->PushToken(tokenId);
    }
}

template <typename TStringVector>
static void AddImpl(