Function SplitIntoTokens

library/cpp/tokenizer/split.cpp:59–68

const TTokenizerSplitParams::THandledMask TTokenizerSplitParams::NOT_PUNCT(NLP_WORD, NLP_INTEGER, NLP_FLOAT, NLP_MARK);

TVector<TUtf16String> SplitIntoTokens(const TUtf16String& text, const TTokenizerSplitParams& params) {
    TVector<TUtf16String> words;

    TSimpleTokenHandler handler(&words, params);
    TNlpTokenizer tokenizer(handler, params.BackwardCompatibility);
    TTokenizerOptions opts { params.SpacePreserve, params.TokenizerLangMask, params.UrlDecode };
    tokenizer.Tokenize(text.data(), text.size(), opts);

    return words;
}

TVector<TUtf16String> SplitIntoSentences(const TUtf16String& text, const TTokenizerSplitParams& params) {
    TVector<TUtf16String> sentences;