Method Tokenize

library/cpp/text_processing/tokenizer/tokenizer.cpp:286–303

}

void NTokenizer::TTokenizer::Tokenize(
    TStringBuf inputString,
    TVector<TString>* tokens,
    TVector<NTokenizer::ETokenType>* tokenTypes
) const {
    Y_ASSERT(tokens);
    tokens->clear();
    if (tokenTypes) {
        tokenTypes->clear();
    }

    if (Options.SeparatorType == NTokenizer::ESeparatorType::BySense) {
        SplitBySense(inputString, Options, Lemmer.Get(), tokens, tokenTypes);
    } else {
        Y_ENSURE(Options.SeparatorType == NTokenizer::ESeparatorType::ByDelimiter, "Unsupported SeparatorType");
        SplitByDelimiter(inputString, Options, Lemmer.Get(), tokens, tokenTypes);
    }
}

TVector<TString> NTokenizer::TTokenizer::Tokenize(TStringBuf inputString) const {
    TVector<TString> tokens;