Method TokenizeWithoutCopy

library/cpp/text_processing/tokenizer/tokenizer.cpp:311–317

}

void NTokenizer::TTokenizer::TokenizeWithoutCopy(TStringBuf inputString, TVector<TStringBuf>* tokens) const {
    Y_ASSERT(!NeedToModifyTokensFlag);
    SplitByDelimiter(inputString, Options.Delimiter, Options.SplitBySet, Options.SkipEmpty, tokens);
    if (Options.NumberProcessPolicy == ETokenProcessPolicy::Skip) {
        FilterNumbers(tokens);
    }
}

TVector<TStringBuf> NTokenizer::TTokenizer::TokenizeWithoutCopy(TStringBuf inputString) const {
    TVector<TStringBuf> tokens;