Method Tokenize

catboost/private/libs/text_processing/tokenizer.cpp:19–29

}

void NCB::TTokenizer::Tokenize(TStringBuf inputString, TTokensWithBuffer* tokens) {
    if (TokenizerImpl.NeedToModifyTokens()) {
        TokenizerImpl.Tokenize(inputString, &tokens->Data);
        tokens->View.clear();
        for (const auto& token: tokens->Data) {
            tokens->View.emplace_back(token);
        }
    } else {
        TokenizerImpl.TokenizeWithoutCopy(inputString, &tokens->View);
    }
}

void NCB::TTokenizer::Save(IOutputStream *stream) const {
    WriteMagic(TokenizerMagic.data(), MagicSize, Alignment, stream);