Method Tokenize

library/cpp/tokenizer/tokenizer.cpp:14–39

#include <util/stream/file.h>

void TNlpTokenizer::Tokenize(const wchar16* str,
                             size_t size,
                             const TTokenizerOptions& opts) {
    bool semicolonBreaksSentence = opts.LangMask == TLangMask(LANG_GRE);
    TSentBreakFilter sentBreakFilter(opts.LangMask);
    THolder<TNlpParser> parser;
    switch (opts.Version) {
        case 2:
            parser = MakeHolder<TVersionedNlpParser<2>>(TokenHandler, sentBreakFilter, Buffer, opts.SpacePreserve,
                    BackwardCompatible, semicolonBreaksSentence, opts.UrlDecode);
            break;
        case 3:
            parser = MakeHolder<TVersionedNlpParser<3>>(TokenHandler, sentBreakFilter, Buffer, opts.SpacePreserve,
                    BackwardCompatible, semicolonBreaksSentence, opts.UrlDecode, opts.KeepAffixes);
            break;
        default:
            parser = MakeHolder<TDefaultNlpParser>(TokenHandler, sentBreakFilter, Buffer, opts.SpacePreserve,
                    BackwardCompatible, semicolonBreaksSentence, opts.UrlDecode);
            break;
    }
    try {
        parser->Execute(str, size, &TextStart);
    } catch (const ITokenHandler::TAllDoneException&) {
        // do nothing
    }
}

#ifndef CATBOOST_OPENSOURCE
void TNlpTokenizer::Tokenize(const char* text,