Method Load

catboost/private/libs/text_processing/tokenizer.cpp:37–41

}

void NCB::TTokenizer::Load(IInputStream *stream) {
    ReadMagic(TokenizerMagic.data(), MagicSize, Alignment, stream);
    Guid.Load(stream);
    TokenizerImpl.Load(stream);
}

NCB::TTokenizerPtr NCB::CreateTokenizer(const NTextProcessing::NTokenizer::TTokenizerOptions& options) {
    return new TTokenizer(options);