| 68 | } |
| 69 | |
| 70 | TVector<TUtf16String> SplitIntoSentences(const TUtf16String& text, const TTokenizerSplitParams& params) { |
| 71 | TVector<TUtf16String> sentences; |
| 72 | |
| 73 | TSimpleSentenceHandler handler(&sentences); |
| 74 | TNlpTokenizer tokenizer(handler, params.BackwardCompatibility); |
| 75 | TTokenizerOptions opts { params.SpacePreserve, params.TokenizerLangMask, params.UrlDecode }; |
| 76 | tokenizer.Tokenize(text.data(), text.size(), opts); |
| 77 | handler.Flush(); |
| 78 | |
| 79 | return sentences; |
| 80 | } |