| 174 | } |
| 175 | |
| 176 | static void SplitBySense( |
| 177 | TStringBuf inputString, |
| 178 | const NTokenizer::TTokenizerOptions& options, |
| 179 | ILemmerImplementation* lemmer, |
| 180 | TVector<TString>* tokens, |
| 181 | TVector<NTokenizer::ETokenType>* tokenTypes |
| 182 | ) { |
| 183 | TTokenHandler handler(tokens, tokenTypes, options, lemmer); |
| 184 | TNlpTokenizer tokenizer(handler); |
| 185 | tokenizer.Tokenize(UTF8ToWide(inputString)); |
| 186 | } |
| 187 | |
| 188 | template <typename StringType> |
| 189 | static void SplitByDelimiter( |
no test coverage detected