| 76 | } |
| 77 | |
| 78 | static void ProcessWordToken(const NTokenizer::TTokenizerOptions& options, ILemmerImplementation* lemmer, TUtf16String* token) { |
| 79 | if (options.Lowercasing) { |
| 80 | ToLower(*token); |
| 81 | } |
| 82 | if (options.Lemmatizing) { |
| 83 | lemmer->Lemmatize(token); |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | static std::pair<const wchar16*, size_t> BuildSubToken(const TWideToken& rawToken, const TCharSpan& subTokenInfo) { |
| 88 | return std::make_pair( |
no test coverage detected