| 259 | } |
| 260 | |
| 261 | void NTokenizer::TTokenizer::Initialize() { |
| 262 | if (TLemmerImplementationFactory::Has(EImplementationType::YandexSpecific)) { |
| 263 | Lemmer.Reset(TLemmerImplementationFactory::Construct(EImplementationType::YandexSpecific, Options.Languages)); |
| 264 | } else { |
| 265 | Y_ENSURE(TLemmerImplementationFactory::Has(EImplementationType::Trivial), |
| 266 | "Lemmer implementation factory should have an open source implementation."); |
| 267 | Y_ENSURE(!Options.Lemmatizing, "Lemmer isn't implemented yet."); |
| 268 | Lemmer.Reset(TLemmerImplementationFactory::Construct(EImplementationType::Trivial, {})); |
| 269 | } |
| 270 | |
| 271 | if (Options.LemmerCacheSize != 0) { |
| 272 | Lemmer.Reset(new TLemmerWithCache(std::move(Lemmer), Options.LemmerCacheSize)); |
| 273 | } |
| 274 | |
| 275 | NeedToModifyTokensFlag |= Options.SeparatorType == NTokenizer::ESeparatorType::BySense; |
| 276 | NeedToModifyTokensFlag |= IsWordChanged(Options); |
| 277 | NeedToModifyTokensFlag |= Options.NumberProcessPolicy == ETokenProcessPolicy::Replace; |
| 278 | } |
| 279 | |
| 280 | NTokenizer::TTokenizer::TTokenizer(const NTokenizer::TTokenizerOptions& options) |
| 281 | : Options(options) |
nothing calls this directly
no test coverage detected