| 70 | } |
| 71 | |
| 72 | void NTextProcessing::NTokenizer::JsonToTokenizerOptions(const NJson::TJsonValue& optionsJson, TTokenizerOptions* options) { |
| 73 | GetOption(optionsJson, LOWERCASING_OPTION_NAME, &options->Lowercasing); |
| 74 | GetOption(optionsJson, LEMMATIZING_OPTION_NAME, &options->Lemmatizing); |
| 75 | GetOption(optionsJson, NUMBER_PROCESS_POLICY_OPTION_NAME, &options->NumberProcessPolicy); |
| 76 | GetOption(optionsJson, NUMBER_TOKEN_OPTION_NAME, &options->NumberToken); |
| 77 | GetOption(optionsJson, SEPARATOR_TYPE_OPTION_NAME, &options->SeparatorType); |
| 78 | GetOption(optionsJson, DELIMITER_OPTION_NAME, &options->Delimiter); |
| 79 | GetOption(optionsJson, SPLIT_BY_SET_OPTION_NAME, &options->SplitBySet); |
| 80 | GetOption(optionsJson, SKIP_EMPTY_OPTION_NAME, &options->SkipEmpty); |
| 81 | GetContainerOption(optionsJson, TOKEN_TYPES_OPTION_NAME, &options->TokenTypes); |
| 82 | GetOption(optionsJson, SUBTOKENS_POLICY_OPTION_NAME, &options->SubTokensPolicy); |
| 83 | TVector<TString> stringLanguages; |
| 84 | GetContainerOption(optionsJson, LANGUAGES_OPTION_NAME, &stringLanguages); |
| 85 | options->Languages.clear(); |
| 86 | for (const auto& language : stringLanguages) { |
| 87 | options->Languages.push_back(LanguageByNameOrDie(language)); |
| 88 | } |
| 89 | GetOption(optionsJson, LEMMER_CACHESIZE_OPTION_NAME, &options->LemmerCacheSize); |
| 90 | } |
| 91 | |
| 92 | NJson::TJsonValue NTextProcessing::NTokenizer::TokenizerOptionsToJson(const TTokenizerOptions& options) { |
| 93 | NJson::TJsonValue optionsJson; |
nothing calls this directly
no test coverage detected