| 49 | } |
| 50 | |
| 51 | void NTextProcessing::NTokenizer::TokenizerOptionsToJson(const TTokenizerOptions& options, NJson::TJsonValue* optionsJson) { |
| 52 | SetOption(options.Lowercasing, LOWERCASING_OPTION_NAME, optionsJson); |
| 53 | SetOption(options.Lemmatizing, LEMMATIZING_OPTION_NAME, optionsJson); |
| 54 | SetOption(options.NumberProcessPolicy, NUMBER_PROCESS_POLICY_OPTION_NAME, optionsJson); |
| 55 | SetOption(options.NumberToken, NUMBER_TOKEN_OPTION_NAME, optionsJson); |
| 56 | SetOption(options.SeparatorType, SEPARATOR_TYPE_OPTION_NAME, optionsJson); |
| 57 | SetOption(options.Delimiter, DELIMITER_OPTION_NAME, optionsJson); |
| 58 | SetOption(options.SplitBySet, SPLIT_BY_SET_OPTION_NAME, optionsJson); |
| 59 | SetOption(options.SkipEmpty, SKIP_EMPTY_OPTION_NAME, optionsJson); |
| 60 | SetContainerOption(options.TokenTypes, TOKEN_TYPES_OPTION_NAME, optionsJson); |
| 61 | SetOption(options.SubTokensPolicy, SUBTOKENS_POLICY_OPTION_NAME, optionsJson); |
| 62 | TVector<TString> stringLanguages; |
| 63 | for (const auto& language : options.Languages) { |
| 64 | stringLanguages.push_back(FullNameByLanguage(language)); |
| 65 | } |
| 66 | SetContainerOption(stringLanguages, LANGUAGES_OPTION_NAME, optionsJson); |
| 67 | if (options.LemmerCacheSize != DEFAULT_LEMMER_CACHE_SIZE) { |
| 68 | SetOption(options.LemmerCacheSize, LEMMER_CACHESIZE_OPTION_NAME, optionsJson); |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | void NTextProcessing::NTokenizer::JsonToTokenizerOptions(const NJson::TJsonValue& optionsJson, TTokenizerOptions* options) { |
| 73 | GetOption(optionsJson, LOWERCASING_OPTION_NAME, &options->Lowercasing); |
nothing calls this directly
no test coverage detected