| 1991 | |
| 1992 | |
| 1993 | static void CreateDictionaries( |
| 1994 | TConstArrayRef<THolder<TStringTextValuesHolder>> textFeatures, |
| 1995 | const TFeaturesLayout& featuresLayout, |
| 1996 | const NCatboostOptions::TRuntimeTextOptions& textOptions, |
| 1997 | TTextDigitizers* textDigitizers |
| 1998 | ) { |
| 1999 | for (ui32 tokenizedFeatureIdx: xrange(textOptions.TokenizedFeatureCount())) { |
| 2000 | const auto& featureDescription = textOptions.GetTokenizedFeatureDescription(tokenizedFeatureIdx); |
| 2001 | const ui32 textFeatureIdx = featureDescription.TextFeatureId; |
| 2002 | |
| 2003 | if (textDigitizers->HasDigitizer(tokenizedFeatureIdx + textFeatures.size()) || |
| 2004 | !featuresLayout.GetInternalFeatureMetaInfo(textFeatureIdx, EFeatureType::Text).IsAvailable) { |
| 2005 | continue; |
| 2006 | } |
| 2007 | |
| 2008 | const auto& srcDenseFeature = dynamic_cast<const TStringTextArrayValuesHolder&>( |
| 2009 | *textFeatures[textFeatureIdx] |
| 2010 | ); |
| 2011 | ITypedArraySubsetPtr<TString> textFeature = srcDenseFeature.GetData(); |
| 2012 | |
| 2013 | const auto& tokenizerOptions = textOptions.GetTokenizerOptions(featureDescription.TokenizerId.Get()); |
| 2014 | TTokenizerPtr tokenizer = CreateTokenizer(tokenizerOptions.TokenizerOptions.Get()); |
| 2015 | |
| 2016 | auto dictionary = CreateDictionary( |
| 2017 | TIterableTextFeature(textFeature), |
| 2018 | textOptions.GetDictionaryOptions(featureDescription.DictionaryId.Get()), |
| 2019 | tokenizer |
| 2020 | ); |
| 2021 | textDigitizers->AddDigitizer(textFeatureIdx, tokenizedFeatureIdx + textFeatures.size(), {tokenizer, dictionary}); |
| 2022 | } |
| 2023 | } |
| 2024 | |
| 2025 | static void AddTokenizedFeaturesToFeatureLayout( |
| 2026 | const NCatboostOptions::TRuntimeTextOptions& textOptions, |
no test coverage detected