| 272 | } |
| 273 | |
| 274 | static TTextDataSetPtr CreateTextDataSet( |
| 275 | const TQuantizedObjectsDataProvider& dataProvider, |
| 276 | ui32 tokenizedTextFeatureIdx, |
| 277 | NPar::ILocalExecutor* localExecutor |
| 278 | ) { |
| 279 | const TTextDigitizers& digitizers = dataProvider.GetQuantizedFeaturesInfo()->GetTextDigitizers(); |
| 280 | auto dictionary = digitizers.GetDigitizer(tokenizedTextFeatureIdx).Dictionary; |
| 281 | const TTokenizedTextValuesHolder* textColumn = *dataProvider.GetTextFeature(tokenizedTextFeatureIdx); |
| 282 | |
| 283 | if (const auto* denseData = dynamic_cast<const TTokenizedTextArrayValuesHolder*>(textColumn)) { |
| 284 | TMaybeOwningArrayHolder<TText> textData = denseData->ExtractValues(localExecutor); |
| 285 | TMaybeOwningConstArrayHolder<TText> constTextData |
| 286 | = TMaybeOwningConstArrayHolder<TText>::CreateOwning(*textData, textData.GetResourceHolder()); |
| 287 | return MakeIntrusive<TTextDataSet>(std::move(constTextData), dictionary); |
| 288 | } |
| 289 | CB_ENSURE_INTERNAL(false, "CreateTextDataSet: unsupported column type"); |
| 290 | } |
| 291 | |
| 292 | static TEmbeddingDataSetPtr CreateEmbeddingDataSet( |
| 293 | const TQuantizedObjectsDataProvider& dataProvider, |
no test coverage detected