| 66 | } |
| 67 | |
| 68 | TTrainingDataProviderPtr GetTrainingData( |
| 69 | TDataProviderPtr srcData, |
| 70 | bool dataCanBeEmpty, |
| 71 | bool isLearnData, |
| 72 | TStringBuf datasetName, |
| 73 | const TMaybe<TString>& bordersFile, |
| 74 | bool unloadCatFeaturePerfectHashFromRam, |
| 75 | bool ensureConsecutiveIfDenseFeaturesDataForCpu, |
| 76 | const TString& tmpDir, |
| 77 | TQuantizedFeaturesInfoPtr quantizedFeaturesInfo, |
| 78 | NCatboostOptions::TCatBoostOptions* params, |
| 79 | TLabelConverter* labelConverter, |
| 80 | TMaybe<float>* targetBorder, |
| 81 | NPar::ILocalExecutor* localExecutor, |
| 82 | TRestorableFastRng64* rand, |
| 83 | TMaybe<TFullModel*> initModel) { |
| 84 | |
| 85 | const ui64 cpuRamLimit = ParseMemorySizeDescription(params->SystemOptions->CpuUsedRamLimit.Get()); |
| 86 | |
| 87 | auto trainingData = MakeIntrusive<TTrainingDataProvider>(); |
| 88 | trainingData->OriginalFeaturesLayout = srcData->MetaInfo.FeaturesLayout; |
| 89 | trainingData->MetaInfo = srcData->MetaInfo; |
| 90 | trainingData->ObjectsGrouping = srcData->ObjectsGrouping; |
| 91 | |
| 92 | if (auto* quantizedObjectsDataProvider |
| 93 | = dynamic_cast<TQuantizedObjectsDataProvider*>(srcData->ObjectsData.Get())) |
| 94 | { |
| 95 | if (params->GetTaskType() == ETaskType::CPU) { |
| 96 | /* |
| 97 | * We need data to be consecutive for efficient blocked permutations |
| 98 | * but there're cases (e.g. CV with many folds) when limiting used CPU RAM is more important |
| 99 | */ |
| 100 | if (ensureConsecutiveIfDenseFeaturesDataForCpu) { |
| 101 | EnsureObjectsDataIsConsecutiveIfQuantized(cpuRamLimit, localExecutor, &srcData); |
| 102 | } |
| 103 | } else { // GPU |
| 104 | /* |
| 105 | * if there're any cat features format should be CPU-compatible to enable final CTR |
| 106 | * calculations. |
| 107 | * TODO(akhropov): compatibility with final CTR calculation should not depend on this flag |
| 108 | */ |
| 109 | CB_ENSURE( |
| 110 | (srcData->MetaInfo.FeaturesLayout->GetCatFeatureCount() == 0) || |
| 111 | quantizedObjectsDataProvider, |
| 112 | "Quantized objects data is not compatible with final CTR calculation" |
| 113 | ); |
| 114 | } |
| 115 | |
| 116 | if (params->DataProcessingOptions.Get().IgnoredFeatures.IsSet()) { |
| 117 | trainingData->ObjectsData = dynamic_cast<TQuantizedObjectsDataProvider*>( |
| 118 | quantizedObjectsDataProvider->GetFeaturesSubset( |
| 119 | params->DataProcessingOptions.Get().IgnoredFeatures, |
| 120 | localExecutor).Get() |
| 121 | ); |
| 122 | } else { |
| 123 | trainingData->ObjectsData = quantizedObjectsDataProvider; |
| 124 | } |
| 125 | } else { |