| 99 | |
| 100 | |
| 101 | static TTrainingDataProviders QuantizePools( |
| 102 | const TPoolLoadParams* poolLoadParams, |
| 103 | const TOutputFilesOptions& outputFileOptions, |
| 104 | const TDataProviders& pools, |
| 105 | TCatBoostOptions* catBoostOptions, |
| 106 | TLabelConverter* labelConverter, |
| 107 | TRestorableFastRng64* rand, |
| 108 | NPar::ILocalExecutor* executor |
| 109 | ) { |
| 110 | const auto learnFeaturesLayout = pools.Learn->MetaInfo.FeaturesLayout; |
| 111 | |
| 112 | // maybe PrepareQuantizationParameters ?? |
| 113 | // create here to possibly load borders |
| 114 | auto quantizedFeaturesInfo = MakeIntrusive<TQuantizedFeaturesInfo>( |
| 115 | *learnFeaturesLayout, |
| 116 | catBoostOptions->DataProcessingOptions->IgnoredFeatures.Get(), |
| 117 | catBoostOptions->DataProcessingOptions->FloatFeaturesBinarization.Get(), |
| 118 | catBoostOptions->DataProcessingOptions->PerFloatFeatureQuantization.Get(), |
| 119 | catBoostOptions->DataProcessingOptions->TextProcessingOptions.Get(), |
| 120 | catBoostOptions->DataProcessingOptions->EmbeddingProcessingOptions.Get(), |
| 121 | /*allowNansInTestOnly*/true |
| 122 | ); |
| 123 | if (poolLoadParams && poolLoadParams->BordersFile) { |
| 124 | LoadBordersAndNanModesFromFromFileInMatrixnetFormat( |
| 125 | poolLoadParams->BordersFile, |
| 126 | quantizedFeaturesInfo.Get() |
| 127 | ); |
| 128 | } |
| 129 | |
| 130 | for (auto testPoolIdx : xrange(pools.Test.size())) { |
| 131 | const auto& testPool = *pools.Test[testPoolIdx]; |
| 132 | if (testPool.GetObjectCount() == 0) { |
| 133 | continue; |
| 134 | } |
| 135 | CheckCompatibleForApply( |
| 136 | *learnFeaturesLayout, |
| 137 | *testPool.MetaInfo.FeaturesLayout, |
| 138 | TStringBuilder() << "test dataset #" << testPoolIdx |
| 139 | ); |
| 140 | } |
| 141 | |
| 142 | TString tmpDir; |
| 143 | if (outputFileOptions.AllowWriteFiles()) { |
| 144 | NCB::NPrivate::CreateTrainDirWithTmpDirIfNotExist(outputFileOptions.GetTrainDir(), &tmpDir); |
| 145 | } |
| 146 | |
| 147 | const bool haveLearnFeaturesInMemory = HaveFeaturesInMemory( |
| 148 | *catBoostOptions, |
| 149 | poolLoadParams ? MakeMaybe(poolLoadParams->LearnSetPath) : Nothing() |
| 150 | ); |
| 151 | |
| 152 | TTrainingDataProviders trainingData = GetTrainingData( |
| 153 | pools, |
| 154 | /*trainDataCanBeEmpty*/ false, |
| 155 | /* borders */ Nothing(), // borders are already loaded to quantizedFeaturesInfo |
| 156 | /*ensureConsecutiveIfDenseLearnFeaturesDataForCpu*/ haveLearnFeaturesInMemory, |
| 157 | outputFileOptions.AllowWriteFiles(), |
| 158 | tmpDir, |
no test coverage detected