| 121 | |
| 122 | |
| 123 | void SetTrainDataFromMaster( |
| 124 | const TTrainingDataProviders& trainData, |
| 125 | ui64 cpuUsedRamLimit, |
| 126 | NPar::ILocalExecutor* localExecutor |
| 127 | ) { |
| 128 | const int workerCount = TMasterEnvironment::GetRef().RootEnvironment->GetSlaveCount(); |
| 129 | |
| 130 | auto learnWorkerParts = Split(*trainData.Learn->ObjectsGrouping, (ui32)workerCount); |
| 131 | |
| 132 | TVector<TVector<TArraySubsetIndexing<ui32>>> testWorkerParts; // [testIdx][workerIdx] |
| 133 | for (auto testIdx : xrange(trainData.Test.size())) { |
| 134 | testWorkerParts.push_back(Split(*trainData.Test[testIdx]->ObjectsGrouping, (ui32)workerCount)); |
| 135 | } |
| 136 | |
| 137 | const bool hasEstimatedData = !!trainData.EstimatedObjectsData.Learn; |
| 138 | |
| 139 | for (int workerIdx = 0; workerIdx < workerCount; ++workerIdx) { |
| 140 | NCB::TTrainingDataProviders workerTrainData; |
| 141 | |
| 142 | GetWorkerPart( |
| 143 | trainData.Learn, |
| 144 | std::move(learnWorkerParts[workerIdx]), |
| 145 | hasEstimatedData ? MakeMaybe(trainData.EstimatedObjectsData.Learn) : Nothing(), |
| 146 | cpuUsedRamLimit, |
| 147 | localExecutor, |
| 148 | &workerTrainData.Learn, |
| 149 | hasEstimatedData ? MakeMaybe(&workerTrainData.EstimatedObjectsData.Learn) : Nothing()); |
| 150 | |
| 151 | workerTrainData.Test.resize(trainData.Test.size()); |
| 152 | if (hasEstimatedData) { |
| 153 | workerTrainData.EstimatedObjectsData.Test.resize(trainData.Test.size()); |
| 154 | } |
| 155 | |
| 156 | for (auto testIdx : xrange(trainData.Test.size())) { |
| 157 | GetWorkerPart( |
| 158 | trainData.Test[testIdx], |
| 159 | std::move(testWorkerParts[testIdx][workerIdx]), |
| 160 | hasEstimatedData ? MakeMaybe(trainData.EstimatedObjectsData.Test[testIdx]) : Nothing(), |
| 161 | cpuUsedRamLimit, |
| 162 | localExecutor, |
| 163 | &workerTrainData.Test[testIdx], |
| 164 | hasEstimatedData ? MakeMaybe(&workerTrainData.EstimatedObjectsData.Test[testIdx]) : Nothing()); |
| 165 | } |
| 166 | |
| 167 | workerTrainData.FeatureEstimators = trainData.FeatureEstimators; |
| 168 | workerTrainData.EstimatedObjectsData.FeatureEstimators |
| 169 | = trainData.EstimatedObjectsData.FeatureEstimators; |
| 170 | workerTrainData.EstimatedObjectsData.QuantizedEstimatedFeaturesInfo |
| 171 | = trainData.EstimatedObjectsData.QuantizedEstimatedFeaturesInfo; |
| 172 | |
| 173 | TMasterEnvironment::GetRef().SharedTrainData->SetContextData( |
| 174 | workerIdx, |
| 175 | new NCatboostDistributed::TTrainData(std::move(workerTrainData)), |
| 176 | NPar::DELETE_RAW_DATA); // only workers |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | void MapBuildPlainFold(TLearnContext* ctx) { |
no test coverage detected