Function MapSetApproxes

catboost/private/libs/distributed/master.cpp:661–784

template <typename TApproxDefs>
void MapSetApproxes(
    const IDerCalcer& error,
    const std::variant<TSplitTree, TNonSymmetricTreeStructure>& splitTree,
    TVector<TVector<double>>* averageLeafValues,
    TVector<double>* sumLeafWeights,
    TLearnContext* ctx) {

    using namespace NCatboostDistributed;
    using TSum = typename TApproxDefs::TSumType;
    using TPairwiseBuckets = typename TApproxDefs::TPairwiseBuckets;
    using TBucketUpdater = typename TApproxDefs::TBucketUpdater;
    using TDeltaUpdater = typename TApproxDefs::TDeltaUpdater;

    Y_ASSERT(ctx->Params.SystemOptions->IsMaster());
    const int workerCount = TMasterEnvironment::GetRef().RootEnvironment->GetSlaveCount();
    ApplyMapper<TCalcApproxStarter>(workerCount, TMasterEnvironment::GetRef().SharedTrainData, splitTree);
    const int gradientIterations = ctx->Params.ObliviousTreeOptions->LeavesEstimationIterations;
    const int approxDimension = ctx->LearnProgress->ApproxDimension;
    const int leafCount = GetLeafCount(splitTree);
    const auto lossFunction = ctx->Params.LossFunctionDescription;
    const auto estimationMethod = ctx->Params.ObliviousTreeOptions->LeavesEstimationMethod;

    if (estimationMethod == ELeavesEstimation::Exact) {
        UpdateLeavesExact<TDeltaUpdater>(error, leafCount, averageLeafValues, ctx);
    } else {
        TVector<TSum> buckets(leafCount, TSum(approxDimension, error.GetHessianType()));
        const auto leafUpdaterFunc = [&] (
            bool recalcLeafWeights,
            const TVector<TVector<double>>& /*approxesPlaceholder*/,
            TVector<TVector<double>>* leafValues
        ) {
            for (auto &bucket : buckets) {
                bucket.SetZeroDers();
            }
            TPairwiseBuckets pairwiseBuckets;
            TApproxDefs::SetPairwiseBucketsSize(leafCount, &pairwiseBuckets);
            const auto bucketsFromAllWorkers = ApplyMapper<TBucketUpdater>(workerCount, TMasterEnvironment::GetRef().SharedTrainData);
            // reduce across workers
            for (const auto& workerBuckets : bucketsFromAllWorkers) {
                const auto& singleBuckets = workerBuckets.first;
                if (singleBuckets.empty()) {
                    continue;
                }
                for (int leafIdx = 0; leafIdx < leafCount; ++leafIdx) {
                    if (estimationMethod == ELeavesEstimation::Gradient) {
                        buckets[leafIdx].AddDerWeight(
                            singleBuckets[leafIdx].SumDer,
                            singleBuckets[leafIdx].SumWeights,
                            recalcLeafWeights);
                    } else {
                        Y_ASSERT(estimationMethod == ELeavesEstimation::Newton);
                        buckets[leafIdx].AddDerDer2(singleBuckets[leafIdx].SumDer, singleBuckets[leafIdx].SumDer2);
                    }
                }
                TApproxDefs::AddPairwiseBuckets(workerBuckets.second, &pairwiseBuckets);
            }
            *leafValues = TApproxDefs::CalcLeafValues(buckets, pairwiseBuckets, *ctx);
        };