| 516 | } |
| 517 | |
| 518 | void CompiledProgram::BCastParamsToDevices(const std::vector<std::string> &vars, |
| 519 | int trainer_id) const { |
| 520 | VLOG(3) << "BCastParamsToDevices"; |
| 521 | // the initializing bcast, all vars would be bcast from device(0). |
| 522 | for (auto &var : vars) { |
| 523 | framework::Variable *main_var = member_->local_scopes_[0]->FindVar(var); |
| 524 | if (main_var == nullptr || !main_var->IsType<DenseTensor>()) { |
| 525 | continue; |
| 526 | } |
| 527 | |
| 528 | auto &main_tensor = main_var->Get<DenseTensor>(); |
| 529 | if (!main_tensor.IsInitialized()) { |
| 530 | VLOG(3) << "one in var not inited, return!"; |
| 531 | continue; |
| 532 | } |
| 533 | auto &dims = main_tensor.dims(); |
| 534 | if (phi::is_gpu_place(main_tensor.place())) { |
| 535 | #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) |
| 536 | std::vector<void *> buffers; |
| 537 | buffers.reserve(member_->places_.size()); |
| 538 | size_t numel = main_tensor.numel(); |
| 539 | auto dtype = framework::TransToProtoVarType(main_tensor.dtype()); |
| 540 | ncclDataType_t data_type = phi::ToNCCLDataType(main_tensor.dtype()); |
| 541 | for (size_t i = 0; i < member_->places_.size(); ++i) { |
| 542 | auto place = member_->places_[i]; |
| 543 | void *buffer; |
| 544 | |
| 545 | if (i == 0 && trainer_id == 0) { |
| 546 | buffer = const_cast<void *>(main_tensor.data()); |
| 547 | } else { |
| 548 | auto local_scope = member_->local_scopes_[i]; |
| 549 | auto *t = local_scope->Var(var)->GetMutable<DenseTensor>(); |
| 550 | t->Resize(dims); |
| 551 | buffer = t->mutable_data(place, main_tensor.dtype()); |
| 552 | } |
| 553 | buffers.push_back(buffer); |
| 554 | } |
| 555 | |
| 556 | PADDLE_ENFORCE_EQ(member_->places_.size(), |
| 557 | buffers.size(), |
| 558 | common::errors::PreconditionNotMet( |
| 559 | "variables' buffer size to bcast is %d, which is " |
| 560 | "NOT equal to places size %d", |
| 561 | buffers.size(), |
| 562 | member_->places_.size())); |
| 563 | if (member_->nccl_ctxs_ != nullptr) { |
| 564 | auto *nccl_ctxs = member_->nccl_ctxs_->DefaultFlatCtx(); |
| 565 | platform::NCCLGroupGuard guard; |
| 566 | for (size_t i = 0; i < member_->places_.size(); ++i) { |
| 567 | auto &nccl_ctx = nccl_ctxs->at(member_->places_[i]); |
| 568 | phi::dynload::ncclBcast(buffers[i], |
| 569 | numel, |
| 570 | data_type, |
| 571 | 0, |
| 572 | nccl_ctx.comm_, |
| 573 | nccl_ctx.stream()); |
| 574 | } |
| 575 | nccl_ctxs->WaitAll(); |
nothing calls this directly
no test coverage detected