| 353 | } |
| 354 | |
| 355 | void ParticleCopyPlan::doHandShakeReduceScatter (const Vector<Long>& Snds, Vector<Long>& Rcvs) |
| 356 | { |
| 357 | #ifdef AMREX_USE_MPI |
| 358 | const int SeqNum = ParallelDescriptor::SeqNum(); |
| 359 | const int NProcs = ParallelContext::NProcsSub(); |
| 360 | |
| 361 | Vector<Long> snd_connectivity(NProcs, 0); |
| 362 | Vector<int > rcv_connectivity(NProcs, 1); |
| 363 | for (int i = 0; i < NProcs; ++i) { if (Snds[i] > 0) { snd_connectivity[i] = 1; } } |
| 364 | |
| 365 | Long num_rcvs = 0; |
| 366 | MPI_Reduce_scatter(snd_connectivity.data(), &num_rcvs, rcv_connectivity.data(), |
| 367 | ParallelDescriptor::Mpi_typemap<Long>::type(), MPI_SUM, |
| 368 | ParallelContext::CommunicatorSub()); |
| 369 | |
| 370 | Vector<MPI_Status> rstats(num_rcvs); |
| 371 | Vector<MPI_Request> rreqs(num_rcvs); |
| 372 | Vector<MPI_Status> sstats; |
| 373 | Vector<MPI_Request> sreqs; |
| 374 | |
| 375 | Vector<Long> num_bytes_rcv(num_rcvs); |
| 376 | for (int i = 0; i < static_cast<int>(num_rcvs); ++i) |
| 377 | { |
| 378 | BL_MPI_REQUIRE(MPI_Irecv( &num_bytes_rcv[i], 1, ParallelDescriptor::Mpi_typemap<Long>::type(), |
| 379 | MPI_ANY_SOURCE, SeqNum, ParallelContext::CommunicatorSub(), &rreqs[i] )); |
| 380 | } |
| 381 | for (int i = 0; i < NProcs; ++i) |
| 382 | { |
| 383 | if (Snds[i] == 0) { continue; } |
| 384 | const Long Cnt = 1; |
| 385 | sreqs.push_back(ParallelDescriptor::Asend( &Snds[i], Cnt, i, SeqNum, ParallelContext::CommunicatorSub()).req()); |
| 386 | } |
| 387 | |
| 388 | sstats.resize(0); |
| 389 | sstats.resize(sreqs.size()); |
| 390 | ParallelDescriptor::Waitall(sreqs, sstats); |
| 391 | ParallelDescriptor::Waitall(rreqs, rstats); |
| 392 | |
| 393 | for (int i = 0; i < num_rcvs; ++i) |
| 394 | { |
| 395 | const auto Who = rstats[i].MPI_SOURCE; |
| 396 | Rcvs[Who] = num_bytes_rcv[i]; |
| 397 | } |
| 398 | #else |
| 399 | amrex::ignore_unused(Snds,Rcvs); |
| 400 | #endif |
| 401 | } |
| 402 | |
| 403 | void ParticleCopyPlan::doHandShakeOneSided (const ParticleContainerBase& pc, |
| 404 | const Vector<Long>& Snds, |