Function StartTeams

Src/Base/AMReX_ParallelDescriptor.cpp:1598–1652

#ifndef BL_AMRPROF
void
StartTeams ()
{
    int nprocs = ParallelDescriptor::NProcs();
    int rank   = ParallelDescriptor::MyProc();

    int team_size = 1;
    int do_team_reduce = 0;

#if defined(BL_USE_MPI3)
    ParmParse pp("amrex.team");
    pp.query("size", team_size);
    pp.query("reduce", do_team_reduce);
    if (team_size <= 0) {
        amrex::Abort("amrex.team.size must be > 0");
    }
    if (nprocs % team_size != 0) {
        amrex::Abort("Number of processes not divisible by team size");
    }
#endif

    m_Team.m_numTeams    = nprocs / team_size;
    m_Team.m_size        = team_size;
    m_Team.m_color       = rank / team_size;
    m_Team.m_lead        = m_Team.m_color * team_size;
    m_Team.m_rankInTeam  = rank - m_Team.m_lead;

    m_Team.m_do_team_reduce = team_size > 0 && do_team_reduce;

#if defined(BL_USE_MPI3)
    {
        MPI_Group grp, team_grp, lead_grp;
        BL_MPI_REQUIRE( MPI_Comm_group(ParallelDescriptor::Communicator(), &grp) );
        Vector<int> team_ranks(team_size);
        for (int i = 0; i < team_size; ++i) {
            team_ranks[i] = MyTeamLead() + i;
        }
        BL_MPI_REQUIRE( MPI_Group_incl(grp, team_size, team_ranks.data(), &team_grp) );
        BL_MPI_REQUIRE( MPI_Comm_create(ParallelDescriptor::Communicator(),
                                        team_grp, &m_Team.m_team_comm) );

        std::vector<int>lead_ranks(m_Team.m_numTeams);
        for (int i = 0; i < lead_ranks.size(); ++i) {
            lead_ranks[i] = i * team_size;
        }
        BL_MPI_REQUIRE( MPI_Group_incl(grp, lead_ranks.size(), lead_ranks.data(), &lead_grp) );
        BL_MPI_REQUIRE( MPI_Comm_create(ParallelDescriptor::Communicator(),
                                        lead_grp, &m_Team.m_lead_comm) );

        BL_MPI_REQUIRE( MPI_Group_free(&grp) );
        BL_MPI_REQUIRE( MPI_Group_free(&team_grp) );
        BL_MPI_REQUIRE( MPI_Group_free(&lead_grp) );
    }
#endif
}
#endif

void