| 295 | } |
| 296 | |
| 297 | void |
| 298 | StartParallel (int* argc, char*** argv, MPI_Comm a_mpi_comm) |
| 299 | { |
| 300 | int sflag(0); |
| 301 | MPI_Initialized(&sflag); |
| 302 | |
| 303 | if ( ! sflag) { |
| 304 | |
| 305 | #ifdef AMREX_MPI_THREAD_MULTIPLE |
| 306 | int requested = MPI_THREAD_MULTIPLE; |
| 307 | int provided = -1; |
| 308 | |
| 309 | MPI_Init_thread(argc, argv, requested, &provided); |
| 310 | #else // |
| 311 | MPI_Init(argc, argv); |
| 312 | #endif |
| 313 | |
| 314 | m_comm = MPI_COMM_WORLD; |
| 315 | call_mpi_finalize = 1; |
| 316 | } else { |
| 317 | MPI_Comm_dup(a_mpi_comm, &m_comm); |
| 318 | call_mpi_finalize = 0; |
| 319 | } |
| 320 | |
| 321 | // It seems that for some MPI implementation, the first call to MPI_Wtime is always 0. That |
| 322 | // sometimes causes problems for amrex::UniqueString function. So we call MPI_Wtime here. |
| 323 | auto tfoo = MPI_Wtime(); |
| 324 | amrex::ignore_unused(tfoo); |
| 325 | |
| 326 | #ifdef AMREX_MPI_THREAD_MULTIPLE |
| 327 | if ( ! sflag) { // we initialized |
| 328 | int requested = MPI_THREAD_MULTIPLE; |
| 329 | int provided = -1; |
| 330 | MPI_Query_thread(&provided); |
| 331 | |
| 332 | if (provided < requested) |
| 333 | { |
| 334 | auto f = ParallelDescriptor::mpi_level_to_string; |
| 335 | std::cout << "MPI provided < requested: " << f(provided) << " < " |
| 336 | << f(requested) << '\n';; |
| 337 | std::abort(); |
| 338 | } |
| 339 | } |
| 340 | #endif |
| 341 | |
| 342 | ParallelContext::push(m_comm); |
| 343 | |
| 344 | if (ParallelDescriptor::NProcs() > 1) |
| 345 | { |
| 346 | #if defined(OPEN_MPI) |
| 347 | int split_type = OMPI_COMM_TYPE_NODE; |
| 348 | #else |
| 349 | int split_type = MPI_COMM_TYPE_SHARED; |
| 350 | #endif |
| 351 | MPI_Comm node_comm; |
| 352 | MPI_Comm_split_type(m_comm, split_type, 0, MPI_INFO_NULL, &node_comm); |
| 353 | MPI_Comm_size(node_comm, &m_nprocs_per_node); |
| 354 | MPI_Comm_rank(node_comm, &m_rank_in_node); |