| 87 | } // namespace lbann |
| 88 | |
| 89 | auto lbann::initialize_lbann(El::mpi::Comm&& c) -> std::unique_ptr<lbann_comm> |
| 90 | { |
| 91 | |
| 92 | // Parse command-line arguments and environment variables |
| 93 | auto& arg_parser = global_argument_parser(); |
| 94 | (void)arg_parser; |
| 95 | |
| 96 | // to ensure that all the necessary infrastructure in Hydrogen and |
| 97 | // Aluminum has been setup. |
| 98 | El::Initialize(); |
| 99 | |
| 100 | // Create a new comm object with provided MPI_Comm |
| 101 | auto comm = std::make_unique<lbann_comm>(0, std::move(c)); |
| 102 | world_comm_ = comm.get(); |
| 103 | |
| 104 | // Install MPI error handler |
| 105 | // MPI_Comm_create_errhandler(lbann_mpi_err_handler, &err_handle); |
| 106 | // MPI_Comm_set_errhandler(MPI_COMM_WORLD, err_handle); |
| 107 | |
| 108 | #if defined(LBANN_TOPO_AWARE) |
| 109 | // Determine the number of NUMA nodes present. |
| 110 | hwloc_topology_t topo; |
| 111 | hwloc_topology_init(&topo); |
| 112 | hwloc_topology_load(topo); |
| 113 | int numa_depth = hwloc_get_type_depth(topo, HWLOC_OBJ_NUMANODE); |
| 114 | if (numa_depth == HWLOC_TYPE_DEPTH_UNKNOWN) { |
| 115 | std::cout << comm->get_rank_in_world() |
| 116 | << ": cannot determine hwloc NUMA-node depth" << std::endl; |
| 117 | } |
| 118 | int num_numa_nodes = hwloc_get_nbobjs_by_depth(topo, numa_depth); |
| 119 | // Warn if there are more NUMA nodes than processes per node. |
| 120 | // It's probably fine if there are more processes than NUMA nodes for now. |
| 121 | // We can adjust that later when we better understand the threaded perf. |
| 122 | int ppn = comm->get_procs_per_node(); |
| 123 | if (num_numa_nodes > ppn) { |
| 124 | if (comm->get_rank_in_world() == 0) { |
| 125 | std::cout << comm->get_rank_in_world() << ": WARNING: node has " |
| 126 | << num_numa_nodes << " NUMA nodes but you have " << ppn |
| 127 | << " processes per node" << std::endl; |
| 128 | } |
| 129 | } |
| 130 | hwloc_topology_destroy(topo); |
| 131 | #endif |
| 132 | |
| 133 | #ifdef LBANN_HAS_CALIPER |
| 134 | initialize_caliper(); |
| 135 | #endif |
| 136 | |
| 137 | #ifdef LBANN_HAS_SHMEM |
| 138 | // Initialize SHMEM |
| 139 | if (arg_parser.get<bool>(LBANN_OPTION_INIT_SHMEM)) { |
| 140 | int threading_level = SHMEM_THREAD_MULTIPLE; |
| 141 | int status = shmem_init_thread(threading_level, &threading_level); |
| 142 | if (status != 0 || threading_level != SHMEM_THREAD_MULTIPLE) { |
| 143 | LBANN_ERROR("error initializing OpenSHMEM"); |
| 144 | } |
| 145 | } |
| 146 | #endif // LBANN_HAS_SHMEM |
nothing calls this directly
no test coverage detected