| 240 | } |
| 241 | |
| 242 | void MeshReader::exchangeGhostlayerMetadata() { |
| 243 | #ifdef USE_MPI |
| 244 | std::unordered_map<int, std::vector<GhostElementMetadata>> sendData; |
| 245 | std::unordered_map<int, std::vector<GhostElementMetadata>> recvData; |
| 246 | |
| 247 | constexpr int Tag = 10; |
| 248 | MPI_Comm comm = seissol::MPI::mpi.comm(); |
| 249 | |
| 250 | std::vector<MPI_Request> requests(m_MPINeighbors.size() * 2); |
| 251 | |
| 252 | // TODO(David): Once a generic MPI type inference module is ready, replace this part here ... |
| 253 | // Maybe. |
| 254 | MPI_Datatype ghostElementType = MPI_DATATYPE_NULL; |
| 255 | |
| 256 | // assume that all vertices are stored contiguously |
| 257 | const int datatypeCount = 3; |
| 258 | const std::vector<int> datatypeBlocklen{12, 1, 1}; |
| 259 | const std::vector<MPI_Aint> datatypeDisplacement{offsetof(GhostElementMetadata, vertices), |
| 260 | offsetof(GhostElementMetadata, group), |
| 261 | offsetof(GhostElementMetadata, globalId)}; |
| 262 | const std::vector<MPI_Datatype> datatypeDatatype{ |
| 263 | MPI_DOUBLE, MPI_INT, PUML::MPITypeInfer<GlobalElemId>::type()}; |
| 264 | |
| 265 | MPI_Type_create_struct(datatypeCount, |
| 266 | datatypeBlocklen.data(), |
| 267 | datatypeDisplacement.data(), |
| 268 | datatypeDatatype.data(), |
| 269 | &ghostElementType); |
| 270 | MPI_Type_commit(&ghostElementType); |
| 271 | |
| 272 | size_t counter = 0; |
| 273 | for (auto it = m_MPINeighbors.begin(); it != m_MPINeighbors.end(); ++it, counter += 2) { |
| 274 | const auto targetRank = it->first; |
| 275 | const auto count = it->second.elements.size(); |
| 276 | |
| 277 | recvData[targetRank].resize(count); |
| 278 | |
| 279 | sendData[targetRank].resize(count); |
| 280 | for (size_t j = 0; j < count; ++j) { |
| 281 | const auto elementIdx = it->second.elements[j].localElement; |
| 282 | const auto& element = m_elements.at(elementIdx); |
| 283 | auto& ghost = sendData[targetRank][j]; |
| 284 | |
| 285 | for (size_t v = 0; v < 4; ++v) { |
| 286 | const auto& vertex = m_vertices[element.vertices[v]]; |
| 287 | ghost.vertices[v][0] = vertex.coords[0]; |
| 288 | ghost.vertices[v][1] = vertex.coords[1]; |
| 289 | ghost.vertices[v][2] = vertex.coords[2]; |
| 290 | } |
| 291 | ghost.group = element.group; |
| 292 | ghost.globalId = element.globalId; |
| 293 | } |
| 294 | |
| 295 | // TODO(David): evaluate, if MPI_Ssend (instead of just MPI_Send) makes sense here? |
| 296 | MPI_Irecv(recvData[targetRank].data(), |
| 297 | count, |
| 298 | ghostElementType, |
| 299 | targetRank, |