--------------------- MPI comm: reduce nb of levels, blocks and origin
| 105 | // --------------------- |
| 106 | // MPI comm: reduce nb of levels, blocks and origin |
| 107 | void GatherInfos(LocalInfo& rankInfo, GlobalInfo& globalInfo) |
| 108 | { |
| 109 | vtkMultiProcessController* controller = vtkMultiProcessController::GetGlobalController(); |
| 110 | |
| 111 | const vtkIdType levels_local = vtkIdType(rankInfo.BlocksPerLevel.size()); |
| 112 | |
| 113 | if (globalInfo.NbOfProcesses == 1) |
| 114 | { |
| 115 | globalInfo.NbOfLevels = levels_local; |
| 116 | std::copy(rankInfo.Origin, rankInfo.Origin + 3, globalInfo.Origin); |
| 117 | } |
| 118 | else if (controller) |
| 119 | { |
| 120 | controller->AllReduce(&levels_local, &globalInfo.NbOfLevels, 1, vtkCommunicator::MAX_OP); |
| 121 | controller->AllReduce(rankInfo.Origin, globalInfo.Origin, 3, vtkCommunicator::MIN_OP); |
| 122 | } |
| 123 | |
| 124 | // need the total number of blocks across all processes |
| 125 | rankInfo.BlocksPerLevel.resize(globalInfo.NbOfLevels, 0); // set the extra values created to 0 |
| 126 | globalInfo.BlocksPerLevelAndRank.resize(globalInfo.NbOfLevels * globalInfo.NbOfProcesses); |
| 127 | // the ordering of the blocks for AMR is first all level 0 blocks, then all level 1 blocks, ... |
| 128 | // at each level we order based on proc rank first and then local id |
| 129 | if (globalInfo.NbOfProcesses == 1) |
| 130 | { |
| 131 | globalInfo.BlocksPerLevelAndRank = rankInfo.BlocksPerLevel; |
| 132 | } |
| 133 | else if (controller) |
| 134 | { |
| 135 | controller->AllGather(rankInfo.BlocksPerLevel.data(), globalInfo.BlocksPerLevelAndRank.data(), |
| 136 | globalInfo.NbOfLevels); |
| 137 | } |
| 138 | |
| 139 | rankInfo.NbOfBlocks = vtkIdType(rankInfo.DomainBlockLevelIds.size()); |
| 140 | globalInfo.NbOfBlocks = std::accumulate( |
| 141 | globalInfo.BlocksPerLevelAndRank.begin(), globalInfo.BlocksPerLevelAndRank.end(), 0); |
| 142 | |
| 143 | // the offset for the start of each block at each level |
| 144 | rankInfo.BlockOffsets.resize(globalInfo.NbOfLevels, 0); |
| 145 | if (globalInfo.NbOfProcesses > 1) |
| 146 | { |
| 147 | for (vtkIdType level = 0; level < globalInfo.NbOfLevels; level++) |
| 148 | { |
| 149 | vtkIdType offset(0); |
| 150 | for (int rank = 0; rank < rankInfo.Rank; rank++) |
| 151 | { |
| 152 | offset += globalInfo.BlocksPerLevelAndRank[level + rank * globalInfo.NbOfLevels]; |
| 153 | } |
| 154 | rankInfo.BlockOffsets[level] = offset; |
| 155 | } |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | // --------------------- |
| 160 | // initialize AMR: each rank has same structure |