| 115 | } |
| 116 | |
| 117 | void DevicePair::compute(const vector<int> devices, vector<DevicePair>* pairs) { |
| 118 | #ifndef CPU_ONLY |
| 119 | vector<int> remaining(devices); |
| 120 | |
| 121 | // Depth for reduction tree |
| 122 | int remaining_depth = static_cast<int>(ceil(log2(remaining.size()))); |
| 123 | |
| 124 | // Group GPUs by board |
| 125 | for (int d = 0; d < remaining_depth; ++d) { |
| 126 | for (int i = 0; i < remaining.size(); ++i) { |
| 127 | for (int j = i + 1; j < remaining.size(); ++j) { |
| 128 | cudaDeviceProp a, b; |
| 129 | CUDA_CHECK(cudaGetDeviceProperties(&a, remaining[i])); |
| 130 | CUDA_CHECK(cudaGetDeviceProperties(&b, remaining[j])); |
| 131 | if (a.isMultiGpuBoard && b.isMultiGpuBoard) { |
| 132 | if (a.multiGpuBoardGroupID == b.multiGpuBoardGroupID) { |
| 133 | pairs->push_back(DevicePair(remaining[i], remaining[j])); |
| 134 | DLOG(INFO) << "GPU board: " << remaining[i] << ":" << remaining[j]; |
| 135 | remaining.erase(remaining.begin() + j); |
| 136 | break; |
| 137 | } |
| 138 | } |
| 139 | } |
| 140 | } |
| 141 | } |
| 142 | ostringstream s; |
| 143 | for (int i = 0; i < remaining.size(); ++i) { |
| 144 | s << (i ? ", " : "") << remaining[i]; |
| 145 | } |
| 146 | DLOG(INFO) << "GPUs paired by boards, remaining: " << s.str(); |
| 147 | |
| 148 | // Group by P2P accessibility |
| 149 | remaining_depth = ceil(log2(remaining.size())); |
| 150 | for (int d = 0; d < remaining_depth; ++d) { |
| 151 | for (int i = 0; i < remaining.size(); ++i) { |
| 152 | for (int j = i + 1; j < remaining.size(); ++j) { |
| 153 | int access; |
| 154 | CUDA_CHECK( |
| 155 | cudaDeviceCanAccessPeer(&access, remaining[i], remaining[j])); |
| 156 | if (access) { |
| 157 | pairs->push_back(DevicePair(remaining[i], remaining[j])); |
| 158 | DLOG(INFO) << "P2P pair: " << remaining[i] << ":" << remaining[j]; |
| 159 | remaining.erase(remaining.begin() + j); |
| 160 | break; |
| 161 | } |
| 162 | } |
| 163 | } |
| 164 | } |
| 165 | s.str(""); |
| 166 | for (int i = 0; i < remaining.size(); ++i) { |
| 167 | s << (i ? ", " : "") << remaining[i]; |
| 168 | } |
| 169 | DLOG(INFO) << "GPUs paired by P2P access, remaining: " << s.str(); |
| 170 | |
| 171 | // Group remaining |
| 172 | remaining_depth = ceil(log2(remaining.size())); |
| 173 | for (int d = 0; d < remaining_depth; ++d) { |
| 174 | for (int i = 0; i < remaining.size(); ++i) { |
nothing calls this directly
no test coverage detected