Function create_gpu_streams

benchmarks/benchmark_allgather.cpp:184–197

Create all needed streams one each GPU

// Create all needed streams one each GPU
static void create_gpu_streams(
    std::vector<std::vector<cudaStream_t>>* streams,
    const int gpus,
    int STREAMS_PER_GPU)
{
  streams->resize(gpus);
  for (int i = 0; i < gpus; ++i) {
    CUDA_CHECK(cudaSetDevice(i));
    (*streams)[i].resize(STREAMS_PER_GPU);
    for (int j = 0; j < STREAMS_PER_GPU; ++j) {
      cudaStreamCreateWithFlags(&((*streams)[i][j]), cudaStreamNonBlocking);
    }
  }
}

// Synchronize all streams on all GPUs
static void sync_all_streams(