| 167 | return int(os.environ.get('SLURM_NTASKS', 1)) |
| 168 | |
| 169 | def dist_init(method='slurm', port='5671'): |
| 170 | assert method == 'slurm' |
| 171 | proc_id = int(os.environ['SLURM_PROCID']) |
| 172 | num_gpus = torch.cuda.device_count() |
| 173 | torch.cuda.set_device(proc_id % num_gpus) |
| 174 | |
| 175 | world_size = get_world_size() |
| 176 | rank = get_rank() |
| 177 | |
| 178 | addr = subprocess.getoutput( |
| 179 | "scontrol show hostname {} | head -n1".format(os.environ["SLURM_NODELIST"]) |
| 180 | ) |
| 181 | os.environ["MASTER_PORT"] = port |
| 182 | os.environ["MASTER_ADDR"] = addr |
| 183 | os.environ["WORLD_SIZE"] = str(world_size) |
| 184 | os.environ["RANK"] = str(rank) |
| 185 | |
| 186 | |
| 187 | dist.init_process_group( |
| 188 | backend="nccl", |
| 189 | world_size=world_size, |
| 190 | rank=rank, |
| 191 | ) |
| 192 | |
| 193 | return rank, world_size |
| 194 | |
| 195 | class DistributedGivenIterationSampler(Sampler): |
| 196 | def __init__(self, dataset, total_iter, batch_size, world_size=None, rank=None, last_iter=-1, |