Args: rank: Unique identifier of each process world_size: Total number of processes
(rank, world_size)
| 11 | |
| 12 | |
| 13 | def ddp_setup(rank, world_size): |
| 14 | """ |
| 15 | Args: |
| 16 | rank: Unique identifier of each process |
| 17 | world_size: Total number of processes |
| 18 | """ |
| 19 | os.environ["MASTER_ADDR"] = "localhost" |
| 20 | os.environ["MASTER_PORT"] = "12355" |
| 21 | torch.cuda.set_device(rank) |
| 22 | init_process_group(backend="nccl", rank=rank, world_size=world_size) |
| 23 | |
| 24 | class Trainer: |
| 25 | def __init__( |