| 13 | |
| 14 | |
| 15 | def init_distributed(n_gpus=-1, rank=-1, auto_detect: bool = True): |
| 16 | print("Initializing Distributed", flush=True) |
| 17 | |
| 18 | if auto_detect: |
| 19 | dist.init_process_group( |
| 20 | backend="nccl", |
| 21 | timeout=datetime.timedelta(weeks=2), |
| 22 | ) |
| 23 | else: |
| 24 | # Set cuda device so everything is done on the right GPU. |
| 25 | if os.environ.get("MASTER_ADDR", None) is None or os.environ.get("MASTER_PORT", None) is None: |
| 26 | os.environ["MASTER_ADDR"] = "localhost" |
| 27 | os.environ["MASTER_PORT"] = "12355" # f'{port_number}' # '12355' |
| 28 | |
| 29 | # Initialize distributed communication |
| 30 | dist.init_process_group( |
| 31 | backend="nccl", |
| 32 | world_size=n_gpus, |
| 33 | rank=rank, |
| 34 | timeout=datetime.timedelta(weeks=2), |
| 35 | ) |
| 36 | |
| 37 | print( |
| 38 | "Done initializing distributed. rank = {}, world size = {}".format( |
| 39 | rank if rank != -1 else dist.get_rank(), |
| 40 | n_gpus if n_gpus != -1 else dist.get_world_size(), |
| 41 | ), |
| 42 | flush=True, |
| 43 | ) |
| 44 | |
| 45 | |
| 46 | def _flatten_dense_tensors(tensors): |