| 27 | multiprocessing_distributed: bool = True |
| 28 | |
| 29 | def init_options(self): |
| 30 | if self.distributed: |
| 31 | if self.dist_init_method == "env://": |
| 32 | if get_master_addr(self.dist_master_addr, self.dist_launcher) is None: |
| 33 | raise RuntimeError( |
| 34 | "--dist_master_addr or MASTER_ADDR must be set " |
| 35 | "if --dist_init_method == 'env://'" |
| 36 | ) |
| 37 | if get_master_port(self.dist_master_port) is None: |
| 38 | raise RuntimeError( |
| 39 | "--dist_master_port or MASTER_PORT must be set " |
| 40 | "if --dist_init_port == 'env://'" |
| 41 | ) |
| 42 | |
| 43 | # About priority order: |
| 44 | # If --dist_* is specified: |
| 45 | # Use the value of --dist_rank and overwrite it environ just in case. |
| 46 | # elif environ is set: |
| 47 | # Use the value of environ and set it to self |
| 48 | self.dist_rank = get_rank(self.dist_rank, self.dist_launcher) |
| 49 | self.dist_world_size = get_world_size( |
| 50 | self.dist_world_size, self.dist_launcher |
| 51 | ) |
| 52 | self.local_rank = get_local_rank(self.local_rank, self.dist_launcher) |
| 53 | |
| 54 | if self.local_rank is not None: |
| 55 | if self.ngpu > 1: |
| 56 | raise RuntimeError(f"Assuming 1GPU in this case: ngpu={self.ngpu}") |
| 57 | if "CUDA_VISIBLE_DEVICES" in os.environ: |
| 58 | cvd = os.environ["CUDA_VISIBLE_DEVICES"] |
| 59 | if self.local_rank >= len(cvd.split(",")): |
| 60 | raise RuntimeError( |
| 61 | f"LOCAL_RANK={self.local_rank} is bigger " |
| 62 | f"than the number of visible devices: {cvd}" |
| 63 | ) |
| 64 | |
| 65 | if ( |
| 66 | self.dist_rank is not None |
| 67 | and self.dist_world_size is not None |
| 68 | and self.dist_rank >= self.dist_world_size |
| 69 | ): |
| 70 | raise RuntimeError( |
| 71 | f"RANK >= WORLD_SIZE: {self.dist_rank} >= {self.dist_world_size}" |
| 72 | ) |
| 73 | |
| 74 | if self.dist_init_method == "env://": |
| 75 | self.dist_master_addr = get_master_addr( |
| 76 | self.dist_master_addr, self.dist_launcher |
| 77 | ) |
| 78 | self.dist_master_port = get_master_port(self.dist_master_port) |
| 79 | if ( |
| 80 | self.dist_master_addr is not None |
| 81 | and self.dist_master_port is not None |
| 82 | ): |
| 83 | self.dist_init_method = ( |
| 84 | f"tcp://{self.dist_master_addr}:{self.dist_master_port}" |
| 85 | ) |
| 86 | |