Args: nnodes: The number of nodes to use for distributed call. nproc_per_node: The number of processes to call on each node. master_addr: Master node (rank 0)'s address, should be either the IP address or the hostname of node 0. master_port:
(
self,
nnodes: int = 1,
nproc_per_node: int = 1,
master_addr: str = "localhost",
master_port: int | None = None,
node_rank: int | None = None,
timeout=60,
init_method=None,
backend: str | None = None,
daemon: bool | None = None,
method: str | None = "spawn",
verbose: bool = False,
)
| 476 | """ |
| 477 | |
| 478 | def __init__( |
| 479 | self, |
| 480 | nnodes: int = 1, |
| 481 | nproc_per_node: int = 1, |
| 482 | master_addr: str = "localhost", |
| 483 | master_port: int | None = None, |
| 484 | node_rank: int | None = None, |
| 485 | timeout=60, |
| 486 | init_method=None, |
| 487 | backend: str | None = None, |
| 488 | daemon: bool | None = None, |
| 489 | method: str | None = "spawn", |
| 490 | verbose: bool = False, |
| 491 | ): |
| 492 | """ |
| 493 | |
| 494 | Args: |
| 495 | nnodes: The number of nodes to use for distributed call. |
| 496 | nproc_per_node: The number of processes to call on each node. |
| 497 | master_addr: Master node (rank 0)'s address, should be either the IP address or the hostname of node 0. |
| 498 | master_port: Master node (rank 0)'s free port. |
| 499 | node_rank: The rank of the node, this could be set via environment variable "NODE_RANK". |
| 500 | timeout: Timeout for operations executed against the process group. |
| 501 | init_method: URL specifying how to initialize the process group. |
| 502 | Default is "env://" or "file:///d:/a_temp" (windows) if unspecified. |
| 503 | If ``"no_init"``, the `dist.init_process_group` must be called within the code to be tested. |
| 504 | backend: The backend to use. Depending on build-time configurations, |
| 505 | valid values include ``mpi``, ``gloo``, and ``nccl``. |
| 506 | daemon: the process’s daemon flag. |
| 507 | When daemon=None, the initial value is inherited from the creating process. |
| 508 | method: set the method which should be used to start a child process. |
| 509 | method can be 'fork', 'spawn' or 'forkserver'. |
| 510 | verbose: whether to print NCCL debug info. |
| 511 | """ |
| 512 | self.nnodes = int(nnodes) |
| 513 | self.nproc_per_node = int(nproc_per_node) |
| 514 | if self.nnodes < 1 or self.nproc_per_node < 1: |
| 515 | raise ValueError( |
| 516 | f"number of nodes and processes per node must be >= 1, got {self.nnodes} and {self.nproc_per_node}" |
| 517 | ) |
| 518 | self.node_rank = int(os.environ.get("NODE_RANK", "0")) if node_rank is None else int(node_rank) |
| 519 | self.master_addr = master_addr |
| 520 | self.master_port = np.random.randint(10000, 20000) if master_port is None else master_port |
| 521 | |
| 522 | if backend is None: |
| 523 | self.backend = "nccl" if torch.distributed.is_nccl_available() and torch.cuda.is_available() else "gloo" |
| 524 | else: |
| 525 | self.backend = backend |
| 526 | self.init_method = init_method |
| 527 | if self.init_method is None and sys.platform == "win32": |
| 528 | self.init_method = "file:///d:/a_temp" |
| 529 | self.timeout = datetime.timedelta(0, timeout) |
| 530 | self.daemon = daemon |
| 531 | self.method = method |
| 532 | self.verbose = verbose |
| 533 | |
| 534 | def run_process(self, func, local_rank, args, kwargs, results): |
| 535 | _env = os.environ.copy() # keep the original system env |