| 1258 | @classmethod |
| 1259 | @typechecked |
| 1260 | def main( |
| 1261 | cls, |
| 1262 | args: Optional[argparse.Namespace] = None, |
| 1263 | cmd: Optional[Sequence[str]] = None, |
| 1264 | ): |
| 1265 | print(get_commandline_args(), file=sys.stderr) |
| 1266 | if args is None: |
| 1267 | parser = cls.get_parser() |
| 1268 | args = parser.parse_args(cmd) |
| 1269 | args.version = __version__ |
| 1270 | if args.pretrain_path is not None: |
| 1271 | raise RuntimeError("--pretrain_path is deprecated. Use --init_param") |
| 1272 | if args.print_config: |
| 1273 | cls.print_config() |
| 1274 | sys.exit(0) |
| 1275 | cls.check_required_command_args(args) |
| 1276 | |
| 1277 | # "distributed" is decided using the other command args |
| 1278 | resolve_distributed_mode(args) |
| 1279 | if not args.distributed or not args.multiprocessing_distributed: |
| 1280 | cls.main_worker(args) |
| 1281 | |
| 1282 | else: |
| 1283 | assert args.ngpu > 1, args.ngpu |
| 1284 | # Multi-processing distributed mode: e.g. 2node-4process-4GPU |
| 1285 | # | Host1 | Host2 | |
| 1286 | # | Process1 | Process2 | <= Spawn processes |
| 1287 | # |Child1|Child2|Child1|Child2| |
| 1288 | # |GPU1 |GPU2 |GPU1 |GPU2 | |
| 1289 | |
| 1290 | # See also the following usage of --multiprocessing-distributed: |
| 1291 | # https://github.com/pytorch/examples/blob/master/imagenet/main.py |
| 1292 | num_nodes = get_num_nodes(args.dist_world_size, args.dist_launcher) |
| 1293 | if num_nodes == 1: |
| 1294 | args.dist_master_addr = "localhost" |
| 1295 | args.dist_rank = 0 |
| 1296 | # Single node distributed training with multi-GPUs |
| 1297 | if ( |
| 1298 | args.dist_init_method == "env://" |
| 1299 | and get_master_port(args.dist_master_port) is None |
| 1300 | ): |
| 1301 | # Get the unused port |
| 1302 | args.dist_master_port = free_port() |
| 1303 | |
| 1304 | # Assume that nodes use same number of GPUs each other |
| 1305 | args.dist_world_size = args.ngpu * num_nodes |
| 1306 | node_rank = get_node_rank(args.dist_rank, args.dist_launcher) |
| 1307 | |
| 1308 | # The following block is copied from: |
| 1309 | # https://github.com/pytorch/pytorch/blob/master/torch/ |
| 1310 | # multiprocessing/spawn.py |
| 1311 | error_files = [] |
| 1312 | processes = [] |
| 1313 | mp = torch.multiprocessing.get_context("spawn") |
| 1314 | for i in range(args.ngpu): |
| 1315 | |
| 1316 | # Each process is assigned a file to write tracebacks to. We |
| 1317 | # use the file being non-empty to indicate an exception |