Parse all the args.
()
| 302 | return parser |
| 303 | |
| 304 | def get_args(): |
| 305 | """Parse all the args.""" |
| 306 | |
| 307 | parser = argparse.ArgumentParser(description='PyTorch BERT Model') |
| 308 | parser = add_model_config_args(parser) |
| 309 | parser = add_fp16_config_args(parser) |
| 310 | parser = add_training_args(parser) |
| 311 | parser = add_evaluation_args(parser) |
| 312 | parser = add_text_generate_args(parser) |
| 313 | parser = add_data_args(parser) |
| 314 | |
| 315 | # Include DeepSpeed configuration arguments |
| 316 | parser = deepspeed.add_config_arguments(parser) |
| 317 | |
| 318 | args = parser.parse_args() |
| 319 | |
| 320 | if not args.train_data and not args.train_data_path: |
| 321 | print('WARNING: No training data specified') |
| 322 | |
| 323 | args.cuda = torch.cuda.is_available() |
| 324 | |
| 325 | args.rank = int(os.getenv('RANK', '0')) |
| 326 | args.world_size = int(os.getenv("WORLD_SIZE", '1')) |
| 327 | |
| 328 | if os.getenv('OMPI_COMM_WORLD_LOCAL_RANK'): |
| 329 | # We are using (OpenMPI) mpirun for launching distributed data parallel processes |
| 330 | local_rank = int(os.getenv('OMPI_COMM_WORLD_LOCAL_RANK')) |
| 331 | local_size = int(os.getenv('OMPI_COMM_WORLD_LOCAL_SIZE')) |
| 332 | |
| 333 | # Possibly running with Slurm |
| 334 | num_nodes = int(os.getenv('SLURM_JOB_NUM_NODES', '1')) |
| 335 | nodeid = int(os.getenv('SLURM_NODEID', '0')) |
| 336 | |
| 337 | args.local_rank = local_rank |
| 338 | args.rank = nodeid*local_size + local_rank |
| 339 | args.world_size = num_nodes*local_size |
| 340 | |
| 341 | args.model_parallel_size = min(args.model_parallel_size, args.world_size) |
| 342 | if args.rank == 0: |
| 343 | print('using world size: {} and model-parallel size: {} '.format( |
| 344 | args.world_size, args.model_parallel_size)) |
| 345 | |
| 346 | args.dynamic_loss_scale = False |
| 347 | if args.loss_scale is None: |
| 348 | args.dynamic_loss_scale = True |
| 349 | if args.rank == 0: |
| 350 | print(' > using dynamic loss scaling') |
| 351 | |
| 352 | # The args fp32_* or fp16_* meant to be active when the |
| 353 | # args fp16 is set. So the default behaviour should all |
| 354 | # be false. |
| 355 | if not args.fp16: |
| 356 | args.fp32_embedding = False |
| 357 | args.fp32_tokentypes = False |
| 358 | args.fp32_layernorm = False |
| 359 | |
| 360 | return args |
no test coverage detected