(args)
| 301 | return parser |
| 302 | |
| 303 | def _adjust_vocab_size(args): |
| 304 | before = args.vocab_size |
| 305 | after = before |
| 306 | multiple = args.make_vocab_size_divisible_by |
| 307 | # you should control args to let it divided by |
| 308 | # mpu.get_model_parallel_world_size() |
| 309 | while (after % multiple) != 0: |
| 310 | after += 1 |
| 311 | if args.rank == 0: |
| 312 | print_rank0('> padded vocab (size: {}) with {} dummy ' |
| 313 | 'tokens (new size: {})'.format( |
| 314 | before, after - before, after)) |
| 315 | |
| 316 | def _simple_init(model_parallel_size=1, seed=0): |
| 317 | '''Necessary initialization for torch.distributed for model-only mode''' |
no test coverage detected