Initialize tokenizer.
(args)
| 69 | |
| 70 | |
| 71 | def build_tokenizer(args): |
| 72 | """Initialize tokenizer.""" |
| 73 | if "tokenizer_path" in args and args.tokenizer_path is not None: |
| 74 | # build huggingface tokenizer |
| 75 | tokenizer = build_hgf_tokenizer(args) |
| 76 | args.padded_vocab_size = _vocab_size_with_padding(tokenizer.vocab_size, args) |
| 77 | return tokenizer |
| 78 | |
| 79 | if args.rank == 0: |
| 80 | print("> building {} tokenizer ...".format(args.tokenizer_type), flush=True) |
| 81 | |
| 82 | # Select and instantiate the tokenizer. |
| 83 | assert args.vocab_file is not None |
| 84 | if args.tokenizer_type == "GPT2BPETokenizer": |
| 85 | assert args.merge_file is not None |
| 86 | tokenizer = _GPT2BPETokenizer(args.vocab_file, args.merge_file) |
| 87 | else: |
| 88 | raise NotImplementedError( |
| 89 | "{} tokenizer is not " "implemented.".format(args.tokenizer_type) |
| 90 | ) |
| 91 | |
| 92 | # Add vocab size. |
| 93 | args.padded_vocab_size = _vocab_size_with_padding(tokenizer.vocab_size, args) |
| 94 | |
| 95 | return tokenizer |
| 96 | |
| 97 | |
| 98 | def _vocab_size_with_padding(orig_vocab_size, args): |
no test coverage detected