(args)
| 36 | return _GLOBAL_TOKENIZER |
| 37 | |
| 38 | def build_tokenizer(args): |
| 39 | |
| 40 | if args.rank == 0: |
| 41 | print('> building {} tokenizer ...'.format(args.patch_tokenizer_type)) |
| 42 | # Select and instantiate the tokenizer. |
| 43 | if args.patch_tokenizer_type == 'JiebaBPETokenizer': |
| 44 | from .jiebabpe_tokenizer import JiebaBPETokenizer |
| 45 | tokenizer = JiebaBPETokenizer(args.patch_vocab_file) |
| 46 | args.padded_vocab_size = _vocab_size_with_padding( |
| 47 | tokenizer.vocab_size, args) |
| 48 | elif args.patch_tokenizer_type == 'BloomTokenizerFromHF': |
| 49 | from transformers import BloomTokenizerFast as BloomTokenizer |
| 50 | if args.load is None: |
| 51 | tokenizer = BloomTokenizer.from_pretrained('bigscience/bloom-560m') |
| 52 | else: |
| 53 | tokenizer = BloomTokenizer.from_pretrained(args.load) |
| 54 | args.padded_vocab_size = 250880 |
| 55 | elif args.patch_tokenizer_type == 'ChatGLMTokenizerFromHF': |
| 56 | tokenizer = AutoTokenizer.from_pretrained('THUDM/chatglm-6b', |
| 57 | trust_remote_code=True) |
| 58 | args.padded_vocab_size = 130528 |
| 59 | elif args.patch_tokenizer_type == 'GLM10BZHTokenizerFromHF': |
| 60 | tokenizer = AutoTokenizer.from_pretrained('THUDM/glm-10b-chinese', |
| 61 | trust_remote_code=True) |
| 62 | args.padded_vocab_size = 50048 |
| 63 | elif args.patch_tokenizer_type == 'IcetkGLM130BTokenizer': |
| 64 | from .icetk_glm130b_tokenizer import _IceTokenizer |
| 65 | tokenizer = _IceTokenizer() |
| 66 | args.padded_vocab_size = 150528 |
| 67 | elif args.patch_tokenizer_type == 'OPTTokenizer': |
| 68 | tokenizer = AutoTokenizer.from_pretrained( |
| 69 | args.load, |
| 70 | model_max_length=args.seq_length, |
| 71 | padding_side='right', |
| 72 | use_fast=False, |
| 73 | ) |
| 74 | DEFAULT_PAD_TOKEN = '<pad>' |
| 75 | DEFAULT_EOS_TOKEN = '</s>' |
| 76 | DEFAULT_BOS_TOKEN = '<s>' |
| 77 | DEFAULT_UNK_TOKEN = '<unk>' |
| 78 | |
| 79 | special_tokens_dict = dict() |
| 80 | if not tokenizer.pad_token: |
| 81 | special_tokens_dict['pad_token'] = DEFAULT_PAD_TOKEN |
| 82 | if not tokenizer.eos_token: |
| 83 | special_tokens_dict['eos_token'] = DEFAULT_EOS_TOKEN |
| 84 | if not tokenizer.bos_token: |
| 85 | special_tokens_dict['bos_token'] = DEFAULT_BOS_TOKEN |
| 86 | if not tokenizer.unk_token: |
| 87 | special_tokens_dict['unk_token'] = DEFAULT_UNK_TOKEN |
| 88 | tokenizer.add_special_tokens(special_tokens_dict) |
| 89 | args.padded_vocab_size = tokenizer.vocab_size + args.extra_vocab_size |
| 90 | |
| 91 | elif args.patch_tokenizer_type == 'LLamaTokenizer': |
| 92 | tokenizer = AutoTokenizer.from_pretrained( |
| 93 | args.load, |
| 94 | model_max_length=args.seq_length, |
| 95 | padding_side="right", |
no test coverage detected