(tokenizer_path, fast_tokenizer=True)
| 76 | return args |
| 77 | |
| 78 | def load_telechat_tokenizer(tokenizer_path, fast_tokenizer=True): |
| 79 | tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, |
| 80 | fast_tokenizer=fast_tokenizer, |
| 81 | padding_side="left", |
| 82 | trust_remote_code=True) |
| 83 | return tokenizer |
| 84 | |
| 85 | def main(): |
| 86 | args = parse_args() |