(args)
| 346 | |
| 347 | |
| 348 | def load_tokenizer(args): |
| 349 | from transformers import AutoTokenizer |
| 350 | |
| 351 | return AutoTokenizer.from_pretrained( |
| 352 | args.tokenizer, |
| 353 | trust_remote_code=True, |
| 354 | local_files_only=bool(args.local_files_only), |
| 355 | ) |
| 356 | |
| 357 | |
| 358 | def load_target_tokenizer(args): |
no outgoing calls
no test coverage detected