(args)
| 407 | return {'text': np.array(tokens), 'pad_mask': pad_mask} |
| 408 | |
| 409 | def get_tokenizer(args): |
| 410 | tokenizer_args = { |
| 411 | 'tokenizer_type': args.tokenizer_type, |
| 412 | 'corpus': None, |
| 413 | 'model_path': args.tokenizer_path, |
| 414 | 'vocab_size': args.vocab_size, |
| 415 | 'model_type': args.tokenizer_model_type, |
| 416 | 'cache_dir': args.cache_dir} |
| 417 | return make_tokenizer(**tokenizer_args) |
| 418 | |
| 419 | def get_eval_data(args): |
| 420 | val_dataloader = None |
no test coverage detected