(model_path, spt_config_path, spt_checkpoint_path)
| 22 | |
| 23 | |
| 24 | def load_tokenizer(model_path, spt_config_path, spt_checkpoint_path): |
| 25 | tokenizer = AutoTokenizer.from_pretrained(model_path) |
| 26 | spt = XY_Tokenizer.load_from_checkpoint( |
| 27 | config_path=spt_config_path, ckpt_path=spt_checkpoint_path |
| 28 | ) |
| 29 | spt.eval() |
| 30 | return tokenizer, spt |
| 31 | |
| 32 | |
| 33 | def process_inputs( |
no test coverage detected