(self, texts, device)
| 580 | name=tokenizer_path, seq_len=text_len, clean='whitespace') |
| 581 | |
| 582 | def __call__(self, texts, device): |
| 583 | ids, mask = self.tokenizer( |
| 584 | texts, return_mask=True, add_special_tokens=True) |
| 585 | ids = ids.to(device) |
| 586 | mask = mask.to(device) |
| 587 | seq_lens = mask.gt(0).sum(dim=1).long() |
| 588 | context = self.model(ids, mask) |
| 589 | return [u[:v] for u, v in zip(context, seq_lens)] |