(self, batch_samples: List[Dict[str, Any]])
| 155 | |
| 156 | # batchify for single format(sample) |
| 157 | def batchify(self, batch_samples: List[Dict[str, Any]]) -> Dict[str, Any]: |
| 158 | batch_text_vec = torch.tensor(pad_sequences( |
| 159 | [sample['text_vec'] for sample in batch_samples], pad_value=self.tokenizer.pad_token_id, padding='left' |
| 160 | ), dtype=torch.long) |
| 161 | return { |
| 162 | 'text_vec': batch_text_vec, |
| 163 | 'text': [sample['text'] for sample in batch_samples] |
| 164 | } |
| 165 | |
| 166 | def batch_generator(self): |
| 167 | while True: |
nothing calls this directly
no test coverage detected