(self, idx)
| 525 | return self.num_samples |
| 526 | |
| 527 | def __getitem__(self, idx): |
| 528 | tokens, targets, loss_mask, attention_mask = self.getidx(idx) |
| 529 | tokens = self.pad_seq(tokens) |
| 530 | targets = self.pad_seq(targets) |
| 531 | loss_mask = self.pad_seq(loss_mask, pad_id=0) |
| 532 | return {'text': np.array(tokens), "target": np.array(targets), "loss_mask": np.array(loss_mask), |
| 533 | "attention_mask": np.array(attention_mask)} |
| 534 | |
| 535 | def getidx(self, idx): |
| 536 | tokens, targets, loss_masks = [], [], [] |