(self, data_idx)
| 682 | return tokens, loss_mask |
| 683 | |
| 684 | def getidx(self, data_idx): |
| 685 | data = self.ds[data_idx] |
| 686 | tokens, loss_masks = data['tokens'], data['loss_masks'] |
| 687 | tokens = tokens + [self.tokenizer.get_command('eos').Id] |
| 688 | loss_masks = loss_masks + [1] |
| 689 | return tokens, loss_masks |
| 690 | |
| 691 | def pad_seq(self, seq, pad_id=None): |
| 692 | total_tokens = self.max_seq_len |
no test coverage detected