(self, seq, pad_id=None)
| 561 | return tokens, targets, loss_masks, attention_mask |
| 562 | |
| 563 | def pad_seq(self, seq, pad_id=None): |
| 564 | total_tokens = self.max_seq_len |
| 565 | num_pad_tokens = max(0, total_tokens - len(seq)) |
| 566 | seq += [self.tokenizer.get_command('pad').Id if pad_id is None else pad_id] * (num_pad_tokens) |
| 567 | return seq |
| 568 | |
| 569 | |
| 570 | class BlockDataset(data.Dataset): |
no test coverage detected