(self, tokens, loss_mask, seq_length)
| 670 | return {'text': np.array(tokens), "loss_mask": np.array(loss_mask)} |
| 671 | |
| 672 | def right_strip_seq(self, tokens, loss_mask, seq_length): |
| 673 | strip_right_tokens = len(tokens) - seq_length |
| 674 | if strip_right_tokens > 0: |
| 675 | while strip_right_tokens < len(tokens) - 1 and not self.contains_sentence_end( |
| 676 | tokens[-strip_right_tokens - 1]): |
| 677 | strip_right_tokens += 1 |
| 678 | if len(tokens) - strip_right_tokens < seq_length // 2: |
| 679 | strip_right_tokens = len(tokens) - seq_length |
| 680 | tokens = tokens[:-strip_right_tokens] |
| 681 | loss_mask = loss_mask[:-strip_right_tokens] |
| 682 | return tokens, loss_mask |
| 683 | |
| 684 | def getidx(self, data_idx): |
| 685 | data = self.ds[data_idx] |
no test coverage detected