Process a sample.
(self, sample: LabelSample)
| 126 | "labels": label |
| 127 | } |
| 128 | def process_sample(self, sample: LabelSample) -> Iterable[Dict[str, List[int]]]: |
| 129 | """ |
| 130 | Process a sample. |
| 131 | """ |
| 132 | prompt_tokens = self._tokenize(sample.prompt) |
| 133 | label = sample.label |
| 134 | |
| 135 | |
| 136 | if len(prompt_tokens) > self._max_seq_len: |
| 137 | if self._discard_overlong: |
| 138 | return |
| 139 | prompt_tokens=prompt_tokens[-self._max_seq_len:] |
| 140 | |
| 141 | yield self.pad_seq(prompt_tokens, label, extra=sample.extra) |
| 142 | |
| 143 | def process_sample_strict(self, sample: LabelSample) -> List[Dict[str, List[int]]]: |
| 144 | """ |
no test coverage detected