Process a sample.
(self, sample: PromptSample)
| 46 | } |
| 47 | |
| 48 | def process_sample(self, sample: PromptSample) -> Iterable[Dict[str, List[int]]]: |
| 49 | """ |
| 50 | Process a sample. |
| 51 | """ |
| 52 | prompt_tokens = self._tokenize(sample.prompt) |
| 53 | code_tokens = self._tokenize(sample.code) |
| 54 | |
| 55 | if self._eod_token is not None: |
| 56 | code_tokens.append(self._eod_token) |
| 57 | |
| 58 | if len(prompt_tokens) + len(code_tokens) > self._max_seq_len: |
| 59 | if self._discard_overlong: |
| 60 | return |
| 61 | for p, t in sliding_window(prompt_tokens, code_tokens, self._max_seq_len, self._sliding_stride, self._sliding_stride): |
| 62 | yield self.pad_seq(p, t) |
| 63 | else: |
| 64 | yield self.pad_seq(prompt_tokens, code_tokens, extra=sample.extra) |
| 65 | |
| 66 | def process_sample_strict(self, sample: PromptSample) -> List[Dict[str, List[int]]]: |
| 67 | """ |
no test coverage detected