Generate a series of (prompt, code) pairs by sliding the window over the code.
(
prompt_tokens: list,
code_tokens: list,
seq_len: int,
sliding_stride: int,
minimum_code_len: int = 1,
)
| 102 | |
| 103 | |
| 104 | def sliding_window( |
| 105 | prompt_tokens: list, |
| 106 | code_tokens: list, |
| 107 | seq_len: int, |
| 108 | sliding_stride: int, |
| 109 | minimum_code_len: int = 1, |
| 110 | ) -> Iterable[Tuple[list, list]]: |
| 111 | """ |
| 112 | Generate a series of (prompt, code) pairs by sliding the window over the code. |
| 113 | """ |
| 114 | prompt_len = len(prompt_tokens) |
| 115 | code_len = len(code_tokens) |
| 116 | total_len = prompt_len + code_len |
| 117 | |
| 118 | start_idx = max(0, prompt_len - seq_len + minimum_code_len) # at least `minimum_code_len` code token should be in the window |
| 119 | end_idx = max(0, total_len - seq_len) |
| 120 | start_idx = min(start_idx, end_idx) |
| 121 | |
| 122 | for i in range(start_idx, end_idx + 1, sliding_stride): |
| 123 | current_prompt = prompt_tokens[i:i + seq_len] |
| 124 | current_code = code_tokens[max(i - prompt_len, 0):i - prompt_len + seq_len] |
| 125 | yield current_prompt, current_code |
| 126 | |
| 127 | if (end_idx - start_idx) % sliding_stride != 0: |
| 128 | current_prompt = prompt_tokens[end_idx:end_idx + seq_len] |
| 129 | current_code = code_tokens[max(end_idx - prompt_len, 0):end_idx - prompt_len + seq_len] |
| 130 | yield current_prompt, current_code |