(input_ids, tokenizer, pad_token=1024, max_channels=8)
| 315 | |
| 316 | |
| 317 | def shifting_inputs(input_ids, tokenizer, pad_token=1024, max_channels=8): |
| 318 | seq_len = input_ids.shape[0] |
| 319 | new_seq_len = seq_len + max_channels - 1 |
| 320 | shifted_input_ids = np.full((new_seq_len, max_channels), pad_token, dtype=np.int64) |
| 321 | shifted_input_ids[:, 0] = np.full( |
| 322 | new_seq_len, tokenizer.pad_token_id, dtype=np.int64 |
| 323 | ) |
| 324 | for i in range(max_channels): |
| 325 | shifted_input_ids[i : (seq_len + i), i] = input_ids[:, i] |
| 326 | return shifted_input_ids |
| 327 | |
| 328 | |
| 329 | def rpadding(input_ids, channels, tokenizer): |
no outgoing calls
no test coverage detected