| 627 | chunks = ids[: n_chunks * seqlen].view(n_chunks, seqlen).contiguous() |
| 628 | |
| 629 | class _DS(torch.utils.data.Dataset): |
| 630 | def __len__(self): return chunks.shape[0] |
| 631 | def __getitem__(self, i): |
| 632 | return {"input_ids": chunks[i], |
| 633 | "attention_mask": torch.ones(seqlen, dtype=torch.long)} |
| 634 | return torch.utils.data.DataLoader(_DS(), batch_size=batch_size, shuffle=False) |
| 635 | |
| 636 |
no outgoing calls
no test coverage detected