(self, indices)
| 101 | yield tokenized |
| 102 | |
| 103 | def sample(self, indices): |
| 104 | n_tokens = (len(self.tokens) // self.context_len) * self.context_len |
| 105 | while self.token_id < n_tokens: |
| 106 | i = next(indices) |
| 107 | start, end = self.token_id, self.token_id + self.context_len |
| 108 | self.token_id += self.context_len |
| 109 | yield {'input_ids': self.buffer[i].to(torch.long)} |
| 110 | self.buffer[i] = torch.tensor(self.tokens[start:end], dtype=self.dtype) |
| 111 | self.token_id = 0 |
| 112 | self.tokens = self.tokens[n_tokens:] |
| 113 | |
| 114 | def randint( |
| 115 | self, |