(self, num_tokens)
| 131 | return self._tokens.numel() |
| 132 | |
| 133 | def _get_split_indices(self, num_tokens): |
| 134 | num_chunks = (num_tokens - 1) // self.max_length |
| 135 | indices = [ |
| 136 | (i * self.max_length, self.max_length + 1) for i in range(num_chunks) |
| 137 | ] |
| 138 | full_chunks = num_chunks * self.max_length + (num_chunks > 0) |
| 139 | remainder = num_tokens - full_chunks |
| 140 | if remainder: |
| 141 | indices.append((full_chunks, remainder + (num_chunks > 0))) |
| 142 | return indices |
| 143 | |
| 144 | def _get_split_indices_lines(self): |
| 145 | indices, start = [], 0 |
no outgoing calls
no test coverage detected