(self)
| 265 | return self |
| 266 | |
| 267 | def __next__(self): |
| 268 | if self._curr_idx >= len(self._block_idxs): |
| 269 | self._load_n_chunks() |
| 270 | # TODO: trigger fetching next next n_chunks if remote |
| 271 | block_idx = self._block_idxs[self._curr_idx] |
| 272 | chunk_id = block_idx // self._n_blocks |
| 273 | buffer = self._buffers[chunk_id] |
| 274 | elem_id = (block_idx % self._n_blocks) * self._block_size |
| 275 | offset = np.dtype(self._dtype).itemsize * elem_id |
| 276 | arr = np.frombuffer( |
| 277 | buffer, dtype=self._dtype, count=self._block_size, offset=offset |
| 278 | ) |
| 279 | self._curr_idx += 1 |
| 280 | return torch.from_numpy(arr.astype(np.int64)) |
| 281 | |
| 282 | |
| 283 | class CombinedDataset(IterableDataset): |
nothing calls this directly
no test coverage detected