(self, filenames, n_chunks, block_size, seed, shuffle, wrap)
| 178 | |
| 179 | class PackedDatasetIterator: |
| 180 | def __init__(self, filenames, n_chunks, block_size, seed, shuffle, wrap): |
| 181 | self._seed = seed |
| 182 | self._shuffle = shuffle |
| 183 | self._rng = np.random.default_rng(seed) if shuffle else None |
| 184 | self._block_idxs = None |
| 185 | |
| 186 | self._wrap = wrap |
| 187 | |
| 188 | # TODO: instead of filenames, we could have a single text stream |
| 189 | # (or text file) with the sequence of all files to be |
| 190 | # fetched/loaded. |
| 191 | self._filenames = filenames |
| 192 | self._file_idx = 0 |
| 193 | |
| 194 | self._n_chunks = n_chunks |
| 195 | |
| 196 | self._dtype = None |
| 197 | self._block_size = block_size |
| 198 | self._n_blocks = None |
| 199 | |
| 200 | self._mmaps = [] |
| 201 | self._buffers = [] |
| 202 | |
| 203 | self._block_idxs = [] |
| 204 | self._curr_idx = 0 |
| 205 | |
| 206 | self._load_n_chunks() |
| 207 | |
| 208 | def _read_header(self, path): |
| 209 | with open(path, 'rb') as f: |
nothing calls this directly
no test coverage detected