(
self,
filenames,
n_chunks,
block_size,
seed=12345,
shuffle=True,
wrap=False,
num_processes=1,
process_rank=0,
)
| 39 | |
| 40 | class PackedDataset(IterableDataset): |
| 41 | def __init__( |
| 42 | self, |
| 43 | filenames, |
| 44 | n_chunks, |
| 45 | block_size, |
| 46 | seed=12345, |
| 47 | shuffle=True, |
| 48 | wrap=False, |
| 49 | num_processes=1, |
| 50 | process_rank=0, |
| 51 | ): |
| 52 | self._filenames = filenames |
| 53 | self._n_chunks = n_chunks |
| 54 | self._block_size = block_size |
| 55 | self._seed = seed |
| 56 | self._shuffle = shuffle |
| 57 | self._wrap = wrap |
| 58 | self._num_processes = num_processes |
| 59 | self._process_rank = process_rank |
| 60 | |
| 61 | def __iter__(self): |
| 62 | worker_info = get_worker_info() |
nothing calls this directly
no outgoing calls
no test coverage detected