(self, path, process_fn, seed, *, shuffle_buffer=1000)
| 76 | |
| 77 | class SimpleDistributedWebDataset(DataPipeline): |
| 78 | def __init__(self, path, process_fn, seed, *, shuffle_buffer=1000): |
| 79 | # set shuffle_buffer = 1 to disable it, model-parallel will be different due to shuffle |
| 80 | try: |
| 81 | from sat.mpu import get_model_parallel_world_size |
| 82 | |
| 83 | if get_model_parallel_world_size() > 1: |
| 84 | shuffle_buffer = 1 |
| 85 | except Exception: |
| 86 | pass |
| 87 | super().__init__( |
| 88 | ConfiguredResampledShards(path, seed), # Lots of shards are recommended, or not evenly |
| 89 | tarfile_to_samples(), |
| 90 | wds.shuffle(shuffle_buffer), |
| 91 | process_fn, |
| 92 | ) |
| 93 | |
| 94 | |
| 95 | def tar_file_iterator_with_meta( |
nothing calls this directly
no test coverage detected