Main data loader loop. Loads batches of data for each training step.
(config, state=None)
| 35 | |
| 36 | |
| 37 | def data_load_loop(config, state=None): |
| 38 | """Main data loader loop. |
| 39 | Loads batches of data for each training step. |
| 40 | """ |
| 41 | _, _, _, _, mesh, _, data_iterator, _, _, _, state = setup_train_loop(config, recorder=None) |
| 42 | data_loader = DataLoader(config, mesh, data_iterator, None) |
| 43 | |
| 44 | example_batch = None |
| 45 | |
| 46 | start = datetime.datetime.now() |
| 47 | start_step = get_first_step(state) |
| 48 | example_batch = data_loader.load_next_batch() |
| 49 | jax.block_until_ready(example_batch) |
| 50 | first_end = datetime.datetime.now() |
| 51 | time_to_load_first_batch = first_end - start |
| 52 | if jax.process_index() == 0: |
| 53 | max_logging.log( |
| 54 | f"STANDALONE DATALOADER : First step completed in {time_to_load_first_batch.seconds} seconds, on host 0" |
| 55 | ) |
| 56 | |
| 57 | for _ in np.arange(start_step + 1, config.steps): |
| 58 | example_batch = data_loader.load_next_batch() |
| 59 | |
| 60 | jax.block_until_ready(example_batch) # wait until the last batch is read |
| 61 | end = datetime.datetime.now() |
| 62 | if jax.process_index() == 0: |
| 63 | max_logging.log(f"STANDALONE DATALOADER : {config.steps} batches loaded in {(end-start).seconds} seconds, on host 0") |
| 64 | return state |
| 65 | |
| 66 | |
| 67 | def main(argv: Sequence[str]) -> None: |
no test coverage detected