| 545 | class JsonVisionDataset(object): |
| 546 | @staticmethod |
| 547 | def get_default_config(updates=None): |
| 548 | config = ConfigDict() |
| 549 | config.path = '' |
| 550 | config.seq_length = 384 |
| 551 | config.batch_size = 4 |
| 552 | config.always_start_with_bos = False |
| 553 | config.start_seek_loc = 0 |
| 554 | config.example_index_at_start = 0 |
| 555 | config.tokens_count_at_start = 0 |
| 556 | config.tokenizer_processes = 1 |
| 557 | config.tokenizer_parallel_chunk_size = 32 |
| 558 | config.tokenizer_parallel_batch_size = 1024 |
| 559 | config.throughput_average_window_size = 200 |
| 560 | config.use_data_sharded_loader = True |
| 561 | config.return_local_batch = False |
| 562 | config.mode = 'pad' |
| 563 | |
| 564 | if updates is not None: |
| 565 | config.update(ConfigDict(updates).copy_and_resolve_references()) |
| 566 | return config |
| 567 | |
| 568 | def __init__(self, config, tokenizer, text_processor, node_info): |
| 569 | self.config = self.get_default_config(config) |