(self)
| 586 | return data |
| 587 | |
| 588 | def json_iterator(self): |
| 589 | index, file_loc = self._index, self._file_loc |
| 590 | with open_file(self.config.path, 'r', block_size=50 * 2 ** 20) as fin: |
| 591 | fin.seek(file_loc) |
| 592 | while True: |
| 593 | line = fin.readline() |
| 594 | file_loc = fin.tell() |
| 595 | if not line: # Reached EOF |
| 596 | index = 0 |
| 597 | fin.seek(0) |
| 598 | continue |
| 599 | if not self.config.use_data_sharded_loader or index % self._node_info['dp_node_size'] == self._node_info['dp_node_rank']: |
| 600 | data = self.parse_json(line) |
| 601 | if data is not None: |
| 602 | # JSON parsing succeeded |
| 603 | yield data, file_loc, index |
| 604 | index += 1 |
| 605 | |
| 606 | def batched(self, iterator, batch_size): |
| 607 | batch = [] |
no test coverage detected