(self)
| 378 | return data |
| 379 | |
| 380 | def json_iterator(self): |
| 381 | index, file_loc = self._index, self._file_loc |
| 382 | with open_file(self.config.path, 'r') as fin: |
| 383 | fin.seek(file_loc) |
| 384 | while True: |
| 385 | line = fin.readline() |
| 386 | file_loc = fin.tell() |
| 387 | if not line: # Reached EOF |
| 388 | index = 0 |
| 389 | fin.seek(0) |
| 390 | continue |
| 391 | |
| 392 | data = self.parse_json(line) |
| 393 | if data is not None and (not self.config.use_data_sharded_loader or index % self._node_info['dp_node_size'] == self._node_info['dp_node_rank']): |
| 394 | # JSON parsing succeeded |
| 395 | yield data, file_loc, index |
| 396 | index += 1 |
| 397 | |
| 398 | def batched(self, iterator, batch_size): |
| 399 | batch = [] |
no test coverage detected