(self, line)
| 576 | self._total_tokens = 0 |
| 577 | |
| 578 | def parse_json(self, line): |
| 579 | if not line or line == '\n': |
| 580 | return None |
| 581 | try: |
| 582 | data = json.loads(line) |
| 583 | except json.decoder.JSONDecodeError: |
| 584 | print(f'Error parsing json line:\n{line}') |
| 585 | return None |
| 586 | return data |
| 587 | |
| 588 | def json_iterator(self): |
| 589 | index, file_loc = self._index, self._file_loc |