(self, line)
| 368 | self._total_tokens = self.config.tokens_count_at_start |
| 369 | |
| 370 | def parse_json(self, line): |
| 371 | if not line or line == '\n': |
| 372 | return None |
| 373 | try: |
| 374 | data = json.loads(line) |
| 375 | except json.decoder.JSONDecodeError: |
| 376 | print(f'Error parsing json line:\n{line}') |
| 377 | return None |
| 378 | return data |
| 379 | |
| 380 | def json_iterator(self): |
| 381 | index, file_loc = self._index, self._file_loc |