(path)
| 17 | return hash |
| 18 | |
| 19 | def read_jsonl(path): |
| 20 | with open(path, 'r', encoding='utf-8') as f: |
| 21 | dataset = [] |
| 22 | lines = f.readlines() |
| 23 | for i, line in enumerate(lines): |
| 24 | try: |
| 25 | dataset.append(json.loads(line)) |
| 26 | except Exception as e: |
| 27 | print('line:', i) |
| 28 | print(e) |
| 29 | print('dataset:', len(dataset)) |
| 30 | return dataset |
| 31 | |
| 32 | def save_jsonl(data, filename): |
| 33 | print('save data:', len(data)) |