(path)
| 95 | |
| 96 | |
| 97 | def read_jsonl(path): |
| 98 | dataset = [] |
| 99 | with open(path, "r", encoding="utf-8") as f: |
| 100 | for i, line in enumerate(f.readlines()): |
| 101 | try: |
| 102 | dataset.append(json.loads(line)) |
| 103 | except Exception as e: |
| 104 | print("line:", i, e) |
| 105 | print("dataset:", len(dataset)) |
| 106 | return dataset |
| 107 | |
| 108 | |
| 109 | def post_process_text(text, enable_think): |