(dataset, output_path: Path)
| 150 | |
| 151 | |
| 152 | def write_train_jsonl(dataset, output_path: Path) -> int: |
| 153 | output_path.parent.mkdir(parents=True, exist_ok=True) |
| 154 | count = 0 |
| 155 | with output_path.open("w", encoding="utf-8") as handle: |
| 156 | for row_number, row in enumerate(dataset, start=1): |
| 157 | converted = normalize_conversations(row) |
| 158 | validate_conversations(converted, row_number) |
| 159 | handle.write(json.dumps(converted, ensure_ascii=False) + "\n") |
| 160 | count += 1 |
| 161 | return count |
| 162 | |
| 163 | |
| 164 | def write_eval_jsonl(dataset, output_path: Path) -> int: |
no test coverage detected