(data, filename)
| 30 | return dataset |
| 31 | |
| 32 | def save_jsonl(data, filename): |
| 33 | print('save data:', len(data)) |
| 34 | os.makedirs(os.path.dirname(filename), exist_ok=True) # Create necessary folders |
| 35 | with open(filename, "a", encoding="utf-8") as f: |
| 36 | for sample in data: |
| 37 | f.write(json.dumps(sample, ensure_ascii=False) + "\n") |
| 38 | |
| 39 | def append_jsonl(path: str, data: dict) -> None: |
| 40 | os.makedirs(os.path.dirname(path), exist_ok=True) # Automatically create directory |
nothing calls this directly
no outgoing calls
no test coverage detected