(path: str, data: dict)
| 37 | f.write(json.dumps(sample, ensure_ascii=False) + "\n") |
| 38 | |
| 39 | def append_jsonl(path: str, data: dict) -> None: |
| 40 | os.makedirs(os.path.dirname(path), exist_ok=True) # Automatically create directory |
| 41 | with open(path, "a", encoding="utf-8") as f: |
| 42 | f.write(json.dumps(data, ensure_ascii=False) + "\n") |
| 43 | |
| 44 | def load_processed_ids(path_save: str) -> set: |
| 45 | if not os.path.exists(path_save): |