(path_save: str)
| 42 | f.write(json.dumps(data, ensure_ascii=False) + "\n") |
| 43 | |
| 44 | def load_processed_ids(path_save: str) -> set: |
| 45 | if not os.path.exists(path_save): |
| 46 | return set() |
| 47 | with open(path_save, encoding="utf-8") as f: |
| 48 | ids = {json.loads(line)["id_ddm"] for line in f} |
| 49 | return ids |
| 50 | |
| 51 | |
| 52 | def construct_dataset(path, data_size): |