(path)
| 101 | rank0_print("Loading data...") |
| 102 | |
| 103 | def load(path): |
| 104 | if not path.endswith(('.json', '.jsonl')): |
| 105 | raise ValueError('need .json or .jsonl') |
| 106 | with open(path, encoding='utf-8') as f: |
| 107 | return json.load(f) if path.endswith('.json') else [json.loads(l) for l in f if l.strip()] |
| 108 | train_json = load(data_args.data_path) |
| 109 | |
| 110 | train_dataset = dataset_cls( |
no outgoing calls
no test coverage detected