| 477 | |
| 478 | |
| 479 | def parse_example_dataset(config): |
| 480 | # config -> .meta.jsonl -> parsed_results |
| 481 | path = config['path'] |
| 482 | |
| 483 | # load sample and get parsed_meta |
| 484 | parsed_meta = {} |
| 485 | if path.endswith('.jsonl'): |
| 486 | with open(path, 'r', encoding='utf-8') as f: |
| 487 | data_item = json.loads(f.readline()) |
| 488 | elif path.endswith('.csv'): |
| 489 | with open(path, 'r', encoding='utf-8') as f: |
| 490 | reader = csv.reader(f) |
| 491 | header = next(reader) |
| 492 | row = next(reader) |
| 493 | data_item = dict(zip(header, row)) |
| 494 | else: |
| 495 | raise ValueError(f'Unsupported ext: {path}, .jsonl or .csv required') |
| 496 | |
| 497 | parsed_meta['path'] = path |
| 498 | input_columns = [i for i in data_item.keys() if i != 'answer'] |
| 499 | parsed_meta['input_columns'] = input_columns |
| 500 | output_column = 'answer' if 'answer' in data_item else None |
| 501 | parsed_meta['output_column'] = output_column |
| 502 | options = [] |
| 503 | for i in range(26): |
| 504 | i = chr(ord('A') + i) |
| 505 | if i in data_item: |
| 506 | options.append(i) |
| 507 | else: |
| 508 | break |
| 509 | parsed_meta['options'] = options |
| 510 | abbr = os.path.basename(path).split('.')[0] |
| 511 | parsed_meta['abbr'] = abbr |
| 512 | parsed_meta['data_type'] = 'mcq' if len(options) > 1 else 'qa' |
| 513 | parsed_meta['infer_method'] = 'gen' |
| 514 | |
| 515 | # try to read meta json |
| 516 | meta_path = config.get('meta_path', path + '.meta.json') |
| 517 | if os.path.exists(meta_path): |
| 518 | with open(meta_path, 'r', encoding='utf-8') as f: |
| 519 | read_from_file_meta = json.load(f) |
| 520 | else: |
| 521 | read_from_file_meta = {} |
| 522 | |
| 523 | # get config meta |
| 524 | config_meta = copy.deepcopy(config) |
| 525 | |
| 526 | # merge meta |
| 527 | meta = {} |
| 528 | meta.update(parsed_meta) |
| 529 | meta.update(read_from_file_meta) |
| 530 | meta.update(config_meta) |
| 531 | |
| 532 | return meta |
| 533 | |
| 534 | |
| 535 | def make_custom_dataset_config(config): |