| 12 | |
| 13 | @staticmethod |
| 14 | def load(path: str): |
| 15 | |
| 16 | with open(path, 'r', encoding='utf-8') as f: |
| 17 | data = json.load(f) |
| 18 | rows = [] |
| 19 | for _, row in enumerate(data): |
| 20 | content = row[0] |
| 21 | content_str = ' '.join( |
| 22 | [''.join(paragraph) for paragraph in content]) |
| 23 | |
| 24 | for question in row[1]: |
| 25 | label = question['choice'].index(question['answer']) |
| 26 | length = len(question['choice']) |
| 27 | if length < 4: |
| 28 | fill_value = question['choice'][0] # 以第一个值为填充值 |
| 29 | fill_count = 4 - length # 需要填充的数量 |
| 30 | question['choice'] += [fill_value] * fill_count # 填充 |
| 31 | |
| 32 | rows.append({ |
| 33 | 'content': content_str, |
| 34 | 'question': question['question'], |
| 35 | 'choices': question['choice'], |
| 36 | 'choice0': question['choice'][0], |
| 37 | 'choice1': question['choice'][1], |
| 38 | 'choice2': question['choice'][2], |
| 39 | 'choice3': question['choice'][3], |
| 40 | 'label': label |
| 41 | }) |
| 42 | |
| 43 | dataset = Dataset.from_dict({ |
| 44 | 'content': [row['content'] for row in rows], |
| 45 | 'question': [row['question'] for row in rows], |
| 46 | 'choice0': [row['choice0'] for row in rows], |
| 47 | 'choice1': [row['choice1'] for row in rows], |
| 48 | 'choice2': [row['choice2'] for row in rows], |
| 49 | 'choice3': [row['choice3'] for row in rows], |
| 50 | 'choices': [row['choices'] for row in rows], |
| 51 | 'label': [row['label'] for row in rows] |
| 52 | }) |
| 53 | return dataset |
| 54 | |
| 55 | |
| 56 | @LOAD_DATASET.register_module() |