| 80 | dataset = load_dataset(path=data_path, name = 'CFA-Easy') |
| 81 | dataset = dataset['test'] |
| 82 | def process_fn2(example, idx): |
| 83 | meta_type = 'CFA' |
| 84 | lang_type = 'EN' |
| 85 | type = 'EASY' |
| 86 | |
| 87 | query = example['query'] |
| 88 | answer = example['answer'] |
| 89 | |
| 90 | pattern = re.compile( |
| 91 | r"Q:\s*(.*?)\s*CHOICES:\s*(.*?)\s*Answer:\s*(.*)", |
| 92 | re.DOTALL |
| 93 | ) |
| 94 | match = pattern.search(query) |
| 95 | |
| 96 | if match: |
| 97 | ex_question = match.group(1).strip() |
| 98 | ex_answer_choices = match.group(2).strip() |
| 99 | ex_answer = match.group(3).strip() |
| 100 | |
| 101 | if ex_question[-1] == ';': |
| 102 | ex_question = ex_question[:-1] |
| 103 | if ex_answer_choices[-1] == ';': |
| 104 | ex_answer_choices = ex_answer_choices[:-1] |
| 105 | new_answer_choices = [] |
| 106 | pattern = r'([A-Z]):\s*(.*?)\s*(?=[A-Z]:|$)' |
| 107 | matches = re.findall(pattern, ex_answer_choices, re.DOTALL) |
| 108 | for match in matches: |
| 109 | key = match[0].strip() |
| 110 | value = match[1] |
| 111 | if value.endswith('.,') or value.endswith('..'): |
| 112 | value = value[:-1] |
| 113 | value = value.strip() |
| 114 | new_answer_choices.append(f"{key}. {value}") |
| 115 | ex_answer_choices = '\n'.join(new_answer_choices) |
| 116 | |
| 117 | # question = f"{ex_question}\n{ex_answer_choices}" |
| 118 | question = (f"Question:\n{ex_question}\n" |
| 119 | f"Choices:\n{ex_answer_choices}") |
| 120 | |
| 121 | example['query'] = question |
| 122 | example['answer'] = answer |
| 123 | example['meta_type'] = meta_type |
| 124 | example['type'] = type |
| 125 | example['lang_type'] = lang_type |
| 126 | return example |
| 127 | else: |
| 128 | raise ValueError("No match found in the question.") |
| 129 | dataset = dataset.map(function=process_fn2, with_indices=True) |
| 130 | datasets.append(dataset) |
| 131 | |