| 11 | dataset = load_dataset(path=data_path, name = 'CFA-Challenge') |
| 12 | dataset = dataset['test'] |
| 13 | def process_fn1(example, idx): |
| 14 | meta_type = 'CFA' |
| 15 | lang_type = 'EN' |
| 16 | |
| 17 | query = example['query'] |
| 18 | answer = example['answer'] |
| 19 | source = example['source'] |
| 20 | |
| 21 | if 'Mock PM' in source: |
| 22 | type = 'PM' |
| 23 | else: |
| 24 | type = 'EXAM' |
| 25 | |
| 26 | pattern = re.compile( |
| 27 | r"Scenario:\s*(.*?)\s*Question:\s*(.*?)\s*Answer Choices:\s*(.*)\s*Answer:\s*(.*)", |
| 28 | re.DOTALL |
| 29 | ) |
| 30 | match = pattern.search(query) |
| 31 | if match: |
| 32 | ex_scenario = match.group(1).strip() |
| 33 | ex_question = match.group(2).strip() |
| 34 | ex_answer_choices = match.group(3).strip() |
| 35 | ex_answer = match.group(4).strip() |
| 36 | |
| 37 | if 'TOPIC: ETHICAL AND PROFESSIONAL STANDARDS' in ex_scenario: |
| 38 | ex_scenario = ex_scenario.replace('TOPIC: ETHICAL AND PROFESSIONAL STANDARDS', '') |
| 39 | if 'TOTAL POINT VALUE OF THIS QUESTION SET IS 12 POINTS' in ex_scenario: |
| 40 | ex_scenario = ex_scenario.replace('TOTAL POINT VALUE OF THIS QUESTION SET IS 12 POINTS', '') |
| 41 | if ex_scenario[-1] == ';': |
| 42 | ex_scenario = ex_scenario[:-1] |
| 43 | ex_scenario = ex_scenario.strip() |
| 44 | |
| 45 | if ex_question[-1] == ';': |
| 46 | ex_question = ex_question[:-1] |
| 47 | |
| 48 | if ex_answer_choices[-1] == ';': |
| 49 | ex_answer_choices = ex_answer_choices[:-1] |
| 50 | new_answer_choices = [] |
| 51 | pattern = r'([A-Z]):\s*(.*?)\s*(?=[A-Z]:|$)' |
| 52 | matches = re.findall(pattern, ex_answer_choices, re.DOTALL) |
| 53 | for match in matches: |
| 54 | key = match[0].strip() |
| 55 | value = match[1] |
| 56 | if value.endswith('.,') or value.endswith('..'): |
| 57 | value = value[:-1] |
| 58 | value = value.strip() |
| 59 | new_answer_choices.append(f"{key}. {value}") |
| 60 | ex_answer_choices = '\n'.join(new_answer_choices) |
| 61 | |
| 62 | question = (f"Context:\n{ex_scenario}\n" |
| 63 | f"Question:\n{ex_question}\n" |
| 64 | f"Choices:\n{ex_answer_choices}") |
| 65 | |
| 66 | example['query'] = question |
| 67 | example['answer'] = answer |
| 68 | example['meta_type'] = meta_type |
| 69 | example['type'] = type |
| 70 | example['lang_type'] = lang_type |