| 178 | dataset = load_dataset(path=data_path, name = 'CRA-CCF') |
| 179 | dataset = dataset['test'] |
| 180 | def process_fn4(example, idx): |
| 181 | meta_type = 'CRA' |
| 182 | lang_type = 'EN' |
| 183 | type = 'CCF' |
| 184 | |
| 185 | query = example['query'] |
| 186 | answer = example['answer'] |
| 187 | |
| 188 | match = re.search(r"(.*?)\s*For instance[,:]\s*(.*?)\s*Text:\s*(.*)\s*Answer:\s*(.*)$", query, re.DOTALL) |
| 189 | if match: |
| 190 | ex_context = match.group(1).strip() |
| 191 | ex_example = match.group(2).strip() |
| 192 | ex_question = match.group(3).strip() |
| 193 | ex_answer = match.group(4).strip() |
| 194 | |
| 195 | maps = { |
| 196 | 'yes': 'A', |
| 197 | 'no': 'B', |
| 198 | } |
| 199 | sorted_maps = sorted(maps.items(), key=lambda x: x[1]) |
| 200 | ex_answer_choices = "\n".join([f"{value}. {key}" for key, value in sorted_maps]) |
| 201 | |
| 202 | ex_question = ex_question + ' should be classified as:' |
| 203 | |
| 204 | question = (f"Context:\n{ex_context}\n" |
| 205 | f"Example:\n{ex_example}\n" |
| 206 | f"Question:\n{ex_question}\n" |
| 207 | f"Choices:\n{ex_answer_choices}") |
| 208 | |
| 209 | answer = maps.get(answer) |
| 210 | |
| 211 | example['query'] = question |
| 212 | example['answer'] = answer |
| 213 | example['meta_type'] = meta_type |
| 214 | example['type'] = type |
| 215 | example['lang_type'] = lang_type |
| 216 | return example |
| 217 | else: |
| 218 | raise ValueError("No match found in the question.") |
| 219 | |
| 220 | dataset = dataset.map(function=process_fn4, with_indices=True) |
| 221 | dataset = dataset.map(function=lambda x: {"type": "CCF"}) |