| 267 | dataset = dataset['test'] |
| 268 | |
| 269 | def process_fn5(example, idx): |
| 270 | meta_type = 'FIQASA' |
| 271 | lang_type = 'EN' |
| 272 | type = 'EXAM' |
| 273 | |
| 274 | query = example['query'] |
| 275 | answer = example['answer'] |
| 276 | |
| 277 | pattern = re.compile( |
| 278 | r"(.*?)\s*Text:\s*(.*?)\s*Answer:\s*(.*)", |
| 279 | re.DOTALL |
| 280 | ) |
| 281 | match = pattern.search(query) |
| 282 | if match: |
| 283 | ex_context = match.group(1).strip() |
| 284 | ex_question = match.group(2).strip() |
| 285 | ex_answer = match.group(3).strip() |
| 286 | |
| 287 | ex_context = ex_context.replace('Positive', 'positive') |
| 288 | ex_context = ex_context.replace('Negative', 'negative') |
| 289 | ex_context = ex_context.replace('Neutral', 'neutral') |
| 290 | |
| 291 | map = { |
| 292 | 'positive': 'A', |
| 293 | 'negative': 'B', |
| 294 | 'neutral': 'C', |
| 295 | } |
| 296 | sorted_map = sorted(map.items(), key=lambda x: x[1]) |
| 297 | ex_answer_choices = "\n".join([f"{value}. {key}" for key, value in sorted_map]) |
| 298 | |
| 299 | answer = map.get(answer) |
| 300 | |
| 301 | question = (f"Context:\n{ex_question}\n" |
| 302 | f"Question:\n{ex_context}\n" |
| 303 | f"Choices:\n{ex_answer_choices}") |
| 304 | |
| 305 | example['query'] = question |
| 306 | example['answer'] = answer |
| 307 | example['meta_type'] = meta_type |
| 308 | example['type'] = type |
| 309 | example['lang_type'] = lang_type |
| 310 | |
| 311 | return example |
| 312 | else: |
| 313 | raise ValueError("No match found in the question.") |
| 314 | |
| 315 | dataset = dataset.map(function=process_fn5, with_indices=True) |
| 316 | dataset = dataset.map(function=lambda x: {"meta_type": "TR", "type": "SENTIMENT"}) |