(pred_str, data_name, use_last_number=True)
| 458 | |
| 459 | #关键提取函数 |
| 460 | def extract_answer(pred_str, data_name, use_last_number=True): |
| 461 | pred_str = pred_str.replace("\u043a\u0438", "") |
| 462 | if data_name in ["mmlu_stem", "sat_math", "aqua", "gaokao2023"]: #skip for math benchmark |
| 463 | # TODO check multiple choice |
| 464 | return choice_answer_clean(pred_str) |
| 465 | |
| 466 | if "final answer is $" in pred_str and "$. I hope" in pred_str: |
| 467 | # minerva_math |
| 468 | tmp = pred_str.split("final answer is $", 1)[1] |
| 469 | pred = tmp.split("$. I hope", 1)[0].strip() |
| 470 | elif "boxed" in pred_str: |
| 471 | ans = pred_str.split("boxed")[-1] |
| 472 | if len(ans) == 0: |
| 473 | return "" |
| 474 | elif ans[0] == "{": |
| 475 | stack = 1 |
| 476 | a = "" |
| 477 | for c in ans[1:]: |
| 478 | if c == "{": |
| 479 | stack += 1 |
| 480 | a += c |
| 481 | elif c == "}": |
| 482 | stack -= 1 |
| 483 | if stack == 0: |
| 484 | break |
| 485 | a += c |
| 486 | else: |
| 487 | a += c |
| 488 | else: |
| 489 | a = ans.split("$")[0].strip() |
| 490 | pred = a |
| 491 | elif "he answer is" in pred_str: |
| 492 | pred = pred_str.split("he answer is")[-1].strip() |
| 493 | elif "final answer is" in pred_str: |
| 494 | pred = pred_str.split("final answer is")[-1].strip() |
| 495 | elif "答案是" in pred_str: |
| 496 | # Handle Chinese few-shot multiple choice problem answer extraction |
| 497 | pred = pred_str.split("答案是")[1].strip().split("\n\n")[0].strip() |
| 498 | else: # use the last number |
| 499 | if use_last_number: |
| 500 | pattern = "-?\d*\.?\d+" |
| 501 | pred = re.findall(pattern, pred_str.replace(",", "")) |
| 502 | if len(pred) >= 1: |
| 503 | pred = pred[-1] |
| 504 | else: |
| 505 | pred = "" |
| 506 | else: |
| 507 | pred = "" |
| 508 | |
| 509 | # choice answer |
| 510 | if ( |
| 511 | data_name in ["sat_math", "aqua"] |
| 512 | or "mmlu" in data_name |
| 513 | ): #false for math benchmark |
| 514 | tmp = re.findall(r"\b(A|B|C|D|E)\b", pred.upper()) |
| 515 | if tmp: |
| 516 | pred = tmp[-1] |
| 517 | else: |
no test coverage detected