Score model response against ground truth. Uses ChemBench's regex patterns to extract answers: - MCQ: extracts letter(s) from [ANSWER]A[/ANSWER] or [ANSWER]A, B[/ANSWER], ignoring trailing text like ". some option text" - Numeric: extracts content from [ANSWER]...[/ANSWER], parses
(
response: str,
target_scores: dict | None,
target: str | None,
)
| 50 | |
| 51 | |
| 52 | def score_answer( |
| 53 | response: str, |
| 54 | target_scores: dict | None, |
| 55 | target: str | None, |
| 56 | ) -> dict: |
| 57 | """Score model response against ground truth. |
| 58 | |
| 59 | Uses ChemBench's regex patterns to extract answers: |
| 60 | - MCQ: extracts letter(s) from [ANSWER]A[/ANSWER] or [ANSWER]A, B[/ANSWER], |
| 61 | ignoring trailing text like ". some option text" |
| 62 | - Numeric: extracts content from [ANSWER]...[/ANSWER], parses as float |
| 63 | |
| 64 | Args: |
| 65 | response: Full model response text (with [ANSWER] tags). |
| 66 | target_scores: MCQ target scores dict, e.g. {"NaCl": 1, "KCl": 0}. |
| 67 | None for numeric questions. |
| 68 | target: Numeric target value as string. None for MCQ questions. |
| 69 | |
| 70 | Returns: |
| 71 | Metrics dict compatible with is_correct(). |
| 72 | """ |
| 73 | if not response: |
| 74 | return {"all_correct": 0} |
| 75 | |
| 76 | if target_scores and isinstance(target_scores, dict): |
| 77 | # MCQ: extract letters using ChemBench's MCQ regex |
| 78 | keys = list(target_scores.keys()) |
| 79 | scores = list(target_scores.values()) |
| 80 | answer_to_score = {chr(65 + i): scores[i] for i in range(len(keys))} |
| 81 | correct_letters = {k for k, v in answer_to_score.items() if v == 1} |
| 82 | |
| 83 | found_letters = set(extract_mcq_answer(response)) |
| 84 | |
| 85 | missed = len(correct_letters - found_letters) |
| 86 | extra = len(found_letters - correct_letters) |
| 87 | hamming = (missed + extra) / len(correct_letters) if correct_letters else 1 |
| 88 | return {"all_correct": 1 if hamming == 0 else 0, "hamming": hamming} |
| 89 | else: |
| 90 | # Numeric: extract content from [ANSWER] tags, parse as float |
| 91 | answer = extract_answer(response) |
| 92 | if not answer: |
| 93 | return {"all_correct": 0} |
| 94 | |
| 95 | target = target or "" |
| 96 | try: |
| 97 | target_val = float(target) |
| 98 | answer_val = float(answer) |
| 99 | mae_val = abs(answer_val - target_val) |
| 100 | tolerance = 0.01 * abs(target_val) if target_val != 0 else 1e-8 |
| 101 | return { |
| 102 | "all_correct": 1 if mae_val <= tolerance else 0, |
| 103 | "mae": mae_val, |
| 104 | } |
| 105 | except (ValueError, TypeError): |
| 106 | return {"all_correct": 1 if answer.strip() == str(target).strip() else 0} |
| 107 | |
| 108 | |
| 109 | def extract_metrics(report: dict[str, Any]) -> dict[str, Any]: |
no test coverage detected