(output, labeled_answer, mode='math', use_llm=False, question=None, extract_answer=False)
| 172 | |
| 173 | |
| 174 | def evaluate_predictions(output, labeled_answer, mode='math', use_llm=False, question=None, extract_answer=False): |
| 175 | final_metric = {"is_valid_answer": False, "acc": 0, "em": 0, "f1": 0, 'math_equal': 0, 'llm_equal': 0} |
| 176 | pred_answer = extract_answer_fn(output, mode=mode, extract_answer=extract_answer) |
| 177 | pred_answer_new = pred_answer |
| 178 | if pred_answer != '': |
| 179 | final_metric["is_valid_answer"] = True |
| 180 | else: |
| 181 | # If no answer was extracted, keep only the last 3 lines |
| 182 | pred_answer_new = '\n'.join(output.replace("\n\n", "\n").strip().split('\n')[-5:]) |
| 183 | |
| 184 | if mode in ['qa']: |
| 185 | def normalize_answer_qa(s): |
| 186 | def remove_articles(text): |
| 187 | return re.sub(r"\b(a|an|the)\b", " ", text) |
| 188 | def white_space_fix(text): |
| 189 | return " ".join(text.strip().split()) |
| 190 | def remove_punc(text): |
| 191 | exclude = set(string.punctuation) |
| 192 | return "".join(ch for ch in text if ch not in exclude) |
| 193 | def lower(text): |
| 194 | return text.lower() |
| 195 | return white_space_fix(remove_articles(remove_punc(lower(s)))) |
| 196 | normalized_pred_answer = normalize_answer_qa(pred_answer_new) |
| 197 | |
| 198 | for answer in labeled_answer: |
| 199 | normalized_ground_truth = normalize_answer_qa(answer) |
| 200 | em = int(normalized_pred_answer == normalized_ground_truth) |
| 201 | acc = int(normalized_ground_truth in normalized_pred_answer) |
| 202 | |
| 203 | prediction_tokens = normalized_pred_answer.split() |
| 204 | ground_truth_tokens = normalized_ground_truth.split() |
| 205 | common = Counter(prediction_tokens) & Counter(ground_truth_tokens) |
| 206 | num_same = sum(common.values()) |
| 207 | if num_same == 0: |
| 208 | continue |
| 209 | precision = 1.0 * num_same / len(prediction_tokens) |
| 210 | recall = 1.0 * num_same / len(ground_truth_tokens) |
| 211 | f1 = (2 * precision * recall) / (precision + recall) |
| 212 | for k in ["em", "acc", "f1"]: |
| 213 | final_metric[k] = max(eval(k), final_metric[k]) |
| 214 | |
| 215 | elif mode in ['math', 'choose']: |
| 216 | def normalize_answer(text): |
| 217 | text = text.lower() |
| 218 | text = " ".join(text.strip().split()) |
| 219 | return text |
| 220 | normalized_pred_answer = normalize_answer(pred_answer_new) |
| 221 | normalized_ground_truth = normalize_answer(labeled_answer) |
| 222 | |
| 223 | em = int(normalized_pred_answer == normalized_ground_truth) |
| 224 | acc = int(normalized_ground_truth in normalized_pred_answer) |
| 225 | |
| 226 | prediction_tokens = normalized_pred_answer.split() |
| 227 | ground_truth_tokens = normalized_ground_truth.split() |
| 228 | common = Counter(prediction_tokens) & Counter(ground_truth_tokens) |
| 229 | num_same = sum(common.values()) |
| 230 | if num_same == 0: |
| 231 | f1 = 0 |
no test coverage detected