| 67 | return pooled_eval_results, example_eval_results |
| 68 | |
| 69 | class QAF1Score: |
| 70 | |
| 71 | metric_name: str = "qa_f1_score" |
| 72 | |
| 73 | def __init__(self): |
| 74 | self.logger = get_logger(__name__) |
| 75 | |
| 76 | def calculate_metric_scores(self, gold_answers: List[List[str]], predicted_answers: List[str], aggregation_fn: Callable = np.max) -> Tuple[Dict[str, float], List[Dict[str, float]]]: |
| 77 | """ |
| 78 | Calculate F1 scores |
| 79 | |
| 80 | Args: |
| 81 | gold_answers: List of standard answers, each element is a list of answers |
| 82 | predicted_answers: List of predicted answers |
| 83 | aggregation_fn: Function to aggregate multiple standard answers |
| 84 | |
| 85 | Returns: |
| 86 | Tuple containing: average F1 score dictionary, list of F1 scores for each sample |
| 87 | """ |
| 88 | assert len(gold_answers) == len(predicted_answers), "Length of gold answers and predicted answers should be the same" |
| 89 | |
| 90 | def compute_f1(gold: str, predicted: str) -> float: |
| 91 | gold_tokens = normalize_answer(gold).split() |
| 92 | predicted_tokens = normalize_answer(predicted).split() |
| 93 | common = Counter(predicted_tokens) & Counter(gold_tokens) |
| 94 | num_same = sum(common.values()) |
| 95 | |
| 96 | if num_same == 0: |
| 97 | return 0.0 |
| 98 | |
| 99 | precision = 1.0 * num_same / len(predicted_tokens) if predicted_tokens else 0.0 |
| 100 | recall = 1.0 * num_same / len(gold_tokens) if gold_tokens else 0.0 |
| 101 | |
| 102 | if precision + recall == 0: |
| 103 | return 0.0 |
| 104 | |
| 105 | return 2 * (precision * recall) / (precision + recall) |
| 106 | |
| 107 | example_eval_results = [] |
| 108 | total_f1 = 0.0 |
| 109 | |
| 110 | for gold_list, predicted in zip(gold_answers, predicted_answers): |
| 111 | f1_scores = [compute_f1(gold, predicted) for gold in gold_list] |
| 112 | aggregated_f1 = aggregation_fn(f1_scores) |
| 113 | example_eval_results.append({"F1": aggregated_f1}) |
| 114 | total_f1 += aggregated_f1 |
| 115 | |
| 116 | avg_f1 = total_f1 / len(gold_answers) if gold_answers else 0.0 |
| 117 | pooled_eval_results = {"F1": avg_f1} |
| 118 | |
| 119 | return pooled_eval_results, example_eval_results |
| 120 | |
| 121 | def find_and_merge_results(root_path: str) -> List[Dict]: |
| 122 | """ |