Calculate comprehensive evaluation metrics for a prediction.
(prediction: str, reference: str)
| 107 | return 0.0 |
| 108 | |
| 109 | def calculate_metrics(prediction: str, reference: str) -> Dict[str, float]: |
| 110 | """Calculate comprehensive evaluation metrics for a prediction.""" |
| 111 | # Handle empty or None values |
| 112 | if not prediction or not reference: |
| 113 | return { |
| 114 | "exact_match": 0, |
| 115 | "f1": 0.0, |
| 116 | "rouge1_f": 0.0, |
| 117 | "rouge2_f": 0.0, |
| 118 | "rougeL_f": 0.0, |
| 119 | "bleu1": 0.0, |
| 120 | "bleu2": 0.0, |
| 121 | "bleu3": 0.0, |
| 122 | "bleu4": 0.0, |
| 123 | "bert_f1": 0.0, |
| 124 | "meteor": 0.0, |
| 125 | "sbert_similarity": 0.0 |
| 126 | } |
| 127 | |
| 128 | # Convert to strings if they're not already |
| 129 | prediction = str(prediction).strip() |
| 130 | reference = str(reference).strip() |
| 131 | |
| 132 | # Calculate exact match |
| 133 | exact_match = int(prediction.lower() == reference.lower()) |
| 134 | |
| 135 | # Calculate token-based F1 score |
| 136 | pred_tokens = set(simple_tokenize(prediction)) |
| 137 | ref_tokens = set(simple_tokenize(reference)) |
| 138 | common_tokens = pred_tokens & ref_tokens |
| 139 | |
| 140 | if not pred_tokens or not ref_tokens: |
| 141 | f1 = 0.0 |
| 142 | else: |
| 143 | precision = len(common_tokens) / len(pred_tokens) |
| 144 | recall = len(common_tokens) / len(ref_tokens) |
| 145 | f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0 |
| 146 | |
| 147 | # Calculate all scores |
| 148 | rouge_scores = calculate_rouge_scores(prediction, reference) |
| 149 | bleu_scores = calculate_bleu_scores(prediction, reference) |
| 150 | bert_scores = calculate_bert_scores(prediction, reference) |
| 151 | meteor = calculate_meteor_score(prediction, reference) |
| 152 | sbert_similarity = calculate_sentence_similarity(prediction, reference) |
| 153 | |
| 154 | # Combine all metrics |
| 155 | metrics = { |
| 156 | "exact_match": exact_match, |
| 157 | "f1": f1, |
| 158 | **rouge_scores, |
| 159 | **bleu_scores, |
| 160 | **bert_scores, |
| 161 | "meteor": meteor, |
| 162 | "sbert_similarity": sbert_similarity |
| 163 | } |
| 164 | |
| 165 | return metrics |
| 166 |
no test coverage detected