Take a single document and the LM results and evaluates, returning a dict where keys are the names of submetrics and values are the values of the metric for that one document :param doc: The document as returned from training_docs, validation_docs, or test_docs.
(self, doc, results)
| 138 | return conts |
| 139 | |
| 140 | def process_results(self, doc, results): |
| 141 | """Take a single document and the LM results and evaluates, returning a |
| 142 | dict where keys are the names of submetrics and values are the values of |
| 143 | the metric for that one document |
| 144 | |
| 145 | :param doc: |
| 146 | The document as returned from training_docs, validation_docs, or test_docs. |
| 147 | :param results: |
| 148 | The results of the requests created in construct_requests. |
| 149 | """ |
| 150 | preds, golds = results, doc["answers"] |
| 151 | max_em = 0 |
| 152 | max_f1 = 0 |
| 153 | for gold_answer in golds: |
| 154 | exact_match, f1_score = self.get_metrics(preds, gold_answer) |
| 155 | if gold_answer[0].strip(): |
| 156 | max_em = max(max_em, exact_match) |
| 157 | max_f1 = max(max_f1, f1_score) |
| 158 | return {"em": max_em, "f1": max_f1} |
| 159 | |
| 160 | def get_metrics(self, predicted, gold): |
| 161 | """ |
nothing calls this directly
no test coverage detected