(self, doc, results)
| 165 | return obs_list |
| 166 | |
| 167 | def process_results(self, doc, results): |
| 168 | # TODO: Calculate a score for extractive spans once a request type for generating |
| 169 | # extractive spans is available |
| 170 | if not results: |
| 171 | return {} |
| 172 | elif len(results) == 1: |
| 173 | [res] = results |
| 174 | elif len(results) == 2: |
| 175 | [ll_yes, ll_no] = results |
| 176 | |
| 177 | # TODO: Handle unanswerability first |
| 178 | # unanswerable_gold = doc["answer_type"] == "unanswerable" |
| 179 | # unanswerable_pred = exp(logprob_unanswerable) |
| 180 | # res_dict["f1_unanswerable"] = (unanswerable_gold, unanswerable_pred) |
| 181 | |
| 182 | res_dict = {} |
| 183 | # Handle yes/no questions |
| 184 | if doc["answer_type"] == "bool": |
| 185 | gold = 1 if doc["answer"] == "yes" else 0 |
| 186 | pred = ll_yes > ll_no |
| 187 | res_dict["f1_yesno"] = (gold, pred) |
| 188 | |
| 189 | # Handle completions |
| 190 | if doc["answer_type"] == "free form answer": |
| 191 | res_dict["f1_abstractive"] = token_f1_score(res, doc["answer"]) |
| 192 | |
| 193 | # TODO: Handle extraction |
| 194 | # if doc["answer_type"] == "extractive_spans": |
| 195 | # res_dict["f1_extractive"] = 0 |
| 196 | return res_dict |
| 197 | |
| 198 | def aggregation(self): |
| 199 | return { |
nothing calls this directly
no test coverage detected