Method process_results

test/general/lm_eval/tasks/hendrycks_math.py:68–80

(self, doc, results)

        return rf.greedy_until(ctx, {"until": ["\n"]})

    def process_results(self, doc, results):
        retval = 0
        indices = [pos for pos, char in enumerate(results[0]) if char == "$"]
        if len(indices) <= 1:
            answer = results[0]
        else:
            answer = results[0][indices[0] + 1 : indices[-1]]

        if self.is_equiv(
            answer, self.remove_boxed(self.last_boxed_only_string(doc["solution"]))
        ):
            retval = 1
        return {"acc": retval}

    def aggregation(self):
        return {"acc": mean}