Function eval_jsonl

eval_output_jsonl.py:18–37

(res_file: str, react: bool = False)

def eval_jsonl(res_file: str, react: bool = False) -> float:
    cnt, total = 0, 0
    with open(res_file, "r") as f:
        for line in tqdm(f):
            d = json.loads(line.strip())
            answer = d["answer"]
            if react:
                # react is the only one solution.
                last_tag = sorted(d["react"].keys(), key=lambda x: len(x), reverse=True)[0]
                prediction = d["react"][last_tag]["final_answer"]
            else:
                if d["react"]["solutions"]:
                    # get top-1
                    prediction = d["react"]["solutions"][0]["final_answer"]
                else:
                    prediction = ""
            if math_is_equiv(answer, prediction):
                cnt += 1
            total += 1
    return cnt / total


if __name__ == '__main__':