(generation_path: str, result_path: str, temp_dir: str)
| 19 | return codelist[0] |
| 20 | |
| 21 | def evaluate_main(generation_path: str, result_path: str, temp_dir: str): |
| 22 | problem_path = (DATA_DIR / f"{version}.jsonl").as_posix() |
| 23 | |
| 24 | print(problem_path) |
| 25 | problems = [json.loads(line) for line in open(problem_path, 'r')] |
| 26 | |
| 27 | id2problems = { x['task_id']: x for x in problems } |
| 28 | |
| 29 | results = [json.loads(line) for line in open(generation_path, 'r')] |
| 30 | for result in results: |
| 31 | if 'task_id' not in result: |
| 32 | result['task_id'] = problems[result['index']]['task_id'] |
| 33 | |
| 34 | if 'generation' not in result: |
| 35 | try: |
| 36 | if 'output' not in result: |
| 37 | result['output'] = result['response'] |
| 38 | if result['output'].startswith("\n "): |
| 39 | func_code = extract_python_code(result['prompt_sft']).strip() |
| 40 | result['generation'] = func_code + '\n' + result['output'] |
| 41 | else: |
| 42 | result['generation'] = extract_python_code(result['output']) |
| 43 | except: |
| 44 | result['generation'] = result['output'] |
| 45 | |
| 46 | with open(result_path, 'w') as fr: |
| 47 | for result in results: |
| 48 | fr.write(json.dumps(result) + "\n") |
| 49 | |
| 50 | score = evaluate_functional_correctness( |
| 51 | input_file=result_path, |
| 52 | tmp_dir=temp_dir, |
| 53 | problem_file=problem_path, |
| 54 | result_path=result_path |
| 55 | ) |
| 56 | |
| 57 | hardness_results = defaultdict(int) |
| 58 | for result in [json.loads(line) for line in open(result_path, 'r')]: |
| 59 | problem = id2problems[result['task_id']] |
| 60 | |
| 61 | hardness = problem['meta']['difficulty'] |
| 62 | hardness_results[hardness] += 1 |
| 63 | hardness_results[hardness + "_correct"] += result['passed'] |
| 64 | |
| 65 | print("="*100) |
| 66 | print("Evaluate {} over.".format(generation_path)) |
| 67 | print("Pass@1: {:.3f}".format(score["pass@1"])) |
| 68 | for key in ["Easy", "Medium", "Hard"]: |
| 69 | if key.endswith("_correct"): |
| 70 | continue |
| 71 | acc = hardness_results[key+"_correct"] / hardness_results[key] |
| 72 | print("{}: {:.3f}({}/{})".format(key, acc, hardness_results[key+"_correct"], hardness_results[key])) |
| 73 | |
| 74 | if __name__ == '__main__': |
| 75 | import argparse |
no test coverage detected