(generation_path: str, result_path: str, temp_dir: str)
| 120 | return codelist[0] |
| 121 | |
| 122 | def evaluate_main(generation_path: str, result_path: str, temp_dir: str): |
| 123 | problem_path = args.input_data |
| 124 | print(problem_path) |
| 125 | problems = [json.loads(line) for line in open(problem_path, 'r',encoding='utf-8')] |
| 126 | |
| 127 | id2problems = { x['task_id']: x for x in problems } |
| 128 | |
| 129 | results = [json.loads(line) for line in open(generation_path, 'r')] |
| 130 | for result in results: |
| 131 | if 'task_id' not in result: |
| 132 | result['task_id'] = problems[result['index']]['task_id'] |
| 133 | |
| 134 | if 'generation' not in result: |
| 135 | try: |
| 136 | if 'output' not in result: |
| 137 | result['output'] = result['response'] |
| 138 | if result['output'].startswith("\n "): |
| 139 | func_code = extract_python_code(result['prompt_sft']).strip() |
| 140 | result['generation'] = func_code + '\n' + result['output'] |
| 141 | else: |
| 142 | result['generation'] = extract_python_code(result['output']) |
| 143 | except: |
| 144 | result['generation'] = result['output'] |
| 145 | |
| 146 | with open(result_path, 'w') as fr: |
| 147 | for result in results: |
| 148 | fr.write(json.dumps(result) + "\n") |
| 149 | |
| 150 | score = evaluate_functional_correctness( |
| 151 | input_file=result_path, |
| 152 | tmp_dir=temp_dir, |
| 153 | problem_file=problem_path, |
| 154 | result_path=result_path |
| 155 | ) |
| 156 | |
| 157 | hardness_results = defaultdict(int) |
| 158 | for result in [json.loads(line) for line in open(result_path, 'r')]: |
| 159 | problem = id2problems[result['task_id']] |
| 160 | |
| 161 | hardness = problem['meta']['difficulty'] |
| 162 | hardness_results[hardness] += 1 |
| 163 | hardness_results[hardness + "_correct"] += result['passed'] |
| 164 | |
| 165 | print("="*100) |
| 166 | print("Evaluate {} over.".format(generation_path)) |
| 167 | print("Pass@1: {:.3f}".format(score["pass@1"])) |
| 168 | for key in ["Easy", "Medium", "Hard"]: |
| 169 | if key.endswith("_correct"): |
| 170 | continue |
| 171 | acc = hardness_results[key+"_correct"] / hardness_results[key] |
| 172 | print("{}: {:.3f}({}/{})".format(key, acc, hardness_results[key+"_correct"], hardness_results[key])) |
| 173 | |
| 174 | score_path = os.path.join(args.save_dir, "result.txt") |
| 175 | with open(score_path, "w") as f: |
| 176 | f.write("Pass@1: {:.3f}\n".format(score["pass@1"])) |
| 177 | for key in ["Easy", "Medium", "Hard"]: |
| 178 | if key.endswith("_correct"): |
| 179 | continue |
no test coverage detected