(set_cover_info: Dict[str, List[str]], sample_eval_dir: str, model: str)
| 138 | |
| 139 | |
| 140 | def gen_report(set_cover_info: Dict[str, List[str]], sample_eval_dir: str, model: str): |
| 141 | tsr_dict = {"ntests": compute_avg_test(set_cover_info), "pass@1": 0} |
| 142 | model_path = os.path.join(sample_eval_dir, f"{model}_temp_0.0", "eval_results.json") |
| 143 | with open(model_path, "r") as f: |
| 144 | mdict = json.load(f) |
| 145 | correct_cnt = 0 |
| 146 | for task_id in task_ids: |
| 147 | legacy_task_id = task_id |
| 148 | if legacy_task_id not in mdict["eval"]: |
| 149 | legacy_task_id = legacy_task_id.replace("/", "_") |
| 150 | if mdict["eval"][legacy_task_id]["base"][0][0] != "success": |
| 151 | continue |
| 152 | correct = True |
| 153 | for plus_id in set_cover_info[task_id]: |
| 154 | index = int(plus_id.split("_")[-1]) |
| 155 | if mdict["eval"][legacy_task_id]["plus"][0][1][index] == False: |
| 156 | correct = False |
| 157 | break |
| 158 | if correct: |
| 159 | correct_cnt += 1 |
| 160 | tsr_dict["pass@1"] = correct_cnt / problem_count |
| 161 | return tsr_dict |
| 162 | |
| 163 | |
| 164 | def dump_humaneval_plus_mini(set_cover_info: Dict[str, List[str]], mini_path: str): |
no test coverage detected
searching dependent graphs…