| 17 | |
| 18 | |
| 19 | def replay_contract(data: dict, tid: int) -> dict: |
| 20 | code = data[tid]["prompt"] + data[tid]["contract"] |
| 21 | exec(code) |
| 22 | func = locals()[data[tid]["entry_point"]] |
| 23 | |
| 24 | new_inputs = [] |
| 25 | for inputs in data[tid]["plus_input"]: |
| 26 | try: |
| 27 | func(*inputs) |
| 28 | new_inputs.append(inputs) |
| 29 | except Exception as e: |
| 30 | assert str(e) == "invalid inputs" |
| 31 | |
| 32 | before, after = len(data[tid]["plus_input"]), len(new_inputs) |
| 33 | data[tid]["plus_input"] = new_inputs |
| 34 | print(f"HumanEval/{tid}: {before} -> {after}") |
| 35 | |
| 36 | return before - after |
| 37 | |
| 38 | |
| 39 | def debug_output(version: str, tasks: List[int]): |