Evaluates the implementation on Human-Eval Python. probably should be written in a dataset-agnostic way but not now
(self, name: str, func: str, test: str, timeout: int = 5)
| 99 | return is_passing, feedback, tuple(state) |
| 100 | |
| 101 | def evaluate(self, name: str, func: str, test: str, timeout: int = 5) -> bool: |
| 102 | """ |
| 103 | Evaluates the implementation on Human-Eval Python. |
| 104 | |
| 105 | probably should be written in a dataset-agnostic way but not now |
| 106 | """ |
| 107 | |
| 108 | code = f"""{func} |
| 109 | |
| 110 | {test} |
| 111 | |
| 112 | check({name}) |
| 113 | """ |
| 114 | try: |
| 115 | function_with_timeout(exec, (code, globals()), timeout) |
| 116 | return True |
| 117 | except Exception: |
| 118 | return False |
| 119 |
nothing calls this directly
no test coverage detected