Evaluates the functional correctness of a completion by running the test suite provided in the problem.
(
task_id: str,
sample: dict,
language_type: str,
timeout: float = 3.0,
tmp_dir: str = None,
completion_id: Optional[int] = None,
)
| 42 | |
| 43 | |
| 44 | def check_correctness( |
| 45 | task_id: str, |
| 46 | sample: dict, |
| 47 | language_type: str, |
| 48 | timeout: float = 3.0, |
| 49 | tmp_dir: str = None, |
| 50 | completion_id: Optional[int] = None, |
| 51 | ) -> Dict: |
| 52 | """ |
| 53 | Evaluates the functional correctness of a completion by running the test |
| 54 | suite provided in the problem. |
| 55 | """ |
| 56 | |
| 57 | def unsafe_execute(tmp_dir): |
| 58 | random_id = random.uniform(1, 1000) |
| 59 | if "python" in language_type.lower(): |
| 60 | with create_tempdir(): |
| 61 | |
| 62 | # These system calls are needed when cleaning up tempdir. |
| 63 | import os |
| 64 | import shutil |
| 65 | rmtree = shutil.rmtree |
| 66 | rmdir = os.rmdir |
| 67 | chdir = os.chdir |
| 68 | |
| 69 | # Disable functionalities that can make destructive changes to the test. |
| 70 | reliability_guard() |
| 71 | |
| 72 | try: |
| 73 | exec_globals = {} |
| 74 | with swallow_io(): |
| 75 | with time_limit(timeout): |
| 76 | # WARNING |
| 77 | # This program exists to execute untrusted model-generated code. Although |
| 78 | # it is highly unlikely that model-generated code will do something overtly |
| 79 | # malicious in response to this test suite, model-generated code may act |
| 80 | # destructively due to a lack of model capability or alignment. |
| 81 | # Users are strongly encouraged to sandbox this evaluation suite so that it |
| 82 | # does not perform destructive actions on their host or network. |
| 83 | # Once you have read this disclaimer and taken appropriate precautions, |
| 84 | # uncomment the following line and proceed at your own risk: |
| 85 | exec(sample["test_code"], exec_globals) |
| 86 | result.append("passed") |
| 87 | except TimeoutException: |
| 88 | result.append("timed out") |
| 89 | except AssertionError as e: |
| 90 | result.append(f"failed: AssertionError") |
| 91 | except BaseException as e: |
| 92 | result.append(f"failed: {e}") |
| 93 | |
| 94 | # Needed for cleaning up. |
| 95 | shutil.rmtree = rmtree |
| 96 | os.rmdir = rmdir |
| 97 | os.chdir = chdir |
| 98 | |
| 99 | elif "go" in language_type.lower(): |
| 100 | assert tmp_dir is not None, "Go should be evaluated in a dir where necessary module files installed." |
| 101 |