(tmp_dir)
| 34 | """ |
| 35 | |
| 36 | def unsafe_execute(tmp_dir): |
| 37 | random_id = random.randint(1, 100000) |
| 38 | if "python" in language_type.lower(): |
| 39 | with create_tempdir(): |
| 40 | |
| 41 | # These system calls are needed when cleaning up tempdir. |
| 42 | import os |
| 43 | import shutil |
| 44 | rmtree = shutil.rmtree |
| 45 | rmdir = os.rmdir |
| 46 | chdir = os.chdir |
| 47 | |
| 48 | # Disable functionalities that can make destructive changes to the test. |
| 49 | reliability_guard() |
| 50 | |
| 51 | try: |
| 52 | exec_globals = {} |
| 53 | with swallow_io(): |
| 54 | with time_limit(timeout): |
| 55 | # WARNING |
| 56 | # This program exists to execute untrusted model-generated code. Although |
| 57 | # it is highly unlikely that model-generated code will do something overtly |
| 58 | # malicious in response to this test suite, model-generated code may act |
| 59 | # destructively due to a lack of model capability or alignment. |
| 60 | # Users are strongly encouraged to sandbox this evaluation suite so that it |
| 61 | # does not perform destructive actions on their host or network. |
| 62 | # Once you have read this disclaimer and taken appropriate precautions, |
| 63 | # uncomment the following line and proceed at your own risk: |
| 64 | exec(sample["test_code"], exec_globals) |
| 65 | result.append("passed") |
| 66 | except TimeoutException: |
| 67 | result.append("timed out") |
| 68 | except AssertionError as e: |
| 69 | result.append(f"failed: AssertionError") |
| 70 | except BaseException as e: |
| 71 | result.append(f"failed: {e}") |
| 72 | #print(sample["test_code"]) |
| 73 | #print(result) |
| 74 | # Needed for cleaning up. |
| 75 | shutil.rmtree = rmtree |
| 76 | os.rmdir = rmdir |
| 77 | os.chdir = chdir |
| 78 | |
| 79 | elif "go" in language_type.lower(): |
| 80 | assert tmp_dir is not None, "Go should be evaluated in a dir where necessary module files installed." |
| 81 | |
| 82 | import os |
| 83 | import shutil |
| 84 | |
| 85 | if "tmp" not in tmp_dir: |
| 86 | tmp_dir = os.path.join(tmp_dir, "tmp") |
| 87 | tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}") |
| 88 | if not os.path.exists(tmp_dir): |
| 89 | os.makedirs(tmp_dir) |
| 90 | origin_path = os.getcwd() |
| 91 | os.chdir(tmp_dir) |
| 92 | open(f"main_test.go", 'w').write(sample["test_code"]) |
| 93 | try: |
nothing calls this directly
no test coverage detected