(sample, problems, example_test=False)
| 25 | |
| 26 | |
| 27 | def process_humaneval_test(sample, problems, example_test=False): |
| 28 | task_id = sample["task_id"] |
| 29 | language = task_id.split("/")[0].lower() |
| 30 | |
| 31 | prompt = sample["prompt"] |
| 32 | if example_test and "example_test" in problems[task_id] and problems[task_id]["example_test"] != "": |
| 33 | test = problems[task_id]["example_test"] |
| 34 | else: |
| 35 | test = problems[task_id]["test"] |
| 36 | code = sample["generation"] |
| 37 | |
| 38 | # Pre-process for different languages |
| 39 | if language == "python": |
| 40 | code_ = [] |
| 41 | for line in code.split("\n"): |
| 42 | if (len(line.strip()) > 0 and line[0] != ' ' and line[0] != '\t'): |
| 43 | break |
| 44 | code_.append(line) |
| 45 | code = "\n".join(code_) |
| 46 | test_setup = "\n".join(IMPORT_HELPER["python"]) + "\n" |
| 47 | test_string = test_setup + prompt + code + "\n" + test + "\n" |
| 48 | elif language == "cpp": |
| 49 | test_set_up = "" |
| 50 | for s in IMPORT_HELPER["cpp"]: |
| 51 | if s not in prompt: |
| 52 | test_set_up += s + "\n" |
| 53 | test_string = test_set_up + "\n" + prompt + code + "\n" + test |
| 54 | elif language == "java": |
| 55 | test_string = prompt + code + "\n" + test |
| 56 | elif language == "js" or language == "javascript": |
| 57 | test_string = prompt + code + "\n" + test |
| 58 | elif language == "go": |
| 59 | import_string = problems[task_id]["import"] |
| 60 | prompt = prompt.replace(import_string, "") |
| 61 | if example_test and "example_test" in problems[task_id]: |
| 62 | test = problems[task_id]["example_test"] |
| 63 | else: |
| 64 | test = problems[task_id]["test"] |
| 65 | test_setup = problems[task_id]["test_setup"] |
| 66 | other_pkgs = [] |
| 67 | for pkg in IMPORT_HELPER["go"]: |
| 68 | if pkg not in test_setup: |
| 69 | p = pkg.split("/")[-1] |
| 70 | if p + "." in code: |
| 71 | other_pkgs.append(f"\"{pkg}\"") |
| 72 | if other_pkgs: |
| 73 | import_other_pkgs = "import (\n" + " ".join([p + "\n" for p in other_pkgs]) + ")" |
| 74 | test_string = test_setup + "\n" + import_other_pkgs + "\n" + prompt + code + "\n" + test |
| 75 | else: |
| 76 | test_string = test_setup + "\n" + prompt + code + "\n" + test |
| 77 | elif language == "rust": |
| 78 | main = "\nfn main(){ \n } \n" |
| 79 | declaration = problems[task_id]["declaration"] |
| 80 | test_string = main + declaration + prompt + code + test |
| 81 | |
| 82 | return test_string |
| 83 | |
| 84 |
no outgoing calls
no test coverage detected