Parameters: * dataset: {None, "HumanEval", "HumanEvalPlus"} * task_id: ralated to dataset * impl: {"canonical", source code} * inputs: {"base_inputs", list} * mode: {"branch"}, will support "line" for coverage-guided LLM test generation
(
dataset: str = "HumanEvalPlus",
task_id: str = "HumanEval/0",
impl: str = "canonical",
inputs: Union[str, List[List[Any]]] = "base_input",
mode: str = "branch",
)
| 126 | |
| 127 | |
| 128 | def test_solution_coverage( |
| 129 | dataset: str = "HumanEvalPlus", |
| 130 | task_id: str = "HumanEval/0", |
| 131 | impl: str = "canonical", |
| 132 | inputs: Union[str, List[List[Any]]] = "base_input", |
| 133 | mode: str = "branch", |
| 134 | ): |
| 135 | """ |
| 136 | Parameters: |
| 137 | * dataset: {None, "HumanEval", "HumanEvalPlus"} |
| 138 | * task_id: ralated to dataset |
| 139 | * impl: {"canonical", source code} |
| 140 | * inputs: {"base_inputs", list} |
| 141 | * mode: {"branch"}, will support "line" for coverage-guided LLM test generation |
| 142 | """ |
| 143 | if "HumanEval" in dataset: |
| 144 | problems, problem = get_human_eval_plus(), None |
| 145 | for p in problems: |
| 146 | if p["task_id"] == task_id: |
| 147 | problem = p |
| 148 | assert problem != None, f"invalid {task_id = }" |
| 149 | entry_point = problem["entry_point"] |
| 150 | code = problem["prompt"] + ( |
| 151 | impl if impl != "canonical" else problem["canonical_solution"] |
| 152 | ) |
| 153 | if inputs == "base_input": |
| 154 | inputs = problem["base_input"] |
| 155 | else: |
| 156 | raise NotImplementedError |
| 157 | |
| 158 | return test_code_coverage(code, inputs, entry_point, mode) |
| 159 | |
| 160 | |
| 161 | if __name__ == "__main__": |
no test coverage detected