Function get_human_eval

evalplus/data/humaneval.py:69–96

Get HumanEval from OpenAI's github repo and return as a list of parsed dicts. Returns: List[Dict[str, str]]: List of dicts with keys "prompt", "test", "entry_point" Notes: "task_id" is the identifier string for the task. "prompt" is the prompt to be used for the tas

()

def get_human_eval() -> Dict[str, Dict]:
    """Get HumanEval from OpenAI's github repo and return as a list of parsed dicts.

    Returns:
        List[Dict[str, str]]: List of dicts with keys "prompt", "test", "entry_point"

    Notes:
        "task_id" is the identifier string for the task.
        "prompt" is the prompt to be used for the task (function signature with docstrings).
        "test" is test-cases wrapped in a `check` function.
        "entry_point" is the name of the function.
    """
    # Check if human eval file exists in CACHE_DIR
    human_eval_path = os.path.join(CACHE_DIR, "HumanEval.jsonl")
    make_cache(
        "https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz",
        human_eval_path,
    )

    human_eval = open(human_eval_path, "r").read().split("\n")
    human_eval = [json.loads(line) for line in human_eval if line]

    # Handle 115_max_fill.py to make its docstring well-formed
    human_eval[115]["prompt"] = "import math\n" + human_eval[115]["prompt"].replace(
        "import math\n", ""
    )

    return {task["task_id"]: task for task in human_eval}