Function main

evalplus/perf/sampling.py:181–289

In the synthesizer file, each line includes a set of input generators for a task. The goal of this script is to use these generators to sample inputs for each task. The generated inputs are expected to be valid.

(input: str, output: str)

def main(input: str, output: str):
    """In the synthesizer file, each line includes a set of input generators for a task.
    The goal of this script is to use these generators to sample inputs for each task.
    The generated inputs are expected to be valid.
    """
    assert output.endswith(".jsonl"), "output must be a .jsonl file"

    id2task = {}
    for task_id, item in get_human_eval_plus().items():
        id2task[task_id] = item

    for task_id, item in get_mbpp_plus().items():
        id2task[task_id] = item

    # loading the synthesizers
    with open(input, "r") as f:
        synthesizers = [json.loads(l) for l in f]

    n_total = 0
    n_parsed = 0
    n_dedup = 0

    for item in synthesizers:
        item["synthesizers"] = [post_process(s) for s in item["synthesizers"]]
        n_total += len(item["synthesizers"])
        item["synthesizers"] = [s for s in item["synthesizers"] if s is not None]
        n_parsed += len(item["synthesizers"])

        dedup_set = set()
        for s in item["synthesizers"]:
            dedup_set.add(
                "\n".join(
                    [l for l in s.splitlines() if l.strip() and not l.startswith("#")]
                )
            )
        item["synthesizers"] = list(dedup_set)
        n_dedup += len(item["synthesizers"])

    print(
        colored(
            f"#Total {n_total} with {n_parsed} parsed => {100 * (1 - n_parsed / n_total) :.1f}% syntax err",
            "green",
        )
    )

    print(
        colored(
            f"#Parsed {n_parsed} with {n_dedup} dedup => {100 * (1 - n_dedup / n_parsed) :.1f}% duplicate",
            "green",
        )
    )

    # resume mode check finished tasks
    finished_tasks = set()
    if os.path.isfile(output):
        with open(output, "r") as f:
            for l in f:
                item = json.loads(l)