Function script

evaluation/evalplus/evalplus/evalperf.py:250–361

(
    samples: str,
    dataset: str,
    output_dir: str,
    profile_n_correct: int = 10,
    max_n_samples: int = 50,
    max_parallelism: int = None,
    lazy_evaluation: bool = False,
    i_just_wanna_run: bool = False,
)

def script(
    samples: str,
    dataset: str,
    output_dir: str,
    profile_n_correct: int = 10,
    max_n_samples: int = 50,
    max_parallelism: int = None,
    lazy_evaluation: bool = False,
    i_just_wanna_run: bool = False,
):
    assert (
        profile_n_correct <= max_n_samples
    ), "profile_n_correct should be no more than max_n_samples"
    assert dataset.endswith(".jsonl") and os.path.isfile(dataset)
    assert samples.endswith(".jsonl") and os.path.isfile(samples)

    console = Console()  # printer

    if lazy_evaluation:
        console.print(
            "Lazy evaluation is enabled: "
            "It's faster but cannot address the order inconsistency on noisy testbed."
        )

    # load evalplus data
    problems, expected_output = get_evalplus_data()

    # load evalperf data
    with open(dataset, "r") as f:
        raw_data = [json.loads(l) for l in f]
        tasks = {task["task_id"]: task for task in raw_data}

    # setup max CPU threads
    max_workers = max(1, multiprocessing.cpu_count() // 4)
    if max_parallelism is not None:
        max_workers = min(max_workers, max_parallelism)

    model_name = os.path.basename(samples).replace(".jsonl", "")
    result_path = os.path.join(output_dir, f"{(model_name + '_results')}.json")

    # resume results
    eval_results = {}
    if not i_just_wanna_run and os.path.exists(result_path):
        eval_results = json.load(open(result_path, "r"))
        # pop tasks that have been evaluated
        for evaluated_task in eval_results:
            tasks.pop(evaluated_task, None)

        console.print(f"Resumed {len(eval_results)} results from {result_path}")

    # load model's solutions
    samples = {
        task["task_id"]: task["solution"][:max_n_samples]
        for task in stream_jsonl(samples)
    }

    # log all tasks
    console.print(f"{len(tasks)} tasks to evaluate :: result path: {result_path}")