Function correctness_worker

evalplus/evalperf.py:115–137

(task_id: str, samples: list, ctask: Dict, expected_output: Dict)

def correctness_worker(task_id: str, samples: list, ctask: Dict, expected_output: Dict):
    assert isinstance(
        samples, list
    ), f"{task_id}: samples is not a list but {type(samples)}"

    results = []

    for solution in samples:
        result, solution = correctness_check(
            solution, task_id.split("/")[0].lower(), ctask, expected_output
        )
        results.append(
            {
                "solution": solution,
                "pass": result[0] == PASS,
                "profiled": False,
                "matching_cluster_idx": None,
                "dps": None,
                "dps_norm": None,
            }
        )

    return task_id, results


def perf_worker(