Function unsafe_execute

evalplus/eval/__init__.py:122–220

(
    dataset: str,
    entry_point: str,
    code: str,
    inputs,
    expected: List,
    time_limits,
    atol,
    fast_check,
    stat,  # Value
    details,  # Array
    progress,  # Value
)

def unsafe_execute(
    dataset: str,
    entry_point: str,
    code: str,
    inputs,
    expected: List,
    time_limits,
    atol,
    fast_check,
    stat,  # Value
    details,  # Array
    progress,  # Value
):
    with create_tempdir():
        # These system calls are needed when cleaning up tempdir.
        import os
        import shutil

        rmtree = shutil.rmtree
        rmdir = os.rmdir
        chdir = os.chdir
        reliability_guard(maximum_memory_bytes=query_maximum_memory_bytes())
        exec_globals = {}
        try:
            with swallow_io():
                exec(code, exec_globals)
                fn = exec_globals[entry_point]

            for i, inp in enumerate(inputs):
                try:
                    with time_limit(time_limits[i]):
                        with swallow_io():
                            out = fn(*inp)

                    exp = expected[i]
                    exact_match = out == exp

                    # ================================================ #
                    # ============== special oracles ================= #
                    if dataset == "mbpp":
                        if "are_equivalent" == entry_point:  # Mbpp/164 special oracle
                            exact_match = exact_match or True
                        elif "sum_div" == entry_point:  # Mbpp/295 special oracle
                            exact_match = exact_match or out == 0
                        elif "surface_Area" == entry_point:  # Mbpp/581 special oracle
                            exact_match = (
                                exact_match or abs(out - _surface_Area(*inp)) <= atol
                            )
                        elif (
                            "digit_distance_nums" == entry_point
                        ):  # Mbpp/558 special oracle
                            exact_match = exact_match or out == _digit_distance_nums(
                                *inp
                            )
                        elif entry_point in MBPP_OUTPUT_SET_EQ_TASKS:
                            exact_match = set(out) == set(exp)
                        elif entry_point in MBPP_OUTPUT_NOT_NONE_TASKS:
                            # exp is True  if not None