Function get_successful_tests_stdio

open_instruct/code_utils/code_utils.py:222–263

Same as above but for stdio format. Parameter: program: a string representation of the python program you want to run tests: a list of (input, output) pairs max_execution_time: the number of second each individual test can run before it is considered failed an

(
    program: str, tests: list[Any], max_execution_time: float = 1.0
)

def get_successful_tests_stdio(
    program: str, tests: list[Any], max_execution_time: float = 1.0
) -> tuple[list[int], list[float]]:
    """Same as above but for stdio format.
    Parameter:
        program: a string representation of the python program you want to run
        tests: a list of (input, output) pairs
        max_execution_time: the number of second each individual test can run before
            it is considered failed and terminated
    Return:
        a tuple of (results, runtimes). results is a list of 0/1 indicating
        passed or not, runtimes is a list of execution times for each test.
    """
    test_ct = len(tests)
    if test_ct == 0:
        return [], []
    if not should_execute(program=program, tests=tests):
        logger.info("Not executing program %s", program)
        return [0] * len(tests), [-1.0] * len(tests)

    stdio_test_results = multiprocessing.Array("i", test_ct)
    stdio_runtimes = multiprocessing.Array("d", test_ct)

    for i in range(test_ct):
        stdio_test_results[i] = 0  # Initialize results to 0 (failure)
        stdio_runtimes[i] = -1.0

    # Total timeout needs to account for all tests running sequentially.
    total_timeout = max_execution_time * test_ct + 5.0

    p = multiprocessing.Process(
        target=run_tests_stdio_helper, args=(program, tests, max_execution_time, stdio_test_results, stdio_runtimes)
    )
    p.start()
    p.join(timeout=total_timeout)

    if p.is_alive():
        p.kill()
        p.join()
    p.close()

    return [stdio_test_results[i] for i in range(test_ct)], [stdio_runtimes[i] for i in range(test_ct)]


# -------------------------------------------------------------