Evaluates the functional correctness of a completion by running the test suite provided in the problem. :param completion_id: an optional completion ID so we can match the results later even if execution finishes asynchronously.
(check_program, timeout, task_id, completion_id)
| 26 | |
| 27 | |
| 28 | def check_correctness(check_program, timeout, task_id, completion_id): |
| 29 | """ |
| 30 | Evaluates the functional correctness of a completion by running the test |
| 31 | suite provided in the problem. |
| 32 | |
| 33 | :param completion_id: an optional completion ID so we can match |
| 34 | the results later even if execution finishes asynchronously. |
| 35 | """ |
| 36 | manager = multiprocessing.Manager() |
| 37 | result = manager.list() |
| 38 | |
| 39 | p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout)) |
| 40 | p.start() |
| 41 | p.join(timeout=timeout + 1) |
| 42 | if p.is_alive(): |
| 43 | p.kill() |
| 44 | |
| 45 | if not result: |
| 46 | result.append("timed out") |
| 47 | |
| 48 | return dict( |
| 49 | task_id=task_id, |
| 50 | passed=result[0] == "passed", |
| 51 | result=result[0], |
| 52 | completion_id=completion_id, |
| 53 | ) |
| 54 | |
| 55 | |
| 56 | def unsafe_execute(check_program, result, timeout): |
nothing calls this directly
no outgoing calls
no test coverage detected
searching dependent graphs…