Evaluates the functional correctness of a completion by running the test suite provided in the problem. :param completion_id: an optional completion ID so we can match the results later even if execution finishes asynchronously.
(check_program, timeout=3)
| 62 | """ |
| 63 | |
| 64 | def check_correctness(check_program, timeout=3): |
| 65 | """ |
| 66 | Evaluates the functional correctness of a completion by running the test |
| 67 | suite provided in the problem. |
| 68 | |
| 69 | :param completion_id: an optional completion ID so we can match |
| 70 | the results later even if execution finishes asynchronously. |
| 71 | """ |
| 72 | manager = multiprocessing.Manager() |
| 73 | result = manager.list() |
| 74 | |
| 75 | p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout)) |
| 76 | p.start() |
| 77 | p.join(timeout=timeout + 1) |
| 78 | if p.is_alive(): |
| 79 | p.kill() |
| 80 | |
| 81 | if not result: |
| 82 | result.append("timed out") |
| 83 | |
| 84 | return result[0] == "passed" |
| 85 | |
| 86 | |
| 87 | def unsafe_execute(check_program, result, timeout): |