| 19 | logger = logging.getLogger(__name__) |
| 20 | |
| 21 | def evaluate_test_cases_multi_solution(prompt, testcases_input, |
| 22 | multi_solutions, timeout=0.1): |
| 23 | logger.info(f'Start evaluation with test code, timeout={timeout}') |
| 24 | |
| 25 | with ProcessPoolExecutor() as executor: |
| 26 | futures = [] |
| 27 | results = [[None for _ in multi_solutions] for _ in testcases_input] |
| 28 | |
| 29 | for i, testcase in enumerate(testcases_input): |
| 30 | for j, solution in enumerate(multi_solutions): |
| 31 | args = (prompt, solution, testcase, timeout) |
| 32 | future = executor.submit(test_case_against_solution, *args) |
| 33 | futures.append((i, j, future)) |
| 34 | logger.info(f'{len(futures)} execution requests are submitted') |
| 35 | |
| 36 | for completed_future in as_completed([f[2] for f in futures]): |
| 37 | for i, j, future in futures: |
| 38 | if future == completed_future: |
| 39 | logger.info('[{}/{}] execution completed'.format( |
| 40 | i * len(multi_solutions) + j + 1, len(futures))) |
| 41 | result = completed_future.result() |
| 42 | results[i][j] = result |
| 43 | break |
| 44 | |
| 45 | return results |
| 46 | |
| 47 | |
| 48 | |