(problems, hashcode, tasks_only_output_not_none)
| 43 | |
| 44 | |
| 45 | def get_groundtruth(problems, hashcode, tasks_only_output_not_none): |
| 46 | cache_file = os.path.join(CACHE_DIR, f"{hashcode}.pkl") |
| 47 | if os.path.exists(cache_file): |
| 48 | print(f"Load from ground-truth from {cache_file}") |
| 49 | with open(cache_file, "rb") as f: |
| 50 | return pickle.load(f) |
| 51 | |
| 52 | os.makedirs(CACHE_DIR, exist_ok=True) |
| 53 | print("Computing expected output...") |
| 54 | tbegin = time.time() |
| 55 | expected_output = {} |
| 56 | for task_id, problem in problems.items(): |
| 57 | oracle = {} |
| 58 | oracle["base"], oracle["base_time"] = trusted_exec( |
| 59 | problem["prompt"] + problem["canonical_solution"], |
| 60 | problem["base_input"], |
| 61 | problem["entry_point"], |
| 62 | record_time=True, |
| 63 | output_not_none=problem["entry_point"] in tasks_only_output_not_none, |
| 64 | ) |
| 65 | |
| 66 | oracle["plus"], oracle["plus_time"] = trusted_exec( |
| 67 | problem["prompt"] + problem["canonical_solution"], |
| 68 | problem["plus_input"], |
| 69 | problem["entry_point"], |
| 70 | record_time=True, |
| 71 | output_not_none=problem["entry_point"] in tasks_only_output_not_none, |
| 72 | ) |
| 73 | expected_output[task_id] = oracle |
| 74 | print(f"Expected outputs computed in {time.time() - tbegin:.2f}s") |
| 75 | |
| 76 | with open(cache_file, "wb") as f: |
| 77 | pickle.dump(expected_output, f) |
| 78 | |
| 79 | return expected_output |
| 80 | |
| 81 | def remove_unindented_lines(code, protect_before, execeptions, trim_tails): |
| 82 | lines = code.splitlines() |
no test coverage detected