(self)
| 82 | |
| 83 | |
| 84 | def process_solutions(self): |
| 85 | count = 0 |
| 86 | total = 0 |
| 87 | for index, prompt in enumerate(self.solutions): |
| 88 | print("Processing solutions for instance #{}".format(index), end = "\r", file = sys.stderr) |
| 89 | new_solutions = [] |
| 90 | for i, solution in enumerate(self.solutions[prompt]): |
| 91 | total += 1 |
| 92 | processor = CodeProcessor(solution, entry_point = self.dataset.prompt2instance[prompt]["entry_point"] if "entry_point" in self.dataset.prompt2instance[prompt] else None, force_rename = True if self.dataset.name in ["mbpp", "openai_humaneval"] else False) |
| 93 | res = processor.run() |
| 94 | if FAILED == res[0]: |
| 95 | count += 1 |
| 96 | new_solutions.append(res) |
| 97 | |
| 98 | self.solutions[prompt] = new_solutions |
| 99 | |
| 100 | print(f"Processing completed for dataset {self.dataset.name}. {count}/{total} ({count/total}) invalid solutions found.") |
| 101 | |
| 102 | |
| 103 | class Evaluator(object): |
nothing calls this directly
no test coverage detected