(self, remove_instance = False, update_solution = False, debug = False, start_index = None, verify_num = None)
| 282 | |
| 283 | |
| 284 | def verify_groundtruth(self, remove_instance = False, update_solution = False, debug = False, start_index = None, verify_num = None): |
| 285 | prompts, overlong_prompts = self.dataset.get_all_prompts() |
| 286 | |
| 287 | max_count = 2 |
| 288 | count = 0 |
| 289 | |
| 290 | success = 0 |
| 291 | |
| 292 | for index, prompt in enumerate(prompts + overlong_prompts): |
| 293 | if len(self.dataset.prompt2groundtruth[prompt]) == 0: |
| 294 | continue |
| 295 | elif len(self.dataset.prompt2testcase[prompt]) == 0: |
| 296 | continue |
| 297 | if start_index != None and index < start_index: |
| 298 | continue |
| 299 | if start_index != None and verify_num != None and index >= start_index + verify_num: |
| 300 | break |
| 301 | if index in [40, 264, 2156, 2200, 3799, 4812] and self.dataset.name == "codeparrot/apps": |
| 302 | self.dataset.prompt2groundtruth[prompt] = [] |
| 303 | continue |
| 304 | succeed = False |
| 305 | if update_solution: |
| 306 | new_solutions = [] |
| 307 | for i, gts in enumerate(self.dataset.prompt2groundtruth[prompt]): |
| 308 | print("Verifying instance #{} solution #{} ".format(index, i), end = '\r', file = sys.stderr) |
| 309 | gt, io = gts |
| 310 | stat, results = self.execute_code(gt, io, self.dataset.prompt2testcase[prompt], check = True, fast_check = True) |
| 311 | |
| 312 | if stat == "fail": |
| 313 | failed_cases = [res for res in results if res["status"] == -1] |
| 314 | if debug: |
| 315 | pass |
| 316 | ''' |
| 317 | print(results) |
| 318 | print("-"*20 + f"Instance#{index},Solution#{i}" + "-"*20) |
| 319 | print(failed_cases) |
| 320 | print(gt) |
| 321 | print([self.dataset.prompt2testcase[prompt][i] for i, res in enumerate(results) if res["status"] == -1]) |
| 322 | print("Groundtruth solution verification failed for instance #{}!".format(index)) |
| 323 | print("Original Solutions:") |
| 324 | if self.dataset.name == "deepmind/code_contests": |
| 325 | raw = self.dataset.prompt2instance[prompt]["solutions"] |
| 326 | solutions = [] |
| 327 | for index, lang in enumerate(raw["language"]): |
| 328 | if lang == 1: |
| 329 | solutions.append(raw["solution"][index]) |
| 330 | for s in solutions: |
| 331 | print(s) |
| 332 | elif self.dataset.name == "codeparrot/apps": |
| 333 | raw = self.dataset.prompt2instance[prompt]["solutions"] |
| 334 | solutions = json.loads(raw) |
| 335 | for solution in solutions: |
| 336 | print(solution) |
| 337 | ''' |
| 338 | if remove_instance: |
| 339 | self.dataset.prompt2testcase[prompt] = [] |
| 340 | print("Instance Removed.") |
| 341 | else: |
no test coverage detected