Method fix_testcases

src/evaluate.py:203–276

(self, old_solution_file)

    def fix_testcases(self, old_solution_file):
        prompts, overlong_prompts = self.dataset.get_all_prompts()

        data = json.load(open(old_solution_file, "r"))

        old_solutions = data["prompt2groundtruth"]
        prompt2ios = data["prompt2io"]

        for index, prompt in enumerate(prompts + overlong_prompts):
            if len(self.dataset.prompt2testcase[prompt]) == 0:
                continue
            elif len(self.dataset.prompt2groundtruth[prompt]) > 0:
                continue
            elif prompt not in self.dataset.prompt2groundtruth:
                continue
            elif prompt not in old_solutions:
                continue

            print(f"Instance # {index}")

            candidates = []
            candidate_gts = []
            for i, gts in enumerate(old_solutions[prompt]):
                print("Verifying instance #{} solution #{}     ".format(index, i), end = '\r', file = sys.stderr)
                gt, io = gts
                try:
                    stat, results = self.execute_code(gt, io, self.dataset.prompt2testcase[prompt], check = False)
                except Exception as e:
                    print(e)
                    continue
                if check_success(results) and len(results) == len(self.dataset.prompt2testcase[prompt]):
                    candidates.append(results)
                    candidate_gts.append(gts)


            equal_groups = []
            for i, c1 in enumerate(candidates):
                print(i, end = '\r', file = sys.stderr)
                if len(equal_groups) == 0:
                    equal_groups.append([i])
                    continue
                equal = False
                for group in equal_groups:
                    if is_all_equal(candidates[group[0]], c1):
                        equal = True
                        group.append(i)
                        break
                if not equal:
                    equal_groups.append([i])
            scores = [len(group) for group in equal_groups]

            if len(scores) == 0:
                continue
            
            max_score = max(scores)
            if max_score > len(candidates) * 0.8:
                candidate = candidates[equal_groups[scores.index(max_score)][0]]
            else: