Method get_expected_outputs

src/evaluate.py:158–200

(self)

    def get_expected_outputs(self):
        prompts, overlong_prompts = self.dataset.get_all_prompts()

        max_count = 1
        count = 0

        for index, prompt in enumerate(prompts + overlong_prompts):
            #if "Write a function to find the nth newman" not in prompt:
            #    continue
            if len(self.dataset.prompt2groundtruth[prompt]) == 0:
                continue
            elif len(self.dataset.prompt2testcase[prompt]) == 0:
                continue
            gt, io = self.dataset.prompt2groundtruth[prompt][0]
            print(index)
            stat, results = self.execute_code(gt, io, self.dataset.prompt2testcase[prompt], check = False)
            
            if stat == "fail":
                print(results)
                print()
                print(self.dataset.prompt2testcase[prompt])
                print()
                print(gt)
                print()
                print(prompt)
                self.dataset.prompt2testcase[prompt] = []
                count += 1
                if count > max_count:
                    raise ValueError("Groundtruth solution verification failed!")
                continue
            if len(results) != len(self.dataset.prompt2testcase[prompt]):
                raise ValueError("Num of returned results is inconsistent with original testcase inputs.")
            for i, res in enumerate(results):
                self.dataset.prompt2testcase[prompt][i]["output"] = res["model_output"]
            try:
                json.dumps(self.dataset.prompt2testcase[prompt])
            except:
                self.dataset.prompt2testcase[prompt] = []
        
        self.dataset.save_testcases()
        self.dataset.load_testcases()
        self.verify_groundtruth(remove_instance=True)
        self.dataset.save_testcases()


    def fix_testcases(self, old_solution_file):