Method verify_predictions

src/evaluate.py:424–487

(self, debug = False, load_temp_data = False, start_index = None, verify_num = None, failed_case = False)

    def verify_predictions(self, debug = False, load_temp_data = False, start_index = None, verify_num = None, failed_case = False):
        prompts, overlong_prompts = self.dataset.get_all_prompts()
        if not load_temp_data:
            passed_solutions = {}
            failed_cases = {}
        else:
            passed_solutions = self.load_temp_data()
            failed_cases = {}
        for index, prompt in enumerate(prompts + overlong_prompts):
            if len(self.dataset.prompt2testcase[prompt]) == 0:
                continue
            if len(self.dataset.prompt2groundtruth[prompt]) == 0 and self.dataset.name != "NTU-NLP-sg/xCodeEval":
                continue
            if start_index != None and index < start_index:
                continue
            if start_index != None and verify_num != None and index >= start_index + verify_num:
                break
            if prompt in passed_solutions:
                continue
            if prompt not in self.solutions:
                continue
            if len(self.solutions[prompt]) == 0:
                continue
            
            correct_count = 0
            passed_solutions[prompt] = []
            failed_cases[prompt] = []
            total_timeout = 10
            if self.dataset.name == "NTU-NLP-sg/xCodeEval":
                instance = self.dataset.prompt2instance[prompt]
                try:
                    total_timeout = len(self.dataset.prompt2testcase[prompt]) * float(instance["time_limit"].replace(" seconds", "").replace(" second", ""))
                except:
                    total_timeout = len(self.dataset.prompt2testcase[prompt])
            for i, solution in enumerate(self.solutions[prompt]):
                print("Verifying prediction #{} for instance #{}         ".format(i, index), end = "\r", file = sys.stderr)
                cur_failed_cases = []
                s, io = solution
                if s == -1:
                    continue
                stat, r = self.execute_code(s, io, self.dataset.prompt2testcase[prompt], check = True, fast_check = True, total_timeout = total_timeout)
                if stat == "pass":
                    correct_count += 1
                    passed_solutions[prompt].append(solution)
                else:
                    if failed_case:
                        for result in r:
                            if result["status"] == FAILED:
                                try:
                                    json.dumps(result)
                                except Exception as e:
                                    del result["model_output"]
                                cur_failed_cases.append(result)
                failed_cases[prompt].append(cur_failed_cases)
                            

            #if index % 200 == 0 and self.dataset.name == "codeparrot/apps" and start_index == None and verify_num == None:
            #    print("Save passed solutions at instance #{}".format(index))