Class Evaluator

src/evaluate.py:103–665

class Evaluator(object):
    def __init__(self, dataset, dataset_repo = "test_datasets", mbpp_helpfile = None):
        self.dataset = Dataset(dataset, data_path = os.path.join(dataset_repo, dataset.replace("/", "_")), testfile_path = mbpp_helpfile)
        self.dataset.load_testcases()
        self.dataset.load_groundtruths()


    def load_solutions(self, solution_file):
        self.solution_file = solution_file
        self.solutions = json.load(open(self.solution_file, "r"))

    def check_element_type(lst, t):
        for l in lst:
            if not isinstance(l, t):
                if t == float and isinstance(l, int):
                    continue
                return False

        return True

    def transform_element_type(lst, t):
        new_lst = []
        for l in lst:
            new_lst.append(t(l))
        return new_lst

    def prepare_testcases(self, solution_io, gt_io, testcases):
        if solution_io == gt_io:
            return testcases
        elif solution_io:
            new_testcases = []
            for testcase in testcases:
                new_testcase = {}
                new_testcase["output"] = testcase["output"]
                if check_element_type(testcase["input"], str):
                    new_testcase["input"] = ["\n".join(testcase["input"]) + "\n"]
                elif check_element_type(testcase["input"], float):
                    new_testcase["input"] = ["\n".join(self.transform_element_type(testcase["input"], str)) + "\n"]
                else:
                    new_testcase["input"] = testcase["input"]
                new_testcases.append(new_testcase)
            return new_testcases
        else:
            new_testcases = []
            for testcase in testcases:
                new_testcase = {}
                new_testcase["output"] = testcase["output"]
                if len(testcase["input"]) == 1 and isinstance(testcase["input"][0], str):
                    new_testcase["input"] = [testcase["input"][0].split("\n")]
                else:
                    new_testcase["input"] = testcase["input"]
                new_testcases.append(new_testcase)
            return new_testcases


    def get_expected_outputs(self):
        prompts, overlong_prompts = self.dataset.get_all_prompts()