Function evaluate

eval/Coding/mbpp/evaluate_mbpp.py:102–146

(dataset)

        exec(compile(code, filename="mbpp", mode='exec'), globals())

def evaluate(dataset):
    correct = 0
    format_error = 0
    exec_error = 0

    for example in dataset.to_dict(orient="records"):
        completion = example["completion"]
        # remove texts
        code = completion.split("\n")
        code_ = []
        for c in code:
            if len(c.lstrip()) == len(c) and not c.startswith("def"):
                continue
            code_.append(c)
        code = "\n".join(code_)

        function = code
        test_cases = "\n".join(example["test_list"]).replace("\/", "/")
        test_run = "\n".join([
            function,
            test_cases,
        ])

        # define function
        try:
            exec_helper(function)
        except Exception as e:
            print(function)
            print("Error",e)
            format_error += 1
            continue           

        try:
            # run test case
            exec_helper(test_cases)
            exec_helper(test_run)
        except:
            exec_error += 1
            continue
        else:
            correct += 1
    print("correct: ", correct)
    print("exec_error: ", exec_error)
    print("format_error: ", format_error)
    return 100 * (correct / len(dataset)), 100 * (exec_error / len(dataset)), 100 * (format_error / len(dataset))


if __name__ == "__main__":