Function inspect_result

codegeex/benchmark/inspect_result.py:73–278

(
        input_dir: str = None,
        input_file: str = None,
        output_dir: str = None,
        pass_at_k_outpath: str = None,
)

def inspect_result(
        input_dir: str = None,
        input_file: str = None,
        output_dir: str = None,
        pass_at_k_outpath: str = None,
):
    if input_dir is not None:
        input_files = glob.glob(input_dir + "/*_results.jsonl")
    else:
        input_files = [input_file]

    if output_dir is not None:
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
    else:
        output_dir = "/"

    pass_at_k_outs = []
    for input_file in input_files:
        result_stats = defaultdict(dict)
        df = None
        incompleted = False
        with open(input_file, "r") as f:
            for i, line in enumerate(f):
                obj = json.loads(line)
                task_id = obj["task_id"]
                language_type = task_id.split("/")[0].lower()
                if language_type not in error_types:
                    if language_type == "humaneval":
                        language_type = "python"
                    elif language_type == "C++":
                        language_type = "cpp"
                    else:
                        incompleted = True
                        break
                if task_id not in result_stats.keys():
                    default_stats = {}
                    default_stats["task_id"] = task_id
                    default_stats["n_sample"] = 0
                    for k in error_types[language_type]:
                        default_stats[k] = 0

                    result_stats[task_id] = default_stats.copy()
                if df is None:
                    df = pd.DataFrame(columns=error_types[language_type])
                result_stats[task_id]["n_sample"] += 1

                if "failed" in obj["result"]:
                    error = obj["result"]
                    if language_type == "python":
                        if "assertionerror" in error.lower():
                            result_stats[task_id]["assertion error"] += 1
                        elif "syntax" in error.lower() or "indent" in error.lower() or "literal" in error.lower():
                            result_stats[task_id]["syntax error"] += 1
                        elif "not defined" in error.lower():
                            result_stats[task_id]["undefined error"] += 1
                        elif "timeout" in error.lower():
                            result_stats[task_id]["timeout error"] += 1