Function run_evaluation

scripts/evaluate/evaluate.py:253–483

(filtered_data, input_list, output_list, task_type, output_dir, output_metrics_path, output_metrics_overall_path, use_llm=False, extract_answer=False, domain_fields=None, api_base_url=None, model_name=None)

def run_evaluation(filtered_data, input_list, output_list, task_type, output_dir, output_metrics_path, output_metrics_overall_path, use_llm=False, extract_answer=False, domain_fields=None, api_base_url=None, model_name=None):
    # Initialize domain metrics dictionary
    domain_metrics = defaultdict(lambda: {
        'total': 0,
        'correct': 0,
        'em': [],
        'acc': [],
        'f1': [],
        'math_equal': [],
        'llm_equal': [],
        'pass@1': []
    })

    # Helper function to get domain from item
    def get_domain(item):
        for field in domain_fields:
            if field in item and item[field] is not None:
                return item[field]
        return 'Unknown'

    if task_type == 'code':
        # Prepare samples and generations for codegen_metrics
        samples_list = []
        generations_list = []
        num_valid_answer = 0

        for item, input_prompt, result in zip(filtered_data, input_list, output_list):
            if type(result) == str:
                item['Output'] = result
            else:
                item['Output'] = result.outputs[0].text

            if item['Output'] == '':
                item['Pred_Answer'] = ''
                item['Question'] = input_prompt
                item['Metrics'] = {'pass@1': 0}
                item['Results'] = {}
                item['Final_metadata'] = {}
                continue

            pred_code = extract_answer_fn(item['Output'], mode='codegen', extract_answer=extract_answer)
            if pred_code != '':
                num_valid_answer += 1

            public_test_cases = json.loads(item.get("test_cases", "{}"))

            inputs = public_test_cases.get("inputs", [])
            outputs = public_test_cases.get("outputs", [])

            sample = {
                "input_output": json.dumps({
                    "inputs": inputs,
                    "outputs": outputs
                }),
            }

            samples_list.append(sample)
            generations_list.append([pred_code])