Function run_infer_eval

light-eval/src/eval_ceval.py:144–175

(model, max_seq_len, data_path, ntrain=-1, few_shot = True)

    return prompt

def run_infer_eval(model, max_seq_len, data_path, ntrain=-1, few_shot = True):

    total_results = {}
    for task in TASK_NAME_MAPPING.keys():

        print('Testing %s ...' % task)
        val_file_path = os.path.join(data_path, "val", f"{task}_val.csv")
        val_df = pd.read_csv(val_file_path)
        dev_file_path = os.path.join(data_path, "dev", f"{task}_dev.csv")
        dev_df = pd.read_csv(dev_file_path)
        few_shot_prompt = generate_few_shot_prompt(task, dev_df, ntrain) if few_shot else []

        results = []
        for _, row in tqdm(val_df.iterrows(), total=len(val_df)):
            prompt = format_example(row, include_answer=False)
            full_prompt = resize_prompt(
                model.tokenizer,
                max_seq_len,
                few_shot_prompt+prompt
            )
            output = model.generate(
                prompts=[full_prompt], 
                images=None,
                max_gen_len=100,
                return_logits=True
            )
            pred = extract_ans_by_logits(tokenizer = model.tokenizer, logits=output)

            results.append(pred == row['answer'])
        total_results[task] = sum(results) / len(results)

    return total_results
            
def cal_ceval(res):
    results = {}