Function main

code/evaluator_series/eval_llama.py:83–134

(
        ckpt_dir: str,
        param_size: int = 13,
        ntrain: int = 5,
        few_shot: bool = False,
        cot: bool = False,
        subject: str = "operating_system",
        max_seq_len: int = 2048,
)

def main(
        ckpt_dir: str,
        param_size: int = 13,
        ntrain: int = 5,
        few_shot: bool = False,
        cot: bool = False,
        subject: str = "operating_system",
        max_seq_len: int = 2048,
):
    evaluator = load(
        ckpt_dir,
        param_size=param_size,
        ntrain=ntrain,
        max_seq_len=max_seq_len
    )
    
    subject_name = subject
    run_date = time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime(time.time()))
    save_result_dir = os.path.join(
        r"logs", f"LLaMA_{param_size}{'_CoT' if cot else ''}_{run_date}")
    os.makedirs(save_result_dir, exist_ok=True)
    
    local_rank, _ = setup_model_parallel()
    if local_rank == 0:
        print(f"Start evaluating {subject_name}")
    
    val_file_path = os.path.join('data/val', f'{subject_name}_val.csv')
    val_df = pd.read_csv(val_file_path)

    if few_shot:
        dev_file_path = os.path.join('data/dev', f'{subject_name}_dev.csv')
        dev_df = pd.read_csv(dev_file_path)
        correct_ratio = evaluator.eval_subject(
            subject_name,
            val_df,
            dev_df,
            few_shot=few_shot,
            save_result_dir=save_result_dir,
            cot=cot,
            **generate_args
        )
    else:
        correct_ratio = evaluator.eval_subject(
            subject_name,
            val_df,
            save_result_dir=save_result_dir,
            cot=cot,
            **generate_args
        )
    
    if local_rank == 0:
        print("Acc:", correct_ratio)


if __name__ == "__main__":