Function eval_zero_shot

lib/eval.py:205–254

(
    model_name,
    model,
    tokenizer,
    task_list=[
        "boolq",
        "rte",
        "hellaswag",
        "winogrande",
        "arc_challenge",
        "openbookqa",
    ],
    num_fewshot=0,
    use_accelerate=False,
    add_special_tokens=False,
    limit=None,
)

def eval_zero_shot(
    model_name,
    model,
    tokenizer,
    task_list=[
        "boolq",
        "rte",
        "hellaswag",
        "winogrande",
        "arc_challenge",
        "openbookqa",
    ],
    num_fewshot=0,
    use_accelerate=False,
    add_special_tokens=False,
    limit=None,
):
    from lm_eval import tasks, evaluator

    def pattern_match(patterns, source_list):
        task_names = set()
        for pattern in patterns:
            for matching in fnmatch.filter(source_list, pattern):
                task_names.add(matching)
        return list(task_names)

    task_names = pattern_match(task_list, tasks.ALL_TASKS)
    model_args = f"pretrained={model_name},cache_dir=./llm_weights"
    if use_accelerate:
        model_args = (
            f"pretrained={model_name},cache_dir=./llm_weights,use_accelerate=True"
        )
    results = evaluator.simple_evaluate(
        model="hf-causal-experimental",
        model_args=model_args,
        tasks=task_names,
        num_fewshot=num_fewshot,
        batch_size=None,
        device=None,
        no_cache=True,
        limit=limit,
        description_dict={},
        decontamination_ngrams_path=None,
        check_integrity=False,
        pretrained_model=model,
        tokenizer=tokenizer,
        add_special_tokens=add_special_tokens,
    )

    return results


def eval_attack(