Instantiate and evaluate a model on a list of tasks. :param model: Union[str, LM] Name of model or LM object, see lm_eval.models.get_model :param model_args: Optional[str] String arguments for each model class, see LM.create_from_arg_string. Ignored if `model` argume
(
model,
model_args: Optional[str] = None,
tasks=None,
num_fewshot: Optional[int] = None,
batch_size: Optional[int] = None,
max_batch_size: Optional[int] = None,
device: Optional[str] = None,
use_cache: Optional[str] = None,
limit: Optional[Union[int, float]] = None,
bootstrap_iters: int = 100000,
check_integrity: bool = False,
decontamination_ngrams_path=None,
write_out: bool = False,
log_samples: bool = True,
gen_kwargs: str = None,
task_manager: TaskManager = None,
verbosity: str = "INFO",
predict_only: bool = False,
random_seed: int = 0,
numpy_random_seed: int = 1234,
torch_random_seed: int = 1234,
# additions
context_length: int = 1000,
answer_length: int = 50,
sequence_length: int = 2048,
context_key: str = "context",
answer_key: list = ["value"],
cutting_context: bool = False,
decode_mode: str = "default",
skip_check_answer: bool = False,
)
| 22 | |
| 23 | @positional_deprecated |
| 24 | def simple_evaluate( |
| 25 | model, |
| 26 | model_args: Optional[str] = None, |
| 27 | tasks=None, |
| 28 | num_fewshot: Optional[int] = None, |
| 29 | batch_size: Optional[int] = None, |
| 30 | max_batch_size: Optional[int] = None, |
| 31 | device: Optional[str] = None, |
| 32 | use_cache: Optional[str] = None, |
| 33 | limit: Optional[Union[int, float]] = None, |
| 34 | bootstrap_iters: int = 100000, |
| 35 | check_integrity: bool = False, |
| 36 | decontamination_ngrams_path=None, |
| 37 | write_out: bool = False, |
| 38 | log_samples: bool = True, |
| 39 | gen_kwargs: str = None, |
| 40 | task_manager: TaskManager = None, |
| 41 | verbosity: str = "INFO", |
| 42 | predict_only: bool = False, |
| 43 | random_seed: int = 0, |
| 44 | numpy_random_seed: int = 1234, |
| 45 | torch_random_seed: int = 1234, |
| 46 | |
| 47 | # additions |
| 48 | context_length: int = 1000, |
| 49 | answer_length: int = 50, |
| 50 | sequence_length: int = 2048, |
| 51 | context_key: str = "context", |
| 52 | answer_key: list = ["value"], |
| 53 | cutting_context: bool = False, |
| 54 | decode_mode: str = "default", |
| 55 | skip_check_answer: bool = False, |
| 56 | ): |
| 57 | """Instantiate and evaluate a model on a list of tasks. |
| 58 | |
| 59 | :param model: Union[str, LM] |
| 60 | Name of model or LM object, see lm_eval.models.get_model |
| 61 | :param model_args: Optional[str] |
| 62 | String arguments for each model class, see LM.create_from_arg_string. |
| 63 | Ignored if `model` argument is a LM object. |
| 64 | :param tasks: list[Union[str, dict, Task]] |
| 65 | List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise. |
| 66 | :param num_fewshot: int |
| 67 | Number of examples in few-shot context |
| 68 | :param batch_size: int or str, optional |
| 69 | Batch size for model |
| 70 | :param max_batch_size: int, optional |
| 71 | Maximal batch size to try with automatic batch size detection |
| 72 | :param device: str, optional |
| 73 | PyTorch device (e.g. "cpu" or "cuda:0") for running models |
| 74 | :param use_cache: str, optional |
| 75 | A path to a sqlite db file for caching model responses. `None` if not caching. |
| 76 | :param limit: int or float, optional |
| 77 | Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples. |
| 78 | :param bootstrap_iters: |
| 79 | Number of iterations for bootstrap statistics |
| 80 | :param check_integrity: bool |
| 81 | Whether to run the relevant part of the test suite for the tasks |
nothing calls this directly
no test coverage detected