| 29 | |
| 30 | |
| 31 | def parse_eval_args() -> argparse.Namespace: |
| 32 | parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) |
| 33 | parser.add_argument("--model", "-m", default="hf", help="Name of model e.g. `hf`") |
| 34 | parser.add_argument( |
| 35 | "--tasks", |
| 36 | "-t", |
| 37 | default=None, |
| 38 | metavar="task1,task2", |
| 39 | help="To get full list of tasks, use the command lm-eval --tasks list", |
| 40 | ) |
| 41 | parser.add_argument( |
| 42 | "--model_args", |
| 43 | "-a", |
| 44 | default="", |
| 45 | help="Comma separated string arguments for model, e.g. `pretrained=EleutherAI/pythia-160m,dtype=float32`", |
| 46 | ) |
| 47 | parser.add_argument( |
| 48 | "--num_fewshot", |
| 49 | "-f", |
| 50 | type=int, |
| 51 | default=None, |
| 52 | metavar="N", |
| 53 | help="Number of examples in few-shot context", |
| 54 | ) |
| 55 | parser.add_argument( |
| 56 | "--batch_size", |
| 57 | "-b", |
| 58 | type=str, |
| 59 | default=1, |
| 60 | metavar="auto|auto:N|N", |
| 61 | help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.", |
| 62 | ) |
| 63 | parser.add_argument( |
| 64 | "--max_batch_size", |
| 65 | type=int, |
| 66 | default=None, |
| 67 | metavar="N", |
| 68 | help="Maximal batch size to try with --batch_size auto.", |
| 69 | ) |
| 70 | parser.add_argument( |
| 71 | "--device", |
| 72 | type=str, |
| 73 | default=None, |
| 74 | help="Device to use (e.g. cuda, cuda:0, cpu).", |
| 75 | ) |
| 76 | parser.add_argument( |
| 77 | "--output_path", |
| 78 | "-o", |
| 79 | default=None, |
| 80 | type=str, |
| 81 | metavar="DIR|DIR/file.json", |
| 82 | help="The path to the output file where the result metrics will be saved. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.", |
| 83 | ) |
| 84 | parser.add_argument( |
| 85 | "--limit", |
| 86 | "-L", |
| 87 | type=float, |
| 88 | default=None, |