| 47 | |
| 48 | |
| 49 | def parse_eval_args() -> argparse.Namespace: |
| 50 | parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) |
| 51 | parser.add_argument("--model", "-m", default="hf", help="Name of model e.g. `hf`") |
| 52 | parser.add_argument( |
| 53 | "--tasks", |
| 54 | "-t", |
| 55 | default=None, |
| 56 | metavar="task1,task2", |
| 57 | help="To get full list of tasks, use the command lm-eval --tasks list", |
| 58 | ) |
| 59 | parser.add_argument( |
| 60 | "--model_args", |
| 61 | "-a", |
| 62 | default="", |
| 63 | help="Comma separated string arguments for model, e.g. `pretrained=EleutherAI/pythia-160m,dtype=float32`", |
| 64 | ) |
| 65 | parser.add_argument( |
| 66 | "--num_fewshot", |
| 67 | "-f", |
| 68 | type=int, |
| 69 | default=None, |
| 70 | metavar="N", |
| 71 | help="Number of examples in few-shot context", |
| 72 | ) |
| 73 | parser.add_argument( |
| 74 | "--batch_size", |
| 75 | "-b", |
| 76 | type=str, |
| 77 | default=1, |
| 78 | metavar="auto|auto:N|N", |
| 79 | help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.", |
| 80 | ) |
| 81 | parser.add_argument( |
| 82 | "--max_batch_size", |
| 83 | type=int, |
| 84 | default=None, |
| 85 | metavar="N", |
| 86 | help="Maximal batch size to try with --batch_size auto.", |
| 87 | ) |
| 88 | parser.add_argument( |
| 89 | "--device", |
| 90 | type=str, |
| 91 | default=None, |
| 92 | help="Device to use (e.g. cuda, cuda:0, cpu).", |
| 93 | ) |
| 94 | parser.add_argument( |
| 95 | "--output_path", |
| 96 | "-o", |
| 97 | default=None, |
| 98 | type=str, |
| 99 | metavar="DIR|DIR/file.json", |
| 100 | help="The path to the output file where the result metrics will be saved. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.", |
| 101 | ) |
| 102 | parser.add_argument( |
| 103 | "--limit", |
| 104 | "-L", |
| 105 | type=float, |
| 106 | default=None, |