| 63 | |
| 64 | |
| 65 | def parse_args(): |
| 66 | parser = argparse.ArgumentParser(description="Run AIME benchmark") |
| 67 | parser.add_argument( |
| 68 | "--workflow-file", |
| 69 | type=str, |
| 70 | default=str(_DEFAULT_WORKFLOW), |
| 71 | help="Path to YAML workflow file", |
| 72 | ) |
| 73 | parser.add_argument( |
| 74 | "--model", |
| 75 | type=str, |
| 76 | default=None, |
| 77 | help="Model to use (overrides workflow config). Defaults to workflow's model.", |
| 78 | ) |
| 79 | parser.add_argument( |
| 80 | "--output-dir", |
| 81 | type=str, |
| 82 | default=_DEFAULT_OUTPUT_DIR, |
| 83 | help="Directory to save results", |
| 84 | ) |
| 85 | parser.add_argument( |
| 86 | "--max-parallel", |
| 87 | type=int, |
| 88 | default=1, |
| 89 | help="Maximum parallel queries (default: 1)", |
| 90 | ) |
| 91 | parser.add_argument( |
| 92 | "--limit", |
| 93 | type=int, |
| 94 | default=None, |
| 95 | help="Limit number of problems to process", |
| 96 | ) |
| 97 | parser.add_argument( |
| 98 | "--problem-ids", |
| 99 | type=str, |
| 100 | nargs="+", |
| 101 | help="Specific problem IDs to run (e.g. 2024-I-1 2025-II-3)", |
| 102 | ) |
| 103 | parser.add_argument( |
| 104 | "--dataset", |
| 105 | type=str, |
| 106 | default="all", |
| 107 | choices=DATASET_CHOICES, |
| 108 | help="Which AIME dataset to run (default: all)", |
| 109 | ) |
| 110 | parser.add_argument( |
| 111 | "--skip-eval", |
| 112 | action="store_true", |
| 113 | help="Skip evaluation after running problems", |
| 114 | ) |
| 115 | parser.add_argument( |
| 116 | "--resume", |
| 117 | action="store_true", |
| 118 | help="Resume from existing output dir, skip completed problems", |
| 119 | ) |
| 120 | parser.add_argument( |
| 121 | "--timeout", |
| 122 | type=int, |