| 157 | |
| 158 | |
| 159 | def parse_args(): |
| 160 | parser = argparse.ArgumentParser(description='Test different optimizers on benchmark tasks') |
| 161 | parser.add_argument("--config", default=os.path.join(root, "configs", "tool_calling_agent.py"), help="config file path") |
| 162 | parser.add_argument("--optimizer", choices=['grpo', 'reinforce_pp', 'reflection'], |
| 163 | default='reflection', help="optimizer to test") |
| 164 | parser.add_argument("--benchmark", default="gpqa", help="benchmark name to test on") |
| 165 | parser.add_argument("--concurrency", type=int, default=4, help="number of concurrent tasks to run") |
| 166 | parser.add_argument("--experiment_file", type =str, default=r'D:\86134\Documents\GitHub\AgentWorld\examples\workdir\results\reflection_gpqa_2026-01-17_14-47-53.json', help="") |
| 167 | |
| 168 | |
| 169 | parser.add_argument( |
| 170 | '--cfg-options', |
| 171 | |
| 172 | nargs='+', |
| 173 | action=DictAction, |
| 174 | help='override some settings in the used config, the key-value pair ' |
| 175 | 'in xxx=yyy format will be merged into config file. If the value to ' |
| 176 | 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' |
| 177 | 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' |
| 178 | 'Note that the quotation marks are necessary and that no white space ' |
| 179 | 'is allowed.') |
| 180 | args = parser.parse_args() |
| 181 | return args |
| 182 | |
| 183 | async def reward_fn(answer: str = None, ground_truth: Any = None): |
| 184 | _, answer = parse_agent_result(answer) |