| 59 | |
| 60 | |
| 61 | def config() -> argparse.Namespace: |
| 62 | parser = argparse.ArgumentParser( |
| 63 | description="Run end-to-end evaluation on the benchmark" |
| 64 | ) |
| 65 | parser.add_argument( |
| 66 | "--render", action="store_true", help="Render the browser" |
| 67 | ) |
| 68 | parser.add_argument( |
| 69 | "--slow_mo", |
| 70 | type=int, |
| 71 | default=0, |
| 72 | help="Slow down the browser by the specified amount", |
| 73 | ) |
| 74 | parser.add_argument( |
| 75 | "--action_set_tag", default="id_accessibility_tree", help="Action type" |
| 76 | ) |
| 77 | parser.add_argument( |
| 78 | "--observation_type", |
| 79 | choices=["accessibility_tree", "html", "image"], |
| 80 | default="accessibility_tree", |
| 81 | help="Observation type", |
| 82 | ) |
| 83 | parser.add_argument( |
| 84 | "--current_viewport_only", |
| 85 | action="store_true", |
| 86 | help="Only use the current viewport for the observation", |
| 87 | ) |
| 88 | parser.add_argument("--viewport_width", type=int, default=1280) |
| 89 | parser.add_argument("--viewport_height", type=int, default=720) |
| 90 | parser.add_argument("--save_trace_enabled", action="store_true") |
| 91 | parser.add_argument("--sleep_after_execution", type=float, default=0.0) |
| 92 | |
| 93 | parser.add_argument("--max_steps", type=int, default=30) |
| 94 | |
| 95 | # agent config |
| 96 | parser.add_argument("--agent_type", type=str, default="prompt") |
| 97 | parser.add_argument( |
| 98 | "--instruction_path", |
| 99 | type=str, |
| 100 | default="agents/prompts/state_action_agent.json", |
| 101 | ) |
| 102 | parser.add_argument( |
| 103 | "--parsing_failure_th", |
| 104 | help="When concesecutive parsing failure exceeds this threshold, the agent will stop", |
| 105 | type=int, |
| 106 | default=3, |
| 107 | ) |
| 108 | parser.add_argument( |
| 109 | "--repeating_action_failure_th", |
| 110 | help="When concesecutive repeating action exceeds this threshold, the agent will stop", |
| 111 | type=int, |
| 112 | default=3, |
| 113 | ) |
| 114 | |
| 115 | # lm config |
| 116 | parser.add_argument("--provider", type=str, default="openai") |
| 117 | parser.add_argument("--model", type=str, default="gpt-3.5-turbo-0613") |
| 118 | parser.add_argument("--mode", type=str, default="chat") |