()
| 1240 | |
| 1241 | |
| 1242 | def main(): |
| 1243 | parser = argparse.ArgumentParser(description="PlotCraft Benchmark Evaluation - Simplified") |
| 1244 | parser.add_argument("--mode", type=str, choices=["generate", "evaluate", "both"], default="both", |
| 1245 | help="Run mode: generate (generation only), evaluate (evaluation only), both (full pipeline)") |
| 1246 | parser.add_argument("--data_dir", type=str, |
| 1247 | default="data", |
| 1248 | help="Data directory path") |
| 1249 | parser.add_argument("--results_dir", type=str, default="results", |
| 1250 | help="Results root directory") |
| 1251 | parser.add_argument("--difficulty", type=str, choices=["simple", "middle", "hard"], default=None, |
| 1252 | help="Specify difficulty level (default: all)") |
| 1253 | |
| 1254 | |
| 1255 | parser.add_argument("--generation_model_name", type=str, default="gpt-4o", |
| 1256 | help="Code generation model name") |
| 1257 | parser.add_argument("--generation_base_url", type=str, default="https://api.openai.com/v1", |
| 1258 | help="Code generation API base URL") |
| 1259 | parser.add_argument("--generation_api_key", type=str, default=None, |
| 1260 | help="Code generation API key (use OPENAI_API_KEY env var if not provided)") |
| 1261 | |
| 1262 | |
| 1263 | parser.add_argument("--evaluation_model_name", type=str, default="gpt-4o", |
| 1264 | help="Evaluation model name") |
| 1265 | parser.add_argument("--evaluation_base_url", type=str, default="https://api.openai.com/v1", |
| 1266 | help="Evaluation API base URL") |
| 1267 | parser.add_argument("--evaluation_api_key", type=str, default=None, |
| 1268 | help="Evaluation API key (use OPENAI_API_KEY env var if not provided)") |
| 1269 | |
| 1270 | |
| 1271 | parser.add_argument("--max_retries", type=int, default=5, |
| 1272 | help="Maximum retry count") |
| 1273 | parser.add_argument("--max_qpm", type=int, default=1000, |
| 1274 | help="Maximum requests per minute") |
| 1275 | parser.add_argument("--max_qps", type=int, default=20, |
| 1276 | help="Maximum requests per second") |
| 1277 | |
| 1278 | |
| 1279 | parser.add_argument("--generation_prompt_path", type=str, |
| 1280 | default="prompts/generate_code_prompt.txt", |
| 1281 | help="Code generation system prompt file path") |
| 1282 | parser.add_argument("--evaluation_prompt_path", type=str, |
| 1283 | default="prompts/eval.txt", |
| 1284 | help="Evaluation system prompt file path") |
| 1285 | parser.add_argument("--user_history_generation_prompt_path", type=str, |
| 1286 | default="prompts/user_history_generate_code_prompt.txt", |
| 1287 | help="User history code generation system prompt file path") |
| 1288 | |
| 1289 | |
| 1290 | parser.add_argument("--max_data_rows", type=int, default=10, |
| 1291 | help="Maximum rows for data preview") |
| 1292 | parser.add_argument("--timeout", type=int, default=120, |
| 1293 | help="Code execution timeout (seconds)") |
| 1294 | parser.add_argument("--diagnose", action="store_true", |
| 1295 | help="Enable diagnostic mode to output detailed information about skipped tasks") |
| 1296 | |
| 1297 | args = parser.parse_args() |
| 1298 | |
| 1299 |
no test coverage detected