()
| 1313 | |
| 1314 | |
| 1315 | def main(): |
| 1316 | parser = argparse.ArgumentParser(description="PlotCraft Benchmark Evaluation - Simplified") |
| 1317 | parser.add_argument("--mode", type=str, choices=["generate", "evaluate", "both"], default="both", |
| 1318 | help="Run mode: generate (generation only), evaluate (evaluation only), both (full pipeline)") |
| 1319 | parser.add_argument("--data_dir", type=str, |
| 1320 | default="data", |
| 1321 | help="Data directory path") |
| 1322 | parser.add_argument("--results_dir", type=str, default="results", |
| 1323 | help="Results root directory") |
| 1324 | parser.add_argument("--difficulty", type=str, choices=["simple", "middle", "hard"], default=None, |
| 1325 | help="Specify difficulty level (default: all)") |
| 1326 | |
| 1327 | |
| 1328 | parser.add_argument("--generation_model_name", type=str, default="gpt-4o", |
| 1329 | help="Code generation model name") |
| 1330 | parser.add_argument("--generation_base_url", type=str, default="https://api.openai.com/v1", |
| 1331 | help="Code generation API base URL") |
| 1332 | parser.add_argument("--generation_api_key", type=str, default=None, |
| 1333 | help="Code generation API key (use OPENAI_API_KEY env var if not provided)") |
| 1334 | |
| 1335 | |
| 1336 | parser.add_argument("--evaluation_model_name", type=str, default="gpt-4o", |
| 1337 | help="Evaluation model name") |
| 1338 | parser.add_argument("--evaluation_base_url", type=str, default="https://api.openai.com/v1", |
| 1339 | help="Evaluation API base URL") |
| 1340 | parser.add_argument("--evaluation_api_key", type=str, default=None, |
| 1341 | help="Evaluation API key (use OPENAI_API_KEY env var if not provided)") |
| 1342 | |
| 1343 | |
| 1344 | parser.add_argument("--max_retries", type=int, default=5, |
| 1345 | help="Maximum retry count") |
| 1346 | parser.add_argument("--max_qpm", type=int, default=1000, |
| 1347 | help="Maximum requests per minute") |
| 1348 | parser.add_argument("--max_qps", type=int, default=20, |
| 1349 | help="Maximum requests per second") |
| 1350 | |
| 1351 | |
| 1352 | parser.add_argument("--generation_prompt_path", type=str, |
| 1353 | default="prompts/generate_code_prompt.txt", |
| 1354 | help="Code generation system prompt file path") |
| 1355 | parser.add_argument("--evaluation_prompt_path", type=str, |
| 1356 | default="prompts/eval.txt", |
| 1357 | help="Evaluation system prompt file path") |
| 1358 | parser.add_argument("--data_evaluation_prompt_path", type=str, |
| 1359 | default="prompts/data_eval.txt", |
| 1360 | help="Evaluation system prompt file path") |
| 1361 | |
| 1362 | |
| 1363 | parser.add_argument("--max_data_rows", type=int, default=10, |
| 1364 | help="Maximum rows for data preview") |
| 1365 | parser.add_argument("--timeout", type=int, default=120, |
| 1366 | help="Code execution timeout (seconds)") |
| 1367 | parser.add_argument("--diagnose", action="store_true", |
| 1368 | help="Enable diagnostic mode to output detailed information about skipped tasks") |
| 1369 | |
| 1370 | args = parser.parse_args() |
| 1371 | |
| 1372 |
no test coverage detected