()
| 462 | print("=" * 60) |
| 463 | |
| 464 | async def main(): |
| 465 | global TARGET_MODEL, TARGET_LANGUAGE, BATCH_SIZE |
| 466 | |
| 467 | parser = argparse.ArgumentParser(description='Test Benchmark Loop') |
| 468 | parser.add_argument("--config", default=os.path.join(root, "configs", "tool_calling_agent.py"), help="config file path") |
| 469 | parser.add_argument("--benchmark", default="leetcode", help="benchmark name to test") |
| 470 | parser.add_argument("--model", default=TARGET_MODEL, help=f"Model to use for inference (default: {TARGET_MODEL})") |
| 471 | parser.add_argument("--language", default=TARGET_LANGUAGE, choices=SUPPORTED_LANGUAGES, |
| 472 | help=f"Programming language for LeetCode solutions (default: {TARGET_LANGUAGE})") |
| 473 | parser.add_argument("--batch-size", type=int, default=BATCH_SIZE, help=f"Batch size for evaluation (default: {BATCH_SIZE})") |
| 474 | parser.add_argument( |
| 475 | '--cfg-options', |
| 476 | nargs='+', |
| 477 | action=DictAction, |
| 478 | help='override settings') |
| 479 | args = parser.parse_args() |
| 480 | |
| 481 | # 更新全局配置 |
| 482 | TARGET_MODEL = args.model |
| 483 | TARGET_LANGUAGE = args.language |
| 484 | BATCH_SIZE = args.batch_size |
| 485 | |
| 486 | config.initialize(config_path=args.config, args=args) |
| 487 | logger.initialize(config=config) |
| 488 | |
| 489 | logger.info("| 🧠 Initializing model manager...") |
| 490 | if hasattr(model_manager, 'initialize'): |
| 491 | await model_manager.initialize() |
| 492 | |
| 493 | benchmark_name = args.benchmark |
| 494 | logger.info(f"| 🛠️ Initializing benchmark manager for {benchmark_name}...") |
| 495 | |
| 496 | await benchmark_manager.initialize(benchmark_names=[benchmark_name]) |
| 497 | |
| 498 | # 设置 benchmark 的语言和批次大小 |
| 499 | benchmark = await benchmark_manager.get(benchmark_name) |
| 500 | if benchmark: |
| 501 | benchmark.language = TARGET_LANGUAGE |
| 502 | benchmark.batch_size = BATCH_SIZE |
| 503 | logger.info(f"| 🔧 Configured benchmark: language={TARGET_LANGUAGE}, batch_size={BATCH_SIZE}") |
| 504 | |
| 505 | await test_leetcode_benchmark(benchmark_name) |
| 506 | |
| 507 | # code = """ |
| 508 | # # |
| 509 | # # @lc app=leetcode id=1 lang=python3 |
| 510 | # # |
| 511 | # # [1] Two Sum |
| 512 | # # |
| 513 | # # @lc code=start |
| 514 | # class Solution: |
| 515 | # def twoSum(self, nums: List[int], target: int) -> List[int]: |
| 516 | # hashmap = {} |
| 517 | # for i, num in enumerate(nums): |
| 518 | # complement = target - num |
| 519 | # if complement in hashmap: |
| 520 | # return [hashmap[complement], i] |
| 521 | # hashmap[num] = i |
no test coverage detected