Function test_leetcode_benchmark

tests/run_leetcode_agent.py:248–397

Test the benchmark manager specifically for LeetCode using a REAL model. Uses PIPELINE mode: inference and evaluation run in parallel. 流水线模式说明： - 推理完成 batch_size 个任务后，立即 push + 评测 - 同时继续推理下一批任务 - 推理失败的任务直接保存结果，不入队列 - 共享一个浏览器实例

(benchmark_name: str = "leetcode")

async def test_leetcode_benchmark(benchmark_name: str = "leetcode"):
    """
    Test the benchmark manager specifically for LeetCode using a REAL model.
    Uses PIPELINE mode: inference and evaluation run in parallel.
    
    流水线模式说明：
    - 推理完成 batch_size 个任务后，立即 push + 评测
    - 同时继续推理下一批任务
    - 推理失败的任务直接保存结果，不入队列
    - 共享一个浏览器实例
    """
    print(f"\n{'='*60}")
    print(f"🧪 LeetCode Benchmark Test (Pipeline Mode)")
    print(f"{'='*60}")
    print(f"🤖 Model: {TARGET_MODEL}")
    print(f"💻 Language: {TARGET_LANGUAGE}")
    print(f"⚡ Max concurrent inference: {MAX_CONCURRENT_INFERENCE}")
    print(f"📦 Batch size for evaluation: {BATCH_SIZE}")
    print(f"{'='*60}\n")
    
    # Define save directory
    save_dir = os.path.join(config.workdir, "benchmark", benchmark_name)
    if not os.path.exists(save_dir):
        os.makedirs(save_dir, exist_ok=True)
        print(f"📁 Created output directory: {save_dir}")
    
    # 1. Reset and collect all tasks
    print(f"🔄 Resetting progress for LeetCode...")
    task = await benchmark_manager.reset(benchmark_name)
    
    if not task:
        logger.warning("⚠️ No tasks available to run (Dataset empty or all finished).")
        summarize_benchmark_results(benchmark_name)
        return

    # ==========================================
    # 收集所有待处理任务
    # ==========================================
    all_tasks: List[Task] = [task]
    while True:
        next_task = await benchmark_manager.step(benchmark_name)
        if next_task is None:
            break
        all_tasks.append(next_task)
    
    total_tasks = len(all_tasks)
    print(f"📋 Collected {total_tasks} tasks for processing\n")
    
    # ==========================================
    # 获取 benchmark 实例
    # ==========================================
    benchmark = await benchmark_manager.get(benchmark_name)
    
    # ==========================================
    # 流水线模式：推理完成直接入队，满5个自动评测
    # ==========================================
    inference_semaphore = asyncio.Semaphore(MAX_CONCURRENT_INFERENCE)