Function run_optimizer_on_benchmark

examples/analysis.py:397–439

Test specified optimizer performance on entire benchmark dataset with concurrency control.

(optimizer_type: str, benchmark_name: str, experiment_file: str, concurrency: int = 4)

async def run_optimizer_on_benchmark(optimizer_type: str, benchmark_name: str, experiment_file: str, concurrency: int = 4):
    """Test specified optimizer performance on entire benchmark dataset with concurrency control."""
    logger.info(f"| 🧪 Testing {optimizer_type.upper()} optimizer on complete benchmark: {benchmark_name}")
    logger.info(f"| ⚡ Using concurrency level: {concurrency}")

    # Get all tasks first
    all_tasks = await get_all_incorrect_tasks(experiment_file, benchmark_name)
    total_tasks = len(all_tasks)

    if total_tasks == 0:
        logger.warning("⚠️ No tasks available to run (Dataset empty or all finished).")
        return

    logger.info(f"| 📋 Total tasks to process: {total_tasks}")

    # Initialize result saver
    model_name = 'openrouter/gemini-3-flash-preview'
    result_saver = ExperimentResultSaver(optimizer_type, benchmark_name, concurrency, total_tasks, model_name)
    logger.info(f"| 💾 Results will be saved to: {result_saver.get_file_path()}")

    # Create semaphore for concurrency control
    semaphore = asyncio.Semaphore(concurrency)
    completed_count = 0

    async def process_with_semaphore(task_data: Dict, task_index: int):
        """Process a task with semaphore control."""
        nonlocal completed_count
        async with semaphore:
            try:
                await process_single_task(optimizer_type, benchmark_name, task_data, task_index, total_tasks, result_saver)
            finally:
                completed_count += 1
                # Progress reporting
                if completed_count % concurrency == 0 or completed_count == total_tasks:
                    progress_msg = f"| 📊 Progress: {completed_count}/{total_tasks} tasks completed"
                    logger.info(progress_msg)
                    print(progress_msg)

    # Create all tasks and run them with semaphore-controlled concurrency
    tasks = [process_with_semaphore(task_data, i) for i, task_data in enumerate(all_tasks)]
    await asyncio.gather(*tasks)

    logger.info(f"| ✅ All {total_tasks} tasks completed for {optimizer_type.upper()} optimizer")


async def main():