Initialize the benchmark context manager.
(self, benchmark_names: Optional[List[str]] = None)
| 65 | self._cleanup_registered = False |
| 66 | |
| 67 | async def initialize(self, benchmark_names: Optional[List[str]] = None): |
| 68 | """Initialize the benchmark context manager.""" |
| 69 | # Register benchmark-related symbols for auto-injection in dynamic code |
| 70 | dynamic_manager.register_symbol("BENCHMARK", BENCHMARK) |
| 71 | dynamic_manager.register_symbol("Benchmark", Benchmark) |
| 72 | |
| 73 | # Register benchmark context provider for automatic import injection |
| 74 | def benchmark_context_provider(): |
| 75 | """Provide benchmark-related imports for dynamic benchmark classes.""" |
| 76 | return { |
| 77 | "BENCHMARK": BENCHMARK, |
| 78 | "Benchmark": Benchmark, |
| 79 | } |
| 80 | dynamic_manager.register_context_provider("benchmark", benchmark_context_provider) |
| 81 | |
| 82 | # Load benchmarks from BENCHMARK registry |
| 83 | benchmark_configs = {} |
| 84 | registry_benchmark_configs: Dict[str, BenchmarkConfig] = await self._load_from_registry() |
| 85 | benchmark_configs.update(registry_benchmark_configs) |
| 86 | |
| 87 | # Load benchmarks from code (JSON file) |
| 88 | code_benchmark_configs: Dict[str, BenchmarkConfig] = await self._load_from_code() |
| 89 | |
| 90 | # Merge code configs with registry configs, only override if code version is strictly greater |
| 91 | for benchmark_name, code_config in code_benchmark_configs.items(): |
| 92 | if benchmark_name in benchmark_configs: |
| 93 | registry_config = benchmark_configs[benchmark_name] |
| 94 | if version_manager.compare_versions(code_config.version, registry_config.version) > 0: |
| 95 | logger.info(f"| 🔄 Overriding benchmark {benchmark_name} from registry (v{registry_config.version}) with code version (v{code_config.version})") |
| 96 | benchmark_configs[benchmark_name] = code_config |
| 97 | else: |
| 98 | logger.info(f"| 📌 Keeping benchmark {benchmark_name} from registry (v{registry_config.version}), code version (v{code_config.version}) is not greater") |
| 99 | if version_manager.compare_versions(code_config.version, registry_config.version) == 0: |
| 100 | if benchmark_name in self._benchmark_history_versions: |
| 101 | self._benchmark_history_versions[benchmark_name][registry_config.version] = registry_config |
| 102 | else: |
| 103 | benchmark_configs[benchmark_name] = code_config |
| 104 | |
| 105 | if benchmark_names is not None: |
| 106 | benchmark_configs = {name: benchmark_configs[name] for name in benchmark_names if name in benchmark_configs} |
| 107 | |
| 108 | # Build all benchmark systems concurrently |
| 109 | names_list = list(benchmark_configs.keys()) |
| 110 | tasks = [self.build(benchmark_configs[name]) for name in names_list] |
| 111 | results = await gather_with_concurrency(tasks, max_concurrency=10, return_exceptions=True) |
| 112 | |
| 113 | for name, result in zip(names_list, results): |
| 114 | if isinstance(result, Exception): |
| 115 | logger.error(f"| ❌ Failed to initialize benchmark {name}: {result}") |
| 116 | continue |
| 117 | self._benchmark_configs[name] = result |
| 118 | logger.info(f"| 🔧 Benchmark {name} initialized") |
| 119 | |
| 120 | # Save to JSON and contract |
| 121 | await self.save_to_json() |
| 122 | await self.save_contract(benchmark_names=names_list) |
| 123 | |
| 124 | # Register cleanup callback |
no test coverage detected