Aggregated benchmark quality data from multiple sources.
| 249 | |
| 250 | @dataclass |
| 251 | class BenchmarkCache: |
| 252 | """Aggregated benchmark quality data from multiple sources.""" |
| 253 | |
| 254 | _sources: dict[str, dict[str, ModelBenchmarkEntry]] = field(default_factory=dict) |
| 255 | _providers: list[BenchmarkProvider] = field(default_factory=list) |
| 256 | _source_weights: dict[str, float] = field(default_factory=dict) |
| 257 | _last_refresh: float = 0.0 |
| 258 | _refresh_thread: threading.Thread | None = field(default=None, init=False, repr=False) |
| 259 | |
| 260 | def __post_init__(self) -> None: |
| 261 | if not self._providers: |
| 262 | self._providers = [ |
| 263 | PinchBenchProvider(), |
| 264 | LocalFileProvider(), |
| 265 | ] |
| 266 | if not self._source_weights: |
| 267 | self._source_weights = { |
| 268 | "pinchbench": 0.6, |
| 269 | "local": 0.8, |
| 270 | } |
| 271 | self._load_cache() |
| 272 | self._load_seed_as_source() |
| 273 | self._build_index() |
| 274 | |
| 275 | def add_provider(self, provider: BenchmarkProvider, weight: float = 0.3) -> None: |
| 276 | self._providers.append(provider) |
| 277 | self._source_weights[provider.source_name] = weight |
| 278 | |
| 279 | async def refresh(self, force: bool = False) -> int: |
| 280 | """Fetch from all providers. Returns number of models updated.""" |
| 281 | total = 0 |
| 282 | for provider in self._providers: |
| 283 | if not force: |
| 284 | source_data = self._sources.get(provider.source_name, {}) |
| 285 | if source_data: |
| 286 | newest = max((e.fetched_at for e in source_data.values()), default=0) |
| 287 | if time.time() - newest < provider.refresh_interval_s: |
| 288 | continue |
| 289 | try: |
| 290 | entries = await provider.fetch() |
| 291 | if entries: |
| 292 | self._sources[provider.source_name] = entries |
| 293 | total += len(entries) |
| 294 | logger.info( |
| 295 | "Benchmark refresh: %d models from %s", |
| 296 | len(entries), |
| 297 | provider.source_name, |
| 298 | ) |
| 299 | except Exception as exc: |
| 300 | logger.warning("Benchmark refresh failed for %s: %s", provider.source_name, exc) |
| 301 | if total > 0: |
| 302 | self._last_refresh = time.time() |
| 303 | self._save_cache() |
| 304 | self._build_index() |
| 305 | return total |
| 306 | |
| 307 | def needs_refresh(self, *, now: float | None = None) -> bool: |
| 308 | """Return whether any configured source is missing or past its TTL.""" |
no outgoing calls