Load benchmark quality from a local JSON file. Supports manual quality overrides or imported benchmark data. File format: {"model_id": {"overall": 0.85, "categories": {...}}, ...}
| 177 | |
| 178 | |
| 179 | class LocalFileProvider(BenchmarkProvider): |
| 180 | """Load benchmark quality from a local JSON file. |
| 181 | |
| 182 | Supports manual quality overrides or imported benchmark data. |
| 183 | File format: {"model_id": {"overall": 0.85, "categories": {...}}, ...} |
| 184 | """ |
| 185 | |
| 186 | source_name = "local" |
| 187 | |
| 188 | def __init__(self, path: Path | None = None) -> None: |
| 189 | self._path = path or (_DATA_DIR / "benchmark_quality.json") |
| 190 | |
| 191 | @property |
| 192 | def refresh_interval_s(self) -> float: |
| 193 | return 300 |
| 194 | |
| 195 | async def fetch(self) -> dict[str, ModelBenchmarkEntry]: |
| 196 | if not self._path.exists(): |
| 197 | return {} |
| 198 | try: |
| 199 | raw = json.loads(self._path.read_text(encoding="utf-8")) |
| 200 | entries: dict[str, ModelBenchmarkEntry] = {} |
| 201 | now = time.time() |
| 202 | for model_id, values in raw.items(): |
| 203 | if isinstance(values, dict): |
| 204 | entries[model_id] = ModelBenchmarkEntry( |
| 205 | overall=float(values.get("overall", values.get("avg", 0.5))), |
| 206 | categories=dict(values.get("categories", {})), |
| 207 | raw=dict(values.get("raw", {})), |
| 208 | fetched_at=now, |
| 209 | ) |
| 210 | elif isinstance(values, (int, float)): |
| 211 | entries[model_id] = ModelBenchmarkEntry( |
| 212 | overall=float(values), fetched_at=now, |
| 213 | ) |
| 214 | return entries |
| 215 | except Exception as exc: |
| 216 | logger.warning("Local benchmark file load failed: %s", exc) |
| 217 | return {} |
| 218 | |
| 219 | |
| 220 | def _load_seed_data() -> dict[str, float]: |