Fetch agent task success rates from PinchBench. PinchBench (https://pinchbench.com) tests LLM models on real-world OpenClaw agent tasks. Results are published via api.pinchbench.com.
| 77 | |
| 78 | |
| 79 | class PinchBenchProvider(BenchmarkProvider): |
| 80 | """Fetch agent task success rates from PinchBench. |
| 81 | |
| 82 | PinchBench (https://pinchbench.com) tests LLM models on real-world |
| 83 | OpenClaw agent tasks. Results are published via api.pinchbench.com. |
| 84 | """ |
| 85 | |
| 86 | source_name = "pinchbench" |
| 87 | |
| 88 | def __init__(self, api_url: str = "https://api.pinchbench.com") -> None: |
| 89 | self._api_url = api_url.rstrip("/") |
| 90 | |
| 91 | async def fetch(self) -> dict[str, ModelBenchmarkEntry]: |
| 92 | try: |
| 93 | async with httpx.AsyncClient(timeout=httpx.Timeout(15.0)) as client: |
| 94 | resp = await client.get( |
| 95 | f"{self._api_url}/api/leaderboard?version=latest", |
| 96 | headers={"user-agent": "uncommon-route/benchmark"}, |
| 97 | ) |
| 98 | if resp.status_code != 200: |
| 99 | logger.warning("PinchBench: HTTP %d", resp.status_code) |
| 100 | return {} |
| 101 | |
| 102 | data = resp.json() |
| 103 | return self._parse_leaderboard(data) |
| 104 | except Exception as exc: |
| 105 | logger.warning("PinchBench fetch failed: %s", exc) |
| 106 | return {} |
| 107 | |
| 108 | def _parse_leaderboard(self, data: dict) -> dict[str, ModelBenchmarkEntry]: |
| 109 | now = time.time() |
| 110 | |
| 111 | raw_entries: dict[str, list[tuple[float, float, int]]] = {} |
| 112 | for item in data.get("leaderboard", []): |
| 113 | if not isinstance(item, dict): |
| 114 | continue |
| 115 | model_id = str(item.get("model", "")).strip() |
| 116 | if not model_id: |
| 117 | continue |
| 118 | runs = int(item.get("submission_count", 0) or 0) |
| 119 | if runs < 2: |
| 120 | continue |
| 121 | # API returns scores as 0-1 fractions despite field name containing "percentage" |
| 122 | best = float(item.get("best_score_percentage", 0)) |
| 123 | avg = float(item.get("average_score_percentage", 0)) |
| 124 | if avg <= 0: |
| 125 | continue |
| 126 | |
| 127 | canonical = self._normalize_model_id(model_id) |
| 128 | raw_entries.setdefault(canonical, []).append((best, avg, runs)) |
| 129 | |
| 130 | entries: dict[str, ModelBenchmarkEntry] = {} |
| 131 | for canonical, scores in raw_entries.items(): |
| 132 | best = max(s[0] for s in scores) |
| 133 | total_runs = sum(s[2] for s in scores) |
| 134 | weighted_avg = sum(s[1] * s[2] for s in scores) / total_runs if total_runs > 0 else best |
| 135 | entries[canonical] = ModelBenchmarkEntry( |
| 136 | overall=weighted_avg, |