| 32 | |
| 33 | |
| 34 | class RoutingMetrics: |
| 35 | def __init__(self) -> None: |
| 36 | self.reset() |
| 37 | |
| 38 | def reset(self) -> None: |
| 39 | self.total_requests = 0 |
| 40 | self.requests_by_tier: dict[int, int] = defaultdict(int) |
| 41 | self.requests_by_model: dict[str, int] = defaultdict(int) |
| 42 | self.cascade_count = 0 |
| 43 | self.conservative_count = 0 |
| 44 | self.escalated_count = 0 |
| 45 | self._confidence_sum = 0.0 |
| 46 | self._signal_agreement_count = 0 |
| 47 | |
| 48 | def record_routing( |
| 49 | self, |
| 50 | tier: int, |
| 51 | model: str, |
| 52 | method: str, |
| 53 | confidence: float, |
| 54 | signals_agreed: bool = True, |
| 55 | ) -> None: |
| 56 | self.total_requests += 1 |
| 57 | self.requests_by_tier[tier] += 1 |
| 58 | self.requests_by_model[model] += 1 |
| 59 | self._confidence_sum += confidence |
| 60 | if method == "cascaded": |
| 61 | self.cascade_count += 1 |
| 62 | elif method == "conservative": |
| 63 | self.conservative_count += 1 |
| 64 | elif method == "escalated": |
| 65 | self.escalated_count += 1 |
| 66 | if signals_agreed: |
| 67 | self._signal_agreement_count += 1 |
| 68 | |
| 69 | def snapshot(self) -> dict[str, Any]: |
| 70 | avg_conf = self._confidence_sum / self.total_requests if self.total_requests else 0.0 |
| 71 | agreement = self._signal_agreement_count / self.total_requests if self.total_requests else 0.0 |
| 72 | return { |
| 73 | "total_requests": self.total_requests, |
| 74 | "requests_by_tier": dict(self.requests_by_tier), |
| 75 | "requests_by_model": dict(self.requests_by_model), |
| 76 | "cascade_count": self.cascade_count, |
| 77 | "conservative_count": self.conservative_count, |
| 78 | "escalated_count": self.escalated_count, |
| 79 | "avg_confidence": round(avg_conf, 4), |
| 80 | "signal_agreement_rate": round(agreement, 4), |
| 81 | } |
no outgoing calls