Classify every record and compute routing and guardrail metrics. Both thresholds are inclusive: a Choice confidence equal to ``min_confidence`` is decided, and a Noul probability equal to ``noul_threshold`` is accepted. Args: records: Records to evaluate. min_confid
(
records: Sequence[RouteRecord | GuardrailRecord],
*,
min_confidence: float,
noul_threshold: float,
routes: Collection[str] | None = None,
)
| 267 | |
| 268 | |
| 269 | def evaluate( |
| 270 | records: Sequence[RouteRecord | GuardrailRecord], |
| 271 | *, |
| 272 | min_confidence: float, |
| 273 | noul_threshold: float, |
| 274 | routes: Collection[str] | None = None, |
| 275 | ) -> EvaluationReport: |
| 276 | """Classify every record and compute routing and guardrail metrics. |
| 277 | |
| 278 | Both thresholds are inclusive: a Choice confidence equal to |
| 279 | ``min_confidence`` is decided, and a Noul probability equal to |
| 280 | ``noul_threshold`` is accepted. |
| 281 | |
| 282 | Args: |
| 283 | records: Records to evaluate. |
| 284 | min_confidence: Minimum Choice confidence for a route to be decided. |
| 285 | noul_threshold: Minimum Noul probability for an output to be accepted. |
| 286 | routes: Known route labels. Defaults to every ``expected_route``. |
| 287 | |
| 288 | Returns: |
| 289 | Counts, rates, usage, latency, and a per-record outcome with its reason. |
| 290 | |
| 291 | Raises: |
| 292 | ValueError: If a threshold is not a finite number in [0, 1]. |
| 293 | """ |
| 294 | _validate_threshold("min_confidence", min_confidence) |
| 295 | _validate_threshold("noul_threshold", noul_threshold) |
| 296 | route_records = [r for r in records if isinstance(r, RouteRecord)] |
| 297 | known_routes = ( |
| 298 | set(routes) if routes is not None else {r.expected_route for r in route_records} |
| 299 | ) |
| 300 | |
| 301 | outcomes: list[RecordOutcome] = [] |
| 302 | false_accepts = false_rejects = fa_denominator = fr_denominator = 0 |
| 303 | for record in records: |
| 304 | if isinstance(record, RouteRecord): |
| 305 | outcome, reason = _classify_route(record, known_routes, min_confidence) |
| 306 | else: |
| 307 | outcome, reason = _classify_guardrail(record, noul_threshold) |
| 308 | if outcome != "failed" and record.expected == "reject": |
| 309 | fa_denominator += 1 |
| 310 | if outcome == "accepted": |
| 311 | false_accepts += 1 |
| 312 | elif outcome != "failed": |
| 313 | fr_denominator += 1 |
| 314 | if outcome == "rejected": |
| 315 | false_rejects += 1 |
| 316 | outcomes.append( |
| 317 | RecordOutcome( |
| 318 | id=record.id, kind=record.kind, outcome=outcome, reason=reason |
| 319 | ) |
| 320 | ) |
| 321 | |
| 322 | route_counts = Counter(o.outcome for o in outcomes if o.kind == "route") |
| 323 | guard_counts = Counter(o.outcome for o in outcomes if o.kind == "guardrail") |
| 324 | route_total = len(route_records) |
| 325 | route_decided = route_counts["correct"] + route_counts["incorrect"] |
| 326 | guard_total = len(records) - route_total |