Function _summarize_exact

scripts/holdout_param_sweep.py:82–123

(
    *,
    config: ExperimentConfig,
    per_row: list[dict[str, Any]],
    errors: list[dict[str, Any]],
    summary: dict[str, Any],
)

def _summarize_exact(
    *,
    config: ExperimentConfig,
    per_row: list[dict[str, Any]],
    errors: list[dict[str, Any]],
    summary: dict[str, Any],
) -> dict[str, Any]:
    total = len(per_row)
    exact_match = sum(1 for row in per_row if row.get("match") is True)
    pass_count = sum(1 for row in per_row if row.get("passed") is True)
    by_benchmark: dict[str, Any] = {}

    grouped: dict[str, list[dict[str, Any]]] = {}
    for row in per_row:
        grouped.setdefault(row["benchmark"], []).append(row)

    for benchmark, rows in grouped.items():
        benchmark_total = len(rows)
        benchmark_exact = sum(1 for row in rows if row.get("match") is True)
        benchmark_pass = sum(1 for row in rows if row.get("passed") is True)
        by_benchmark[benchmark] = {
            "total": benchmark_total,
            "exact_match": benchmark_exact,
            "pass_count": benchmark_pass,
            "tier_match_accuracy": benchmark_exact / benchmark_total if benchmark_total else 0.0,
            "pass_rate": benchmark_pass / benchmark_total if benchmark_total else 0.0,
        }

    return {
        "name": _config_name(config),
        "config": asdict(config),
        "changed_dimensions": _changed_dimensions(config),
        "samples": total,
        "exact_match": exact_match,
        "pass_count": pass_count,
        "tier_match_accuracy": exact_match / total if total else 0.0,
        "pass_rate": pass_count / total if total else 0.0,
        "api_errors": len(errors),
        "cost_savings_score": summary["section_11"]["cost_savings_score"],
        "overall_score_percent": summary["router_accounting"]["overall_score_percent"],
        "by_benchmark": _sort_benchmarks(by_benchmark),
    }


def _attach_delta(result: dict[str, Any], baseline: dict[str, Any]) -> dict[str, Any]: