Function build_eval_summary

src/benchmarks/chembench/evaluate.py:243–284

Build a complete evaluation summary dict. Args: results: List of result dicts. mode: "baseline" or "agent". model: Model name string. topic_key: Key for topic field in results. extra: Additional fields to include (e.g., usage stats). Returns:

(
    results: list[dict[str, Any]],
    mode: str,
    model: str,
    topic_key: str = "topic",
    extra: dict[str, Any] | None = None,
)

def build_eval_summary(
    results: list[dict[str, Any]],
    mode: str,
    model: str,
    topic_key: str = "topic",
    extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
    """Build a complete evaluation summary dict.

    Args:
        results: List of result dicts.
        mode: "baseline" or "agent".
        model: Model name string.
        topic_key: Key for topic field in results.
        extra: Additional fields to include (e.g., usage stats).

    Returns:
        Complete evaluation summary dict.
    """
    topic_stats = compute_topic_stats(results, topic_key=topic_key)
    overall = compute_overall_stats(topic_stats)

    # Collect wrong question identifiers
    wrong_names = []
    for r in results:
        report = r.get("report", r)
        if not is_correct(report):
            name = r.get("name", report.get("name", ""))
            wrong_names.append(name)

    summary = {
        "mode": mode,
        "model": model,
        **overall,
        "topic_stats": topic_stats,
        "wrong_names": wrong_names,
    }

    if extra:
        summary.update(extra)

    return summary


def save_eval_summary(