Function run_eval

code_review_graph/eval/runner.py:133–211

Run evaluation benchmarks across repositories. Args: repos: List of repo config names to evaluate (None = all). benchmarks: List of benchmark names to run (None = all). output_dir: Directory for CSV output files. Returns: Dict mapping ``{repo}_{benchmark}``

(
    repos: list[str] | None = None,
    benchmarks: list[str] | None = None,
    output_dir: str | Path | None = None,
)

def run_eval(
    repos: list[str] | None = None,
    benchmarks: list[str] | None = None,
    output_dir: str | Path | None = None,
) -> dict[str, list[dict]]:
    """Run evaluation benchmarks across repositories.

    Args:
        repos: List of repo config names to evaluate (None = all).
        benchmarks: List of benchmark names to run (None = all).
        output_dir: Directory for CSV output files.

    Returns:
        Dict mapping ``{repo}_{benchmark}`` to list of result dicts.
    """
    output_dir = Path(output_dir) if output_dir else DEFAULT_OUTPUT
    output_dir.mkdir(parents=True, exist_ok=True)

    if repos:
        configs = [load_config(r) for r in repos]
    else:
        configs = load_all_configs()

    benchmark_names = benchmarks or list(BENCHMARK_REGISTRY.keys())
    all_results: dict[str, list[dict]] = {}
    today = date.today().isoformat()

    for config in configs:
        name = config["name"]
        logger.info("Evaluating %s...", name)

        # Resolve the repo path to an absolute Path before handing it to
        # full_build / get_db_path so the stored qualified_names match what
        # the CLI/MCP layer produces (those paths go through _get_store ->
        # _validate_repo_root which .resolve()s). Without this, a later
        # ``code-review-graph update --repo <relative>`` writes the same
        # function under a new absolute-prefixed qualified_name, leaving the
        # graph with duplicate nodes for the same source location.
        repo_path = clone_or_update(config).resolve()

        # Build graph
        from code_review_graph.graph import GraphStore
        from code_review_graph.incremental import full_build, get_db_path
        from code_review_graph.postprocessing import run_post_processing

        db_path = get_db_path(repo_path)
        store = GraphStore(db_path)

        full_build(repo_path, store)
        # full_build is the parsing-only primitive; the higher-level CLI/MCP
        # wrappers run postprocessing on top. The eval framework bypasses
        # those, so call it directly here. Without this, FTS5 stays empty
        # and downstream benchmarks (token_efficiency, search_quality)
        # silently produce useless results. See: search.rebuild_fts_index.
        pp_result = run_post_processing(store)
        for warning in pp_result.get("warnings", []):
            logger.warning("  postprocessing: %s", warning)