Function evaluate

src/benchmarks/writing_bench/evaluate.py:169–310

Run rubric-based evaluation over completed writing results. Args: runs_dir: Directory containing run result JSON files. benchmark_path: Path to benchmark_all.jsonl. eval_dir: Directory to save evaluation results. judge_model: Model to use for judging. for

(
    runs_dir: str,
    benchmark_path: str,
    eval_dir: str = "evals",
    judge_model: str = "claude-sonnet-4-5",
    force: bool = False,
    max_parallel: int = 1,
)

def evaluate(
    runs_dir: str,
    benchmark_path: str,
    eval_dir: str = "evals",
    judge_model: str = "claude-sonnet-4-5",
    force: bool = False,
    max_parallel: int = 1,
) -> dict:
    """Run rubric-based evaluation over completed writing results.

    Args:
        runs_dir: Directory containing run result JSON files.
        benchmark_path: Path to benchmark_all.jsonl.
        eval_dir: Directory to save evaluation results.
        judge_model: Model to use for judging.
        force: If True, re-evaluate even if cached results exist.
        max_parallel: Max parallel evaluation threads (default: 1, sequential).
    """
    runs_path = Path(runs_dir)
    output_dir = Path(eval_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    benchmark = load_benchmark_queries(Path(benchmark_path))

    # Support both directory layouts:
    #   New: runs/{index}/result.json  (per-instance directories)
    #   Old: runs/{index}.json         (flat layout, backward compat)
    json_files = sorted(runs_path.glob("*/result.json"))
    if not json_files:
        json_files = sorted(runs_path.glob("*.json"))
    if not json_files:
        print(f"No result JSON files in {runs_path}")
        return {}

    client = LLMClient()

    # Build (json_path, eval_path) pairs
    eval_tasks = []
    for json_path in json_files:
        if json_path.name == "result.json":
            eval_key = json_path.parent.name
        else:
            eval_key = json_path.stem
        eval_path = output_dir / f"{eval_key}_eval.json"
        eval_tasks.append((json_path, eval_path))

    all_scores: List[dict] = []

    if max_parallel > 1:
        print(
            f"Evaluating {len(eval_tasks)} results with max {max_parallel} parallel judges"
        )
        with ThreadPoolExecutor(max_workers=max_parallel) as executor:
            future_to_key = {
                executor.submit(
                    _evaluate_single,
                    json_path,
                    eval_path,