Function judge_dir

benchmarking/vllm/judge_eval.py:161–200

Judge all samples in an eval directory. Returns {doc_id: correct}.

(eval_dir: str, client: AsyncOpenAI, sem: asyncio.Semaphore)

async def judge_dir(eval_dir: str, client: AsyncOpenAI, sem: asyncio.Semaphore) -> dict[int, bool]:
    """Judge all samples in an eval directory. Returns {doc_id: correct}."""
    samples_path = find_samples(eval_dir)
    if not samples_path:
        print(f"No samples JSONL found in {eval_dir}", file=sys.stderr)
        return {}

    samples = load_samples(samples_path)
    print(f"Judging {len(samples)} samples from {samples_path}")

    results_path = samples_path.parent / samples_path.name.replace("samples_", "judge_")
    existing = load_existing_results(results_path)
    if existing:
        print(
            f"  Resuming: {len(existing)} already judged, {len(samples) - len(existing)} remaining"
        )

    pending = []
    for sample in samples:
        doc_id = sample["doc_id"]
        if doc_id in existing:
            continue
        ref = extract_reference(sample["doc"]["answer"])
        pending.append(judge_one(sample, ref, client, sem))

    n_existing = len(existing)
    done = 0
    total = n_existing + len(pending)

    with open(results_path, "a") as f:
        for coro in asyncio.as_completed(pending):
            r = await coro
            f.write(json.dumps(r) + "\n")
            f.flush()
            existing[r["doc_id"]] = r
            done += 1
            if done % 100 == 0:
                print(f"  {n_existing + done}/{total} judged...")

    return {doc_id: r["correct"] for doc_id, r in existing.items()}


async def judge_one(