Function paired_bootstrap

benchmarking/vllm/judge_eval.py:104–158

Paired bootstrap test: is there a significant difference between two variants? Both variants answer the same set of questions, so we use a *paired* test: compute per-question differences diff[i] = A[i] - B[i] (each is +1, -1, or 0), then bootstrap the mean of that difference vector. Th

(
    results_a: dict[int, bool],
    results_b: dict[int, bool],
    name_a: str,
    name_b: str,
    rng: np.random.Generator,
)

def paired_bootstrap(
    results_a: dict[int, bool],
    results_b: dict[int, bool],
    name_a: str,
    name_b: str,
    rng: np.random.Generator,
):
    """Paired bootstrap test: is there a significant difference between two variants?

    Both variants answer the same set of questions, so we use a *paired* test:
    compute per-question differences diff[i] = A[i] - B[i] (each is +1, -1, or 0),
    then bootstrap the mean of that difference vector.  This cancels out shared
    question difficulty — if both variants get the same 150 hard questions wrong
    and only differ on 3 borderline ones, the paired test sees the tiny signal
    clearly, while an unpaired test would be swamped by shared noise.

    Concretely:
    1. Align on shared doc_ids → two boolean arrays of length n.
    2. diff = A - B.  Observed accuracy gap = mean(diff).
    3. Draw BOOTSTRAP_N resamples (with replacement) of size n from diff,
       compute the mean of each → distribution of plausible gaps.
    4. 95% CI = 2.5th / 97.5th percentiles of bootstrap means.
    5. Two-sided p-value = 2 * fraction of bootstrap means with opposite sign
       to the observed gap (clamped to 1.0).
    """
    shared_ids = sorted(set(results_a) & set(results_b))
    if not shared_ids:
        print(f"No shared doc_ids between {name_a} and {name_b}")
        return

    arr_a = np.array([results_a[d] for d in shared_ids], dtype=np.float64)
    arr_b = np.array([results_b[d] for d in shared_ids], dtype=np.float64)
    diff = arr_a - arr_b  # per-question difference

    observed_diff = diff.mean()
    n = len(diff)
    idx = rng.integers(0, n, size=(BOOTSTRAP_N, n))
    boot_diffs = diff[idx].mean(axis=1)
    lo, hi = np.percentile(boot_diffs, [2.5, 97.5])

    # Two-sided p-value: fraction of bootstrap samples where sign flips
    if observed_diff >= 0:
        p_value = (boot_diffs <= 0).mean()
    else:
        p_value = (boot_diffs >= 0).mean()
    p_value = min(2 * p_value, 1.0)  # two-sided

    sig = "" if p_value < 0.05 else " (not significant)"
    print(
        f"{name_a} vs {name_b}: "
        f"diff = {100 * observed_diff:+.1f}pp, "
        f"95% CI [{100 * lo:+.1f}pp, {100 * hi:+.1f}pp], "
        f"p = {p_value:.3f}{sig}"
    )
    print(f"  ({len(shared_ids)} shared questions, {BOOTSTRAP_N:,} bootstrap samples)")


async def judge_dir(eval_dir: str, client: AsyncOpenAI, sem: asyncio.Semaphore) -> dict[int, bool]: