Function _call_judge

src/benchmarks/writing_bench/evaluate.py:55–90

Score a single (response, criteria) pair via the judge model. Matches the official WritingBench evaluation protocol: - Criteria passed as raw dict (str() representation via .format()) - temperature=1.0, max_tokens=2048 - Up to 5 retries per criterion

(
    client: LLMClient,
    query: str,
    response: str,
    criteria: dict,
    model: str,
    max_retries: int = 5,
)

def _call_judge(
    client: LLMClient,
    query: str,
    response: str,
    criteria: dict,
    model: str,
    max_retries: int = 5,
) -> Optional[dict]:
    """Score a single (response, criteria) pair via the judge model.

    Matches the official WritingBench evaluation protocol:
    - Criteria passed as raw dict (str() representation via .format())
    - temperature=1.0, max_tokens=2048
    - Up to 5 retries per criterion
    """
    prompt = EVALUATE_PROMPT.format(query=query, response=response, criteria=criteria)

    for _ in range(max_retries):
        try:
            completion = client.completion(
                model=model,
                messages=[
                    {"role": "system", "content": EVALUATE_SYSTEM},
                    {"role": "user", "content": prompt},
                ],
                temperature=1.0,
                max_tokens=2048,
            )
            text = completion.choices[0].message.content or ""
            result = _parse_score_json(text)
            if result is not None:
                return result
        except Exception as e:
            print(f"  Judge API error: {e}")

    return None


def load_benchmark_queries(jsonl_path: Path) -> Dict[int, dict]: