Function llm_evaluate

python/agents/invoice-processing/eval/eval.py:621–662

Run a single LLM call to get holistic alignment verdict.

(
    model: "GenerativeModel",
    ground_truth: dict,
    agent_output: dict,
    mismatches: list[str],
    domain_name: str = "invoice processing",
)

def llm_evaluate(
    model: "GenerativeModel",
    ground_truth: dict,
    agent_output: dict,
    mismatches: list[str],
    domain_name: str = "invoice processing",
) -> dict | None:
    """Run a single LLM call to get holistic alignment verdict."""
    prompt = LLM_EVAL_PROMPT_TEMPLATE.format(
        domain_name=domain_name,
        ground_truth=json.dumps(ground_truth, indent=2),
        agent_output=json.dumps(agent_output, indent=2),
        mismatches="\n".join(f"- {m}" for m in mismatches)
        if mismatches
        else "None detected",
    )

    try:
        response = model.generate_content(prompt)
        text = response.text.strip()

        # Strip markdown code fences if present
        if text.startswith("```"):
            match = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL)
            if match:
                text = match.group(1).strip()

        result = json.loads(text)

        # Validate verdict value
        verdict = result.get("verdict", "").upper()
        if verdict not in ("ALIGNED", "PARTIALLY_ALIGNED", "NOT_ALIGNED"):
            result["verdict"] = "NOT_ALIGNED"
            result["reason"] = (
                result.get("reason", "")
                + f" (original verdict '{verdict}' was invalid)"
            )

        return result

    except Exception as e:
        return {"verdict": "ERROR", "reason": str(e)}


# ============================================================================