Function evaluate_case

python/agents/invoice-processing/eval/eval.py:836–889

Evaluate a single case by comparing ground truth and agent output. When master data is provided, uses schema-driven comparison groups. Otherwise falls back to the hardcoded invoice-specific comparators.

(
    case_id: str,
    gt_file: Path,
    agent_file: Path,
    tolerance: float,
    llm_model: Any | None = None,
    master: MasterData | None = None,
)

def evaluate_case(
    case_id: str,
    gt_file: Path,
    agent_file: Path,
    tolerance: float,
    llm_model: Any | None = None,
    master: MasterData | None = None,
) -> dict:
    """Evaluate a single case by comparing ground truth and agent output.

    When master data is provided, uses schema-driven comparison groups.
    Otherwise falls back to the hardcoded invoice-specific comparators.
    """

    # Load files
    try:
        gt = json.loads(gt_file.read_text(encoding="utf-8"))
    except Exception as e:
        return {
            "case_id": case_id,
            "status": "ERROR",
            "error": f"Failed to load ground truth: {e}",
        }

    try:
        agent = json.loads(agent_file.read_text(encoding="utf-8"))
    except Exception as e:
        return {
            "case_id": case_id,
            "status": "ERROR",
            "error": f"Failed to load agent output: {e}",
        }

    # Decision comparison (always schema-driven when master data available)
    decision = compare_decision(gt, agent, master)

    # Field group comparisons — schema-driven or legacy fallback
    if master:
        result_dict = _evaluate_schema_driven(
            case_id, gt, agent, decision, master, tolerance
        )
    else:
        result_dict = _evaluate_legacy(case_id, gt, agent, decision, tolerance)

    # LLM alignment verdict (optional)
    llm_verdict = None
    if llm_model is not None:
        domain_name = master.display_name if master else "invoice processing"
        llm_verdict = llm_evaluate(
            llm_model, gt, agent, result_dict["mismatches"], domain_name
        )
    result_dict["llm_verdict"] = llm_verdict

    return result_dict


# ============================================================================