Function _run_evaluation_loop

python/agents/invoice-processing/eval/eval.py:1317–1372

Run evaluation on each case and return the list of results.

(
    case_ids: list[str],
    gt_dir: Path,
    agent_dir: Path,
    tolerance: float,
    llm_model: Any | None,
    master: MasterData | None,
)

def _run_evaluation_loop(
    case_ids: list[str],
    gt_dir: Path,
    agent_dir: Path,
    tolerance: float,
    llm_model: Any | None,
    master: MasterData | None,
) -> list[dict]:
    """Run evaluation on each case and return the list of results."""
    results = []
    for idx, case_id in enumerate(case_ids, 1):
        gt_file = gt_dir / case_id / "Postprocessing_Data.json"
        agent_file = agent_dir / case_id / "Postprocessing_Data.json"

        if not gt_file.exists():
            results.append(
                {
                    "case_id": case_id,
                    "status": "ERROR",
                    "error": "Ground truth file not found",
                }
            )
            print(
                f"  [{idx}/{len(case_ids)}] {case_id}: ERROR (no ground truth)"
            )
            continue
        if not agent_file.exists():
            results.append(
                {
                    "case_id": case_id,
                    "status": "ERROR",
                    "error": "Agent output file not found",
                }
            )
            print(
                f"  [{idx}/{len(case_ids)}] {case_id}: ERROR (no agent output)"
            )
            continue

        result = evaluate_case(
            case_id, gt_file, agent_file, tolerance, llm_model, master
        )
        results.append(result)

        status_icon = "PASS" if result["all_correct"] else "FAIL"
        llm_tag = ""
        if result.get("llm_verdict"):
            llm_tag = f" [{result['llm_verdict']['verdict']}]"
        detail = ""
        if not result["all_correct"]:
            detail = f"  ({', '.join(result['mismatches'][:2])})"
        print(
            f"  [{idx}/{len(case_ids)}] {case_id}: {status_icon}{llm_tag}{detail}"
        )

    return results