Collect and return the list of case IDs to evaluate.
(
args: argparse.Namespace, gt_dir: Path, agent_dir: Path
)
| 1282 | |
| 1283 | |
| 1284 | def _collect_case_ids( |
| 1285 | args: argparse.Namespace, gt_dir: Path, agent_dir: Path |
| 1286 | ) -> list[str]: |
| 1287 | """Collect and return the list of case IDs to evaluate.""" |
| 1288 | if args.case: |
| 1289 | case_ids = [args.case] |
| 1290 | else: |
| 1291 | gt_cases = {d.name for d in gt_dir.iterdir() if d.is_dir()} |
| 1292 | agent_cases = {d.name for d in agent_dir.iterdir() if d.is_dir()} |
| 1293 | common = sorted(gt_cases & agent_cases) |
| 1294 | |
| 1295 | if not common: |
| 1296 | print("No matching case folders found between:") |
| 1297 | print(f" Ground truth: {gt_dir} ({len(gt_cases)} folders)") |
| 1298 | print(f" Agent output: {agent_dir} ({len(agent_cases)} folders)") |
| 1299 | if gt_cases - agent_cases: |
| 1300 | print( |
| 1301 | f" In ground truth only: {sorted(gt_cases - agent_cases)[:5]}" |
| 1302 | ) |
| 1303 | if agent_cases - gt_cases: |
| 1304 | print( |
| 1305 | f" In agent output only: {sorted(agent_cases - gt_cases)[:5]}" |
| 1306 | ) |
| 1307 | sys.exit(1) |
| 1308 | |
| 1309 | case_ids = common |
| 1310 | |
| 1311 | if args.num_cases > 0: |
| 1312 | case_ids = case_ids[: args.num_cases] |
| 1313 | |
| 1314 | return case_ids |
| 1315 | |
| 1316 | |
| 1317 | def _run_evaluation_loop( |