()
| 684 | |
| 685 | |
| 686 | def main() -> int: |
| 687 | parser = argparse.ArgumentParser() |
| 688 | parser.add_argument("--mode", choices=("preview", "live"), default="preview") |
| 689 | parser.add_argument("--timeout", type=int, default=90) |
| 690 | parser.add_argument("--limit", type=int, default=0) |
| 691 | parser.add_argument("--case", action="append", default=[], help="Run cases whose name contains this text.") |
| 692 | parser.add_argument("--sleep", type=float, default=0.0) |
| 693 | args = parser.parse_args() |
| 694 | |
| 695 | selected = cases() |
| 696 | if len(selected) != 50: |
| 697 | raise SystemExit(f"expected 50 cases, got {len(selected)}") |
| 698 | if args.case: |
| 699 | wanted = [part.lower() for part in args.case] |
| 700 | selected = [case for case in selected if any(part in case.name.lower() for part in wanted)] |
| 701 | if args.limit: |
| 702 | selected = selected[: args.limit] |
| 703 | |
| 704 | runner = route_preview if args.mode == "preview" else live_request |
| 705 | rows: list[tuple[Case, dict[str, Any], bool, list[str]]] = [] |
| 706 | for case in selected: |
| 707 | result = runner(case, args.timeout) |
| 708 | ok, failures = evaluate(case, result) |
| 709 | rows.append((case, result, ok, failures)) |
| 710 | if args.sleep: |
| 711 | time.sleep(args.sleep) |
| 712 | |
| 713 | print(f"mode={args.mode} total={len(rows)} ok={sum(1 for _, _, ok, _ in rows if ok)}") |
| 714 | for case, result, ok, failures in rows: |
| 715 | marker = "OK" if ok else "!!" |
| 716 | print( |
| 717 | f"{marker} {case.name:24} exp={case.expected_tier:7} got={str(result.get('tier')):7} " |
| 718 | f"q={str(result.get('quality')):8} lane={str(result.get('lane')):20} " |
| 719 | f"model={result.get('model')} status={result.get('status')}" |
| 720 | ) |
| 721 | if failures: |
| 722 | print(" failures:", "; ".join(failures)) |
| 723 | reasoning = str(result.get("reasoning") or result.get("error") or "") |
| 724 | if reasoning: |
| 725 | print(" detail:", reasoning[:700]) |
| 726 | |
| 727 | print("tiers", dict(Counter(result.get("tier") for _, result, _, _ in rows))) |
| 728 | print("qualities", dict(Counter(result.get("quality") for _, result, _, _ in rows))) |
| 729 | print("models", Counter(result.get("model") for _, result, _, _ in rows).most_common(12)) |
| 730 | |
| 731 | return 0 if all(ok for _, _, ok, _ in rows) else 1 |
| 732 | |
| 733 | |
| 734 | if __name__ == "__main__": |
no test coverage detected