| 506 | |
| 507 | |
| 508 | def build_parser() -> argparse.ArgumentParser: |
| 509 | parser = argparse.ArgumentParser(description=__doc__) |
| 510 | parser.add_argument("--log", default="INFO", help="log level (default: INFO)") |
| 511 | sub = parser.add_subparsers(dest="command", required=True) |
| 512 | |
| 513 | run_p = sub.add_parser("run", help="run the LLM extractor and write a summary") |
| 514 | run_p.add_argument("--label", required=True, help="human-readable run label") |
| 515 | run_p.add_argument( |
| 516 | "--model", default=DEFAULT_MODEL, help=f"LLM model (default: {DEFAULT_MODEL})" |
| 517 | ) |
| 518 | run_p.add_argument( |
| 519 | "--batch", |
| 520 | type=int, |
| 521 | default=None, |
| 522 | help="use provider batch API with this batch size (e.g. 50)", |
| 523 | ) |
| 524 | run_p.add_argument( |
| 525 | "--jobs", |
| 526 | "-j", |
| 527 | type=int, |
| 528 | default=1, |
| 529 | help="number of concurrent batch jobs (default: 1)", |
| 530 | ) |
| 531 | run_p.add_argument( |
| 532 | "--description", |
| 533 | "-d", |
| 534 | default="", |
| 535 | help="short description of changes for this run", |
| 536 | ) |
| 537 | run_p.add_argument("--corpus", default=str(DEFAULT_CORPUS), help="corpus file path") |
| 538 | run_p.add_argument("--output", help="run output directory") |
| 539 | run_p.add_argument( |
| 540 | "paths", nargs="*", help="optional .gz files / dirs (overrides --corpus)" |
| 541 | ) |
| 542 | run_p.set_defaults(func=run_bench) |
| 543 | |
| 544 | cmp_p = sub.add_parser("compare", help="compare two LLM eval runs") |
| 545 | cmp_p.add_argument("baseline", help="baseline run directory") |
| 546 | cmp_p.add_argument("current", help="current run directory") |
| 547 | cmp_p.add_argument( |
| 548 | "--fail-on-suspicious", |
| 549 | action="store_true", |
| 550 | help="exit non-zero when suspicious structural changes are detected", |
| 551 | ) |
| 552 | cmp_p.set_defaults(func=compare_runs) |
| 553 | |
| 554 | list_p = sub.add_parser("list", help="list saved LLM eval runs") |
| 555 | list_p.add_argument( |
| 556 | "--runs-dir", |
| 557 | default=str(DEFAULT_RUNS_DIR), |
| 558 | help=f"runs directory (default: {DEFAULT_RUNS_DIR})", |
| 559 | ) |
| 560 | list_p.set_defaults(func=list_runs) |
| 561 | |
| 562 | return parser |
| 563 | |
| 564 | |
| 565 | def main(argv: list[str] | None = None) -> int: |