()
| 48 | |
| 49 | |
| 50 | async def main(): |
| 51 | args = parse_args() |
| 52 | api_key = os.environ.get("OPENROUTER_API_KEY") |
| 53 | if not api_key: |
| 54 | print("Error: OPENROUTER_API_KEY env var not set", file=sys.stderr) |
| 55 | sys.exit(1) |
| 56 | |
| 57 | client = AsyncOpenAI(base_url=JUDGE_BASE_URL, api_key=api_key) |
| 58 | sem = asyncio.Semaphore(CONCURRENCY) |
| 59 | |
| 60 | # {name: {doc_id: bool}} |
| 61 | all_results = {} |
| 62 | for eval_dir in args.eval_dirs: |
| 63 | name = Path(eval_dir).name |
| 64 | all_results[name] = await judge_dir(eval_dir, client, sem) |
| 65 | |
| 66 | print() |
| 67 | print_results(all_results) |
| 68 | |
| 69 | |
| 70 | def print_results(all_results: dict[str, dict[int, bool]]): |
no test coverage detected