()
| 76 | |
| 77 | |
| 78 | async def main(): |
| 79 | args = parse_args() |
| 80 | output_dir = Path(args.output_dir).expanduser() |
| 81 | output_dir.mkdir(parents=True, exist_ok=True) |
| 82 | agent_names = [name for name, num in zip(args.agent_names, args.agent_nums) for _ in range(num)] |
| 83 | kwargs = get_kwargs(args.mode, len(agent_names)) |
| 84 | kv_config = KVCommConfig.from_env().apply_overrides( |
| 85 | threshold=args.kv_threshold, |
| 86 | max_anchor_num=args.kv_max_anchor_num, |
| 87 | window_size=args.kv_window_size, |
| 88 | thread_pool_workers=args.kv_thread_workers, |
| 89 | worker_timeout=args.kv_worker_timeout, |
| 90 | ) |
| 91 | |
| 92 | graph = Graph( |
| 93 | domain=args.domain, |
| 94 | llm_name=args.llm_name, |
| 95 | agent_names=agent_names, |
| 96 | decision_method=args.decision_method, |
| 97 | kv_config=kv_config, |
| 98 | **kwargs, |
| 99 | ) |
| 100 | |
| 101 | download() |
| 102 | dataset_val = MMLUDataset("val") |
| 103 | limit_questions = 153 |
| 104 | eval_kwargs = {} |
| 105 | if args.execution_mode == "allow_kv_reuse": |
| 106 | eval_kwargs = { |
| 107 | "prefix": args.prefix, |
| 108 | "output_dir": str(output_dir), |
| 109 | } |
| 110 | |
| 111 | configure_logging(log_path=output_dir / "logs/log.txt") |
| 112 | timestamp = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) |
| 113 | score = await evaluate( |
| 114 | graph=graph, |
| 115 | dataset=dataset_val, |
| 116 | limit_questions=limit_questions, |
| 117 | eval_batch_size=args.batch_size, |
| 118 | mode=args.execution_mode, |
| 119 | **eval_kwargs, |
| 120 | ) |
| 121 | logger.opt(colors=True).info("<blue>[MMLU SCORE]</blue> {:.4f}", score) |
| 122 | result_file = output_dir / f"{args.domain}_{args.llm_name}_{timestamp}.json" |
| 123 | result_file.touch(exist_ok=True) |
| 124 | payload = { |
| 125 | "score": score, |
| 126 | "execution_mode": args.execution_mode, |
| 127 | "agent_names": args.agent_names, |
| 128 | "agent_nums": args.agent_nums, |
| 129 | "timestamp": timestamp, |
| 130 | } |
| 131 | with open(result_file, "w", encoding="utf-8") as handle: |
| 132 | json.dump(payload, handle, ensure_ascii=False, indent=2) |
| 133 | logger.opt(colors=True).info("<blue>[RESULT SAVED]</blue> {}", str(result_file)) |
| 134 | |
| 135 |
no test coverage detected