(args)
| 85 | |
| 86 | |
| 87 | def main(args): |
| 88 | dataset = DataLoader(args.dataset, seed=args.seed).load(sample_size=args.sample_size) |
| 89 | if args.method == 'direct': |
| 90 | method = IO(model_name=args.base_lm) |
| 91 | eval = Evaluator(args.dataset, dataset, method) |
| 92 | elif args.method == 'cot': |
| 93 | method = CoT(model_name=args.base_lm, fewshot=DEFAULT_EXEMPLARS_COT[args.dataset]) |
| 94 | eval = Evaluator(args.dataset, dataset, method) |
| 95 | elif args.method == 'react': |
| 96 | if args.dataset in ['hotpot_qa', 'trivia_qa']: |
| 97 | method = ReactBase(model_name=args.base_lm, fewshot=DEFAULT_EXEMPLARS_REACT[args.dataset], verbose=False) |
| 98 | else: |
| 99 | method = ReactExtraTool(model_name=args.base_lm, available_tools=args.toolset, |
| 100 | fewshot=DEFAULT_EXEMPLARS_REACT[args.dataset], verbose=False) |
| 101 | eval = Evaluator(args.dataset, dataset, method) |
| 102 | elif args.method == 'rewoo': |
| 103 | if args.planner_lm is None: |
| 104 | args.planner_lm = args.base_lm |
| 105 | if args.solver_lm is None: |
| 106 | args.solver_lm = args.base_lm |
| 107 | method = PWS_Base(planner_model=args.planner_lm, solver_model=args.solver_lm, |
| 108 | fewshot=DEFAUL_EXEMPLARS_PWS[args.dataset], available_tools=args.toolset) |
| 109 | eval = Evaluator(args.dataset, dataset, method) |
| 110 | else: |
| 111 | raise NotImplementedError |
| 112 | |
| 113 | responses, data = eval.run() |
| 114 | if args.save_result: |
| 115 | save_data(dataset, data, f'./results/eval_{args.dataset}_{args.method}_{args.base_lm}.csv') |
| 116 | print(responses) |
| 117 | |
| 118 | |
| 119 | if __name__ == '__main__': |
no test coverage detected