| 34 | abs_dir = os.path.split(__file__)[0] |
| 35 | |
| 36 | def parse_args(): |
| 37 | parser = argparse.ArgumentParser() |
| 38 | parser.add_argument('--converted_answer_path', type=str, default="", required=True, help='result save path') |
| 39 | parser.add_argument('--reference_model', type=str, default="gpt-4-0613_dfs", required=False, help='ref model predictions path') |
| 40 | parser.add_argument('--output_model', type=str, default="toolllama-2-0830-thought", required=False, help='output model predictions path') |
| 41 | parser.add_argument('--test_ids', type=str, default="", required=True, help='test query ids path') |
| 42 | parser.add_argument('--save_path', type=str, default="preference_results", required=False, help='preference results save path') |
| 43 | parser.add_argument('--pass_rate_result_path', type=str, default="pass_rate_results", required=False, help='pass rate results save path') |
| 44 | parser.add_argument('--max_eval_threads', type=int, default=3, required=False, help='max threads nums') |
| 45 | parser.add_argument('--evaluator',default='tooleval_gpt-3.5-turbo_default', required=False, help='which evaluator to use.') |
| 46 | parser.add_argument('--use_pass_rate',default='false',help='to use existed pass rate result or compare preference from scratch.') |
| 47 | parser.add_argument('--evaluate_times',default=2,help='how many times to predict with the evaluator for each solution path.') |
| 48 | |
| 49 | return parser.parse_args() |
| 50 | |
| 51 | def get_pass_rate_results(filename: str) -> dict: |
| 52 | csv_reader = csv.reader(open(filename), delimiter="\t") |