(query_id, task_status, answer_statuss, ref_example, output_example)
| 133 | evaluators = [load_registered_automatic_evaluator(evaluator_name=args.evaluator, evaluators_cfg_path=os.path.join(abs_dir,'evaluators')) for _ in range(args.max_eval_threads)] |
| 134 | |
| 135 | def get_preference(query_id, task_status, answer_statuss, ref_example, output_example): |
| 136 | global evaluators |
| 137 | evaluator = random.choice(evaluators) |
| 138 | |
| 139 | preference = evaluator.annotate_preference( |
| 140 | ref_example['query'], |
| 141 | ref_example['available_tools'], |
| 142 | [ref_example['answer'], output_example['answer']], |
| 143 | task_status=task_status, answer_statuss=answer_statuss |
| 144 | ) |
| 145 | if preference == 0: |
| 146 | return query_id, "ref" |
| 147 | elif preference == 1: |
| 148 | return query_id, "output" |
| 149 | else: |
| 150 | return query_id, "equal" |
| 151 | |
| 152 | reference_model = args.reference_model |
| 153 | output_model = args.output_model |
nothing calls this directly
no test coverage detected