(qid,query,tools,ref_ans,ans,)
| 78 | evaluators = [load_registered_automatic_evaluator(vars(args)) for _ in range(args.max_eval_threads)] |
| 79 | |
| 80 | def get_preference(qid,query,tools,ref_ans,ans,): |
| 81 | global evaluators |
| 82 | evaluator = random.choice(evaluators) |
| 83 | ret = evaluator.annotate_preference( |
| 84 | query, |
| 85 | tools, |
| 86 | [ref_ans,ans]) |
| 87 | return qid,ret |
| 88 | def get_most_preferred(d:list)->np.ndarray: |
| 89 | if np.iterable(d): |
| 90 | d = np.asanyarray(d) |
nothing calls this directly
no test coverage detected