(evaluator_cfg)
| 44 | return pearsonr(x,y)[0] |
| 45 | |
| 46 | def test_on_annotated_data(evaluator_cfg)->List[List[int]]: |
| 47 | evaluators = [load_registered_automatic_evaluator(evaluator_cfg) for _ in range(NUM_WORKERS)] |
| 48 | def get_preference(idx): |
| 49 | data = annotated_data[idx] |
| 50 | def process_tools(tools:list): |
| 51 | for tool in tools: |
| 52 | tool.pop('description',None) |
| 53 | tool.pop('parameters',None) |
| 54 | return tools |
| 55 | |
| 56 | tools = process_tools(data['available_tools']) |
| 57 | ret = evaluators[idx%NUM_WORKERS].annotate_preference( |
| 58 | data['query'], |
| 59 | tools, |
| 60 | data['answers'],multisample=True) |
| 61 | return idx,ret |
| 62 | prefer_dict = {} |
| 63 | with ThreadPoolExecutor(NUM_WORKERS) as pool: |
| 64 | # future = [pool.submit(get_preference,idx) for idx in range(100)] |
| 65 | future = [pool.submit(get_preference,idx) for idx in range(len(annotated_data))] |
| 66 | for thd in tqdm(as_completed(future),total=len(future),ncols=100): |
| 67 | if thd.exception() is not None: |
| 68 | pool.shutdown(cancel_futures=True) |
| 69 | raise thd.exception() |
| 70 | exit(-1) |
| 71 | idx,preference = thd.result() |
| 72 | prefer_dict[idx] = preference |
| 73 | prefer = [prefer_dict[idx] for idx in range(len(future))] |
| 74 | return prefer |
| 75 | |
| 76 | def get_popped_and_rest(d:list,index:int): |
| 77 | l = copy.deepcopy(d) |
no test coverage detected