(idx)
| 46 | def test_on_annotated_data(evaluator_cfg)->List[List[int]]: |
| 47 | evaluators = [load_registered_automatic_evaluator(evaluator_cfg) for _ in range(NUM_WORKERS)] |
| 48 | def get_preference(idx): |
| 49 | data = annotated_data[idx] |
| 50 | def process_tools(tools:list): |
| 51 | for tool in tools: |
| 52 | tool.pop('description',None) |
| 53 | tool.pop('parameters',None) |
| 54 | return tools |
| 55 | |
| 56 | tools = process_tools(data['available_tools']) |
| 57 | ret = evaluators[idx%NUM_WORKERS].annotate_preference( |
| 58 | data['query'], |
| 59 | tools, |
| 60 | data['answers'],multisample=True) |
| 61 | return idx,ret |
| 62 | prefer_dict = {} |
| 63 | with ThreadPoolExecutor(NUM_WORKERS) as pool: |
| 64 | # future = [pool.submit(get_preference,idx) for idx in range(100)] |
nothing calls this directly
no test coverage detected