(filename:str, prefer_dict: dict, reference_model: str, output_model: str, reference_examples: dict, output_examples: dict)
| 104 | return return_dict |
| 105 | |
| 106 | def write_results(filename:str, prefer_dict: dict, reference_model: str, output_model: str, reference_examples: dict, output_examples: dict) -> None: |
| 107 | with open(filename, 'w', newline='') as file: |
| 108 | writer = csv.writer(file, delimiter="\t") |
| 109 | writer.writerow(["query", "available_tools", "ref_model_intermediate_steps", "ref_model_final_step", "output_model_intermediate_steps", "output_model_final_step", "preference_label", "query_id", "ref_model", "output_model"]) |
| 110 | |
| 111 | for query_id in prefer_dict: |
| 112 | ref_example = reference_examples[query_id] |
| 113 | output_example = output_examples[query_id] |
| 114 | tool_names = [] |
| 115 | for tool_dict in ref_example['available_tools']: |
| 116 | tool_name = tool_dict["name"] |
| 117 | tool_names.append(tool_name) |
| 118 | ref_steps, ref_final_step = get_steps(ref_example) |
| 119 | output_steps, output_final_step = get_steps(output_example) |
| 120 | |
| 121 | if prefer_dict[query_id][reference_model] > prefer_dict[query_id][output_model]: |
| 122 | preference = 1 |
| 123 | elif prefer_dict[query_id][reference_model] < prefer_dict[query_id][output_model]: |
| 124 | preference = 2 |
| 125 | else: |
| 126 | preference = 3 |
| 127 | writer.writerow([ref_example['query'], str(tool_names), ref_steps, ref_final_step, output_steps, output_final_step, str(preference), query_id, reference_model, output_model]) |
| 128 | return None |
| 129 | |
| 130 | |
| 131 | if __name__=='__main__': |
no test coverage detected