| 107 | |
| 108 | |
| 109 | def calc_llm_index(ass_predict, ass_truth, chatrounds): |
| 110 | global function_call_sum, function_call_correct, function_call_fail, function_call_fail_functioncall, function_call_fail_name, function_call_fail_name_illusion, function_call_fail_param |
| 111 | |
| 112 | chatrounds_functionname_list = [] |
| 113 | for function_dict in chatrounds.get("functions", []): |
| 114 | chatrounds_functionname_list.append(function_dict["name"]) |
| 115 | |
| 116 | if "function_call" in ass_truth: |
| 117 | function_call_sum += 1 |
| 118 | if "function_call" not in ass_predict: |
| 119 | function_call_fail += 1 |
| 120 | function_call_fail_functioncall += 1 |
| 121 | elif ass_predict["function_call"]["name"] not in chatrounds_functionname_list: |
| 122 | # 模型幻觉 |
| 123 | function_call_fail += 1 |
| 124 | function_call_fail_name += 1 |
| 125 | function_call_fail_name_illusion += 1 |
| 126 | else: |
| 127 | function_call_name_label = False |
| 128 | function_call_args_label = False |
| 129 | if ass_predict["function_call"]["name"] == ass_truth["function_call"]["name"]: |
| 130 | function_call_name_label = True |
| 131 | if cmp_arguments(ass_predict["function_call"]["arguments"], ass_truth["function_call"]["arguments"]): |
| 132 | function_call_args_label = True |
| 133 | else: |
| 134 | function_call_fail_param += 1 |
| 135 | else: |
| 136 | function_call_fail_name += 1 |
| 137 | |
| 138 | if function_call_name_label and function_call_args_label: |
| 139 | function_call_correct += 1 |
| 140 | else: |
| 141 | function_call_fail += 1 |
| 142 | else: |
| 143 | calc_relevancy(ass_predict, ass_truth, chatrounds) |
| 144 | |
| 145 | |
| 146 | def print_result(): |