(exec_results, diff_json_path)
| 94 | |
| 95 | |
| 96 | def compute_acc_by_diff(exec_results, diff_json_path): |
| 97 | num_queries = len(exec_results) |
| 98 | results = [res['res'] for res in exec_results] |
| 99 | contents = load_json(diff_json_path) |
| 100 | simple_results, moderate_results, challenging_results = [], [], [] |
| 101 | |
| 102 | for i, content in enumerate(contents): |
| 103 | if content['difficulty'] == 'simple': |
| 104 | simple_results.append(exec_results[i]) |
| 105 | |
| 106 | if content['difficulty'] == 'moderate': |
| 107 | moderate_results.append(exec_results[i]) |
| 108 | |
| 109 | if content['difficulty'] == 'challenging': |
| 110 | challenging_results.append(exec_results[i]) |
| 111 | |
| 112 | simple_acc = sum([res['res'] for res in simple_results]) / len(simple_results) |
| 113 | moderate_acc = sum([res['res'] for res in moderate_results]) / len(moderate_results) |
| 114 | # try except is for debug |
| 115 | try: |
| 116 | challenging_acc = sum([res['res'] for res in challenging_results]) / len(challenging_results) |
| 117 | except: |
| 118 | challenging_acc = 0 |
| 119 | all_acc = sum(results) / num_queries |
| 120 | count_lists = [len(simple_results), len(moderate_results), len(challenging_results), num_queries] |
| 121 | return simple_acc * 100, moderate_acc * 100, challenging_acc * 100, all_acc * 100, count_lists |
| 122 | |
| 123 | |
| 124 | def compute_acc_by_diff_sample(exec_results, query_pairs, diff_json_path, sample=True, test_id=None, fold="1", whether_print=False, print_file_name=""): |
nothing calls this directly
no test coverage detected