(folder, max_cnt=100)
| 128 | |
| 129 | |
| 130 | def eval_majority_voting(folder, max_cnt=100): |
| 131 | lines = [] |
| 132 | paths = [os.path.join(folder, f'raw_generation_0.7_{i}_test.json') for i in range(max_cnt)] |
| 133 | idx = 0 |
| 134 | for path in paths: |
| 135 | if os.path.exists(path): |
| 136 | with open(path, 'r') as f: |
| 137 | now_lines = f.readlines() |
| 138 | lines.append(now_lines) |
| 139 | idx += 1 |
| 140 | if not lines: |
| 141 | paths = [os.path.join(folder, f'raw_generation_0.7sampled_on_test_seed_{i}_shard_SHARD.json') for i in range(max_cnt)] |
| 142 | for path in paths: |
| 143 | now_lines = [] |
| 144 | for SHARD in range(8): |
| 145 | p = path.replace('SHARD', str(SHARD)) |
| 146 | if os.path.exists(p): |
| 147 | with open(p, 'r') as f: |
| 148 | now_shard_lines = f.readlines() |
| 149 | now_lines.extend(now_shard_lines) |
| 150 | if len(now_lines) == 1319: |
| 151 | lines.append(now_lines) |
| 152 | idx += 1 |
| 153 | if not lines: |
| 154 | return |
| 155 | |
| 156 | def maj(lst): |
| 157 | lst = [x for x in lst if x != INVALID_ANS] |
| 158 | if not lst: |
| 159 | return INVALID_ANS |
| 160 | # Count the occurrences of each string in the list |
| 161 | counts = Counter(lst) |
| 162 | |
| 163 | # Find the string with the highest count |
| 164 | most_common = max(counts, key=counts.get) |
| 165 | |
| 166 | return most_common |
| 167 | |
| 168 | pred_ans_multiple = [parse(prediction) for prediction in lines] |
| 169 | pred_ans = [maj([prediction[i] for prediction in pred_ans_multiple]) for i in range(len(pred_ans_multiple[0]))] |
| 170 | |
| 171 | if not pred_ans: |
| 172 | return |
| 173 | |
| 174 | with open(f'./data/test_use.jsonl', 'r') as f: |
| 175 | lines = f.readlines() |
| 176 | gold_ans = parse_gold(lines) |
| 177 | |
| 178 | cor = 0 |
| 179 | rg = range(min(len(pred_ans), len(gold_ans))) |
| 180 | for i in rg: |
| 181 | if pred_ans[i] != INVALID_ANS and abs(float(pred_ans[i]) - float(gold_ans[i])) < 1e-4: |
| 182 | cor += 1 |
| 183 | print(folder, cor, cor/len(list(rg)) * 100, len(rg), f'Ensemble count: {idx}') |
| 184 | return pred_ans |
| 185 | |
| 186 | if __name__ == "__main__": |
| 187 | import argparse |
nothing calls this directly
no test coverage detected