(json_path, mode='test')
| 87 | return all_ans |
| 88 | |
| 89 | def eval_json(json_path, mode='test'): |
| 90 | if json_path.endswith('/') or not json_path.endswith('json'): |
| 91 | origin_json_path = json_path |
| 92 | json_path = os.path.join(json_path, 'raw_generation_greedy.json') |
| 93 | if not os.path.exists(json_path): |
| 94 | lines = [] |
| 95 | for i in range(8): |
| 96 | path = os.path.join(origin_json_path, f'raw_generation_greedy_shard_{i}.json') |
| 97 | if os.path.exists(path): |
| 98 | with open(path, 'r') as f: |
| 99 | now_lines = f.readlines() |
| 100 | lines.extend(now_lines) |
| 101 | |
| 102 | if not lines: |
| 103 | for i in range(8): |
| 104 | path = os.path.join(origin_json_path, f'raw_generation_greedy_on_{mode}_shard_{i}.json') |
| 105 | if os.path.exists(path): |
| 106 | with open(path, 'r') as f: |
| 107 | now_lines = f.readlines() |
| 108 | lines.extend(now_lines) |
| 109 | else: |
| 110 | with open(json_path, 'r') as f: |
| 111 | lines = f.readlines() |
| 112 | |
| 113 | pred_ans = parse(lines) |
| 114 | if not pred_ans: |
| 115 | return |
| 116 | |
| 117 | with open(f'./{mode}_use.jsonl', 'r') as f: |
| 118 | lines = f.readlines() |
| 119 | gold_ans = parse_gold(lines) |
| 120 | |
| 121 | cor = 0 |
| 122 | rg = range(min(len(pred_ans), len(gold_ans))) |
| 123 | for i in rg: |
| 124 | if pred_ans[i] != INVALID_ANS and abs(float(pred_ans[i]) - float(gold_ans[i])) < 1e-4: |
| 125 | cor += 1 |
| 126 | print(json_path, cor, cor/len(list(rg)) * 100, len(rg)) |
| 127 | return pred_ans |
| 128 | |
| 129 | |
| 130 | def eval_majority_voting(folder, max_cnt=100): |
no test coverage detected