(preds: List[str], gold_file: str = "classical_test.pkl", verbose: bool = True,
num_processes: int = NUM_PROCESSES, subset: str = 'full', use_cache: bool = True)
| 97 | |
| 98 | |
| 99 | def main(preds: List[str], gold_file: str = "classical_test.pkl", verbose: bool = True, |
| 100 | num_processes: int = NUM_PROCESSES, subset: str = 'full', use_cache: bool = True) -> List[bool]: |
| 101 | gold_dicts = pkl.load(open(gold_file, 'rb')) |
| 102 | if subset != 'full': |
| 103 | gold_dicts = [d for d in gold_dicts if d['db_path'] == 'database/{db_id}/{db_id}.sqlite'.format(db_id=subset)] |
| 104 | assert len(gold_dicts) == len(preds), 'number of gold and prediction should be equal' |
| 105 | group_name2idxes = defaultdict(list) |
| 106 | |
| 107 | for idx, gold_dict in enumerate(gold_dicts): |
| 108 | group_name2idxes[gold_dict['db_id']].append(idx) |
| 109 | |
| 110 | with Pool(num_processes) as pool: |
| 111 | result = list(tqdm.tqdm(pool.imap(judge, zip(gold_dicts, preds, repeat(use_cache, len(preds)))), total=len(gold_dicts))) |
| 112 | |
| 113 | if verbose: |
| 114 | print('overall accuracy: ', acc(result)) |
| 115 | for group, idxes in group_name2idxes.items(): |
| 116 | print('accuracy for ', group, acc(result, idxes)) |
| 117 | return result |
| 118 | |
| 119 | |
| 120 | if __name__ == "__main__": |
no test coverage detected