| 8 | config = Config() |
| 9 | |
| 10 | def evaluate(pred_dir, method, testset, only_S_MAE=False, epoch=0): |
| 11 | filename = os.path.join('evaluation', 'eval-{}.txt'.format(method)) |
| 12 | if os.path.exists(filename): |
| 13 | id_suffix = 1 |
| 14 | filename = filename.rstrip('.txt') + '_{}.txt'.format(id_suffix) |
| 15 | while os.path.exists(filename): |
| 16 | id_suffix += 1 |
| 17 | filename = filename.replace('_{}.txt'.format(id_suffix-1), '_{}.txt'.format(id_suffix)) |
| 18 | gt_paths = sorted([ |
| 19 | os.path.join(config.data_root_dir, config.task, testset, 'gt', p) |
| 20 | for p in os.listdir(os.path.join(config.data_root_dir, config.task, testset, 'gt')) |
| 21 | ]) |
| 22 | pred_paths = sorted([os.path.join(pred_dir, method, testset, p) for p in os.listdir(os.path.join(pred_dir, method, testset))]) |
| 23 | with open(filename, 'a+') as file_to_write: |
| 24 | tb = pt.PrettyTable() |
| 25 | field_names = [ |
| 26 | "Dataset", "Method", "maxFm", "wFmeasure", 'MAE', "Smeasure", "meanEm", "maxEm", "meanFm", |
| 27 | "adpEm", "adpFm", 'HCE' |
| 28 | ] |
| 29 | tb.field_names = [name for name in field_names if not only_S_MAE or all(metric not in name for metric in ['Em', 'Fm'])] |
| 30 | em, sm, fm, mae, wfm, hce = evaluator( |
| 31 | gt_paths=gt_paths[:], |
| 32 | pred_paths=pred_paths[:], |
| 33 | metrics=['S', 'MAE', 'E', 'F', 'HCE'][:10*(not only_S_MAE) + 2], # , 'WF' |
| 34 | verbose=config.verbose_eval, |
| 35 | ) |
| 36 | e_max, e_mean, e_adp = em['curve'].max(), em['curve'].mean(), em['adp'].mean() |
| 37 | f_max, f_mean, f_wfm, f_adp = fm['curve'].max(), fm['curve'].mean(), wfm, fm['adp'] |
| 38 | tb.add_row( |
| 39 | [ |
| 40 | method+str(epoch), testset, f_max.round(3), f_wfm.round(3), mae.round(3), sm.round(3), |
| 41 | e_mean.round(3), e_max.round(3), f_mean.round(3), em['adp'].round(3), f_adp.round(3), hce.round(3) |
| 42 | ] if not only_S_MAE else [method, testset, mae.round(3), sm.round(3)] |
| 43 | ) |
| 44 | print(tb) |
| 45 | file_to_write.write(str(tb).replace('+', '|')+'\n') |
| 46 | file_to_write.close() |
| 47 | return {'e_max': e_max, 'e_mean': e_mean, 'e_adp': e_adp, 'sm': sm, 'mae': mae, 'f_max': f_max, 'f_mean': f_mean, 'f_wfm': f_wfm, 'f_adp': f_adp, 'hce': hce} |
| 48 | |
| 49 | |
| 50 | def main(): |