| 34 | |
| 35 | |
| 36 | def validate_model(model, test_loaders, epoch): |
| 37 | num_image_testset_all = {'DIS-VD': 470, 'DIS-TE1': 500, 'DIS-TE2': 500, 'DIS-TE3': 500, 'DIS-TE4': 500} |
| 38 | num_image_testset = {} |
| 39 | for testset in testsets.split('+'): |
| 40 | if 'DIS-TE' in testset: |
| 41 | num_image_testset[testset] = num_image_testset_all[testset] |
| 42 | weighted_scores = {'f_max': 0, 'sm': 0, 'e_max': 0, 'mae': 0} |
| 43 | len_all_data_loaders = 0 |
| 44 | model.epoch = epoch |
| 45 | for testset, data_loader_test in test_loaders.items(): |
| 46 | print('Validating {}...'.format(testset)) |
| 47 | performance_dict = valid( |
| 48 | model, |
| 49 | data_loader_test, |
| 50 | pred_dir='.', |
| 51 | method=ckpt_dir.split('/')[-1] if ckpt_dir.split('/')[-1].strip('.').strip('/') else 'tmp_val', |
| 52 | testset=testset, |
| 53 | only_S_MAE=config.only_S_MAE, |
| 54 | device=device |
| 55 | ) |
| 56 | print('Test set: {}:'.format(testset)) |
| 57 | if config.only_S_MAE: |
| 58 | print('Smeasure: {:.4f}, MAE: {:.4f}'.format( |
| 59 | performance_dict['sm'], performance_dict['mae'] |
| 60 | )) |
| 61 | else: |
| 62 | print('Fmax: {:.4f}, Fwfm: {:.4f}, Smeasure: {:.4f}, Emean: {:.4f}, MAE: {:.4f}'.format( |
| 63 | performance_dict['f_max'], performance_dict['f_wfm'], performance_dict['sm'], performance_dict['e_mean'], performance_dict['mae'] |
| 64 | )) |
| 65 | if '-TE' in testset: |
| 66 | for metric in ['sm', 'mae'] if config.only_S_MAE else ['f_max', 'f_wfm', 'sm', 'e_mean', 'mae']: |
| 67 | weighted_scores[metric] += performance_dict[metric] * len(data_loader_test) |
| 68 | len_all_data_loaders += len(data_loader_test) |
| 69 | print('Weighted Scores:') |
| 70 | for metric, score in weighted_scores.items(): |
| 71 | if score: |
| 72 | print('\t{}: {:.4f}.'.format(metric, score / len_all_data_loaders)) |
| 73 | |
| 74 | @torch.no_grad() |
| 75 | def main(): |