(tree: ProtoTree,
test_loader: DataLoader,
device,
log: Log = None,
progress_prefix: str = 'Fidelity'
)
| 71 | |
| 72 | @torch.no_grad() |
| 73 | def eval_fidelity(tree: ProtoTree, |
| 74 | test_loader: DataLoader, |
| 75 | device, |
| 76 | log: Log = None, |
| 77 | progress_prefix: str = 'Fidelity' |
| 78 | ) -> dict: |
| 79 | tree = tree.to(device) |
| 80 | |
| 81 | # Keep an info dict about the procedure |
| 82 | info = dict() |
| 83 | |
| 84 | # Make sure the model is in evaluation mode |
| 85 | tree.eval() |
| 86 | # Show progress on progress bar |
| 87 | test_iter = tqdm(enumerate(test_loader), |
| 88 | total=len(test_loader), |
| 89 | desc=progress_prefix, |
| 90 | ncols=0) |
| 91 | |
| 92 | distr_samplemax_fidelity = 0 |
| 93 | distr_greedy_fidelity = 0 |
| 94 | # Iterate through the test set |
| 95 | for i, (xs, ys) in test_iter: |
| 96 | xs, ys = xs.to(device), ys.to(device) |
| 97 | |
| 98 | # Use the model to classify this batch of input data, with 3 types of routing |
| 99 | out_distr, _ = tree.forward(xs, 'distributed') |
| 100 | ys_pred_distr = torch.argmax(out_distr, dim=1) |
| 101 | |
| 102 | out_samplemax, _ = tree.forward(xs, 'sample_max') |
| 103 | ys_pred_samplemax = torch.argmax(out_samplemax, dim=1) |
| 104 | |
| 105 | out_greedy, _ = tree.forward(xs, 'greedy') |
| 106 | ys_pred_greedy = torch.argmax(out_greedy, dim=1) |
| 107 | |
| 108 | # Calculate fidelity |
| 109 | distr_samplemax_fidelity += torch.sum(torch.eq(ys_pred_samplemax, ys_pred_distr)).item() |
| 110 | distr_greedy_fidelity += torch.sum(torch.eq(ys_pred_greedy, ys_pred_distr)).item() |
| 111 | # Update the progress bar |
| 112 | test_iter.set_postfix_str( |
| 113 | f'Batch [{i + 1}/{len(test_iter)}]' |
| 114 | ) |
| 115 | del out_distr |
| 116 | del out_samplemax |
| 117 | del out_greedy |
| 118 | |
| 119 | distr_samplemax_fidelity = distr_samplemax_fidelity/float(len(test_loader.dataset)) |
| 120 | distr_greedy_fidelity = distr_greedy_fidelity/float(len(test_loader.dataset)) |
| 121 | info['distr_samplemax_fidelity'] = distr_samplemax_fidelity |
| 122 | info['distr_greedy_fidelity'] = distr_greedy_fidelity |
| 123 | log.log_message("Fidelity between standard distributed routing and sample_max routing: "+str(distr_samplemax_fidelity)) |
| 124 | log.log_message("Fidelity between standard distributed routing and greedy routing: "+str(distr_greedy_fidelity)) |
| 125 | return info |
| 126 | |
| 127 | @torch.no_grad() |
| 128 | def eval_ensemble(trees: list, test_loader: DataLoader, device, log: Log, args: argparse.Namespace, sampling_strategy: str = 'distributed', progress_prefix: str = 'Eval Ensemble'): |
no test coverage detected