(trees: list, test_loader: DataLoader, device, log: Log, args: argparse.Namespace, sampling_strategy: str = 'distributed', progress_prefix: str = 'Eval Ensemble')
| 126 | |
| 127 | @torch.no_grad() |
| 128 | def eval_ensemble(trees: list, test_loader: DataLoader, device, log: Log, args: argparse.Namespace, sampling_strategy: str = 'distributed', progress_prefix: str = 'Eval Ensemble'): |
| 129 | # Keep an info dict about the procedure |
| 130 | info = dict() |
| 131 | # Build a confusion matrix |
| 132 | cm = np.zeros((trees[0]._num_classes, trees[0]._num_classes), dtype=int) |
| 133 | |
| 134 | # Show progress on progress bar |
| 135 | test_iter = tqdm(enumerate(test_loader), |
| 136 | total=len(test_loader), |
| 137 | desc=progress_prefix, |
| 138 | ncols=0) |
| 139 | |
| 140 | # Iterate through the test set |
| 141 | for i, (xs, ys) in test_iter: |
| 142 | xs, ys = xs.to(device), ys.to(device) |
| 143 | outs = [] |
| 144 | for tree in trees: |
| 145 | # Make sure the model is in evaluation mode |
| 146 | tree.eval() |
| 147 | tree = tree.to(device) |
| 148 | # Use the model to classify this batch of input data |
| 149 | out, _ = tree.forward(xs, sampling_strategy) |
| 150 | outs.append(out) |
| 151 | del out |
| 152 | stacked = torch.stack(outs, dim=0) |
| 153 | ys_pred = torch.argmax(torch.mean(stacked, dim=0), dim=1) |
| 154 | |
| 155 | for y_pred, y_true in zip(ys_pred, ys): |
| 156 | cm[y_true][y_pred] += 1 |
| 157 | |
| 158 | test_iter.set_postfix_str( |
| 159 | f'Batch [{i + 1}/{len(test_iter)}]' |
| 160 | ) |
| 161 | del outs |
| 162 | |
| 163 | info['confusion_matrix'] = cm |
| 164 | info['test_accuracy'] = acc_from_cm(cm) |
| 165 | log.log_message("Ensemble accuracy with %s routing: %s"%(sampling_strategy, str(info['test_accuracy']))) |
| 166 | return info |
| 167 | |
| 168 | def acc_from_cm(cm: np.ndarray) -> float: |
| 169 | """ |
no test coverage detected