Evaluate this benchmark with all registered methods.
(self, name: str, on_error="raise")
| 74 | ) |
| 75 | |
| 76 | def evaluate(self, name: str, on_error="raise"): |
| 77 | """Evaluate this benchmark with all registered methods.""" |
| 78 | |
| 79 | if name not in self.names(): |
| 80 | raise ValueError(f"{name} is not registered. Valid names are {self.names()}") |
| 81 | if on_error not in ("ignore", "return", "raise"): |
| 82 | raise ValueError(f"on_error got an undefined value: {on_error}") |
| 83 | mean_avg_precision = float("nan") |
| 84 | root_mean_squared_error = float("nan") |
| 85 | try: |
| 86 | predictions = self.get_predictions(name) |
| 87 | predictions = self._validate_predictions(name, predictions) |
| 88 | mean_avg_precision = self.compute_pose_map(predictions) |
| 89 | root_mean_squared_error = self.compute_pose_rmse(predictions) |
| 90 | except Exception as exception: |
| 91 | if on_error == "ignore": |
| 92 | # ignore the exception and continue with the next evaluation, without |
| 93 | # yielding a result value. |
| 94 | return |
| 95 | elif on_error == "return": |
| 96 | # return the result value, with NaN as the result for all metrics that |
| 97 | # could not be computed due to the error. |
| 98 | pass |
| 99 | elif on_error == "raise": |
| 100 | # raise the error and stop evaluation |
| 101 | raise BenchmarkEvaluationError(f"Error during benchmark evaluation for model {name}") from exception |
| 102 | else: |
| 103 | raise NotImplementedError() from exception |
| 104 | return Result( |
| 105 | code=self.code, |
| 106 | method_name=name, |
| 107 | benchmark_name=self.name, |
| 108 | mean_avg_precision=mean_avg_precision, |
| 109 | root_mean_squared_error=root_mean_squared_error, |
| 110 | ) |
| 111 | |
| 112 | def _validate_predictions(self, name: str, predictions: dict) -> dict: |
| 113 | """Validates the submitted predictions object Checks that there is a prediction |