Like np.testing.assert_allclose, but allows outlier percentiles to be specified. `tolerance_map` is mapping of percentile values (between 0 and 1) to `Tolerance` objects. Each entry defines the acceptable tolerance for a certain percentile of elements in the difference `abs(
(self, actual, desired, *, tolerance_map: dict[float, Tolerance])
| 275 | self.assertEqual(a_value.dtype, b_value.dtype) |
| 276 | |
| 277 | def assertAllCloseWithOutliers(self, actual, desired, *, tolerance_map: dict[float, Tolerance]): |
| 278 | """Like np.testing.assert_allclose, but allows outlier percentiles to be specified. |
| 279 | |
| 280 | `tolerance_map` is mapping of percentile values (between 0 and 1) to `Tolerance` objects. |
| 281 | Each entry defines the acceptable tolerance for a certain percentile of elements in the |
| 282 | difference `abs(actual - desired)`. The specified tolerance should be met within the given |
| 283 | percentile of total elements in `actual` or `desired`. |
| 284 | |
| 285 | Example: |
| 286 | ```python |
| 287 | self.assertAllCloseWithOutliers(x, y, tolerance_map={ |
| 288 | 1.0: Tolerance(atol=0.2), |
| 289 | 0.95: Tolerance(atol=0.05), |
| 290 | }) |
| 291 | ``` |
| 292 | This example asserts 100% elements of `abs(x - y)` should be within atol=0.2, and 95% |
| 293 | elements of `abs(x - y)` should be within atol=0.05. |
| 294 | """ |
| 295 | assert len(tolerance_map) > 0 |
| 296 | self.assertEqual(actual.shape, desired.shape) |
| 297 | self.assertEqual(actual.dtype, desired.dtype) |
| 298 | actual = actual.astype(np.float32) |
| 299 | desired = desired.astype(np.float32) |
| 300 | diff = np.abs(actual - desired) |
| 301 | for percentile, tol in tolerance_map.items(): |
| 302 | percentile = 1 - percentile |
| 303 | tolerance = tol.atol + tol.rtol * np.abs(desired) |
| 304 | expected_num_ele = round(diff.size * percentile) |
| 305 | actual_num_ele = np.count_nonzero(diff > tolerance) |
| 306 | actual_percent = actual_num_ele / diff.size |
| 307 | self.assertLessEqual( |
| 308 | actual_num_ele, |
| 309 | expected_num_ele, |
| 310 | msg=f"Expected the number of elements over {tol} to be less than {percentile:.3%}" |
| 311 | f" of total elements (or {expected_num_ele}), but got {actual_percent:.3%} " |
| 312 | f"(or {actual_num_ele}). These differences are {diff[diff > tolerance]}. " |
| 313 | f"Max difference = {diff.max()}", |
| 314 | ) |
| 315 | |
| 316 | def assertNestedAllCloseWithOutliers( |
| 317 | self, actual: NestedTensor, desired: NestedTensor, *, tolerance_map: dict[float, Tolerance] |
no test coverage detected