Args: results (OrderedDict[dict]): task_name -> {metric -> score} Returns: bool: whether the verification succeeds or not
(cfg, results)
| 26 | |
| 27 | |
| 28 | def verify_results(cfg, results): |
| 29 | """ |
| 30 | Args: |
| 31 | results (OrderedDict[dict]): task_name -> {metric -> score} |
| 32 | |
| 33 | Returns: |
| 34 | bool: whether the verification succeeds or not |
| 35 | """ |
| 36 | expected_results = cfg.TEST.EXPECTED_RESULTS |
| 37 | if not len(expected_results): |
| 38 | return True |
| 39 | |
| 40 | ok = True |
| 41 | for task, metric, expected, tolerance in expected_results: |
| 42 | actual = results[task].get(metric, None) |
| 43 | if actual is None: |
| 44 | ok = False |
| 45 | continue |
| 46 | if not np.isfinite(actual): |
| 47 | ok = False |
| 48 | continue |
| 49 | diff = abs(actual - expected) |
| 50 | if diff > tolerance: |
| 51 | ok = False |
| 52 | |
| 53 | logger = logging.getLogger(__name__) |
| 54 | if not ok: |
| 55 | logger.error("Result verification failed!") |
| 56 | logger.error("Expected Results: " + str(expected_results)) |
| 57 | logger.error("Actual Results: " + pprint.pformat(results)) |
| 58 | |
| 59 | sys.exit(1) |
| 60 | else: |
| 61 | logger.info("Results verification passed.") |
| 62 | return ok |
| 63 | |
| 64 | |
| 65 | def flatten_results_dict(results): |