Function diff_metrics

benchmark/reconstruction/evaluation/utils.py:1200–1243

Computes difference between two sets of metrics. Raises exception if the metrics are inconsistent.

(
    metrics_a: MetricsByDatasetByCatByScene,
    metrics_b: MetricsByDatasetByCatByScene,
)

def diff_metrics(
    metrics_a: MetricsByDatasetByCatByScene,
    metrics_b: MetricsByDatasetByCatByScene,
):
    """Computes difference between two sets of metrics.

    Raises exception if the metrics are inconsistent.
    """
    metrics_diff = copy.deepcopy(metrics_a)
    for dataset, category_metrics_a in metrics_a.items():
        if dataset not in metrics_b:
            raise ValueError(f"Dataset {dataset} not found in metrics_b")
        category_metrics_b = metrics_b[dataset]
        for category, scene_metrics_a in category_metrics_a.items():
            if category not in category_metrics_b:
                raise ValueError(f"Category {category} not found in metrics_b")
            scene_metrics_b = category_metrics_b[category]
            for scene, metrics_a_item in scene_metrics_a.items():
                if scene not in scene_metrics_b:
                    raise ValueError(f"Scene {scene} not found in metrics_b")
                metrics_b_item = scene_metrics_b[scene]
                if (
                    metrics_a_item.error_type != metrics_b_item.error_type
                    or not np.all(
                        metrics_a_item.error_thresholds
                        == metrics_b_item.error_thresholds
                    )
                ):
                    raise ValueError("Inconsistent error thresholds or types")
                metrics_diff[dataset][category][scene] = Metrics(
                    aucs=metrics_a_item.aucs - metrics_b_item.aucs,
                    recalls=metrics_a_item.recalls - metrics_b_item.recalls,
                    error_thresholds=metrics_a_item.error_thresholds,
                    error_type=metrics_a_item.error_type,
                    num_images=metrics_a_item.num_images
                    - metrics_b_item.num_images,
                    num_reg_images=metrics_a_item.num_reg_images
                    - metrics_b_item.num_reg_images,
                    num_components=metrics_a_item.num_components
                    - metrics_b_item.num_components,
                    largest_component=metrics_a_item.largest_component
                    - metrics_b_item.largest_component,
                )
    return metrics_diff


def create_result_table(