Function load_metrics

paper/common.py:170–238

(
    scenes: Iterable[Scene],
    methods: Iterable[Method],
    metrics: Iterable[Metric],
)

def load_metrics(
    scenes: Iterable[Scene],
    methods: Iterable[Method],
    metrics: Iterable[Metric],
) -> pd.DataFrame:
    metrics = tuple(metrics)

    data = defaultdict(list)

    for method in methods:
        for scene in scenes:
            # Load the scene's metrics.
            try:
                with (
                    METRICS_PATH
                    / f"{METRICS_PREFIX}{method.tag}_{scene.tag}/metrics.json"
                ).open("r") as f:
                    scene_metrics = json.load(f)
            except FileNotFoundError:
                scene_metrics = {}

            # Load the runtime separately.
            try:
                # NoPE-NeRF runtimes are different since it combines view synthesis with
                # camera pose estimation.
                if method == METHOD_NOPENERF:
                    with (
                        METRICS_PATH
                        / f"{METRICS_PREFIX}{method.tag}_{scene.tag}/runtime.json"
                    ).open("r") as f:
                        scene_metrics["runtime"] = json.load(f)["runtime"]
                else:
                    with (RESULTS_PATH / method.tag / scene.tag / "runtime.json").open(
                        "r"
                    ) as f:
                        scene_metrics["runtime"] = json.load(f)["runtime"]

                # Convert seconds to minutes.
                scene_metrics["runtime"] = scene_metrics["runtime"] / 60
            except FileNotFoundError:
                pass

            if METRIC_COLMAP_ATE in metrics:
                if method in (METHOD_COLMAP, METHOD_MVSCOLMAP):
                    scene_metrics["ate"] = None
                else:
                    try:
                        gt = load_trajectory(METHOD_COLMAP, scene)
                        hat = load_trajectory(method, scene)
                        scene_metrics["ate"] = compute_ate(gt, hat)[0]
                    except FileNotFoundError:
                        scene_metrics["ate"] = None

            # Select the metrics we care about.
            for metric in metrics:
                data["scene_tag"].append(scene.tag)
                data["scene_full_name"].append(scene.full_name)