Function main

eval_code/recons/monodepth/eval.py:20–130

(hydra_cfg: DictConfig)

@hydra.main(version_base="1.2", config_path="../configs", config_name="eval")
def main(hydra_cfg: DictConfig):
    # setup_debug(hydra_cfg.debug)

    logger = logging.getLogger("monodepth-eval")

    if hydra_cfg.invariant == "median-scale":
        align_with_scale = True
    elif hydra_cfg.invariant == "scale":
        align_with_scale = False
    else:
        raise NotImplementedError(f"Unknown invariant {hydra_cfg.invariant}")
    logger.info(f"Evaluating with invariant depthmap type: {hydra_cfg.invariant}")

    all_eval_models: ListConfig   = hydra_cfg.eval_models    # see configs/evaluation/monodepth.yaml
    all_eval_datasets: ListConfig = hydra_cfg.eval_datasets  # see configs/evaluation/monodepth.yaml
    all_data_info: DictConfig     = hydra_cfg.data           # see configs/data

    for idx_model, model_keyname in enumerate(all_eval_models, start=1):
        # 0. decide the model name (to save)
        model_logger = logging.getLogger(f"monodepth-eval-{model_keyname}")
        model_logger.info(f"[{idx_model}/{len(all_eval_models)}] Start evaluating {model_keyname} on {len(all_eval_datasets)} datasets...")
        
        for idx_dataset, dataset_name in enumerate(all_eval_datasets, start=1):
            # 1. look up dataset config from configs/data, decide the dataset name
            if dataset_name not in all_data_info:
                raise ValueError(f"Unknown dataset: {dataset_name}")
            dataset_info = all_data_info[dataset_name]

            # 2. get gt and pred depth pathes
            output_root = osp.join(hydra_cfg.output_dir, model_keyname, dataset_name)
            if dataset_info.type == "video":
                # most of the datasets have many sequences of video
                seq_list = get_all_sequences(dataset_info)
                gt_paths = {
                    seq: list_depths_a_sequence(dataset_info, seq)
                    for seq in seq_list
                }
                pred_paths = {
                    seq: sorted(glob.glob(f"{output_root}/{seq}/*.npy"))
                    for seq in seq_list
                }
            elif dataset_info.type == "mono":
                seq_list = [dataset_name]
                # some datasets (like nyu-v2) have only a set of images, only for monodepth
                gt_paths = {dataset_name: list_depths_a_sequence(dataset_info, seq=None)}
                pred_paths = {dataset_name: sorted(glob.glob(f"{output_root}/*.npy"))}
            else:
                raise ValueError(f"Unknown dataset type: {dataset_info.type}")
            
            # 3. get depth read function and evaluation kwargs
            mono_metadata = EVAL_DEPTH_METADATA.get(dataset_name, None)
            if mono_metadata is None:
                raise ValueError(f"Dataset {dataset_name} doesn't have monodepth metadata")
            depth_read_func = mono_metadata["depth_read_func"]
            depth_evaluation_kwargs = mono_metadata["depth_evaluation_kwargs"]
            depth_evaluation_kwargs["align_with_scale"] = align_with_scale

            model_logger.info(f"[{idx_dataset}/{len(all_eval_datasets)}] Start evaluating dataset: {dataset_name}, {len(seq_list)} sequences in total")