Function main

eval_code/recons/videodepth/eval.py:19–153

(hydra_cfg: DictConfig)

@hydra.main(version_base="1.2", config_path="../configs", config_name="eval")
def main(hydra_cfg: DictConfig):
    # setup_debug(hydra_cfg.debug)
    logger = logging.getLogger("videodepth-eval")
    
    all_eval_models: ListConfig   = hydra_cfg.eval_models    # see configs/evaluation/videodepth.yaml
    all_eval_datasets: ListConfig = hydra_cfg.eval_datasets  # see configs/evaluation/videodepth.yaml
    all_data_info: DictConfig     = hydra_cfg.data           # see configs/data

    for idx_model, model_keyname in enumerate(all_eval_models, start=1):
        # 0. decide the model name (to save)
        model_logger = logging.getLogger(f"monodepth-eval-{model_keyname}")
        model_logger.info(f"[{idx_model}/{len(all_eval_models)}] Start evaluating {model_keyname} on {len(all_eval_datasets)} datasets...")

        for idx_dataset, dataset_name in enumerate(all_eval_datasets, start=1):
            # 1. look up dataset config from configs/data, decide the dataset name
            if dataset_name not in all_data_info:
                raise ValueError(f"Unknown dataset in global data information: {dataset_name}")
            dataset_info = all_data_info[dataset_name]

            # 2. get gt and pred depth pathes
            output_root = osp.join(hydra_cfg.output_dir, model_keyname, dataset_name)
            if dataset_info.type == "video":
                # most of the datasets have many sequences of video
                seq_list = get_all_sequences(dataset_info)
                gt_paths = {
                    seq: list_depths_a_sequence(dataset_info, seq)
                    for seq in seq_list
                }
                pred_paths = {
                    seq: sorted(glob.glob(f"{output_root}/{seq}/*.npy"))
                    for seq in seq_list
                }
                for seq in seq_list:
                    if len(pred_paths[seq]) != len(gt_paths[seq]):
                        raise ValueError(f"Number of prediction and ground truth depth maps are not equal: {len(pred_paths[seq])} vs {len(gt_paths[seq])}")
            else:
                raise ValueError(f"Unknown dataset type: {dataset_info.type}")
            
            # 3. get depth read function and evaluation kwargs
            videodepth_metadata = EVAL_DEPTH_METADATA.get(dataset_name, None)
            if videodepth_metadata is None:
                raise ValueError(f"Dataset {dataset_name} doesn't have video depth metadata")
            depth_read_func = videodepth_metadata["depth_read_func"]
            depth_evaluation_kwargs = videodepth_metadata["depth_evaluation_kwargs"]
            if hydra_cfg.align == "scale&shift":
                depth_evaluation_kwargs["align_with_lad2"] = True
            elif hydra_cfg.align == "scale":
                depth_evaluation_kwargs["align_with_scale"] = True
            elif hydra_cfg.align == "metric":
                depth_evaluation_kwargs["metric_scale"] = True
            else:
                raise ValueError(f"Unknown alignment method: {hydra_cfg.align}")

            gathered_depth_metrics = []
            all_fps = []
            total_time_per_seq = []
            infer_time_per_seq = []