Evaluates given checkpoint on a datamodule testset. This method is wrapped in optional @task_wrapper decorator, that controls the behavior during failure. Useful for multiruns, saving info about the crash, etc. :param cfg: DictConfig configuration composed by Hydra. :return: Tuple[
(cfg: DictConfig)
| 46 | |
| 47 | @task_wrapper |
| 48 | def evaluate(cfg: DictConfig) -> Tuple[Dict[str, Any], Dict[str, Any]]: |
| 49 | """Evaluates given checkpoint on a datamodule testset. |
| 50 | |
| 51 | This method is wrapped in optional @task_wrapper decorator, that controls the behavior during |
| 52 | failure. Useful for multiruns, saving info about the crash, etc. |
| 53 | |
| 54 | :param cfg: DictConfig configuration composed by Hydra. |
| 55 | :return: Tuple[dict, dict] with metrics and dict with all instantiated objects. |
| 56 | """ |
| 57 | assert cfg.ckpt_path, "Please provide a checkpoint path to evaluate!" |
| 58 | assert os.path.exists(cfg.ckpt_path), f"Checkpoint path {cfg.ckpt_path} does not exist!" |
| 59 | |
| 60 | # set seed for random number generators in pytorch, numpy and python.random |
| 61 | if cfg.get("seed"): |
| 62 | L.seed_everything(cfg.seed, workers=True) |
| 63 | |
| 64 | log.info( |
| 65 | f"Setting `float32_matmul_precision` to {cfg.model.cfg.task.float32_matmul_precision}." |
| 66 | ) |
| 67 | torch.set_float32_matmul_precision(precision=cfg.model.cfg.task.float32_matmul_precision) |
| 68 | |
| 69 | log.info(f"Instantiating datamodule <{cfg.data._target_}>") |
| 70 | datamodule: LightningDataModule = hydra.utils.instantiate(cfg.data, stage="test") |
| 71 | |
| 72 | # Establish model input arguments |
| 73 | with open_dict(cfg): |
| 74 | if cfg.model.cfg.task.start_time == "auto": |
| 75 | cfg.model.cfg.task.start_time = 1.0 |
| 76 | else: |
| 77 | cfg.model.cfg.task.start_time = float(cfg.model.cfg.task.start_time) |
| 78 | |
| 79 | log.info(f"Instantiating model <{cfg.model._target_}>") |
| 80 | model: LightningModule = hydra.utils.instantiate(cfg.model) |
| 81 | |
| 82 | log.info("Instantiating loggers...") |
| 83 | logger: List[Logger] = instantiate_loggers(cfg.get("logger")) |
| 84 | |
| 85 | plugins = None |
| 86 | if "_target_" in cfg.environment: |
| 87 | log.info(f"Instantiating environment <{cfg.environment._target_}>") |
| 88 | plugins: ClusterEnvironment = hydra.utils.instantiate(cfg.environment) |
| 89 | |
| 90 | strategy = getattr(cfg.trainer, "strategy", None) |
| 91 | if "_target_" in cfg.strategy: |
| 92 | log.info(f"Instantiating strategy <{cfg.strategy._target_}>") |
| 93 | strategy: Strategy = hydra.utils.instantiate(cfg.strategy) |
| 94 | if ( |
| 95 | "mixed_precision" in strategy.__dict__ |
| 96 | and getattr(strategy, "mixed_precision", None) is not None |
| 97 | ): |
| 98 | strategy.mixed_precision.param_dtype = ( |
| 99 | resolve_omegaconf_variable(cfg.strategy.mixed_precision.param_dtype) |
| 100 | if getattr(cfg.strategy.mixed_precision, "param_dtype", None) is not None |
| 101 | else None |
| 102 | ) |
| 103 | strategy.mixed_precision.reduce_dtype = ( |
| 104 | resolve_omegaconf_variable(cfg.strategy.mixed_precision.reduce_dtype) |
| 105 | if getattr(cfg.strategy.mixed_precision, "reduce_dtype", None) is not None |
no test coverage detected