( self )
| 123 | self.table = self.load() |
| 124 | |
| 125 | def load( self ): # noqa |
| 126 | model_cfgs = self.cfg['models'] |
| 127 | dataset_cfgs = self.cfg['datasets'] |
| 128 | work_dir = self.cfg['work_dir'] |
| 129 | |
| 130 | # pick up results |
| 131 | raw_results = {} |
| 132 | parsed_results = {} |
| 133 | dataset_metrics = {} |
| 134 | |
| 135 | model_abbrs = [model_abbr_from_cfg(model) for model in model_cfgs] |
| 136 | for model in model_cfgs: |
| 137 | model_abbr = model_abbr_from_cfg(model) |
| 138 | parsed_results[model_abbr] = {} |
| 139 | raw_results[model_abbr] = {} |
| 140 | for dataset in dataset_cfgs: |
| 141 | dataset_abbr = dataset_abbr_from_cfg(dataset) |
| 142 | filepath = get_infer_output_path(model, dataset, osp.join(work_dir, 'results')) |
| 143 | if not osp.exists(filepath): |
| 144 | continue |
| 145 | result = mmengine.load(filepath) |
| 146 | raw_results[model_abbr][dataset_abbr] = result |
| 147 | if 'error' in result: |
| 148 | self.logger.debug(f'error in {model_abbr} {dataset_abbr} {result["error"]}') |
| 149 | continue |
| 150 | else: |
| 151 | parsed_results[model_abbr][dataset_abbr] = [] |
| 152 | dataset_metrics[dataset_abbr] = [] |
| 153 | for metric, score in result.items(): |
| 154 | if metric not in METRIC_BLACKLIST and isinstance(score, (int, float)): |
| 155 | parsed_results[model_abbr][dataset_abbr].append(score) |
| 156 | dataset_metrics[dataset_abbr].append(metric) |
| 157 | else: |
| 158 | continue |
| 159 | if len(parsed_results[model_abbr][dataset_abbr]) == 0: |
| 160 | self.logger.warning(f'unknown result format: {result}, continue') |
| 161 | del parsed_results[model_abbr][dataset_abbr] |
| 162 | del dataset_metrics[dataset_abbr] |
| 163 | continue |
| 164 | indice = sorted( |
| 165 | list(range(len(dataset_metrics[dataset_abbr]))), |
| 166 | key=lambda i: ( |
| 167 | METRIC_WHITELIST.index(dataset_metrics[dataset_abbr][i]) |
| 168 | if dataset_metrics[dataset_abbr][i] in METRIC_WHITELIST |
| 169 | else len(METRIC_WHITELIST) |
| 170 | ) |
| 171 | ) |
| 172 | parsed_results[model_abbr][dataset_abbr] = [parsed_results[model_abbr][dataset_abbr][i] for i in indice] |
| 173 | dataset_metrics[dataset_abbr] = [dataset_metrics[dataset_abbr][i] for i in indice] |
| 174 | |
| 175 | # parse eval mode |
| 176 | dataset_eval_mode = {} |
| 177 | for dataset in dataset_cfgs: |
| 178 | inferencer = dataset.get('infer_cfg', {}).get('inferencer', {}).get('type', '') |
| 179 | inferencer = inferencer if isinstance(inferencer, str) else inferencer.__name__ |
| 180 | dataset_abbr = dataset_abbr_from_cfg(dataset) |
| 181 | if 'GenInferencer' in inferencer: |
| 182 | dataset_eval_mode[dataset_abbr] = 'gen' |
no test coverage detected