()
| 154 | |
| 155 | |
| 156 | def main(): |
| 157 | logger = get_logger('RUN') |
| 158 | rank, world_size = get_rank_and_world_size() |
| 159 | args = parse_args() |
| 160 | use_config, cfg = False, None |
| 161 | if args.config is not None: |
| 162 | assert args.data is None and args.model is None, '--data and --model should not be set when using --config' |
| 163 | use_config, cfg = True, load(args.config) |
| 164 | args.model = list(cfg['model'].keys()) |
| 165 | args.data = list(cfg['data'].keys()) |
| 166 | else: |
| 167 | assert len(args.data), '--data should be a list of data files' |
| 168 | |
| 169 | if rank == 0: |
| 170 | if not args.reuse: |
| 171 | logger.warning('--reuse is not set, will not reuse previous (before one day) temporary files') |
| 172 | else: |
| 173 | logger.warning('--reuse is set, will reuse the latest prediction & temporary pickle files') |
| 174 | |
| 175 | if 'MMEVAL_ROOT' in os.environ: |
| 176 | args.work_dir = os.environ['MMEVAL_ROOT'] |
| 177 | |
| 178 | if not use_config: |
| 179 | for k, v in supported_VLM.items(): |
| 180 | if hasattr(v, 'keywords') and 'retry' in v.keywords and args.retry is not None: |
| 181 | v.keywords['retry'] = args.retry |
| 182 | supported_VLM[k] = v |
| 183 | if hasattr(v, 'keywords') and 'verbose' in v.keywords and args.verbose is not None: |
| 184 | v.keywords['verbose'] = args.verbose |
| 185 | supported_VLM[k] = v |
| 186 | |
| 187 | if world_size > 1: |
| 188 | local_rank = os.environ.get('LOCAL_RANK', 0) |
| 189 | torch.cuda.set_device(int(local_rank)) |
| 190 | dist.init_process_group( |
| 191 | backend='nccl', |
| 192 | timeout=datetime.timedelta(seconds=int(os.environ.get('DIST_TIMEOUT', 3600))) |
| 193 | ) |
| 194 | |
| 195 | for _, model_name in enumerate(args.model): |
| 196 | model = None |
| 197 | date, commit_id = timestr('day'), githash(digits=8) |
| 198 | eval_id = f"T{date}_G{commit_id}" |
| 199 | |
| 200 | pred_root = osp.join(args.work_dir, model_name, eval_id) |
| 201 | pred_root_meta = osp.join(args.work_dir, model_name) |
| 202 | os.makedirs(pred_root_meta, exist_ok=True) |
| 203 | |
| 204 | prev_pred_roots = ls(osp.join(args.work_dir, model_name), mode='dir') |
| 205 | if len(prev_pred_roots) and args.reuse: |
| 206 | prev_pred_roots.sort() |
| 207 | |
| 208 | if not osp.exists(pred_root): |
| 209 | os.makedirs(pred_root, exist_ok=True) |
| 210 | |
| 211 | if use_config: |
| 212 | model = build_model_from_config(cfg['model'], model_name) |
| 213 |
no test coverage detected