(self)
| 77 | return template.format(task_cmd=command) |
| 78 | |
| 79 | def run(self): |
| 80 | # model_cfg can be a list of model configs |
| 81 | for model_cfg, dataset_cfgs in zip(self.model_cfgs, self.dataset_cfgs): |
| 82 | for dataset_cfg in dataset_cfgs: |
| 83 | # Load Dataset |
| 84 | eval_cfg = dataset_cfg.get('eval_cfg') |
| 85 | output_column = dataset_cfg['reader_cfg']['output_column'] |
| 86 | out_path = get_infer_output_path( |
| 87 | deal_with_judge_model_abbr(model_cfg, self.judge_cfg, |
| 88 | self.meta), dataset_cfg, |
| 89 | osp.join(self.work_dir, 'results')) |
| 90 | if osp.exists(out_path): |
| 91 | continue |
| 92 | |
| 93 | self._score(model_cfg, dataset_cfg, eval_cfg, output_column, |
| 94 | self.meta) |
| 95 | |
| 96 | @property |
| 97 | def name(self) -> str: |
no test coverage detected