OpenICL Evaluation Task. This task is used to evaluate the metric between predictions and references.
| 23 | |
| 24 | @TASKS.register_module() |
| 25 | class OpenICLEvalTask(BaseTask): |
| 26 | """OpenICL Evaluation Task. |
| 27 | |
| 28 | This task is used to evaluate the metric between predictions and |
| 29 | references. |
| 30 | """ |
| 31 | |
| 32 | name_prefix = 'OpenICLEval' |
| 33 | log_subdir = 'logs/eval' |
| 34 | output_subdir = 'results' |
| 35 | |
| 36 | def __init__(self, cfg: ConfigDict): |
| 37 | super().__init__(cfg) |
| 38 | self.logger = get_logger() |
| 39 | self.num_gpus = max( |
| 40 | max( |
| 41 | c.get('eval_cfg', {}).get('num_gpus', 0), |
| 42 | c.get('eval_cfg', {}).get('evaluator', {}).get( |
| 43 | 'judge_cfg', {}).get('run_cfg', {}).get('num_gpus', 0), |
| 44 | ) for c in sum(self.dataset_cfgs, [])) |
| 45 | self.num_procs = max( |
| 46 | c.get('eval_cfg', {}).get('evaluator', {}).get( |
| 47 | 'judge_cfg', {}).get('run_cfg', {}).get('num_procs', 1) |
| 48 | for c in sum(self.dataset_cfgs, [])) |
| 49 | self.dump_details = (cfg.get('eval', {}).get('runner', {}).get( |
| 50 | 'task', {}).get('dump_details', False)) |
| 51 | self.cal_extract_rate = (cfg.get('eval', {}).get('runner', {}).get( |
| 52 | 'task', {}).get('cal_extract_rate', False)) |
| 53 | |
| 54 | def get_command(self, cfg_path, template): |
| 55 | sys.path.append(os.getcwd()) |
| 56 | script_path = __file__ |
| 57 | if self.num_gpus > 1: |
| 58 | port = random.randint(12000, 32000) |
| 59 | command = (f'torchrun --master_port={port} ' |
| 60 | f'--nproc_per_node {self.num_procs} ' |
| 61 | f'{script_path} {cfg_path}') |
| 62 | else: |
| 63 | python = sys.executable |
| 64 | command = f'{python} {script_path} {cfg_path}' |
| 65 | return template.format(task_cmd=command) |
| 66 | |
| 67 | def run(self): |
| 68 | for model_cfg, dataset_cfgs in zip(self.model_cfgs, self.dataset_cfgs): |
| 69 | for dataset_cfg in dataset_cfgs: |
| 70 | self.model_cfg = model_cfg |
| 71 | self.dataset_cfg = dataset_cfg |
| 72 | |
| 73 | # Load Dataset |
| 74 | self.eval_cfg = copy.deepcopy(dataset_cfg.get('eval_cfg')) |
| 75 | self.output_column = copy.deepcopy( |
| 76 | dataset_cfg['reader_cfg']['output_column']) |
| 77 | |
| 78 | out_path = get_infer_output_path( |
| 79 | self.model_cfg, |
| 80 | self.dataset_cfg, |
| 81 | osp.join(self.work_dir, 'results'), |
| 82 | ) |