Subjective Evaluation Task. This task is used to evaluate the metric between predictions and references. Args: cfg (ConfigDict): The configuration of the entire evaluation task.
| 20 | |
| 21 | |
| 22 | class SubjectiveEvalTask(BaseTask): |
| 23 | """Subjective Evaluation Task. |
| 24 | |
| 25 | This task is used to evaluate the metric between predictions and |
| 26 | references. |
| 27 | |
| 28 | Args: |
| 29 | cfg (ConfigDict): The configuration of the entire evaluation task. |
| 30 | """ |
| 31 | |
| 32 | name_prefix = 'SubjectiveEval' |
| 33 | log_subdir = 'logs/eval' |
| 34 | output_subdir = 'results' |
| 35 | |
| 36 | def __init__(self, cfg: ConfigDict): |
| 37 | super().__init__(cfg) |
| 38 | self.logger = get_logger() |
| 39 | judge_cfg = cfg.get('judge_model', None) |
| 40 | meta_judge_cfg = cfg.get('meta_judge_model', None) |
| 41 | judge_models = cfg.get('judge_models', None) |
| 42 | |
| 43 | if judge_cfg is None and meta_judge_cfg is None: |
| 44 | assert judge_cfg is not None, 'Both judge_cfg and meta_judge_cfg are None, but judge_models must be provided.' |
| 45 | |
| 46 | if meta_judge_cfg is not None: |
| 47 | assert judge_models is not None, 'meta_judge_cfg is provided, but judge_models are missing.' |
| 48 | judge_cfg = meta_judge_cfg # Relpace judge_cfg to meta_judge_cfg when it is not None |
| 49 | self.meta = True |
| 50 | else: |
| 51 | self.meta = False |
| 52 | run_cfg = judge_cfg.get('run_cfg', {}) |
| 53 | self.num_gpus = run_cfg.get('num_gpus', 0) |
| 54 | self.num_procs = run_cfg.get('num_procs', 1) |
| 55 | self.judge_cfg = copy.deepcopy(judge_cfg) |
| 56 | self.judge_models = judge_models |
| 57 | self.infer_order = cfg.get('infer_order') |
| 58 | self.given_pred = cfg['datasets'][0][0].get('given_pred', []) |
| 59 | |
| 60 | def get_command(self, cfg_path, template): |
| 61 | """Get the command template for the task. |
| 62 | |
| 63 | Args: |
| 64 | cfg_path (str): The path to the config file of the task. |
| 65 | template (str): The template which have '{task_cmd}' to format |
| 66 | the command. |
| 67 | """ |
| 68 | script_path = __file__ |
| 69 | if self.num_gpus > 0: |
| 70 | port = random.randint(12000, 32000) |
| 71 | command = (f'torchrun --master_port={port} ' |
| 72 | f'--nproc_per_node {self.num_procs} ' |
| 73 | f'{script_path} {cfg_path}') |
| 74 | else: |
| 75 | command = f'python {script_path} {cfg_path}' |
| 76 | |
| 77 | return template.format(task_cmd=command) |
| 78 | |
| 79 | def run(self): |