(self,
model_cfg,
dataset_cfg,
eval_cfg,
output_column,
meta=False)
| 352 | } |
| 353 | |
| 354 | def _score(self, |
| 355 | model_cfg, |
| 356 | dataset_cfg, |
| 357 | eval_cfg, |
| 358 | output_column, |
| 359 | meta=False): |
| 360 | test_set = build_dataset_from_cfg(dataset_cfg).test |
| 361 | # Postprocess dataset if necessary |
| 362 | if 'dataset_postprocessor' in eval_cfg: |
| 363 | proc = TEXT_POSTPROCESSORS.get( |
| 364 | eval_cfg['dataset_postprocessor']['type']) |
| 365 | |
| 366 | def postprocess(sample): |
| 367 | s = sample[output_column] |
| 368 | sample[output_column] = proc(s) |
| 369 | return sample |
| 370 | |
| 371 | test_set = test_set.map(postprocess) |
| 372 | # Get out_path |
| 373 | out_path = get_infer_output_path( |
| 374 | deal_with_judge_model_abbr(model_cfg, self.judge_cfg, self.meta), |
| 375 | dataset_cfg, osp.join(self.work_dir, 'results')) |
| 376 | if meta: |
| 377 | model_preds = self._load_model_pred(model_cfg, dataset_cfg, |
| 378 | eval_cfg, self.given_pred) |
| 379 | model_judges = self._load_model_judgements(model_cfg, dataset_cfg, |
| 380 | eval_cfg, |
| 381 | self.judge_models) |
| 382 | else: |
| 383 | model_preds = self._load_model_pred(model_cfg, dataset_cfg, |
| 384 | eval_cfg, self.given_pred) |
| 385 | model_judges = None |
| 386 | if not self.judge_cfg: |
| 387 | raise ValueError('missing "eval.judge_cfg"') |
| 388 | eval_cfg['evaluator']['judge_cfg'] = self.judge_cfg |
| 389 | eval_cfg['evaluator']['dataset_cfg'] = dataset_cfg |
| 390 | eval_cfg['evaluator']['output_path'] = out_path |
| 391 | icl_evaluator = ICL_EVALUATORS.build(eval_cfg['evaluator']) |
| 392 | references = (test_set[output_column] if output_column else None) |
| 393 | if 'error' not in model_preds: |
| 394 | result = icl_evaluator.score(predictions=model_preds, |
| 395 | judgements=model_judges, |
| 396 | references=references, |
| 397 | meta=meta, |
| 398 | infer_order=self.infer_order) |
| 399 | else: |
| 400 | result = model_preds |
| 401 | |
| 402 | if 'error' in result: |
| 403 | self.logger.error( |
| 404 | f'Task {task_abbr_from_cfg(self.cfg)}: {result["error"]}') |
| 405 | return |
| 406 | else: |
| 407 | self.logger.info( |
| 408 | f'Task {task_abbr_from_cfg(self.cfg)}') #: {result}') |
| 409 | |
| 410 | # Save result |
| 411 | mkdir_or_exist(osp.split(out_path)[0]) |
no test coverage detected