| 82 | |
| 83 | |
| 84 | class Evaluator: |
| 85 | def __init__(self, config, global_config, prompt): |
| 86 | self.config = config |
| 87 | self.global_config = global_config |
| 88 | self.prompt = prompt |
| 89 | |
| 90 | def evaluate(self, outputs, ds, args): |
| 91 | base_result_dir = f'results/{args.pretrained_path[0].split("ckpts")[-1].replace("/", "_")}' |
| 92 | os.makedirs(base_result_dir, exist_ok=True) |
| 93 | os.makedirs('vqa_logs', exist_ok=True) |
| 94 | time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime()) |
| 95 | results_file = f'{base_result_dir}/{ds}_{time_prefix}_{args.seed}.json' |
| 96 | |
| 97 | if self.config[ds]['metric'] == 'vqa_score': |
| 98 | vqa = VQA(self.config[ds]['annotation'], |
| 99 | self.config[ds]['question']) |
| 100 | |
| 101 | json.dump(outputs, open(results_file, 'w'), |
| 102 | ensure_ascii=False) |
| 103 | results = vqa.loadRes( |
| 104 | resFile=results_file, |
| 105 | quesFile=self.config[ds]['question']) |
| 106 | vqa_scorer = VQAEval(vqa, results, n=2) |
| 107 | vqa_scorer.evaluate() |
| 108 | |
| 109 | print(vqa_scorer.accuracy) |
| 110 | save_result(args, vqa_scorer.accuracy, self.prompt, self.global_config, self.config, results_file, ds) |
| 111 | elif self.config[ds]['metric'] == 'mme_score': |
| 112 | base_mme_dir = f'{base_result_dir}/MME_results' |
| 113 | os.makedirs(base_mme_dir, exist_ok=True) |
| 114 | # MME evaluation |
| 115 | eval_type_dict = { |
| 116 | "Perception": ["existence", "count", "position", "color", "posters", "celebrity", "scene", |
| 117 | "landmark", "artwork", "OCR"], |
| 118 | "Cognition": ["commonsense_reasoning", "numerical_calculation", "text_translation", |
| 119 | "code_reasoning"] |
| 120 | } |
| 121 | pred_by_category = {} |
| 122 | for pred in outputs: |
| 123 | cate = None |
| 124 | for c in eval_type_dict['Perception'] + eval_type_dict['Cognition']: |
| 125 | if c in pred['image_path']: |
| 126 | cate = c |
| 127 | if cate is None: |
| 128 | raise ValueError |
| 129 | |
| 130 | if cate not in pred_by_category: |
| 131 | pred_by_category[cate] = [pred] |
| 132 | else: |
| 133 | pred_by_category[cate].append(pred) |
| 134 | |
| 135 | for k, v in pred_by_category.items(): |
| 136 | v.sort(key=lambda x: x['question_id']) |
| 137 | |
| 138 | out_datas = [ |
| 139 | f"{data['image_path']}\t{data['question']}\t{data['gt_answers']}\t{data['answer']}" |
| 140 | for data in v |
| 141 | ] |