()
| 137 | } |
| 138 | |
| 139 | def main(): |
| 140 | parser = argparse.ArgumentParser(description='Calculate Mind2Web Scores') |
| 141 | parser.add_argument('--standard_files', nargs='+', required=False, help='Path to standard answer files (jsonl)', default="./resources/agents/mind2web_topk20_100_eval.jsonl") |
| 142 | parser.add_argument('--model_files', nargs='+', required=False, help='Path to model output files (jsonl)', default="./examples/results/eval_wrong_cases.jsonl") |
| 143 | |
| 144 | args = parser.parse_args() |
| 145 | |
| 146 | # Load Standard Data |
| 147 | print("Loading standard files...") |
| 148 | standard_items = load_data(args.standard_files) |
| 149 | |
| 150 | if not standard_items: |
| 151 | print("No standard items loaded.") |
| 152 | return |
| 153 | |
| 154 | # Map question -> standard_item and group by task |
| 155 | std_by_task = defaultdict(list) |
| 156 | |
| 157 | for item in standard_items: |
| 158 | task_id = item.get('task_id') |
| 159 | if task_id: |
| 160 | std_by_task[task_id].append(item) |
| 161 | |
| 162 | print(f"Loaded {len(standard_items)} standard items.") |
| 163 | print(f"Found {len(std_by_task)} unique tasks.") |
| 164 | |
| 165 | # Process each model file separately |
| 166 | if not args.model_files: |
| 167 | print("No model files provided.") |
| 168 | return |
| 169 | |
| 170 | # Handle if args.model_files is a string (should be list due to nargs='+') |
| 171 | model_files_list = args.model_files if isinstance(args.model_files, list) else [args.model_files] |
| 172 | |
| 173 | for model_file in model_files_list: |
| 174 | print(f"\nEvaluating file: {model_file}") |
| 175 | model_items = load_data([model_file]) |
| 176 | |
| 177 | if not model_items: |
| 178 | print(f"No items loaded from {model_file}") |
| 179 | continue |
| 180 | |
| 181 | print(f"Loaded {len(model_items)} items from {model_file}") |
| 182 | |
| 183 | results = evaluate_model(model_items, std_by_task) |
| 184 | |
| 185 | print("-" * 30) |
| 186 | print(f"File: {model_file}") |
| 187 | print(f"Select Success (Element Match): {results['select_success_rate']:.2f}% ({results['select_correct_count']}/{results['select_total_count']})") |
| 188 | print(f"Task Success (All Steps Correct): {results['task_success_rate']:.2f}% ({results['task_success_count']}/{results['total_tasks']})") |
| 189 | if results['step_acc_total_count'] > 0: |
| 190 | print(f"Step Acc (Direct from Model Identifier): {results['step_acc_rate']:.2f}% ({results['step_acc_correct_count']}/{results['step_acc_total_count']})") |
| 191 | else: |
| 192 | print("Step Acc (Direct from Model Identifier): N/A (No identifier found in model output)") |
| 193 | print("-" * 30) |
| 194 | |
| 195 | if __name__ == "__main__": |
| 196 | main() |
no test coverage detected