MCPcopy Create free account
hub / github.com/Bairong-Xdynamics/MistakeNotebookLearning / main

Function main

resources/agents/caculate_scores.py:139–193  ·  view source on GitHub ↗
()

Source from the content-addressed store, hash-verified

137 }
138
139def main():
140 parser = argparse.ArgumentParser(description='Calculate Mind2Web Scores')
141 parser.add_argument('--standard_files', nargs='+', required=False, help='Path to standard answer files (jsonl)', default="./resources/agents/mind2web_topk20_100_eval.jsonl")
142 parser.add_argument('--model_files', nargs='+', required=False, help='Path to model output files (jsonl)', default="./examples/results/eval_wrong_cases.jsonl")
143
144 args = parser.parse_args()
145
146 # Load Standard Data
147 print("Loading standard files...")
148 standard_items = load_data(args.standard_files)
149
150 if not standard_items:
151 print("No standard items loaded.")
152 return
153
154 # Map question -> standard_item and group by task
155 std_by_task = defaultdict(list)
156
157 for item in standard_items:
158 task_id = item.get('task_id')
159 if task_id:
160 std_by_task[task_id].append(item)
161
162 print(f"Loaded {len(standard_items)} standard items.")
163 print(f"Found {len(std_by_task)} unique tasks.")
164
165 # Process each model file separately
166 if not args.model_files:
167 print("No model files provided.")
168 return
169
170 # Handle if args.model_files is a string (should be list due to nargs='+')
171 model_files_list = args.model_files if isinstance(args.model_files, list) else [args.model_files]
172
173 for model_file in model_files_list:
174 print(f"\nEvaluating file: {model_file}")
175 model_items = load_data([model_file])
176
177 if not model_items:
178 print(f"No items loaded from {model_file}")
179 continue
180
181 print(f"Loaded {len(model_items)} items from {model_file}")
182
183 results = evaluate_model(model_items, std_by_task)
184
185 print("-" * 30)
186 print(f"File: {model_file}")
187 print(f"Select Success (Element Match): {results['select_success_rate']:.2f}% ({results['select_correct_count']}/{results['select_total_count']})")
188 print(f"Task Success (All Steps Correct): {results['task_success_rate']:.2f}% ({results['task_success_count']}/{results['total_tasks']})")
189 if results['step_acc_total_count'] > 0:
190 print(f"Step Acc (Direct from Model Identifier): {results['step_acc_rate']:.2f}% ({results['step_acc_correct_count']}/{results['step_acc_total_count']})")
191 else:
192 print("Step Acc (Direct from Model Identifier): N/A (No identifier found in model output)")
193 print("-" * 30)
194
195if __name__ == "__main__":
196 main()

Callers 1

caculate_scores.pyFile · 0.70

Calls 3

load_dataFunction · 0.85
evaluate_modelFunction · 0.85
getMethod · 0.45

Tested by

no test coverage detected