(data_dir, prediction_dir, llm, split, n_tool, metric, tool_desc, tool_map, tool_output_type_map, tool_map_reverse, all_metric_dict, dependency_type, alignment = None)
| 253 | metric_json.write(json.dumps(all_metric_dict, indent=2)) |
| 254 | |
| 255 | def evaluate(data_dir, prediction_dir, llm, split, n_tool, metric, tool_desc, tool_map, tool_output_type_map, tool_map_reverse, all_metric_dict, dependency_type, alignment = None): |
| 256 | if f"{split}_{n_tool}" in all_metric_dict: |
| 257 | metric_dict = all_metric_dict[f"{split}_{n_tool}"] |
| 258 | else: |
| 259 | metric_dict = {} |
| 260 | all_metric_dict[f"{split}_{n_tool}"] = metric_dict |
| 261 | |
| 262 | label_rf = open(f"{data_dir}/data.json", "r") |
| 263 | |
| 264 | alignment_ids = None |
| 265 | if alignment is not None: |
| 266 | if alignment == "human": |
| 267 | label_rf = open(f"{data_dir}/data.json", "r") |
| 268 | logger.info(f"Alignment Mode: {alignment} ({len(label_rf.readlines())})") |
| 269 | else: |
| 270 | alignment_file = open(f"{data_dir}/alignment_ids.json", "r") |
| 271 | alignment_ids = json.load(alignment_file) |
| 272 | alignment_ids = list(itertools.chain(*alignment_ids[f"{alignment}_alignment_id"].values())) |
| 273 | logger.info(f"Alignment Mode: {alignment} ({len(alignment_ids)})") |
| 274 | |
| 275 | predcition_rf = open(f"{data_dir}/{prediction_dir}/{llm}.json", "r") |
| 276 | |
| 277 | predcitions = {} |
| 278 | labels = {} |
| 279 | label_rf = open(f"{data_dir}/data.json", "r") |
| 280 | for line in label_rf: |
| 281 | data = json.loads(line) |
| 282 | real_tool_num = len(data["task_nodes"]) |
| 283 | if alignment_ids is None or data["id"] in alignment_ids: |
| 284 | if split == "overall" or data["type"] == split: |
| 285 | if n_tool == "overall" or str(real_tool_num) == n_tool: |
| 286 | id = data["id"] |
| 287 | labels[id] = data |
| 288 | |
| 289 | for line in predcition_rf: |
| 290 | try: |
| 291 | data = json.loads(line) |
| 292 | except Exception as e: |
| 293 | print(e) |
| 294 | print(line) |
| 295 | exit() |
| 296 | id = data["id"] |
| 297 | predcitions[id] = data |
| 298 | |
| 299 | ids = set(labels.keys()).intersection(set(predcitions.keys())) |
| 300 | labels = {id: labels[id] for id in ids} |
| 301 | predcitions = {id: predcitions[id] for id in ids} |
| 302 | |
| 303 | predcition_task_steps = [] |
| 304 | label_task_steps = [] |
| 305 | predcition_names = [] |
| 306 | label_names = [] |
| 307 | label_graphs = [] |
| 308 | predcition_graphs = [] |
| 309 | label_links = [] |
| 310 | predcition_links = [] |
| 311 | label_task_arg_names = [] |
| 312 | predcition_task_arg_names = [] |
no test coverage detected