()
| 237 | json.dump(answer_extraction_results, f, ensure_ascii=False, indent=2) |
| 238 | |
| 239 | def main(): |
| 240 | parser = argparse.ArgumentParser(description="QA Evaluation Tool - Automatically merge results and calculate EM and F1 metrics") |
| 241 | parser.add_argument("root_path", help="Root directory path containing results.json files") |
| 242 | parser.add_argument("--output", "-o", help="Output directory path for results", default=None) |
| 243 | |
| 244 | args = parser.parse_args() |
| 245 | |
| 246 | logger = get_logger(__name__) |
| 247 | |
| 248 | if not os.path.exists(args.root_path): |
| 249 | logger.error(f"Path does not exist: {args.root_path}") |
| 250 | return |
| 251 | |
| 252 | if args.output: |
| 253 | output_path = args.output |
| 254 | else: |
| 255 | output_path = os.path.join(args.root_path, "evaluation_results") |
| 256 | |
| 257 | os.makedirs(output_path, exist_ok=True) |
| 258 | |
| 259 | logger.info("Starting QA evaluation process...") |
| 260 | |
| 261 | logger.info("Step 1: Merging all results.json files") |
| 262 | merged_results = find_and_merge_results(args.root_path) |
| 263 | |
| 264 | if not merged_results: |
| 265 | logger.error("No valid results.json files found or files are empty") |
| 266 | return |
| 267 | |
| 268 | merged_path = os.path.join(output_path, "merged_results.json") |
| 269 | with open(merged_path, 'w', encoding='utf-8') as f: |
| 270 | json.dump(merged_results, f, ensure_ascii=False, indent=2) |
| 271 | logger.info(f"Merged results saved to: {merged_path}") |
| 272 | |
| 273 | logger.info("Step 2: Extracting standard answers and predicted answers") |
| 274 | gold_answers, predicted_answers = extract_answers_from_results(merged_results) |
| 275 | |
| 276 | if len(gold_answers) != len(predicted_answers): |
| 277 | logger.error(f"Number of standard answers ({len(gold_answers)}) does not match number of predicted answers ({len(predicted_answers)})") |
| 278 | return |
| 279 | |
| 280 | logger.info(f"Successfully extracted {len(gold_answers)} answer pairs") |
| 281 | |
| 282 | logger.info("Step 3: Calculating evaluation metrics") |
| 283 | |
| 284 | em_metric = QAExactMatch() |
| 285 | em_pooled, em_examples = em_metric.calculate_metric_scores(gold_answers, predicted_answers) |
| 286 | |
| 287 | f1_metric = QAF1Score() |
| 288 | f1_pooled, f1_examples = f1_metric.calculate_metric_scores(gold_answers, predicted_answers) |
| 289 | |
| 290 | pooled_results = {**em_pooled, **f1_pooled} |
| 291 | example_results = [] |
| 292 | for em_ex, f1_ex in zip(em_examples, f1_examples): |
| 293 | example_results.append({**em_ex, **f1_ex}) |
| 294 | |
| 295 | logger.info("Step 4: Saving evaluation results") |
| 296 | save_detailed_results(output_path, pooled_results, example_results, merged_results, predicted_answers) |
no test coverage detected