(scores, etype, include_turn_acc=True)
| 448 | |
| 449 | |
| 450 | def print_scores(scores, etype, include_turn_acc=True): |
| 451 | turns = ['turn 1', 'turn 2', 'turn 3', 'turn 4', 'turn > 4'] |
| 452 | levels = ['easy', 'medium', 'hard', 'extra', 'all'] |
| 453 | if include_turn_acc: |
| 454 | levels.append('joint_all') |
| 455 | partial_types = ['select', 'select(no AGG)', 'where', 'where(no OP)', 'group(no Having)', |
| 456 | 'group', 'order', 'and/or', 'IUEN', 'keywords'] |
| 457 | |
| 458 | print_formated_s("", levels, '{:20}') |
| 459 | counts = [scores[level]['count'] for level in levels] |
| 460 | print_formated_s("count", counts, '{:<20d}') |
| 461 | |
| 462 | if etype in ["all", "exec"]: |
| 463 | print('===================== EXECUTION ACCURACY =====================') |
| 464 | exec_scores = [scores[level]['exec'] for level in levels] |
| 465 | print_formated_s("execution", exec_scores, '{:<20.3f}') |
| 466 | |
| 467 | if etype in ["all", "match"]: |
| 468 | print('\n====================== EXACT MATCHING ACCURACY =====================') |
| 469 | exact_scores = [scores[level]['exact'] for level in levels] |
| 470 | print_formated_s("exact match", exact_scores, '{:<20.3f}') |
| 471 | print('\n---------------------PARTIAL MATCHING ACCURACY----------------------') |
| 472 | for type_ in partial_types: |
| 473 | this_scores = [scores[level]['partial'][type_]['acc'] for level in levels] |
| 474 | print_formated_s(type_, this_scores, '{:<20.3f}') |
| 475 | |
| 476 | print('---------------------- PARTIAL MATCHING RECALL ----------------------') |
| 477 | for type_ in partial_types: |
| 478 | this_scores = [scores[level]['partial'][type_]['rec'] for level in levels] |
| 479 | print_formated_s(type_, this_scores, '{:<20.3f}') |
| 480 | |
| 481 | print('---------------------- PARTIAL MATCHING F1 --------------------------') |
| 482 | for type_ in partial_types: |
| 483 | this_scores = [scores[level]['partial'][type_]['f1'] for level in levels] |
| 484 | print_formated_s(type_, this_scores, '{:<20.3f}') |
| 485 | |
| 486 | if include_turn_acc: |
| 487 | print() |
| 488 | print() |
| 489 | print_formated_s("", turns, '{:20}') |
| 490 | counts = [scores[turn]['count'] for turn in turns] |
| 491 | print_formated_s("count", counts, "{:<20d}") |
| 492 | |
| 493 | if etype in ["all", "exec"]: |
| 494 | print('===================== TURN EXECUTION ACCURACY =====================') |
| 495 | exec_scores = [scores[turn]['exec'] for turn in turns] |
| 496 | print_formated_s("execution", exec_scores, '{:<20.3f}') |
| 497 | |
| 498 | if etype in ["all", "match"]: |
| 499 | print('\n====================== TURN EXACT MATCHING ACCURACY =====================') |
| 500 | exact_scores = [scores[turn]['exact'] for turn in turns] |
| 501 | print_formated_s("exact match", exact_scores, '{:<20.3f}') |
| 502 | |
| 503 | |
| 504 | def evaluate(gold, predict, db_dir, etype, kmaps, plug_value, keep_distinct, progress_bar_for_each_datapoint, whether_print=False, print_file_name=""): |
no test coverage detected