A simple evaluator
| 354 | |
| 355 | |
| 356 | class Evaluator: |
| 357 | """A simple evaluator""" |
| 358 | |
| 359 | def __init__(self): |
| 360 | self.partial_scores = None |
| 361 | |
| 362 | def eval_hardness(self, sql): |
| 363 | count_comp1_ = count_component1(sql) |
| 364 | count_comp2_ = count_component2(sql) |
| 365 | count_others_ = count_others(sql) |
| 366 | |
| 367 | if count_comp1_ <= 1 and count_others_ == 0 and count_comp2_ == 0: |
| 368 | return "easy" |
| 369 | elif (count_others_ <= 2 and count_comp1_ <= 1 and count_comp2_ == 0) or \ |
| 370 | (count_comp1_ <= 2 and count_others_ < 2 and count_comp2_ == 0): |
| 371 | return "medium" |
| 372 | elif (count_others_ > 2 and count_comp1_ <= 2 and count_comp2_ == 0) or \ |
| 373 | (2 < count_comp1_ <= 3 and count_others_ <= 2 and count_comp2_ == 0) or \ |
| 374 | (count_comp1_ <= 1 and count_others_ == 0 and count_comp2_ <= 1): |
| 375 | return "hard" |
| 376 | else: |
| 377 | return "extra" |
| 378 | |
| 379 | def eval_exact_match(self, pred, label): |
| 380 | partial_scores = self.eval_partial_match(pred, label) |
| 381 | self.partial_scores = partial_scores |
| 382 | |
| 383 | for key, score in partial_scores.items(): |
| 384 | if score['f1'] != 1: |
| 385 | return 0 |
| 386 | |
| 387 | if len(label['from']['table_units']) > 0: |
| 388 | label_tables = sorted(label['from']['table_units']) |
| 389 | pred_tables = sorted(pred['from']['table_units']) |
| 390 | return label_tables == pred_tables |
| 391 | return 1 |
| 392 | |
| 393 | def eval_partial_match(self, pred, label): |
| 394 | res = {} |
| 395 | |
| 396 | label_total, pred_total, cnt, cnt_wo_agg = eval_sel(pred, label) |
| 397 | acc, rec, f1 = get_scores(cnt, pred_total, label_total) |
| 398 | res['select'] = {'acc': acc, 'rec': rec, 'f1': f1, 'label_total': label_total, 'pred_total': pred_total} |
| 399 | acc, rec, f1 = get_scores(cnt_wo_agg, pred_total, label_total) |
| 400 | res['select(no AGG)'] = {'acc': acc, 'rec': rec, 'f1': f1, 'label_total': label_total, 'pred_total': pred_total} |
| 401 | |
| 402 | label_total, pred_total, cnt, cnt_wo_agg = eval_where(pred, label) |
| 403 | acc, rec, f1 = get_scores(cnt, pred_total, label_total) |
| 404 | res['where'] = {'acc': acc, 'rec': rec, 'f1': f1, 'label_total': label_total, 'pred_total': pred_total} |
| 405 | acc, rec, f1 = get_scores(cnt_wo_agg, pred_total, label_total) |
| 406 | res['where(no OP)'] = {'acc': acc, 'rec': rec, 'f1': f1, 'label_total': label_total, 'pred_total': pred_total} |
| 407 | |
| 408 | label_total, pred_total, cnt = eval_group(pred, label) |
| 409 | acc, rec, f1 = get_scores(cnt, pred_total, label_total) |
| 410 | res['group(no Having)'] = {'acc': acc, 'rec': rec, 'f1': f1, 'label_total': label_total, 'pred_total': pred_total} |
| 411 | |
| 412 | label_total, pred_total, cnt = eval_having(pred, label) |
| 413 | acc, rec, f1 = get_scores(cnt, pred_total, label_total) |
no outgoing calls
no test coverage detected