(args: Tuple[Dict[str, Any], str, bool])
| 42 | # the input is a tuple of gold_dict, model prediction and whether to use cache |
| 43 | # and teh output is whether the model prediction passes the entire test suite |
| 44 | def judge(args: Tuple[Dict[str, Any], str, bool]) -> bool: |
| 45 | gold_dict, pred, use_cache = args |
| 46 | |
| 47 | testsuite_paths = gold_dict['testsuite'] |
| 48 | gold_query = gold_dict['query'] |
| 49 | order_matters = 'order by' in gold_query.lower() |
| 50 | db_path = gold_dict['db_path'] |
| 51 | |
| 52 | # if already computed sometime before |
| 53 | # and cache allowed, directly return the result |
| 54 | k = (db_path, gold_query, pred) |
| 55 | if use_cache and k in cache: |
| 56 | return cache[k] |
| 57 | |
| 58 | pass_all_testcase = True |
| 59 | for testcase_path in testsuite_paths: |
| 60 | |
| 61 | start = time.time() |
| 62 | flg, gold_result = exec_on_db(testcase_path, gold_query, timeout=GOLD_TIMEOUT) |
| 63 | duration = time.time() - start |
| 64 | timeout = ADDITIVE_OVERHEAD + MULTIPLICATIVE_OVERHEAD * duration |
| 65 | |
| 66 | if flg != 'result': |
| 67 | print('Warning: executing gold query results in an exception') |
| 68 | continue |
| 69 | flg, pred_result = exec_on_db(testcase_path, pred, timeout=int(timeout)) |
| 70 | if flg != 'result': |
| 71 | pass_all_testcase = False |
| 72 | break |
| 73 | if not result_eq(gold_result, pred_result, order_matters): |
| 74 | pass_all_testcase = False |
| 75 | break |
| 76 | |
| 77 | # save the results in the cache |
| 78 | if use_cache: |
| 79 | cache[k] = pass_all_testcase |
| 80 | return pass_all_testcase |
| 81 | |
| 82 | |
| 83 | # cache is a dictionary |
nothing calls this directly
no test coverage detected