(table, gold, predict, single_equal=False, mode=None)
| 291 | return sql, sql_sels |
| 292 | |
| 293 | def evaluate_NL2SQL(table, gold, predict, single_equal=False, mode=None): |
| 294 | scores = {} |
| 295 | scores_novalue = {} |
| 296 | |
| 297 | # load db |
| 298 | with open(table) as ifs: |
| 299 | table_list = json.load(ifs) |
| 300 | table_dict = {} |
| 301 | for table in table_list: |
| 302 | table_dict[table['db_id']] = table |
| 303 | |
| 304 | # load qa |
| 305 | with open(gold, 'r', encoding='utf-8')as f1, open(predict, 'r', encoding='utf-8')as f2: |
| 306 | gold_list = [l.strip().split('\t') for l in f1 if len(l.strip()) > 0] |
| 307 | gold_dict = dict([(x[0], x[1:]) for x in gold_list]) |
| 308 | |
| 309 | pred_list = [l.strip().split('\t') for l in f2 if len(l.strip()) > 0] |
| 310 | pred_dict = dict([(x[0], x[1]) for x in pred_list if len(x) >= 2]) |
| 311 | |
| 312 | right = total = 0 |
| 313 | cnt_sel = 0 |
| 314 | cnt_cond = cnt_conn = 0 |
| 315 | |
| 316 | def compare_set(gold, pred): |
| 317 | _pred = copy.deepcopy(pred) |
| 318 | _gold = copy.deepcopy(gold) |
| 319 | |
| 320 | pred_total = len(_pred) |
| 321 | gold_total = len(_gold) |
| 322 | cnt = 0 |
| 323 | |
| 324 | for unit in _pred: |
| 325 | if unit in _gold: |
| 326 | cnt += 1 |
| 327 | _gold.remove(unit) |
| 328 | return cnt, pred_total, gold_total |
| 329 | |
| 330 | for qid, item in gold_dict.items(): |
| 331 | total += 1 |
| 332 | if qid not in pred_dict: |
| 333 | continue |
| 334 | sql_gold, db_id = ''.join(item[0:-1]), item[-1] |
| 335 | |
| 336 | db = table_dict[db_id] |
| 337 | cols = [i[1] for i in db["column_names"]] |
| 338 | |
| 339 | sql_pred = pred_dict[qid] |
| 340 | |
| 341 | try: |
| 342 | sql_gold = sql_gold.replace('==', '=') |
| 343 | sql_pred = sql_pred.replace('==', '=') |
| 344 | components_gold, sels_gold = query2sql(sql_gold, cols, single_equal=single_equal) |
| 345 | components_pred, sels_pred = query2sql(sql_pred, cols, single_equal=single_equal) |
| 346 | |
| 347 | cnt, pred_total, gold_total = compare_set(sels_gold, sels_pred) |
| 348 | score_sels, _, _ = get_scores(cnt, pred_total, gold_total) |
| 349 | cnt, pred_total, gold_total = compare_set(components_gold["conds"], components_pred["conds"]) |
| 350 | score_conds, _, _ = get_scores(cnt, pred_total, gold_total) |
no test coverage detected