Args: Returns:
(string, cols, single_equal=False, math=True)
| 61 | return 0, 0, 0 |
| 62 | |
| 63 | def tokenize_NL2SQL(string, cols, single_equal=False, math=True): |
| 64 | """ |
| 65 | Args: |
| 66 | |
| 67 | Returns: |
| 68 | """ |
| 69 | |
| 70 | string = string.replace("\'", "\"").lower() |
| 71 | assert string.count('"') % 2 == 0, "Unexpected quote" |
| 72 | |
| 73 | re_cols = [i.lower() for i in cols] |
| 74 | |
| 75 | def _extract_value(string): |
| 76 | """extract values in sql""" |
| 77 | fields = string.split('"') |
| 78 | for idx, tok in enumerate(fields): |
| 79 | if idx % 2 == 1: |
| 80 | fields[idx] = '"%s"' % (tok) |
| 81 | return fields |
| 82 | |
| 83 | def _resplit(tmp_tokens, fn_split, fn_omit): |
| 84 | """resplit""" |
| 85 | new_tokens = [] |
| 86 | for token in tmp_tokens: |
| 87 | token = token.strip() |
| 88 | if fn_omit(token): |
| 89 | new_tokens.append(token) |
| 90 | elif re.match(r'\d\d\d\d-\d\d(-\d\d)?', token): |
| 91 | new_tokens.append('"%s"' % (token)) |
| 92 | else: |
| 93 | new_tokens.extend(fn_split(token)) |
| 94 | return new_tokens |
| 95 | |
| 96 | def _split_aggs(tmp_tokens): |
| 97 | """split aggs in select""" |
| 98 | new_toks = [] |
| 99 | for i, tok in enumerate(tmp_tokens): |
| 100 | if tok in ('from', 'where'): |
| 101 | new_toks.extend(tmp_tokens[i:]) |
| 102 | break |
| 103 | if not ((tok.endswith(')') or tok.endswith('),')) and len(tok) > 5): |
| 104 | new_toks.extend(tok.split(',')) |
| 105 | continue |
| 106 | |
| 107 | extra = '' |
| 108 | if tok.endswith(','): |
| 109 | extra = ',' |
| 110 | tok = tok[:-1] |
| 111 | |
| 112 | if tok[:4] in ('sum(', 'avg(', 'max(', 'min('): |
| 113 | new_toks.extend([tok[:3], '(', tok[4:-1], ')']) |
| 114 | elif tok[:6] == 'count(': |
| 115 | new_toks.extend(['count', '(', tok[6:-1], ')']) |
| 116 | else: |
| 117 | new_toks.append(tok) |
| 118 | |
| 119 | if extra: |
| 120 | new_toks.append(extra) |
no test coverage detected