| 143 | |
| 144 | |
| 145 | def tokenize(string): |
| 146 | string = str(string) |
| 147 | string = string.replace("\'", "\"") # ensures all string values wrapped by "" problem?? |
| 148 | quote_idxs = [idx for idx, char in enumerate(string) if char == '"'] |
| 149 | assert len(quote_idxs) % 2 == 0, "Unexpected quote" |
| 150 | |
| 151 | # keep string value as token |
| 152 | vals = {} |
| 153 | for i in range(len(quote_idxs) - 1, -1, -2): |
| 154 | qidx1 = quote_idxs[i - 1] |
| 155 | qidx2 = quote_idxs[i] |
| 156 | val = string[qidx1: qidx2 + 1] |
| 157 | key = "__val_{}_{}__".format(qidx1, qidx2) |
| 158 | string = string[:qidx1] + key + string[qidx2 + 1:] |
| 159 | vals[key] = val |
| 160 | |
| 161 | toks = [word.lower() for word in word_tokenize(string)] |
| 162 | # replace with string value token |
| 163 | for i in range(len(toks)): |
| 164 | if toks[i] in vals: |
| 165 | toks[i] = vals[toks[i]] |
| 166 | |
| 167 | # find if there exists !=, >=, <= |
| 168 | eq_idxs = [idx for idx, tok in enumerate(toks) if tok == "="] |
| 169 | eq_idxs.reverse() |
| 170 | prefix = ('!', '>', '<') |
| 171 | for eq_idx in eq_idxs: |
| 172 | pre_tok = toks[eq_idx - 1] |
| 173 | if pre_tok in prefix: |
| 174 | toks = toks[:eq_idx - 1] + [pre_tok + "="] + toks[eq_idx + 1:] |
| 175 | |
| 176 | return toks |
| 177 | |
| 178 | |
| 179 | def scan_alias(toks): |