| 114 | |
| 115 | |
| 116 | def tokenize(string): |
| 117 | string = str(string) |
| 118 | string = string.replace("\'", "\"") # ensures all string values wrapped by "" problem?? |
| 119 | quote_idxs = [idx for idx, char in enumerate(string) if char == '"'] |
| 120 | assert len(quote_idxs) % 2 == 0, "Unexpected quote" |
| 121 | |
| 122 | # keep string value as token |
| 123 | vals = {} |
| 124 | for i in range(len(quote_idxs)-1, -1, -2): |
| 125 | qidx1 = quote_idxs[i-1] |
| 126 | qidx2 = quote_idxs[i] |
| 127 | val = string[qidx1: qidx2+1] |
| 128 | key = "__val_{}_{}__".format(qidx1, qidx2) |
| 129 | string = string[:qidx1] + key + string[qidx2+1:] |
| 130 | vals[key] = val |
| 131 | |
| 132 | toks = [word.lower() for word in word_tokenize(string)] |
| 133 | # replace with string value token |
| 134 | for i in range(len(toks)): |
| 135 | if toks[i] in vals: |
| 136 | toks[i] = vals[toks[i]] |
| 137 | |
| 138 | # find if there exists !=, >=, <= |
| 139 | eq_idxs = [idx for idx, tok in enumerate(toks) if tok == "="] |
| 140 | eq_idxs.reverse() |
| 141 | prefix = ('!', '>', '<') |
| 142 | for eq_idx in eq_idxs: |
| 143 | pre_tok = toks[eq_idx-1] |
| 144 | if pre_tok in prefix: |
| 145 | toks = toks[:eq_idx-1] + [pre_tok + "="] + toks[eq_idx+1: ] |
| 146 | |
| 147 | return toks |
| 148 | |
| 149 | |
| 150 | def scan_alias(toks): |