Function tokenize

evaluation/test-suite-sql-eval/parse.py:14–16

(query: str)

def tokenize(query: str) -> List[Token]:
    tokens = list([Token(t.ttype, t.value) for t in sqlparse.parse(query)[0].flatten()])
    return tokens


def join_tokens(tokens: List[Token]) -> str: