ParseBytesWithTokens is like ParseBytes but also returns the preprocessed token slice ([]models.TokenWithSpan) for callers that need both.
(input []byte)
| 117 | // ParseBytesWithTokens is like ParseBytes but also returns the preprocessed |
| 118 | // token slice ([]models.TokenWithSpan) for callers that need both. |
| 119 | func ParseBytesWithTokens(input []byte) (*ast.AST, []models.TokenWithSpan, error) { |
| 120 | tkz := tokenizer.GetTokenizer() |
| 121 | defer tokenizer.PutTokenizer(tkz) |
| 122 | |
| 123 | tokens, err := tkz.Tokenize(input) |
| 124 | if err != nil { |
| 125 | return nil, nil, fmt.Errorf("tokenization error: %w", err) |
| 126 | } |
| 127 | |
| 128 | if len(tokens) == 0 { |
| 129 | return nil, nil, goerrors.IncompleteStatementError(models.Location{}, "") |
| 130 | } |
| 131 | |
| 132 | p := GetParser() |
| 133 | defer PutParser(p) |
| 134 | |
| 135 | preprocessed := preprocessTokens(tokens) |
| 136 | astResult, parseErr := p.parseTokens(preprocessed) |
| 137 | if parseErr != nil { |
| 138 | return nil, nil, parseErr |
| 139 | } |
| 140 | |
| 141 | return astResult, preprocessed, nil |
| 142 | } |
| 143 | |
| 144 | // ValidateWithDialect checks whether the given SQL string is syntactically valid |
| 145 | // using the specified SQL dialect for keyword recognition. |