Method parseTokens

pkg/sql/parser/parser.go:267–320

parseTokens is the core parsing routine. It takes a preprocessed []models.TokenWithSpan (already normalised by preprocessTokens) and returns the parsed AST.

(tokens []models.TokenWithSpan)

// []models.TokenWithSpan (already normalised by preprocessTokens) and returns
// the parsed AST.
func (p *Parser) parseTokens(tokens []models.TokenWithSpan) (*ast.AST, error) {
	p.tokens = tokens
	p.currentPos = 0
	if len(tokens) > 0 {
		p.currentToken = tokens[0]
	}

	// Get a pre-allocated AST from the pool
	result := ast.NewAST()

	// Pre-allocate statements slice based on a reasonable estimate
	estimatedStmts := 1 // Most SQL queries have just one statement
	if len(tokens) > 100 {
		estimatedStmts = 2 // For larger inputs, allocate more
	}
	result.Statements = make([]ast.Statement, 0, estimatedStmts)

	// Parse statements using Type (int) comparisons for speed
	for p.currentPos < len(tokens) && !p.isType(models.TokenTypeEOF) {
		// Skip semicolons between statements
		if p.isType(models.TokenTypeSemicolon) {
			if err := p.checkStrictEmptySemicolon(); err != nil {
				ast.ReleaseAST(result)
				return nil, err
			}
			p.advance()
			continue
		}

		stmt, err := p.parseStatement()
		if err != nil {
			// Clean up the AST on error
			ast.ReleaseAST(result)
			return nil, err
		}
		result.Statements = append(result.Statements, stmt)

		// Optionally consume semicolon after statement
		if p.isType(models.TokenTypeSemicolon) {
			p.advance()
		}
	}

	// Check if we got any statements
	if len(result.Statements) == 0 {
		ast.ReleaseAST(result)
		if err := p.checkStrictEmpty(); err != nil {
			return nil, err
		}
		return nil, goerrors.IncompleteStatementError(models.Location{}, "")
	}

	return result, nil
}

// ParseFromModelTokens parses tokenizer output ([]models.TokenWithSpan) directly into an AST.
//