Function Tokens

pkg/sql/parser/scanner.go:38–52

Tokens decomposes the input into lexical tokens.

(sql string)

// Tokens decomposes the input into lexical tokens.
func Tokens(sql string) (tokens []TokenString, ok bool) {
	s := makeSQLScanner(sql)
	for {
		var lval = &sqlSymType{}
		s.Scan(lval)
		if lval.ID() == lexbase.ERROR {
			return nil, false
		}
		if lval.ID() == 0 {
			break
		}
		tokens = append(tokens, TokenString{TokenID: lval.ID(), Str: lval.Str()})
	}
	return tokens, true
}

// TokenString is the unit value returned by Tokens.
type TokenString struct {